├── .eslintignore ├── .github ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DISCUSSION_TEMPLATE │ ├── feature-request.yml │ └── help.yml ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ └── config.yml ├── SECURITY.md ├── pull_request_template.md ├── renovate.json └── workflows │ ├── cla.yml │ ├── codeql.yml │ ├── coverage.yml │ ├── dependency-review.yml │ ├── docs.yml │ ├── lock-threads.yml │ ├── publish.yml │ ├── stale.yml │ └── test.yml ├── .gitignore ├── .npmignore ├── .prettierignore ├── .prettierrc.js ├── LICENSE ├── README.md ├── coverage └── .keep ├── example └── index.ts ├── package.json ├── pnpm-lock.yaml ├── src ├── index.ts ├── lib │ ├── cloudflare.ts │ ├── consumer.ts │ └── fetch.ts ├── types.ts └── utils │ ├── emitter.ts │ ├── errors.ts │ ├── logger.ts │ └── validation.ts ├── test ├── fixtures │ ├── ackMessagesRequest.json │ ├── ackMessagesResponse.json │ ├── pullMessagesRequest.json │ └── pullMessagesResponse.json └── unit │ ├── lib │ ├── cloudflare.test.ts │ ├── consumer.test.ts │ └── fetch.test.ts │ └── utils │ ├── emitter.test.ts │ ├── errors.test.ts │ └── validation.test.ts ├── tsconfig.cjs.json ├── tsconfig.esm.json ├── tsconfig.json └── typedoc.json /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | dist -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # All changes should be reviewed by codeowners 2 | * @bbc/detl-engineers -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct (CoC) 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. 8 | 9 | ## Our Standards 10 | 11 | Examples of behaviour that contributes to a positive environment for our community include: 12 | 13 | - Respect different opinions, perspectives, and experiences 14 | - Giving and appreciating constructive feedback 15 | - Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience 16 | - Focusing on what is best for us as individuals and the overall community 17 | - Demonstrating kindness toward other people 18 | 19 | Examples of unacceptable behaviour include: 20 | 21 | - The use of sexualised language or imagery, and sexual attention or advances of any kind 22 | - Trolling, insulting or derogatory comments, and personal or political attacks 23 | - Public or private harassment 24 | - Publishing others’ private information, such as a physical or email address, without their explicit permission 25 | - Other conduct which could reasonably be considered inappropriate in a professional setting 26 | 27 | ## Enforcement Responsibilities 28 | 29 | Project maintainers are responsible for clarifying and enforcing our standards of acceptable behaviour and will take appropriate and fair corrective action in response to any behaviour that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. 32 | 33 | ## Scope 34 | 35 | This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. 36 | 37 | ## Enforcement 38 | 39 | Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported to the community leaders responsible for enforcement. All complaints will be reviewed and investigated promptly and fairly. 40 | 41 | [Project maintainers](https://github.com/bbc/cloudflare-queue-consumber/blob/main/.github/CODEOWNERS) are obligated to respect the privacy and security of the reporter of any incident. 42 | 43 | ## Enforcement Guidelines 44 | 45 | ### 1. Correction 46 | 47 | Community Impact: Use of inappropriate language or other behaviour deemed unprofessional or unwelcome in the community. 48 | 49 | Consequence: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behaviour was inappropriate. A public apology may be requested. 50 | 51 | ### 2. Warning 52 | 53 | Community Impact: A violation through a single incident or series of actions. 54 | 55 | Consequence: A warning with consequences for continued behaviour. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. 56 | 57 | ### 3. Temporary Ban 58 | 59 | Community Impact: A serious violation of community standards, including sustained inappropriate behaviour. 60 | 61 | Consequence: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. 62 | 63 | ### 4. Permanent Ban 64 | 65 | Community Impact: Demonstrating a pattern of violation of community standards, including sustained inappropriate behaviour, harassment of an individual, or aggression toward or disparagement of classes of individuals. 66 | 67 | Consequence: A permanent ban from any sort of public interaction within the community. 68 | 69 | ## Attribution 70 | 71 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][version] 72 | 73 | [homepage]: https://contributor-covenant.org 74 | [version]: https://www.contributor-covenant.org/version/2/0/code_of_conduct/ 75 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for your interest in contributing to the cloudflare-queue-consumer. 4 | 5 | - If you're unsure if a feature would make a good addition, you can always [create an issue](https://github.com/bbc/cloudflare-queue-consumer/issues/new) first. Raising an issue before creating a pull request is recommended. 6 | - We aim for 100% test coverage. Please write tests for any new functionality or changes. 7 | - Any API changes should be fully documented. 8 | - Make sure your code meets our linting standards. Run `npm run lint` to check your code. 9 | - Maintain the existing coding style. 10 | - Be mindful of others when making suggestions and/or code reviewing. 11 | 12 | ## Reporting Issues 13 | 14 | Before opening a new issue, first check that there is not already an [open issue or Pull Request](https://github.com/bbc/cloudflare-queue-consumer/issues?utf8=%E2%9C%93&q=is%3Aopen) that addresses it. 15 | 16 | If there is, make relevant comments and add your reaction. Use a reaction in place of a "+1" comment: 17 | 18 | - 👍 - upvote 19 | - 👎 - downvote 20 | 21 | If you cannot find an existing issue that describes your bug or feature, create a new issue using the guidelines below. 22 | 23 | 1. Pick an appropriate template for the type of issue [from here](https://github.com/bbc/cloudflare-queue-consumer/issues/choose) 24 | 2. Provide as much detail as possible 25 | 3. Follow your issue in the issue tracking workflow 26 | 27 | ## Contributing Code 28 | 29 | If you do not have push access to the repository, please [fork it](https://help.github.com/en/articles/fork-a-repo). You should then work on your own `main` branch. 30 | 31 | Otherwise, you may clone this repository and create a working branch with a _kebab-case_ name reflecting what you are working on (e.g. `fix-the-thing`). 32 | 33 | Follow the setup instructions in the [README](../README.md). 34 | 35 | Ensure all your code is thoroughly tested and that this testing is detailed in the pull request. 36 | 37 | ## Contributors Licence Agreement 38 | 39 | In order to accept contributions, we need all contributors grant Us a licence to the intellectual 40 | property rights in their Contributions. This Agreement (“Agreement”) is intended to protect your 41 | rights as a contributor, and to help ensure that the intellectual property contained 42 | within is available to the whole community, to use and build on. 43 | 44 | When you raise a pull request and you haven't previously signed a CLA, the bot will automatically 45 | ask you to do this. You must complete this step in order for your PR to be merged. 46 | 47 | ## Pull Request Process 48 | 49 | 1. Make sure you have opened an issue and it was approved by a project maintainer before working on a PR 50 | 2. Read and complete all relevant sections of the PR template 51 | 3. Wait for the PR get approved 52 | -------------------------------------------------------------------------------- /.github/DISCUSSION_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | body: 2 | - type: textarea 3 | attributes: 4 | label: Background 5 | description: Why do you think this feature is needed? Are there current alternatives? 6 | validations: 7 | required: true 8 | - type: textarea 9 | attributes: 10 | label: Objectives 11 | description: What should this feature request aim to address? 12 | value: | 13 | 1. 14 | 2. 15 | 3. 16 | validations: 17 | required: true -------------------------------------------------------------------------------- /.github/DISCUSSION_TEMPLATE/help.yml: -------------------------------------------------------------------------------- 1 | body: 2 | - type: textarea 3 | attributes: 4 | label: Summary 5 | description: What do you need help with? 6 | validations: 7 | required: true 8 | - type: input 9 | attributes: 10 | label: Example 11 | description: A link to a minimal reproduction is helpful for debugging! 12 | validations: 13 | required: false 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: 'Bug report' 2 | title: "[Bug]: " 3 | labels: ["bug", "triage"] 4 | description: Report a reproducible bug or regression 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thank you for reporting an issue! 10 | 11 | This issue tracker is for reporting reproducible bugs or regression's found in this package, if you have a question or feature request, please report it within the [Discussions tab](https://github.com/bbc/cloudflare-queue-consumer/discussions) instead. 12 | 13 | Before submitting a new bug/issue, please check the links below to see if there is a solution or question posted there already: 14 | 15 | - [Discussions](https://github.com/bbc/cloudflare-queue-consumer/discussions) 16 | - [Open Issues](https://github.com/bbc/cloudflare-queue-consumer/issues?q=is%3Aopen+is%3Aissue) 17 | - [Closed Issues](https://github.com/bbc/cloudflare-queue-consumer/issues?q=is%3Aissue+is%3Aclosed) 18 | 19 | The more information you fill in, the better the community can help you. 20 | - type: textarea 21 | id: description 22 | attributes: 23 | label: Describe the bug 24 | description: Provide a clear and concise description of what the bug is. 25 | validations: 26 | required: true 27 | - type: input 28 | id: link 29 | attributes: 30 | label: Your minimal, reproducible example 31 | description: | 32 | Please add a link to a minimal reproduction. 33 | Note: 34 | - Please keep your example as simple and reproduceable as possible, try leaving out dependencies that are not required for reproduction. 35 | - To create a shareable code example for web, you can use Stackblitz (https://stackblitz.com/edit/cloudflare-queue-consumer-starter). 36 | - Please make sure the example is complete and runnable - e.g. avoid localhost URLs. 37 | placeholder: | 38 | e.g. Code Sandbox, Stackblitz, Expo Snack or TypeScript playground 39 | validations: 40 | required: true 41 | - type: textarea 42 | id: steps 43 | attributes: 44 | label: Steps to reproduce 45 | description: Describe the steps we have to take to reproduce the behavior. 46 | placeholder: | 47 | 1. Go to '...' 48 | 2. Click on '....' 49 | 3. Scroll down to '....' 50 | 4. See error 51 | validations: 52 | required: true 53 | - type: textarea 54 | id: expected 55 | attributes: 56 | label: Expected behavior 57 | description: Provide a clear and concise description of what you expected to happen. 58 | placeholder: | 59 | As a user, I expected ___ behavior but i am seeing ___ 60 | validations: 61 | required: true 62 | - type: dropdown 63 | attributes: 64 | label: How often does this bug happen? 65 | description: | 66 | Following the repro steps above, how easily are you able to reproduce this bug? 67 | options: 68 | - Every time 69 | - Often 70 | - Sometimes 71 | - Only once 72 | - type: textarea 73 | id: screenshots_or_videos 74 | attributes: 75 | label: Screenshots or Videos 76 | description: | 77 | If applicable, add screenshots or a video to help explain your problem. 78 | For more information on the supported file image/file types and the file size limits, please refer 79 | to the following link: https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/attaching-files 80 | placeholder: | 81 | You can drag your video or image files inside of this editor ↓ 82 | - type: textarea 83 | id: platform 84 | attributes: 85 | label: Platform 86 | description: | 87 | Please let us know which Operating System and Node version you were using when the issue occurred. 88 | placeholder: | 89 | - OS: [e.g. macOS, Windows, Linux, iOS, Android] 90 | - Node Version: [e.g. 16.6.0] 91 | validations: 92 | required: true 93 | - type: input 94 | id: package-version 95 | attributes: 96 | label: Package version 97 | description: | 98 | Please let us know the exact version of the package you were using when the issue occurred. Please don't just put in "latest", as this is subject to change. 99 | placeholder: | 100 | e.g. v6.0.0 101 | validations: 102 | required: true 103 | - type: input 104 | id: ts-version 105 | attributes: 106 | label: AWS SDK version 107 | description: | 108 | Please include what version of the AWS SDK you are using 109 | placeholder: | 110 | e.g. v3.226.0 111 | - type: textarea 112 | id: additional 113 | attributes: 114 | label: Additional context 115 | description: Add any other context about the problem here. 116 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Feature Requests & Questions 4 | url: https://github.com/bbc/cloudflare-queue-consumer/discussions 5 | about: Please ask and answer questions here. -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Our full security policy and vulnerability reporting procedure is documented on [this external website](https://www.bbc.com/backstage/security-disclosure-policy/#reportingavulnerability). 6 | 7 | Please note that this is a general BBC process. Communication will not be direct with the team responsible for this repo. 8 | 9 | If you would like to, you can also open an issue in this repo regarding your disclosure, but please never share any details of the vulnerability in the GitHub issue. 10 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Resolves #NUMBER 2 | 3 | **Description:** 4 | _A very high-level summary of easily-reproducible changes that can be understood by non-devs._ 5 | 6 | **Type of change:** 7 | 8 | - [ ] Bug fix (non-breaking change which fixes an issue) 9 | - [ ] New feature (non-breaking change which adds functionality) 10 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 11 | 12 | **Why is this change required?:** 13 | _A simple explanation of what the problem is and how this PR solves it_ 14 | 15 | **Code changes:** 16 | 17 | - _A bullet point list of key code changes that have been made._ 18 | - _When describing code changes, try to communicate **how** and **why** you implemented something a specific way, not just **what** has changed._ 19 | 20 | --- 21 | 22 | **Checklist:** 23 | 24 | - [ ] My code follows the code style of this project. 25 | - [ ] My change requires a change to the documentation. 26 | - [ ] I have updated the documentation accordingly. 27 | - [ ] I have read the **CONTRIBUTING** document. 28 | - [ ] I have added tests to cover my changes. 29 | - [ ] All new and existing tests passed. 30 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base", 5 | ":combinePatchMinorReleases", 6 | ":dependencyDashboard", 7 | ":enableVulnerabilityAlertsWithLabel(vulnerability)", 8 | ":prConcurrentLimit10", 9 | ":prHourlyLimit4", 10 | ":prNotPending", 11 | ":preserveSemverRanges", 12 | ":rebaseStalePrs", 13 | ":semanticCommits", 14 | ":semanticPrefixFixDepsChoreOthers", 15 | ":label(dependencies)", 16 | ":timezone(Europe/London)", 17 | "docker:enableMajor", 18 | "docker:pinDigests", 19 | "group:postcss", 20 | "group:linters", 21 | "group:monorepos", 22 | "npm:unpublishSafe", 23 | "regexManagers:dockerfileVersions", 24 | "replacements:all" 25 | ], 26 | "rangeStrategy": "update-lockfile", 27 | "supportPolicy": ["lts_latest"], 28 | "dependencyDashboardAutoclose": true, 29 | "platformAutomerge": true, 30 | "vulnerabilityAlerts": { 31 | "labels": ["security"], 32 | "automerge": true 33 | }, 34 | "stabilityDays": 3, 35 | "packageRules": [ 36 | { 37 | "matchUpdateTypes": ["major"], 38 | "labels": ["dependencies"] 39 | }, 40 | { 41 | "matchUpdateTypes": ["minor", "patch", "pin", "digest"], 42 | "labels": ["dependencies"], 43 | "automerge": true 44 | }, 45 | { 46 | "groupName": "docker", 47 | "matchDatasources": ["docker"], 48 | "labels": ["dependencies"], 49 | "automerge": true 50 | } 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /.github/workflows/cla.yml: -------------------------------------------------------------------------------- 1 | name: "CLA Check" 2 | on: 3 | issue_comment: 4 | types: [created] 5 | pull_request_target: 6 | types: [opened,closed,synchronize] 7 | 8 | permissions: 9 | contents: write 10 | issues: write 11 | pull-requests: write 12 | 13 | jobs: 14 | CLAAssistant: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: "CLA Check" 18 | if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' 19 | uses: contributor-assistant/github-action@v2.3.2 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} 23 | with: 24 | path-to-signatures: 'cloudflare-queue-consumer/v1/cla.json' 25 | remote-organization-name: 'bbc' 26 | remote-repository-name: 'cla-signatures' 27 | path-to-document: 'https://bbc.github.io/cla-signatures/cla/v1/cla.html' 28 | branch: 'main' 29 | allowlist: renovate* 30 | custom-allsigned-prcomment: '**CLA CHECK** All Contributors have signed the CLA' 31 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main" ] 20 | schedule: 21 | - cron: '43 21 * * 2' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'javascript' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Use only 'java' to analyze code written in Java, Kotlin or both 38 | # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both 39 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v4 44 | 45 | # Initializes the CodeQL tools for scanning. 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@v3 48 | with: 49 | languages: ${{ matrix.language }} 50 | # If you wish to specify custom queries, you can do so here or in a config file. 51 | # By default, queries listed here will override any specified in a config file. 52 | # Prefix the list here with "+" to use these queries and those in the config file. 53 | 54 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 55 | # queries: security-extended,security-and-quality 56 | 57 | 58 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 59 | # If this step fails, then you should remove it and run the build manually (see below) 60 | - name: Autobuild 61 | uses: github/codeql-action/autobuild@v3 62 | 63 | # ℹ️ Command-line programs to run using the OS shell. 64 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 65 | 66 | # If the Autobuild fails above, remove it and uncomment the following three lines. 67 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 68 | 69 | # - run: | 70 | # echo "Run, Build Application using script" 71 | # ./location_of_script_within_repo/buildscript.sh 72 | 73 | - name: Perform CodeQL Analysis 74 | uses: github/codeql-action/analyze@v3 75 | with: 76 | category: "/language:${{matrix.language}}" 77 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Report Coverage 2 | on: 3 | pull_request: 4 | branches: 5 | - 'main' 6 | push: 7 | branches: 8 | - main 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | node-version: [22.x] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - uses: pnpm/action-setup@v3 23 | with: 24 | version: 9 25 | 26 | - name: Use Node.js ${{ matrix.node-version }} 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: ${{ matrix.node-version }} 30 | cache: 'pnpm' 31 | 32 | - name: Install Node Modules 33 | run: pnpm install --frozen-lockfile 34 | 35 | - name: Run Coverage 36 | run: pnpm run lcov 37 | 38 | - name: Report Coverage 39 | uses: paambaati/codeclimate-action@v5.0.0 40 | env: 41 | CC_TEST_REPORTER_ID: 03d3b753352184e77df4c96aaef330a330eca28118a2a6da010b305f12216f45 42 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | name: 'Dependency Review' 2 | on: [pull_request] 3 | 4 | permissions: 5 | contents: read 6 | 7 | jobs: 8 | dependency-review: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: 'Checkout Repository' 12 | uses: actions/checkout@v4 13 | - name: 'Dependency Review' 14 | uses: actions/dependency-review-action@v4 15 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | # Simple workflow for deploying static content to GitHub Pages 2 | name: Deploy static content to Pages 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow one concurrent deployment 19 | concurrency: 20 | group: "pages" 21 | cancel-in-progress: true 22 | 23 | jobs: 24 | # Single deploy job since we're just deploying 25 | deploy: 26 | environment: 27 | name: github-pages 28 | url: ${{ steps.deployment.outputs.page_url }} 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v4 33 | 34 | - uses: pnpm/action-setup@v3 35 | with: 36 | version: 9 37 | 38 | - name: Setup Node.js 39 | uses: actions/setup-node@v4 40 | with: 41 | node-version: 18.x 42 | cache: 'pnpm' 43 | 44 | - name: Install Node Modules 45 | run: pnpm install --frozen-lockfile 46 | 47 | - name: Build Docs 48 | run: pnpm run generate-docs 49 | 50 | - name: Setup Pages 51 | uses: actions/configure-pages@v4 52 | 53 | - name: Upload artifact 54 | uses: actions/upload-pages-artifact@v3 55 | with: 56 | path: './public' 57 | 58 | - name: Deploy to GitHub Pages 59 | id: deployment 60 | uses: actions/deploy-pages@v4 61 | -------------------------------------------------------------------------------- /.github/workflows/lock-threads.yml: -------------------------------------------------------------------------------- 1 | name: "Lock Threads" 2 | 3 | on: 4 | schedule: 5 | - cron: "0 * * * *" # Once a day, at midnight UTC 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | concurrency: 13 | group: lock 14 | 15 | jobs: 16 | action: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: dessant/lock-threads@v5 20 | with: 21 | github-token: ${{ secrets.GITHUB_TOKEN }} 22 | issue-inactive-days: "30" # Lock issues after 30 days of being closed 23 | pr-inactive-days: "5" # Lock closed PRs after 5 days. This ensures that issues that stem from a PR are opened as issues, rather than comments on the recently merged PR. 24 | add-issue-labels: "outdated" 25 | exclude-issue-created-before: "2023-01-01" 26 | issue-comment: > 27 | This issue has been closed for more than 30 days. If this issue is still occuring, please open a new issue with more recent context. 28 | pr-comment: > 29 | This pull request has already been merged/closed. If you experience issues related to these changes, please open a new issue referencing this pull request. 30 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Package to npmjs 2 | on: 3 | workflow_dispatch: 4 | jobs: 5 | build: 6 | runs-on: ubuntu-latest 7 | permissions: 8 | contents: write 9 | issues: write 10 | pull-requests: write 11 | id-token: write 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - uses: pnpm/action-setup@v3 16 | with: 17 | version: 9 18 | 19 | - uses: actions/setup-node@v4 20 | with: 21 | node-version: '20.x' 22 | cache: 'pnpm' 23 | registry-url: 'https://registry.npmjs.org' 24 | 25 | - run: pnpm install --frozen-lockfile 26 | 27 | - name: Release 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 31 | run: npx semantic-release 32 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: 'Close stale issues and PRs' 2 | on: 3 | schedule: 4 | - cron: '30 1 * * *' 5 | 6 | jobs: 7 | stale: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/stale@v9 11 | with: 12 | stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' 13 | stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' 14 | close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' 15 | close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.' 16 | days-before-issue-stale: 30 17 | days-before-pr-stale: 45 18 | days-before-issue-close: 5 19 | days-before-pr-close: 10 20 | operations-per-run: 90 21 | exempt-issue-labels: keep 22 | exempt-pr-labels: keep 23 | exempt-all-assignees: true 24 | exempt-all-milestones: true 25 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | on: 3 | pull_request: 4 | branches: 5 | - 'main' 6 | push: 7 | branches: 8 | - main 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | node-version: [22.x] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - uses: pnpm/action-setup@v3 23 | with: 24 | version: 9 25 | 26 | - name: Use Node.js ${{ matrix.node-version }} 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: ${{ matrix.node-version }} 30 | cache: 'pnpm' 31 | 32 | - name: Install Node Modules 33 | run: pnpm install --frozen-lockfile 34 | 35 | - name: Run Tests and Linting 36 | run: pnpm run test 37 | 38 | - uses: actions/upload-artifact@v4 39 | with: 40 | name: test-reports-${{ matrix.node-version }} 41 | path: test/reports/ 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # Snowpack dependency directory (https://snowpack.dev/) 45 | web_modules/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Optional stylelint cache 57 | .stylelintcache 58 | 59 | # Microbundle cache 60 | .rpt2_cache/ 61 | .rts2_cache_cjs/ 62 | .rts2_cache_es/ 63 | .rts2_cache_umd/ 64 | 65 | # Optional REPL history 66 | .node_repl_history 67 | 68 | # Output of 'npm pack' 69 | *.tgz 70 | 71 | # Yarn Integrity file 72 | .yarn-integrity 73 | 74 | # dotenv environment variable files 75 | .env 76 | .env.development.local 77 | .env.test.local 78 | .env.production.local 79 | .env.local 80 | 81 | # parcel-bundler cache (https://parceljs.org/) 82 | .cache 83 | .parcel-cache 84 | 85 | # Next.js build output 86 | .next 87 | out 88 | 89 | # Nuxt.js build / generate output 90 | .nuxt 91 | dist 92 | 93 | # Gatsby files 94 | .cache/ 95 | # Comment in the public line in if your project uses Gatsby and not Next.js 96 | # https://nextjs.org/blog/next-9-1#public-directory-support 97 | # public 98 | 99 | # vuepress build output 100 | .vuepress/dist 101 | 102 | # vuepress v2.x temp and cache directory 103 | .temp 104 | .cache 105 | 106 | # Docusaurus cache and generated files 107 | .docusaurus 108 | 109 | # Serverless directories 110 | .serverless/ 111 | 112 | # FuseBox cache 113 | .fusebox/ 114 | 115 | # DynamoDB Local files 116 | .dynamodb/ 117 | 118 | # TernJS port file 119 | .tern-port 120 | 121 | # Stores VSCode versions used for testing VSCode extensions 122 | .vscode-test 123 | 124 | # yarn v2 125 | .yarn/cache 126 | .yarn/unplugged 127 | .yarn/build-state.yml 128 | .yarn/install-state.gz 129 | .pnp.* 130 | 131 | public 132 | 133 | coverage/* 134 | !coverage/.keep -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | *.log 4 | .vscode 5 | example 6 | test 7 | public 8 | .nyc_output 9 | scripts 10 | .* 11 | renovate.json 12 | typedoc.json -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | dist 4 | public 5 | test/reports -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | /** @type {import("prettier").Config} */ 2 | 3 | export const config = { 4 | singleQuote: true, 5 | arrowParens: "always", 6 | trailingComma: "none", 7 | }; 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2024-present British Broadcasting Corporation 2 | 3 | All rights reserved 4 | 5 | (http://www.bbc.co.uk), sqs-consumer and cloudflare-queue-consumer Contributors 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Please contact us for an alternative licence 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | See the License for the specific language governing permissions and 19 | limitations under the License. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cloudflare-queue-consumer 2 | 3 | [![NPM downloads](https://img.shields.io/npm/dm/@bbc/cloudflare-queue-consumer.svg?style=flat)](https://npmjs.org/package/@bbc/cloudflare-queue-consumer) 4 | [![Build Status](https://github.com/bbc/cloudflare-queue-consumer/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/bbc/cloudflare-queue-consumer/actions/workflows/test.yml) 5 | [![Maintainability](https://api.codeclimate.com/v1/badges/a0fcd77021e4f54ffdd4/maintainability)](https://codeclimate.com/github/bbc/cloudflare-queue-consumer/maintainability) 6 | [![Test Coverage](https://api.codeclimate.com/v1/badges/a0fcd77021e4f54ffdd4/test_coverage)](https://codeclimate.com/github/bbc/cloudflare-queue-consumer/test_coverage) 7 | 8 | Build [Cloudflare Queues](https://developers.cloudflare.com/queues/) applications without the boilerplate. Just define an async function that handles the message processing. 9 | 10 | Based on [sqs-consumer](https://github.com/bbc/sqs-consumer). 11 | 12 | > **Note:** This package is still in development and should be used with caution. 13 | 14 | ## Installation 15 | 16 | To install this package, simply enter the following command into your terminal (or the variant of whatever package manager you are using): 17 | 18 | ```bash 19 | npm install @bbc/cloudflare-queue-consumer 20 | ``` 21 | 22 | ## Documentation 23 | 24 | Visit [https://bbc.github.io/cloudflare-queue-consumer/](https://bbc.github.io/cloudflare-queue-consumer/) for the full API documentation. 25 | 26 | ## Usage 27 | 28 | ```js 29 | import { Consumer } from "@bbc/cloudflare-queue-consumer"; 30 | 31 | const consumer = new Consumer({ 32 | accountId: process.env.ACCOUNT_ID, // Your Cloudflare account ID 33 | queueId: process.env.QUEUE_ID, // The Queue ID that you want to use. 34 | handleMessage: async (message) => { 35 | // Your message handling code... 36 | }, 37 | }); 38 | 39 | consumer.on("error", (err) => { 40 | console.error(err.message); 41 | }); 42 | 43 | consumer.on("processing_error", (err) => { 44 | console.error(err.message); 45 | }); 46 | 47 | consumer.start(); 48 | ``` 49 | 50 | Some implementation notes: 51 | 52 | - [Pull consumers](https://developers.cloudflare.com/queues/reference/pull-consumers/) are designed to use a "short polling" approach, this means that the API from Cloudflare will respond immediately with any messages that are available, or an empty response if there are no messages available, this is different from SQS which will wait an amount of time before responding with an empty response. 53 | - `handleMessage` will send one message to the handler at a time, if you would prefer to receive multiple messages at once, use the `handleMessageBatch` method instead. 54 | - It is important to await any processing that you are doing to ensure that the next action only happens after your processing has completed. 55 | - By default, messages that are sent to the functions will be considered as processed if they return without an error. 56 | - To acknowledge, you can return a promise that resolves the message or messages that you want to acknowledge. 57 | - Returning an empty object or an empty array will be considered an acknowledgment of no message(s). If you would like to change this behaviour, you can set the `alwaysAcknowledge` option to `true`. 58 | - By default, if an object or an array is not returned, all messages will be acknowledged. 59 | - Any message that errors will not be retried until the end of the visibility timeout, if you would like to trigger an immediate retry, you can set the `retryMessagesOnError` option to `true`. 60 | - You can set a delay for this retry with the `retryMessageDelay` option. 61 | 62 | ### Credentials 63 | 64 | In order to authenticate with the Cloudflare API, you will need to create an API token with read and write access to Cloudflare Queues, more information can be found [here](https://developers.cloudflare.com/queues/reference/pull-consumers/#create-api-tokens). 65 | 66 | Copy that token and set it as the value for an environment variable named `QUEUES_API_TOKEN`. 67 | 68 | ### Example project 69 | 70 | You'll also find an example project in the folder `./example`, set the variables `ACCOUNT_ID` and `QUEUE_ID` and then run this with the command `pnpm dev`. 71 | 72 | ## API 73 | 74 | ### `Consumer.create(options)` 75 | 76 | Creates a new SQS consumer using the [defined options](https://bbc.github.io/cloudflare-queue-consumer/interfaces/ConsumerOptions.html). 77 | 78 | ### `consumer.start()` 79 | 80 | Start polling the queue for messages. 81 | 82 | ### `consumer.stop(options)` 83 | 84 | Stop polling the queue for messages. [You can find the options definition here](https://bbc.github.io/cloudflare-queue-consumer/interfaces/StopOptions.html). 85 | 86 | By default, the value of `abort` is set to `false` which means pre existing requests to Cloudflare will still be made until they have concluded. If you would like to abort these requests instead, pass the abort value as `true`, like so: 87 | 88 | `consumer.stop({ abort: true })` 89 | 90 | ### `consumer.status` 91 | 92 | Returns the current status of the consumer. 93 | 94 | - `isRunning` - `true` if the consumer has been started and not stopped, `false` if was not started or if it was stopped. 95 | - `isPolling` - `true` if the consumer is actively polling, `false` if it is not. 96 | 97 | ### `consumer.updateOption(option, value)` 98 | 99 | Updates the provided option with the provided value. 100 | 101 | Please note that any update of the option `pollingWaitTimeMs` will take effect only on next polling cycle. 102 | 103 | You can [find out more about this here](https://bbc.github.io/cloudflare-queue-consumer/classes/Consumer.html#updateOption). 104 | 105 | ### Events 106 | 107 | Each consumer is an [`EventEmitter`](https://nodejs.org/api/events.html) and [emits these events](https://bbc.github.io/cloudflare-queue-consumer/interfaces/Events.html). 108 | 109 | ## Contributing 110 | 111 | We welcome and appreciate contributions for anyone who would like to take the time to fix a bug or implement a new feature. 112 | 113 | But before you get started, [please read the contributing guidelines](https://github.com/bbc/cloudflare-queue-consumer/blob/main/.github/CONTRIBUTING.md) and [code of conduct](https://github.com/bbc/cloudflare-queue-consumer/blob/main/.github/CODE_OF_CONDUCT.md). 114 | 115 | ## License 116 | 117 | Cloudflare Queue Consumer is distributed under the Apache License, Version 2.0, see [LICENSE](https://github.com/bbc/cloudflare-queue-consumer/blob/main/LICENSE) for more information. 118 | -------------------------------------------------------------------------------- /coverage/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bbc/cloudflare-queue-consumer/db5aaec42540218a11af7300d2c95d6e18d85226/coverage/.keep -------------------------------------------------------------------------------- /example/index.ts: -------------------------------------------------------------------------------- 1 | import { Consumer } from "../dist/esm/index.js"; 2 | 3 | function start() { 4 | const consumer = new Consumer({ 5 | accountId: process.env.ACCOUNT_ID, 6 | queueId: process.env.QUEUE_ID, 7 | handleMessage: async (message) => { 8 | // eslint-disable-next-line no-console 9 | console.log(message); 10 | 11 | return message; 12 | }, 13 | }); 14 | 15 | consumer.start(); 16 | } 17 | 18 | start(); 19 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@bbc/cloudflare-queue-consumer", 3 | "version": "0.0.5", 4 | "description": "Build Cloudflare Queue applications without the boilerplate", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/bbc/cloudflare-queue-consumer.git" 8 | }, 9 | "bugs": { 10 | "url": "https://github.com/bbc/cloudflare-queue-consumer/issues" 11 | }, 12 | "homepage": "https://bbc.github.io/cloudflare-queue-consumer/", 13 | "keywords": [ 14 | "cloudflare", 15 | "queue", 16 | "consumer" 17 | ], 18 | "license": "Apache-2.0", 19 | "publishConfig": { 20 | "provenance": false 21 | }, 22 | "type": "module", 23 | "exports": { 24 | ".": { 25 | "types": "./dist/types/index.d.ts", 26 | "require": "./dist/cjs/index.js", 27 | "import": "./dist/esm/index.js", 28 | "default": "./dist/esm/index.js" 29 | } 30 | }, 31 | "engines": { 32 | "node": ">=18.0.0" 33 | }, 34 | "scripts": { 35 | "clean": "rm -fr dist/*", 36 | "compile": "tsc -b ./tsconfig.cjs.json ./tsconfig.esm.json", 37 | "build": "pnpm run clean && pnpm run compile", 38 | "watch": "tsc --watch", 39 | "prepublishOnly": "pnpm run build", 40 | "lint": "eslint . --ext .ts", 41 | "lint:fix": "eslint . --fix", 42 | "format": "prettier --log-level warn --write \"**/*.{js,json,jsx,md,ts,tsx,html}\"", 43 | "format:check": "prettier --check \"**/*.{js,json,jsx,md,ts,tsx,html}\"", 44 | "test:unit": "node --import tsx --test ./test/unit/**/*.test.ts", 45 | "test:unit:watch": "node --import tsx --test --watch ./test/unit/*.test.ts", 46 | "test": "pnpm run test:unit && pnpm run lint && pnpm run format:check", 47 | "lcov": "node --import tsx --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=coverage/lcov.info ./test/unit/*.test.ts", 48 | "generate-docs": "typedoc", 49 | "dev": "DEBUG=cloudflare-queue-consumer tsx ./example/index.ts" 50 | }, 51 | "devDependencies": { 52 | "@types/node": "22.7.4", 53 | "@types/sinon": "17.0.3", 54 | "chai": "5.1.1", 55 | "chai-nock": "1.3.0", 56 | "eslint": "8.57.0", 57 | "eslint-config-iplayer": "9.2.0", 58 | "eslint-config-prettier": "9.1.0", 59 | "nock": "14.0.0-beta.5", 60 | "p-event": "6.0.1", 61 | "prettier": "3.3.3", 62 | "sinon": "17.0.1", 63 | "tsx": "4.19.1", 64 | "typedoc": "0.26.7", 65 | "typescript": "5.6.2" 66 | }, 67 | "dependencies": { 68 | "debug": "^4.3.7" 69 | }, 70 | "eslintConfig": { 71 | "extends": [ 72 | "iplayer/base", 73 | "iplayer/ts", 74 | "prettier" 75 | ], 76 | "parserOptions": { 77 | "sourceType": "module" 78 | }, 79 | "rules": { 80 | "@typescript-eslint/naming-convention": [ 81 | "error", 82 | { 83 | "selector": "variable", 84 | "format": [ 85 | "camelCase", 86 | "UPPER_CASE", 87 | "PascalCase" 88 | ], 89 | "leadingUnderscore": "allow" 90 | } 91 | ] 92 | } 93 | }, 94 | "packageManager": "pnpm@9.10.0+sha512.73a29afa36a0d092ece5271de5177ecbf8318d454ecd701343131b8ebc0c1a91c487da46ab77c8e596d6acf1461e3594ced4becedf8921b074fbd8653ed7051c" 95 | } 96 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { Consumer } from "./lib/consumer.js"; 2 | export * from "./types.js"; 3 | -------------------------------------------------------------------------------- /src/lib/cloudflare.ts: -------------------------------------------------------------------------------- 1 | import { ProviderError } from "../utils/errors.js"; 2 | import { throwErrorIfResponseNotOk } from "./fetch.js"; 3 | 4 | const CLOUDFLARE_HOST = "https://api.cloudflare.com/client/v4"; 5 | const MAX_RETRIES = 5; 6 | 7 | export function getCredentials() { 8 | const QUEUES_API_TOKEN = process.env.QUEUES_API_TOKEN; 9 | 10 | if (!QUEUES_API_TOKEN) { 11 | throw new Error( 12 | "Missing Cloudflare credentials, please set a QUEUES_API_TOKEN in the environment variables.", 13 | ); 14 | } 15 | 16 | return { 17 | QUEUES_API_TOKEN, 18 | }; 19 | } 20 | 21 | function calculateDelay(attempt: number): number { 22 | return Math.pow(2, attempt) * 100 + Math.random() * 100; 23 | } 24 | 25 | function shouldRetry( 26 | error: unknown, 27 | attempt: number, 28 | maxRetries: number, 29 | ): boolean { 30 | return ( 31 | error instanceof ProviderError && 32 | error.message.includes("429") && 33 | attempt < maxRetries 34 | ); 35 | } 36 | 37 | async function exponentialBackoff( 38 | fn: () => Promise, 39 | maxRetries: number, 40 | ): Promise { 41 | let attempt = 0; 42 | while (attempt < maxRetries) { 43 | try { 44 | return await fn(); 45 | } catch (error) { 46 | if (shouldRetry(error, attempt, maxRetries)) { 47 | attempt++; 48 | const delay = calculateDelay(attempt); 49 | await new Promise((resolve) => setTimeout(resolve, delay)); 50 | } else { 51 | throw error; 52 | } 53 | } 54 | } 55 | throw new ProviderError("Max retries reached"); 56 | } 57 | 58 | export async function queuesClient({ 59 | path, 60 | method, 61 | body, 62 | accountId, 63 | queueId, 64 | signal, 65 | }: { 66 | path: string; 67 | method: string; 68 | body?: Record; 69 | accountId: string; 70 | queueId: string; 71 | signal?: AbortSignal; 72 | }): Promise { 73 | const { QUEUES_API_TOKEN } = getCredentials(); 74 | 75 | const url = `${CLOUDFLARE_HOST}/accounts/${accountId}/queues/${queueId}/${path}`; 76 | const options = { 77 | method, 78 | headers: { 79 | "content-type": "application/json", 80 | authorization: `Bearer ${QUEUES_API_TOKEN}`, 81 | }, 82 | body: JSON.stringify(body), 83 | signal, 84 | }; 85 | 86 | async function fetchWithBackoff() { 87 | const response = await fetch(url, options); 88 | 89 | if (!response) { 90 | throw new ProviderError("No response from Cloudflare Queues API"); 91 | } 92 | 93 | if (response.status === 429) { 94 | throw new ProviderError("429 Too Many Requests"); 95 | } 96 | 97 | throwErrorIfResponseNotOk(response); 98 | 99 | const data = (await response.json()) as T; 100 | 101 | return data; 102 | } 103 | 104 | return exponentialBackoff(fetchWithBackoff, MAX_RETRIES); 105 | } 106 | -------------------------------------------------------------------------------- /src/lib/consumer.ts: -------------------------------------------------------------------------------- 1 | import { TypedEventEmitter } from "../utils/emitter.js"; 2 | import type { 3 | ConsumerOptions, 4 | Message, 5 | PullMessagesResponse, 6 | AckMessageResponse, 7 | UpdatableOptions, 8 | StopOptions, 9 | } from "../types.js"; 10 | import { 11 | assertOptions, 12 | hasMessages, 13 | validateOption, 14 | } from "../utils/validation.js"; 15 | import { queuesClient } from "./cloudflare.js"; 16 | import { logger } from "../utils/logger.js"; 17 | import { 18 | toProviderError, 19 | ProviderError, 20 | toStandardError, 21 | toTimeoutError, 22 | TimeoutError, 23 | } from "../utils/errors.js"; 24 | 25 | /** 26 | * [Usage](https://bbc.github.io/cloudflare-queue-consumer/index.html#usage) 27 | */ 28 | export class Consumer extends TypedEventEmitter { 29 | private accountId: string; 30 | private queueId: string; 31 | private handleMessage: (message: Message) => Promise; 32 | private handleMessageBatch: (message: Message[]) => Promise; 33 | private preReceiveMessageCallback?: () => Promise; 34 | private postReceiveMessageCallback?: () => Promise; 35 | private batchSize: number; 36 | private visibilityTimeoutMs: number; 37 | private retryMessagesOnError: boolean; 38 | private pollingWaitTimeMs: number; 39 | private pollingTimeoutId: NodeJS.Timeout; 40 | private stopped = true; 41 | private isPolling = false; 42 | private handleMessageTimeout: number; 43 | private alwaysAcknowledge: number; 44 | private retryMessageDelay: number; 45 | private shouldDeleteMessages: boolean; 46 | public abortController: AbortController; 47 | 48 | /** 49 | * Create a new consumer 50 | * @param options The options for the consumer 51 | */ 52 | constructor(options) { 53 | super(); 54 | assertOptions(options); 55 | this.accountId = options.accountId; 56 | this.queueId = options.queueId; 57 | this.handleMessage = options.handleMessage; 58 | this.handleMessageBatch = options.handleMessageBatch; 59 | this.preReceiveMessageCallback = options.preReceiveMessageCallback; 60 | this.postReceiveMessageCallback = options.postReceiveMessageCallback; 61 | this.batchSize = options.batchSize ?? 10; 62 | this.visibilityTimeoutMs = options.visibilityTimeoutMs ?? 1000; 63 | this.retryMessagesOnError = options.retryMessagesOnError || false; 64 | this.pollingWaitTimeMs = options.pollingWaitTimeMs ?? 1000; 65 | this.handleMessageTimeout = options.handleMessageTimeout; 66 | this.alwaysAcknowledge = options.alwaysAcknowledge ?? false; 67 | this.retryMessageDelay = options.retryMessageDelay ?? 10; 68 | this.shouldDeleteMessages = options.shouldDeleteMessages ?? true; 69 | } 70 | 71 | /** 72 | * Creates a new consumer. 73 | */ 74 | public static create(options: ConsumerOptions): Consumer { 75 | return new Consumer(options); 76 | } 77 | 78 | /** 79 | * Start polling the queue. 80 | */ 81 | public start(): void { 82 | if (!this.stopped) { 83 | return; 84 | } 85 | // Create a new abort controller each time the consumer is started 86 | this.abortController = new AbortController(); 87 | logger.debug("starting"); 88 | this.stopped = false; 89 | this.emit("started"); 90 | this.poll(); 91 | } 92 | 93 | /** 94 | * A reusable options object for queue.sending that's used to avoid duplication. 95 | */ 96 | private get fetchOptions(): { signal: AbortSignal } { 97 | return { 98 | // return the current abortController signal or a fresh signal that has not been aborted. 99 | // This effectively defaults the signal sent to not aborted 100 | signal: this.abortController?.signal || new AbortController().signal, 101 | }; 102 | } 103 | 104 | /** 105 | * Stop polling the queue. 106 | */ 107 | public stop(options?: StopOptions): void { 108 | if (this.stopped) { 109 | logger.debug("already_stopped"); 110 | return; 111 | } 112 | 113 | logger.debug("stopping"); 114 | this.stopped = true; 115 | 116 | if (this.pollingTimeoutId) { 117 | clearTimeout(this.pollingTimeoutId); 118 | this.pollingTimeoutId = undefined; 119 | } 120 | 121 | if (options?.abort) { 122 | logger.debug("aborting"); 123 | this.abortController.abort(); 124 | this.emit("aborted"); 125 | } 126 | 127 | this.emit("stopped"); 128 | } 129 | 130 | /** 131 | * Returns the current status of the consumer. 132 | * This includes whether it is running or currently polling. 133 | */ 134 | public get status(): { 135 | isRunning: boolean; 136 | isPolling: boolean; 137 | } { 138 | return { 139 | isRunning: !this.stopped, 140 | isPolling: this.isPolling, 141 | }; 142 | } 143 | 144 | /** 145 | * Poll the queue for messages. 146 | */ 147 | private async poll(): Promise { 148 | if (this.stopped) { 149 | logger.debug("cancelling_poll", { 150 | detail: 151 | "Poll was called while consumer was stopped, cancelling poll...", 152 | }); 153 | return; 154 | } 155 | 156 | logger.debug("polling"); 157 | 158 | this.isPolling = true; 159 | 160 | const currentPollingTimeout: number = this.pollingWaitTimeMs; 161 | 162 | this.receiveMessage() 163 | .then((output: PullMessagesResponse) => this.handleQueueResponse(output)) 164 | .then((): void => { 165 | if (this.pollingTimeoutId) { 166 | clearTimeout(this.pollingTimeoutId); 167 | } 168 | this.pollingTimeoutId = setTimeout( 169 | () => this.poll(), 170 | currentPollingTimeout, 171 | ); 172 | }) 173 | .catch((err): void => { 174 | // This error handling should be expanded and improved to handle auth timeouts 175 | // https://github.com/bbc/cloudflare-queue-consumer/issues/19 176 | this.emit("error", err); 177 | setTimeout(() => this.poll(), this.pollingWaitTimeMs); 178 | }) 179 | .finally((): void => { 180 | this.isPolling = false; 181 | }); 182 | } 183 | 184 | /** 185 | * Send a request to Cloudflare Queues to retrieve messages 186 | * @param params The required params to receive messages from Cloudflare Queues 187 | */ 188 | private async receiveMessage(): Promise { 189 | try { 190 | if (this.preReceiveMessageCallback) { 191 | await this.preReceiveMessageCallback(); 192 | } 193 | 194 | const result = queuesClient({ 195 | ...this.fetchOptions, 196 | path: "messages/pull", 197 | method: "POST", 198 | body: { 199 | batch_size: this.batchSize, 200 | visibility_timeout_ms: this.visibilityTimeoutMs, 201 | }, 202 | accountId: this.accountId, 203 | queueId: this.queueId, 204 | }); 205 | 206 | if (this.postReceiveMessageCallback) { 207 | await this.postReceiveMessageCallback(); 208 | } 209 | 210 | return result; 211 | } catch (err) { 212 | throw toProviderError(err, `Receive message failed: ${err.message}`); 213 | } 214 | } 215 | 216 | /** 217 | * Handles the response from Cloudflare, determining if we should proceed to 218 | * the message handler. 219 | * @param response The output from Cloudflare 220 | */ 221 | private async handleQueueResponse( 222 | response: PullMessagesResponse, 223 | ): Promise { 224 | if (!response.success) { 225 | this.emit("error", new Error("Failed to pull messages")); 226 | this.isPolling = false; 227 | return; 228 | } 229 | 230 | if (hasMessages(response)) { 231 | if (this.handleMessageBatch) { 232 | await this.processMessageBatch(response.result.messages); 233 | } else { 234 | await Promise.all( 235 | response.result.messages.map((message: Message) => 236 | this.processMessage(message), 237 | ), 238 | ); 239 | } 240 | 241 | this.emit("response_processed"); 242 | } else if (response) { 243 | this.emit("empty"); 244 | } 245 | } 246 | 247 | /** 248 | * Process a message that has been received from Cloudflare Queues. This will execute the message 249 | * handler and delete the message once complete. 250 | * @param message The message that was delivered from Cloudflare 251 | */ 252 | private async processMessage(message: Message): Promise { 253 | try { 254 | this.emit("message_received", message); 255 | 256 | // At the moment, we don't extend timeouts on a heartbeat 257 | // https://github.com/bbc/cloudflare-queue-consumer/issues/20 258 | 259 | const ackedMessage: Message = await this.executeHandler(message); 260 | 261 | if (ackedMessage?.id === message.id) { 262 | if (this.shouldDeleteMessages) { 263 | // We should probably batch these up to reduce API rate limits 264 | // https://github.com/bbc/cloudflare-queue-consumer/issues/21 265 | await this.acknowledgeMessage([message], []); 266 | } 267 | 268 | this.emit("message_processed", message); 269 | } 270 | } catch (err) { 271 | this.emitError(err, message); 272 | 273 | if (this.retryMessagesOnError) { 274 | // We should probably batch these up to reduce API rate limits 275 | // https://github.com/bbc/cloudflare-queue-consumer/issues/21 276 | await this.acknowledgeMessage([], [message]); 277 | } 278 | } 279 | } 280 | 281 | /** 282 | * Process a batch of messages from the SQS queue. 283 | * @param messages The messages that were delivered from SQS 284 | */ 285 | private async processMessageBatch(messages: Message[]): Promise { 286 | try { 287 | messages.forEach((message: Message): void => { 288 | this.emit("message_received", message); 289 | }); 290 | 291 | // At the moment, we don't extend timeouts on a heartbeat 292 | // https://github.com/bbc/cloudflare-queue-consumer/issues/20 293 | 294 | const ackedMessages: Message[] = await this.executeBatchHandler(messages); 295 | 296 | if (ackedMessages?.length > 0) { 297 | if (this.shouldDeleteMessages) { 298 | await this.acknowledgeMessage(ackedMessages, []); 299 | } 300 | 301 | ackedMessages.forEach((message: Message): void => { 302 | this.emit("message_processed", message); 303 | }); 304 | } 305 | } catch (err) { 306 | this.emit("error", err, messages); 307 | 308 | if (this.retryMessagesOnError) { 309 | await this.acknowledgeMessage([], messages); 310 | } 311 | } 312 | } 313 | 314 | /** 315 | * Trigger the applications handleMessage function 316 | * @param message The message that was received from Cloudflare 317 | */ 318 | private async executeHandler(message: Message): Promise { 319 | let handleMessageTimeoutId: NodeJS.Timeout | undefined = undefined; 320 | 321 | try { 322 | let result; 323 | 324 | if (this.handleMessageTimeout) { 325 | const pending: Promise = new Promise((_, reject): void => { 326 | handleMessageTimeoutId = setTimeout((): void => { 327 | reject(new TimeoutError()); 328 | }, this.handleMessageTimeout); 329 | }); 330 | result = await Promise.race([this.handleMessage(message), pending]); 331 | } else { 332 | result = await this.handleMessage(message); 333 | } 334 | 335 | return !this.alwaysAcknowledge && result instanceof Object 336 | ? result 337 | : message; 338 | } catch (err) { 339 | if (err instanceof TimeoutError) { 340 | throw toTimeoutError( 341 | err, 342 | `Message handler timed out after ${this.handleMessageTimeout}ms: Operation timed out.`, 343 | ); 344 | } else if (err instanceof Error) { 345 | throw toStandardError( 346 | err, 347 | `Unexpected message handler failure: ${err.message}`, 348 | ); 349 | } 350 | throw err; 351 | } finally { 352 | if (handleMessageTimeoutId) { 353 | clearTimeout(handleMessageTimeoutId); 354 | } 355 | } 356 | } 357 | 358 | /** 359 | * Execute the application's message batch handler 360 | * @param messages The messages that should be forwarded from the SQS queue 361 | */ 362 | private async executeBatchHandler(messages: Message[]): Promise { 363 | try { 364 | const result: void | Message[] = await this.handleMessageBatch(messages); 365 | 366 | return !this.alwaysAcknowledge && result instanceof Object 367 | ? result 368 | : messages; 369 | } catch (err) { 370 | if (err instanceof Error) { 371 | throw toStandardError( 372 | err, 373 | `Unexpected message handler failure: ${err.message}`, 374 | ); 375 | } 376 | throw err; 377 | } 378 | } 379 | 380 | /** 381 | * Acknowledge a message that has been processed by the consumer 382 | * @param acks The message(s) to acknowledge 383 | * @param retries The message(s) to retry 384 | */ 385 | private async acknowledgeMessage( 386 | acks: Message[], 387 | retries: Message[], 388 | ): Promise { 389 | try { 390 | // This is pretty hacky, is there a better way to do this? 391 | // https://github.com/bbc/cloudflare-queue-consumer/issues/22 392 | const retriesWithDelay = retries.map((message) => ({ 393 | ...message, 394 | delay_seconds: this.retryMessageDelay, 395 | })); 396 | const input = { acks, retries: retriesWithDelay }; 397 | this.emit("acknowledging_messages", acks, retriesWithDelay); 398 | 399 | const result = await queuesClient({ 400 | ...this.fetchOptions, 401 | path: "messages/ack", 402 | method: "POST", 403 | body: input, 404 | accountId: this.accountId, 405 | queueId: this.queueId, 406 | }); 407 | 408 | if (!result.success) { 409 | throw new Error("Message Acknowledgement did not succeed."); 410 | } 411 | 412 | this.emit("acknowledged_messages", result.result); 413 | 414 | return result; 415 | } catch (err) { 416 | this.emit( 417 | "error", 418 | toProviderError(err, `Error acknowledging messages: ${err.message}`), 419 | ); 420 | } 421 | } 422 | 423 | /** 424 | * Validates and then updates the provided option to the provided value. 425 | * @param option The option to validate and then update 426 | * @param value The value to set the provided option to 427 | */ 428 | public updateOption( 429 | option: UpdatableOptions, 430 | value: ConsumerOptions[UpdatableOptions], 431 | ): void { 432 | validateOption(option, value, true); 433 | 434 | this[option] = value; 435 | 436 | this.emit("option_updated", option, value); 437 | } 438 | 439 | /** 440 | * Emit one of the consumer's error events depending on the error received. 441 | * @param err The error object to forward on 442 | * @param message The message that the error occurred on 443 | */ 444 | private emitError(err: Error, message?: Message): void { 445 | if (!message) { 446 | this.emit("error", err); 447 | } else if (err.name === ProviderError.name) { 448 | this.emit("error", err, message); 449 | } else if (err instanceof TimeoutError) { 450 | this.emit("timeout_error", err, message); 451 | } else { 452 | this.emit("processing_error", err, message); 453 | } 454 | } 455 | } 456 | -------------------------------------------------------------------------------- /src/lib/fetch.ts: -------------------------------------------------------------------------------- 1 | import { toProviderError, ProviderError } from "../utils/errors.js"; 2 | 3 | export function throwErrorIfResponseNotOk(response: Response): void { 4 | if (!response?.ok) { 5 | const message = response?.status 6 | ? `[${response?.status} - ${response?.statusText}] ${response?.url}` 7 | : `[${response?.statusText}] ${response?.url}`; 8 | 9 | const error = new ProviderError(message); 10 | error.code = "ResponseNotOk"; 11 | Object.defineProperty(error, "response", { value: response }); 12 | 13 | throw toProviderError(error, message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The options for the consumer. 3 | */ 4 | export interface ConsumerOptions { 5 | /** 6 | * The number of messages to request from Cloudflare when polling (default `10`). 7 | * @defaultvalue `10` 8 | */ 9 | batchSize?: number; 10 | /** 11 | * The duration (in milliseconds) that the received messages are hidden from subsequent 12 | * retrieve requests after being retrieved by a ReceiveMessage request. 13 | * @defaultvalue 1000 14 | */ 15 | visibilityTimeoutMs?: number; 16 | /** 17 | * If the Consumer should trigger the message(s) to be retired on 18 | * @defaultvalue false 19 | */ 20 | retryMessagesOnError?: boolean; 21 | /** 22 | * You Cloudflare account id 23 | */ 24 | accountId: string; 25 | /** 26 | * The ID of the queue you want to receive messages from. 27 | */ 28 | queueId: string; 29 | /** 30 | * An `async` function (or function that returns a `Promise`) to be called whenever 31 | * a message is received. 32 | * 33 | * In the case that you need to acknowledge the message, return an object containing 34 | * the MessageId that you'd like to acknowledge. 35 | */ 36 | handleMessage?(message: Message): Promise; 37 | /** 38 | * An `async` function (or function that returns a `Promise`) to be called whenever 39 | * a batch of messages is received. Similar to `handleMessage` but will receive the 40 | * list of messages, not each message individually, this is preferred to reduce API 41 | * rate limits. 42 | * 43 | * **If both are set, `handleMessageBatch` overrides `handleMessage`**. 44 | * 45 | * In the case that you need to ack only some of the messages, return an array with 46 | * the successful messages only. 47 | */ 48 | handleMessageBatch?(messages: Message[]): Promise; 49 | /** 50 | * An `async` function (or function that returns a `Promise`) to be called right 51 | * before the consumer sends a receive message command. 52 | */ 53 | preReceiveMessageCallback?(): Promise; 54 | /** 55 | * An `async` function (or function that returns a `Promise`) to be called right 56 | * after the consumer sends a receive message command. 57 | */ 58 | postReceiveMessageCallback?(): Promise; 59 | /** 60 | * Time in ms to wait for `handleMessage` to process a message before timing out. 61 | * 62 | * Emits `timeout_error` on timeout. By default, if `handleMessage` times out, 63 | * the unprocessed message returns to the end of the queue. 64 | */ 65 | handleMessageTimeout?: number; 66 | /** 67 | * By default, the consumer will treat an empty object or array from either of the 68 | * handlers as a acknowledgement of no messages and will not delete those messages as 69 | * a result. Set this to `true` to always acknowledge all messages no matter the returned 70 | * value. 71 | * @defaultvalue `false` 72 | */ 73 | alwaysAcknowledge?: boolean; 74 | /** 75 | * The amount of time to delay a message for before retrying (in seconds) 76 | * @defaultvalue 10 77 | */ 78 | retryMessageDelay?: number; 79 | /** 80 | * The duration (in milliseconds) to wait before repolling the queue. 81 | * (Note: As Cloudflare uses short polling, you probably shouldn't set this too low) 82 | * @defaultvalue `1000` 83 | */ 84 | pollingWaitTimeMs?: number; 85 | /** 86 | * If the consumer should delete messages after they have been processed. 87 | * @defaultvalue `true` 88 | */ 89 | shouldDeleteMessages?: boolean; 90 | } 91 | 92 | /** 93 | * A subset of the ConsumerOptions that can be updated at runtime. 94 | */ 95 | export type UpdatableOptions = 96 | | "visibilityTimeoutMs" 97 | | "batchSize" 98 | | "pollingWaitTimeMs"; 99 | 100 | /** 101 | * The options for the stop method. 102 | */ 103 | export interface StopOptions { 104 | /** 105 | * Default to `false`, if you want the stop action to also abort requests to SQS 106 | * set this to `true`. 107 | * @defaultvalue `false` 108 | */ 109 | abort?: boolean; 110 | } 111 | 112 | export type Message = { 113 | body: string; 114 | id: string; 115 | timestamp_ms: number; 116 | attempts: number; 117 | lease_id: string; 118 | metadata: { 119 | "CF-sourceMessageSource": string; 120 | "CF-Content-Type": "json" | "text"; 121 | }; 122 | }; 123 | 124 | export type CloudFlareError = { 125 | code: number; 126 | message: string; 127 | }; 128 | 129 | export type CloudFlareResultInfo = { 130 | page: number; 131 | per_page: number; 132 | total_pages: number; 133 | count: number; 134 | total_count: number; 135 | }; 136 | 137 | export type PullMessagesResponse = { 138 | errors: CloudFlareError[]; 139 | messages: CloudFlareError[]; 140 | result: { 141 | messages: Message[]; 142 | }; 143 | success: boolean; 144 | result_info: CloudFlareResultInfo; 145 | }; 146 | 147 | export type AckMessageResponse = { 148 | errors: CloudFlareError[]; 149 | messages: CloudFlareError[]; 150 | result: { 151 | ackCount: number; 152 | retryCount: number; 153 | warnings: string[]; 154 | }; 155 | success: boolean; 156 | result_info: CloudFlareResultInfo; 157 | }; 158 | 159 | /** 160 | * These are the events that the consumer emits. 161 | */ 162 | export interface Events { 163 | /** 164 | * Fired after one batch of items (up to `batchSize`) has been successfully processed. 165 | */ 166 | response_processed: []; 167 | /** 168 | * Fired when the queue is empty (All messages have been consumed). 169 | */ 170 | empty: []; 171 | /** 172 | * Fired when a message is received. 173 | */ 174 | message_received: [Message]; 175 | /** 176 | * Fired when a message is successfully processed and removed from the queue. 177 | */ 178 | message_processed: [Message]; 179 | /** 180 | * Fired when an error occurs interacting with the queue. 181 | * 182 | * If the error correlates to a message, that message is included in Params 183 | */ 184 | error: [Error, void | Message | Message[]]; 185 | /** 186 | * Fired when `handleMessageTimeout` is supplied as an option and if 187 | * `handleMessage` times out. 188 | */ 189 | timeout_error: [Error, Message]; 190 | /** 191 | * Fired when an error occurs processing the message. 192 | */ 193 | processing_error: [Error, Message]; 194 | /** 195 | * Fired when requests to Cloudflare were aborted. 196 | */ 197 | aborted: []; 198 | /** 199 | * Fired when the consumer starts its work.. 200 | */ 201 | started: []; 202 | /** 203 | * Fired when the consumer finally stops its work. 204 | */ 205 | stopped: []; 206 | /** 207 | * Fired when messages are acknowledging 208 | */ 209 | acknowledging_messages: [ 210 | { 211 | lease_id: string; 212 | }[], 213 | { 214 | lease_id: string; 215 | delay_seconds: number; 216 | }[], 217 | ]; 218 | /** 219 | * Fired when messages have been acknowledged 220 | */ 221 | acknowledged_messages: [ 222 | { 223 | ackCount: number; 224 | retryCount: number; 225 | warnings: string[]; 226 | }, 227 | ]; 228 | /** 229 | * Fired when an option is updated 230 | */ 231 | option_updated: [UpdatableOptions, ConsumerOptions[UpdatableOptions]]; 232 | } 233 | -------------------------------------------------------------------------------- /src/utils/emitter.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from "node:events"; 2 | 3 | import { logger } from "./logger.js"; 4 | import type { Events } from "../types.js"; 5 | 6 | export class TypedEventEmitter extends EventEmitter { 7 | /** 8 | * Trigger a listener on all emitted events 9 | * @param event The name of the event to listen to 10 | * @param listener A function to trigger when the event is emitted 11 | */ 12 | on( 13 | event: E, 14 | listener: (...args: Events[E]) => void, 15 | ): this { 16 | return super.on(event, listener); 17 | } 18 | /** 19 | * Trigger a listener only once for an emitted event 20 | * @param event The name of the event to listen to 21 | * @param listener A function to trigger when the event is emitted 22 | */ 23 | once( 24 | event: E, 25 | listener: (...args: Events[E]) => void, 26 | ): this { 27 | return super.once(event, listener); 28 | } 29 | /** 30 | * Emits an event with the provided arguments 31 | * @param event The name of the event to emit 32 | */ 33 | emit(event: E, ...args: Events[E]): boolean { 34 | logger.debug(event, ...args); 35 | return super.emit(event, ...args); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/utils/errors.ts: -------------------------------------------------------------------------------- 1 | export class ProviderError extends Error { 2 | stack: string; 3 | code: string; 4 | time: Date; 5 | status: number; 6 | statusText?: string; 7 | url?: string; 8 | 9 | constructor(message: string) { 10 | super(message); 11 | this.name = this.constructor.name; 12 | } 13 | } 14 | 15 | interface ErrorWithResponse extends Error { 16 | code: string; 17 | response?: Response; 18 | } 19 | 20 | /** 21 | * Formats a provider's error the the ProviderError type. 22 | * @param err The error object that was received. 23 | * @param message The message to send with the error. 24 | */ 25 | export function toProviderError( 26 | err: ErrorWithResponse, 27 | message: string, 28 | ): ProviderError { 29 | const error = new ProviderError(message); 30 | error.code = err.code; 31 | error.stack = err?.stack; 32 | error.time = new Date(); 33 | 34 | if (err.response) { 35 | error.status = err.response.status; 36 | error.statusText = err.response.statusText; 37 | error.url = err.response.url; 38 | } 39 | 40 | return error; 41 | } 42 | 43 | export class StandardError extends Error { 44 | cause: Error; 45 | time: Date; 46 | 47 | constructor(message = "An unexpected error occurred:") { 48 | super(message); 49 | this.message = message; 50 | this.name = "StandardError"; 51 | } 52 | } 53 | 54 | /** 55 | * Formats an Error to the StandardError type. 56 | * @param err The error object that was received. 57 | * @param message The message to send with the error. 58 | */ 59 | export function toStandardError(err: Error, message: string): StandardError { 60 | const error = new StandardError(message); 61 | error.cause = err; 62 | error.time = new Date(); 63 | 64 | return error; 65 | } 66 | 67 | export class TimeoutError extends Error { 68 | cause: Error; 69 | time: Date; 70 | 71 | constructor(message = "Operation timed out.") { 72 | super(message); 73 | this.message = message; 74 | this.name = "TimeoutError"; 75 | } 76 | } 77 | 78 | /** 79 | * Formats an Error to the TimeoutError type. 80 | * @param err The error object that was received. 81 | * @param message The message to send with the error. 82 | */ 83 | export function toTimeoutError( 84 | err: TimeoutError, 85 | message: string, 86 | ): TimeoutError { 87 | const error = new TimeoutError(message); 88 | error.cause = err; 89 | error.time = new Date(); 90 | 91 | return error; 92 | } 93 | -------------------------------------------------------------------------------- /src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | import createDebug from "debug"; 2 | const debug = createDebug("cloudflare-queue-consumer"); 3 | 4 | export const logger = { 5 | debug, 6 | }; 7 | -------------------------------------------------------------------------------- /src/utils/validation.ts: -------------------------------------------------------------------------------- 1 | import type { ConsumerOptions, PullMessagesResponse } from "../types.js"; 2 | 3 | const requiredOptions = [ 4 | "accountId", 5 | "queueId", 6 | // only one of handleMessage / handleMessagesBatch is required 7 | "handleMessage|handleMessageBatch", 8 | ]; 9 | 10 | function validateOption(option: string, value: number, strict?: boolean): void { 11 | switch (option) { 12 | case "batchSize": 13 | if (value > 100 || value < 1) { 14 | throw new Error("batchSize must be between 1 and 100"); 15 | } 16 | break; 17 | case "visibilityTimeoutMs": 18 | if (value > 43200000) { 19 | throw new Error("visibilityTimeoutMs must be less than 43200000"); 20 | } 21 | break; 22 | case "retryMessageDelay": 23 | if (value > 42300) { 24 | throw new Error("retryMessageDelay must be less than 42300"); 25 | } 26 | break; 27 | case "pollingWaitTimeMs": 28 | if (value < 0) { 29 | throw new Error("pollingWaitTimeMs must be greater than 0."); 30 | } 31 | break; 32 | default: 33 | if (strict) { 34 | throw new Error(`The update ${option} cannot be updated`); 35 | } 36 | break; 37 | } 38 | } 39 | 40 | /** 41 | * Ensure that the required options have been set. 42 | * @param options The options that have been set by the application. 43 | */ 44 | function assertOptions(options: ConsumerOptions): void { 45 | requiredOptions.forEach((option) => { 46 | const possibilities = option.split("|"); 47 | if (!possibilities.find((p) => options[p])) { 48 | throw new Error( 49 | `Missing consumer option [ ${possibilities.join(" or ")} ].`, 50 | ); 51 | } 52 | }); 53 | 54 | if (options.batchSize) { 55 | validateOption("batchSize", options.batchSize); 56 | } 57 | 58 | if (options.visibilityTimeoutMs) { 59 | validateOption("visibilityTimeoutMs", options.visibilityTimeoutMs); 60 | } 61 | 62 | if (options.retryMessageDelay) { 63 | validateOption("retryMessageDelay", options.retryMessageDelay); 64 | } 65 | } 66 | 67 | /** 68 | * Determine if the response has messages in it. 69 | * @param response The response from Cloudflare. 70 | */ 71 | function hasMessages(response: PullMessagesResponse): boolean { 72 | return ( 73 | Array.isArray(response?.result?.messages) && 74 | response.result.messages.length > 0 75 | ); 76 | } 77 | 78 | export { assertOptions, validateOption, hasMessages }; 79 | -------------------------------------------------------------------------------- /test/fixtures/ackMessagesRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "acks": [ 3 | { 4 | "body": "body", 5 | "id": "123", 6 | "timestamp_ms": 1234567890, 7 | "attempts": 1, 8 | "lease_id": "lease-id", 9 | "metadata": { 10 | "CF-sourceMessageSource": "test", 11 | "CF-Content-Type": "text" 12 | } 13 | } 14 | ], 15 | "retries": [] 16 | } 17 | -------------------------------------------------------------------------------- /test/fixtures/ackMessagesResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "errors": [], 3 | "messages": [], 4 | "result": { 5 | "ackCount": 1, 6 | "retryCount": 0, 7 | "warnings": [] 8 | }, 9 | "success": true, 10 | "result_info": {} 11 | } 12 | -------------------------------------------------------------------------------- /test/fixtures/pullMessagesRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "batch_size": 10, 3 | "visibility_timeout_ms": 1000 4 | } 5 | -------------------------------------------------------------------------------- /test/fixtures/pullMessagesResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "errors": {}, 3 | "messages": [], 4 | "result": { 5 | "messages": [ 6 | { 7 | "body": "body", 8 | "id": "123", 9 | "timestamp_ms": 1234567890, 10 | "attempts": 1, 11 | "lease_id": "lease-id", 12 | "metadata": { 13 | "CF-sourceMessageSource": "test", 14 | "CF-Content-Type": "text" 15 | } 16 | } 17 | ] 18 | }, 19 | "success": true, 20 | "result_info": {} 21 | } 22 | -------------------------------------------------------------------------------- /test/unit/lib/cloudflare.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, beforeEach, afterEach } from "node:test"; 2 | import { assert } from "chai"; 3 | import nock from "nock"; 4 | import sinon from "sinon"; 5 | 6 | import { queuesClient } from "../../../src/lib/cloudflare"; 7 | import { ProviderError } from "../../../src/utils/errors"; 8 | 9 | const CLOUDFLARE_HOST = "https://api.cloudflare.com/client/v4"; 10 | const ACCOUNT_ID = "test-account-id"; 11 | const QUEUE_ID = "test-queue-id"; 12 | const QUEUES_API_TOKEN = "test-queues-api-token"; 13 | 14 | describe("queuesClient", () => { 15 | let sandbox: sinon.SinonSandbox; 16 | 17 | beforeEach(() => { 18 | sandbox = sinon.createSandbox(); 19 | process.env.QUEUES_API_TOKEN = QUEUES_API_TOKEN; 20 | }); 21 | 22 | afterEach(() => { 23 | sandbox.restore(); 24 | delete process.env.QUEUES_API_TOKEN; 25 | nock.cleanAll(); 26 | }); 27 | 28 | it("should successfully fetch data from Cloudflare Queues API", async () => { 29 | const path = "messages"; 30 | const method = "GET"; 31 | const responseBody = { success: true, result: [] }; 32 | 33 | nock(CLOUDFLARE_HOST) 34 | .get(`/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/${path}`) 35 | .reply(200, responseBody); 36 | 37 | const result = await queuesClient({ 38 | path, 39 | method, 40 | accountId: ACCOUNT_ID, 41 | queueId: QUEUE_ID, 42 | }); 43 | 44 | assert.deepEqual(result, responseBody); 45 | }); 46 | 47 | it("should throw an error if the API token is missing", async () => { 48 | delete process.env.QUEUES_API_TOKEN; 49 | 50 | try { 51 | await queuesClient({ 52 | path: "messages", 53 | method: "GET", 54 | accountId: ACCOUNT_ID, 55 | queueId: QUEUE_ID, 56 | }); 57 | assert.fail("Expected error to be thrown"); 58 | } catch (error) { 59 | assert.instanceOf(error, Error); 60 | assert.equal( 61 | error.message, 62 | "Missing Cloudflare credentials, please set a QUEUES_API_TOKEN in the environment variables.", 63 | ); 64 | } 65 | }); 66 | 67 | it("should retry on 429 Too Many Requests", async () => { 68 | const path = "messages"; 69 | const method = "GET"; 70 | const responseBody = { success: true, result: [] }; 71 | 72 | nock(CLOUDFLARE_HOST) 73 | .get(`/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/${path}`) 74 | .reply(429, "Too Many Requests") 75 | .get(`/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/${path}`) 76 | .reply(200, responseBody); 77 | 78 | const result = await queuesClient({ 79 | path, 80 | method, 81 | accountId: ACCOUNT_ID, 82 | queueId: QUEUE_ID, 83 | }); 84 | 85 | assert.deepEqual(result, responseBody); 86 | }); 87 | 88 | it("should throw ProviderError after max retries", async () => { 89 | const path = "messages"; 90 | const method = "GET"; 91 | 92 | nock(CLOUDFLARE_HOST) 93 | .get(`/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/${path}`) 94 | .times(5) 95 | .reply(429, "Too Many Requests"); 96 | 97 | try { 98 | await queuesClient({ 99 | path, 100 | method, 101 | accountId: ACCOUNT_ID, 102 | queueId: QUEUE_ID, 103 | }); 104 | assert.fail("Expected error to be thrown"); 105 | } catch (error) { 106 | assert.instanceOf(error, ProviderError); 107 | assert.equal(error.message, "Max retries reached"); 108 | } 109 | }); 110 | }); 111 | -------------------------------------------------------------------------------- /test/unit/lib/consumer.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-warning-comments */ 2 | import { beforeEach, afterEach, describe, it } from "node:test"; 3 | import { assert, expect } from "chai"; 4 | import * as chai from "chai"; 5 | import chaiNock from "chai-nock"; 6 | import * as sinon from "sinon"; 7 | import { pEvent } from "p-event"; 8 | import nock from "nock"; 9 | 10 | import { Consumer } from "../../../src/lib/consumer"; 11 | import { logger } from "../../../src/utils/logger"; 12 | 13 | import pullMessagesResponse from "../../fixtures/pullMessagesResponse.json"; 14 | import ackMessagesResponse from "../../fixtures/ackMessagesResponse.json"; 15 | 16 | chai.use(chaiNock); 17 | nock.disableNetConnect(); 18 | 19 | const ACCOUNT_ID = "023e105f4ecef8ad9ca31a8372d0c353"; 20 | const QUEUE_ID = "023e105f4ecef8ad9ca31a8372d0c353"; 21 | const CLOUDFLARE_HOST = "https://api.cloudflare.com/client/v4"; 22 | const PULL_MESSAGES_ENDPOINT = `/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/messages/pull`; 23 | const ACK_MESSAGES_ENDPOINT = `/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/messages/ack`; 24 | const QUEUES_API_TOKEN = "queues_token"; 25 | 26 | const currentProcessEnv = { ...process.env }; 27 | 28 | const sandbox = sinon.createSandbox(); 29 | 30 | type MockRequest = { 31 | timeout?: number; 32 | persist?: boolean; 33 | status?: number; 34 | response?: Record; 35 | }; 36 | 37 | function mockPullRequest({ 38 | timeout, 39 | persist = false, 40 | status = 200, 41 | response = pullMessagesResponse, 42 | }: MockRequest) { 43 | const request = nock(CLOUDFLARE_HOST, { 44 | reqheaders: { 45 | authorization: `Bearer ${QUEUES_API_TOKEN}`, 46 | }, 47 | }) 48 | .persist(persist) 49 | .post(PULL_MESSAGES_ENDPOINT); 50 | 51 | if (timeout) { 52 | request.delayConnection(timeout); 53 | } 54 | 55 | return request.reply(status, response); 56 | } 57 | 58 | function mockAckRequest({ 59 | timeout, 60 | persist = false, 61 | status = 200, 62 | response = ackMessagesResponse, 63 | }: MockRequest) { 64 | const request = nock(CLOUDFLARE_HOST, { 65 | reqheaders: { 66 | authorization: `Bearer ${QUEUES_API_TOKEN}`, 67 | }, 68 | }) 69 | .persist(persist) 70 | .post(ACK_MESSAGES_ENDPOINT); 71 | 72 | if (timeout) { 73 | request.delayConnection(timeout); 74 | } 75 | 76 | return request.reply(status, response); 77 | } 78 | 79 | describe("Consumer", () => { 80 | let consumer; 81 | let clock; 82 | let handleMessage; 83 | let handleMessageBatch; 84 | 85 | beforeEach(() => { 86 | clock = sinon.useFakeTimers(); 87 | 88 | process.env.QUEUES_API_TOKEN = QUEUES_API_TOKEN; 89 | 90 | handleMessage = sandbox.stub().resolves(null); 91 | handleMessageBatch = sandbox.stub().resolves(null); 92 | 93 | if (!nock.isActive()) { 94 | nock.activate(); 95 | } 96 | nock.cleanAll(); 97 | 98 | consumer = new Consumer({ 99 | accountId: ACCOUNT_ID, 100 | queueId: QUEUE_ID, 101 | handleMessage, 102 | }); 103 | }); 104 | 105 | afterEach(() => { 106 | if (consumer) { 107 | consumer.stop(); 108 | } 109 | process.env = currentProcessEnv; 110 | clock.restore(); 111 | sandbox.restore(); 112 | nock.cleanAll(); 113 | nock.restore(); 114 | }); 115 | 116 | describe("Options Validation", () => { 117 | beforeEach(() => { 118 | mockPullRequest({}); 119 | mockAckRequest({}); 120 | }); 121 | 122 | it("requires an accountId to be set", () => { 123 | assert.throws( 124 | () => new Consumer({}), 125 | "Missing consumer option [ accountId ].", 126 | ); 127 | }); 128 | 129 | it("requires a queueId to be set", () => { 130 | assert.throws( 131 | () => new Consumer({ accountId: "123" }), 132 | "Missing consumer option [ queueId ].", 133 | ); 134 | }); 135 | 136 | it("require a handleMessage or handleMessageBatch function to be set", () => { 137 | assert.throws( 138 | () => new Consumer({ accountId: "123", queueId: "123" }), 139 | "Missing consumer option [ handleMessage or handleMessageBatch ].", 140 | ); 141 | }); 142 | 143 | it("requires batchSize to be no greater than 100", () => { 144 | assert.throws( 145 | () => 146 | new Consumer({ 147 | accountId: "123", 148 | queueId: "123", 149 | handleMessage: async (message) => { 150 | return message; 151 | }, 152 | batchSize: 101, 153 | }), 154 | "batchSize must be between 1 and 100", 155 | ); 156 | }); 157 | 158 | it("requires batchSize to be more than or equal to 1", () => { 159 | assert.throws( 160 | () => 161 | new Consumer({ 162 | accountId: "123", 163 | queueId: "123", 164 | handleMessage: async (message) => { 165 | return message; 166 | }, 167 | batchSize: -1, 168 | }), 169 | "batchSize must be between 1 and 100", 170 | ); 171 | }); 172 | 173 | it("requires visibilityTimeoutMs to be less than 43200000", () => { 174 | assert.throws( 175 | () => 176 | new Consumer({ 177 | accountId: "123", 178 | queueId: "123", 179 | handleMessage: async (message) => { 180 | return message; 181 | }, 182 | visibilityTimeoutMs: 43200001, 183 | }), 184 | "visibilityTimeoutMs must be less than 43200000", 185 | ); 186 | }); 187 | 188 | it("requires retryMessageDelay to be less than 42300", () => { 189 | assert.throws( 190 | () => 191 | new Consumer({ 192 | accountId: "123", 193 | queueId: "123", 194 | handleMessage: async (message) => { 195 | return message; 196 | }, 197 | retryMessageDelay: 42301, 198 | }), 199 | "retryMessageDelay must be less than 42300", 200 | ); 201 | }); 202 | }); 203 | 204 | describe(".create", () => { 205 | beforeEach(() => { 206 | mockPullRequest({}); 207 | mockAckRequest({}); 208 | }); 209 | 210 | it("creates a new Consumer instance", () => { 211 | const instance = Consumer.create({ 212 | accountId: "123", 213 | queueId: "123", 214 | handleMessage: async (message) => { 215 | return message; 216 | }, 217 | }); 218 | 219 | assert(instance instanceof Consumer); 220 | }); 221 | }); 222 | 223 | describe(".start", () => { 224 | it("fires an event when the consumer starts", async () => { 225 | mockPullRequest({}); 226 | mockAckRequest({}); 227 | 228 | const handleStart = sandbox.stub().returns(null); 229 | consumer.on("started", handleStart); 230 | 231 | consumer.start(); 232 | await pEvent(consumer, "message_processed"); 233 | consumer.stop(); 234 | 235 | sandbox.assert.calledOnce(handleStart); 236 | }); 237 | 238 | it("calls the handleMessage function when a message is received", async () => { 239 | mockPullRequest({}); 240 | mockAckRequest({}); 241 | 242 | consumer.start(); 243 | await pEvent(consumer, "message_processed"); 244 | consumer.stop(); 245 | 246 | sandbox.assert.calledWith( 247 | handleMessage, 248 | pullMessagesResponse.result.messages[0], 249 | ); 250 | }); 251 | 252 | it("calls the preReceiveMessageCallback and postReceiveMessageCallback function before receiving a message", async () => { 253 | mockPullRequest({}); 254 | mockAckRequest({}); 255 | const preReceiveMessageCallbackStub = sandbox.stub().resolves(null); 256 | const postReceiveMessageCallbackStub = sandbox.stub().resolves(null); 257 | 258 | consumer = new Consumer({ 259 | accountId: ACCOUNT_ID, 260 | queueId: QUEUE_ID, 261 | handleMessage, 262 | preReceiveMessageCallback: preReceiveMessageCallbackStub, 263 | postReceiveMessageCallback: postReceiveMessageCallbackStub, 264 | }); 265 | 266 | consumer.start(); 267 | await pEvent(consumer, "message_processed"); 268 | consumer.stop(); 269 | 270 | sandbox.assert.calledOnce(preReceiveMessageCallbackStub); 271 | sandbox.assert.calledOnce(postReceiveMessageCallbackStub); 272 | }); 273 | 274 | it("deletes the message when the handleMessage function is called", async () => { 275 | mockPullRequest({}); 276 | const mockedAckRequest = mockAckRequest({}); 277 | handleMessage.resolves(); 278 | 279 | consumer.start(); 280 | await pEvent(consumer, "message_processed"); 281 | consumer.stop(); 282 | 283 | expect(mockedAckRequest).to.have.been.requested; 284 | }); 285 | 286 | it("does not delete the message if shouldDeleteMessages is false", async () => { 287 | mockPullRequest({}); 288 | const mockedAckRequest = mockAckRequest({}); 289 | 290 | consumer = new Consumer({ 291 | accountId: ACCOUNT_ID, 292 | queueId: QUEUE_ID, 293 | handleMessage, 294 | shouldDeleteMessages: false, 295 | }); 296 | 297 | handleMessage.resolves(); 298 | 299 | consumer.start(); 300 | await pEvent(consumer, "message_processed"); 301 | consumer.stop(); 302 | 303 | expect(mockedAckRequest).to.not.have.been.requested; 304 | }); 305 | 306 | it("doesn't delete the message when a processing error is reported", async () => { 307 | mockPullRequest({}); 308 | const mockedAckRequest = mockAckRequest({}); 309 | handleMessage.rejects(new Error("Processing error")); 310 | 311 | consumer.start(); 312 | await pEvent(consumer, "processing_error"); 313 | consumer.stop(); 314 | 315 | expect(mockedAckRequest).to.not.have.been.requested; 316 | }); 317 | 318 | it.skip("consumes another message once one is processed", async () => { 319 | const mockedPullRequest = mockPullRequest({}); 320 | mockAckRequest({}); 321 | handleMessage.resolves(); 322 | 323 | consumer.start(); 324 | await clock.runToLastAsync(); 325 | consumer.stop(); 326 | 327 | // TODO: Should this actually be 11? Seems a bit arbitrary 328 | // Also, this doesn't work anyway 329 | const mockedPullRequestCount = mockedPullRequest.activeMocks().length; 330 | expect(mockedPullRequestCount).to.equal(11); 331 | }); 332 | 333 | it.skip("doesn't consume more messages when called multiple times", () => { 334 | const mockedPullRequest = mockPullRequest({ 335 | timeout: 100, 336 | }); 337 | mockAckRequest({}); 338 | consumer.start(); 339 | consumer.start(); 340 | consumer.start(); 341 | consumer.start(); 342 | consumer.start(); 343 | consumer.stop(); 344 | 345 | expect(mockedPullRequest).to.have.been.requested; 346 | }); 347 | 348 | it.skip("doesn't consume more messages when called multiple times after stopped", async () => { 349 | const mockedPullRequest = mockPullRequest({ 350 | timeout: 100, 351 | }); 352 | consumer.start(); 353 | consumer.stop(); 354 | 355 | consumer.start(); 356 | consumer.start(); 357 | consumer.start(); 358 | consumer.start(); 359 | 360 | expect(mockedPullRequest).to.have.been.requested; 361 | }); 362 | 363 | it.skip("consumes multiple messages", async () => { 364 | const mockedPullRequest = mockPullRequest({ 365 | response: { 366 | ...pullMessagesResponse, 367 | result: { 368 | messages: [ 369 | ...pullMessagesResponse.result.messages, 370 | { 371 | body: "body-2", 372 | id: "124", 373 | timestamp_ms: 1234567890, 374 | attempts: 1, 375 | lease_id: "lease-id-2", 376 | metadata: { 377 | "CF-sourceMessageSource": "test", 378 | "CF-Content-Type": "text", 379 | }, 380 | }, 381 | { 382 | body: "body-3", 383 | id: "125", 384 | timestamp_ms: 1234567890, 385 | attempts: 1, 386 | lease_id: "lease-id-3", 387 | metadata: { 388 | "CF-sourceMessageSource": "test", 389 | "CF-Content-Type": "text", 390 | }, 391 | }, 392 | ], 393 | }, 394 | }, 395 | }); 396 | mockAckRequest({}); 397 | 398 | consumer.start(); 399 | await pEvent(consumer, "message_received"); 400 | consumer.stop(); 401 | 402 | sandbox.assert.callCount(handleMessage, 3); 403 | expect(mockedPullRequest).to.have.been.requested; 404 | }); 405 | 406 | it("fires an emptyQueue event when all messages have been consumed", async () => { 407 | mockPullRequest({ 408 | response: { 409 | ...pullMessagesResponse, 410 | result: { messages: [] }, 411 | }, 412 | }); 413 | mockAckRequest({}); 414 | const handleEmpty = sandbox.stub().returns(null); 415 | 416 | consumer.on("empty", handleEmpty); 417 | 418 | consumer.start(); 419 | await pEvent(consumer, "empty"); 420 | consumer.stop(); 421 | 422 | sandbox.assert.calledOnce(handleEmpty); 423 | }); 424 | 425 | it("prefers handleMessagesBatch over handleMessage when both are set", async () => { 426 | mockPullRequest({}); 427 | mockAckRequest({}); 428 | 429 | consumer = new Consumer({ 430 | accountId: ACCOUNT_ID, 431 | queueId: QUEUE_ID, 432 | handleMessageBatch, 433 | handleMessage, 434 | }); 435 | 436 | consumer.start(); 437 | await pEvent(consumer, "response_processed"); 438 | consumer.stop(); 439 | 440 | sandbox.assert.callCount(handleMessageBatch, 1); 441 | sandbox.assert.callCount(handleMessage, 0); 442 | }); 443 | 444 | it("calls the handleMessagesBatch function when a batch of messages is received", async () => { 445 | mockPullRequest({}); 446 | mockAckRequest({}); 447 | 448 | consumer = new Consumer({ 449 | accountId: ACCOUNT_ID, 450 | queueId: QUEUE_ID, 451 | handleMessageBatch, 452 | }); 453 | 454 | consumer.start(); 455 | await pEvent(consumer, "response_processed"); 456 | consumer.stop(); 457 | 458 | sandbox.assert.callCount(handleMessageBatch, 1); 459 | }); 460 | 461 | it("handles unexpected exceptions thrown by the handler batch function", async () => { 462 | mockPullRequest({}); 463 | mockAckRequest({}); 464 | 465 | consumer = new Consumer({ 466 | accountId: ACCOUNT_ID, 467 | queueId: QUEUE_ID, 468 | handleMessageBatch: () => { 469 | throw new Error("unexpected parsing error"); 470 | }, 471 | }); 472 | 473 | consumer.start(); 474 | const err: any = await pEvent(consumer, "error"); 475 | consumer.stop(); 476 | 477 | assert.ok(err); 478 | assert.equal( 479 | err.message, 480 | "Unexpected message handler failure: unexpected parsing error", 481 | ); 482 | }); 483 | 484 | it("handles non-standard objects thrown by the handler batch function", async () => { 485 | mockPullRequest({}); 486 | mockAckRequest({}); 487 | 488 | class CustomError { 489 | private _message: string; 490 | 491 | constructor(message) { 492 | this._message = message; 493 | } 494 | 495 | get message() { 496 | return this._message; 497 | } 498 | } 499 | 500 | consumer = new Consumer({ 501 | accountId: ACCOUNT_ID, 502 | queueId: QUEUE_ID, 503 | handleMessageBatch: () => { 504 | throw new CustomError("unexpected parsing error"); 505 | }, 506 | batchSize: 2, 507 | }); 508 | 509 | consumer.start(); 510 | const err: any = await pEvent(consumer, "error"); 511 | consumer.stop(); 512 | 513 | assert.ok(err); 514 | assert.equal(err.message, "unexpected parsing error"); 515 | }); 516 | 517 | it("handles non-standard exceptions thrown by the handler batch function", async () => { 518 | mockPullRequest({}); 519 | mockAckRequest({}); 520 | 521 | const customError = new Error(); 522 | Object.defineProperty(customError, "message", { 523 | get: () => "unexpected parsing error", 524 | }); 525 | 526 | consumer = new Consumer({ 527 | accountId: ACCOUNT_ID, 528 | queueId: QUEUE_ID, 529 | handleMessageBatch: () => { 530 | throw customError; 531 | }, 532 | batchSize: 2, 533 | }); 534 | 535 | consumer.start(); 536 | const err: any = await pEvent(consumer, "error"); 537 | consumer.stop(); 538 | 539 | assert.ok(err); 540 | assert.equal( 541 | err.message, 542 | "Unexpected message handler failure: unexpected parsing error", 543 | ); 544 | }); 545 | 546 | it.skip("acknowledges the message if handleMessage returns void", async () => { 547 | mockPullRequest({}); 548 | const mockedAckRequest = mockAckRequest({}); 549 | 550 | consumer = new Consumer({ 551 | accountId: ACCOUNT_ID, 552 | queueId: QUEUE_ID, 553 | handleMessage: async () => {}, 554 | }); 555 | 556 | consumer.start(); 557 | await pEvent(consumer, "message_processed"); 558 | consumer.stop(); 559 | 560 | expect(mockedAckRequest).to.have.been.requestedWith({ 561 | acks: [pullMessagesResponse.result.messages[0].id], 562 | retries: [], 563 | }); 564 | }); 565 | 566 | it.skip("acknowledges the message if handleMessage returns a message with the same ID", async () => { 567 | mockPullRequest({}); 568 | const mockedAckRequest = mockAckRequest({}); 569 | 570 | consumer = new Consumer({ 571 | accountId: ACCOUNT_ID, 572 | queueId: QUEUE_ID, 573 | handleMessage: async () => { 574 | return { 575 | MessageId: pullMessagesResponse.result.messages[0].id, 576 | }; 577 | }, 578 | }); 579 | 580 | consumer.start(); 581 | await pEvent(consumer, "message_processed"); 582 | consumer.stop(); 583 | 584 | expect(mockedAckRequest).to.have.been.requestedWith({ 585 | acks: [pullMessagesResponse.result.messages[0].id], 586 | retries: [], 587 | }); 588 | }); 589 | 590 | it("does not acknowledge the message if handleMessage returns an empty object", async () => { 591 | mockPullRequest({}); 592 | const mockedAckRequest = mockAckRequest({}); 593 | 594 | consumer = new Consumer({ 595 | accountId: ACCOUNT_ID, 596 | queueId: QUEUE_ID, 597 | handleMessage: async () => { 598 | return {}; 599 | }, 600 | }); 601 | 602 | consumer.start(); 603 | await pEvent(consumer, "response_processed"); 604 | consumer.stop(); 605 | 606 | expect(mockedAckRequest).to.not.have.been.requested; 607 | }); 608 | 609 | it("does not acknowledge the message if handleMessage returns a different ID", async () => { 610 | mockPullRequest({}); 611 | const mockedAckRequest = mockAckRequest({}); 612 | 613 | consumer = new Consumer({ 614 | accountId: ACCOUNT_ID, 615 | queueId: QUEUE_ID, 616 | handleMessage: async () => { 617 | return { 618 | MessageId: "143", 619 | }; 620 | }, 621 | }); 622 | 623 | consumer.start(); 624 | await pEvent(consumer, "response_processed"); 625 | consumer.stop(); 626 | 627 | expect(mockedAckRequest).to.not.have.been.requested; 628 | }); 629 | 630 | it("acknowledges the message if alwaysAcknowledge is `true` and handleMessage returns an empty object", async () => { 631 | mockPullRequest({}); 632 | const mockedAckRequest = mockAckRequest({}); 633 | 634 | consumer = new Consumer({ 635 | accountId: ACCOUNT_ID, 636 | queueId: QUEUE_ID, 637 | handleMessage: async () => { 638 | return {}; 639 | }, 640 | alwaysAcknowledge: true, 641 | }); 642 | 643 | consumer.start(); 644 | await pEvent(consumer, "response_processed"); 645 | consumer.stop(); 646 | 647 | expect(mockedAckRequest).to.have.been.requestedWith({ 648 | acks: [pullMessagesResponse.result.messages[0].id], 649 | retries: [], 650 | }); 651 | }); 652 | 653 | it("does not acknowledge if handleMessagesBatch returns an empty array", async () => { 654 | mockPullRequest({}); 655 | const mockedAckRequest = mockAckRequest({}); 656 | 657 | consumer = new Consumer({ 658 | accountId: ACCOUNT_ID, 659 | queueId: QUEUE_ID, 660 | handleMessageBatch: async () => [], 661 | batchSize: 2, 662 | }); 663 | 664 | consumer.start(); 665 | await pEvent(consumer, "response_processed"); 666 | consumer.stop(); 667 | 668 | expect(mockedAckRequest).to.not.have.been.requested; 669 | }); 670 | 671 | // TODO: Batch needs more messages to properly validate these 672 | 673 | it.skip("acknowledges the messages if alwaysAcknowledge is `true` and handleMessagesBatch returns an empty array", async () => { 674 | mockPullRequest({}); 675 | const mockedAckRequest = mockAckRequest({}); 676 | 677 | consumer = new Consumer({ 678 | accountId: ACCOUNT_ID, 679 | queueId: QUEUE_ID, 680 | handleMessageBatch: async () => [], 681 | batchSize: 2, 682 | alwaysAcknowledge: true, 683 | }); 684 | 685 | consumer.start(); 686 | await pEvent(consumer, "response_processed"); 687 | consumer.stop(); 688 | 689 | expect(mockedAckRequest).to.have.been.requestedWith({ 690 | acks: [pullMessagesResponse.result.messages[0].id], 691 | retries: [], 692 | }); 693 | }); 694 | 695 | it.skip("acknowledges all messages if handleMessageBatch returns void", async () => { 696 | mockPullRequest({}); 697 | const mockedAckRequest = mockAckRequest({}); 698 | 699 | consumer = new Consumer({ 700 | accountId: ACCOUNT_ID, 701 | queueId: QUEUE_ID, 702 | handleMessageBatch: async () => {}, 703 | batchSize: 2, 704 | }); 705 | 706 | consumer.start(); 707 | await pEvent(consumer, "response_processed"); 708 | consumer.stop(); 709 | 710 | expect(mockedAckRequest).to.have.been.requestedWith({ 711 | acks: [pullMessagesResponse.result.messages[0].id], 712 | retries: [], 713 | }); 714 | }); 715 | 716 | it.skip("acknowledges only returned messages if handleMessagesBatch returns an array", async () => { 717 | mockPullRequest({}); 718 | const mockedAckRequest = mockAckRequest({}); 719 | 720 | consumer = new Consumer({ 721 | accountId: ACCOUNT_ID, 722 | queueId: QUEUE_ID, 723 | handleMessageBatch: async () => [ 724 | { MessageId: "123", ReceiptHandle: "receipt-handle" }, 725 | ], 726 | batchSize: 2, 727 | }); 728 | 729 | consumer.start(); 730 | await pEvent(consumer, "response_processed"); 731 | consumer.stop(); 732 | 733 | expect(mockedAckRequest).to.have.been.requestedWith({ 734 | acks: [pullMessagesResponse.result.messages[0].id], 735 | retries: [], 736 | }); 737 | }); 738 | 739 | // TODO: End requirement for more messages 740 | 741 | it.skip("it retries the message on error", async () => {}); 742 | 743 | it.skip("it retries the message on error with a custom retryMessageDelay", async () => {}); 744 | 745 | it.skip("it retries multiple messages on error", async () => {}); 746 | 747 | // TODO: There are a few error cases to handle here, also test batch and non batch 748 | // TODO: Errors from the pull and ack requests 749 | // TODO: Errors from the handler 750 | // TODO: Non standard errors from handler 751 | // TODO: Non staandard exceptions from handler 752 | // TODO: Timeout errors 753 | // TODO: processing_error 754 | it.skip("it emits an error event when an error occurs", async () => {}); 755 | 756 | it.skip("fires a message_received event when a message is received", async () => { 757 | mockPullRequest({}); 758 | mockAckRequest({}); 759 | 760 | const handleMessageReceived = sandbox.stub().returns(null); 761 | consumer.on("message_received", handleMessageReceived); 762 | 763 | consumer.start(); 764 | await pEvent(consumer, "message_received"); 765 | consumer.stop(); 766 | 767 | sandbox.assert.calledWith( 768 | handleMessageReceived, 769 | pullMessagesResponse.result.messages[0], 770 | ); 771 | }); 772 | 773 | it("fires a message_processed event when a message is processed", async () => { 774 | mockPullRequest({}); 775 | mockAckRequest({}); 776 | 777 | const handleMessageProcessed = sandbox.stub().returns(null); 778 | consumer.on("message_processed", handleMessageProcessed); 779 | 780 | consumer.start(); 781 | await pEvent(consumer, "message_processed"); 782 | consumer.stop(); 783 | 784 | sandbox.assert.calledWith( 785 | handleMessageProcessed, 786 | pullMessagesResponse.result.messages[0], 787 | ); 788 | }); 789 | 790 | it.skip("Waits before re polling when an authentication error occurs", async () => {}); 791 | 792 | it.skip("Waits before re polling when a 403 error occurs", async () => {}); 793 | 794 | it.skip("Wait before re polling when a polling timeout is set", async () => {}); 795 | }); 796 | 797 | describe.skip(".stop", () => { 798 | beforeEach(() => { 799 | mockPullRequest({}); 800 | mockAckRequest({}); 801 | }); 802 | 803 | it("stops the consumer polling for messages", async () => { 804 | const handleStop = sandbox.stub().returns(null); 805 | 806 | consumer.on("stopped", handleStop); 807 | 808 | consumer.start(); 809 | consumer.stop(); 810 | 811 | await clock.runAllAsync(); 812 | 813 | sandbox.assert.calledOnce(handleStop); 814 | sandbox.assert.calledOnce(handleMessage); 815 | }); 816 | 817 | it("clears the polling timeout when stopped", async () => { 818 | sinon.spy(clock, "clearTimeout"); 819 | 820 | consumer.start(); 821 | await clock.tickAsync(0); 822 | consumer.stop(); 823 | 824 | await clock.runAllAsync(); 825 | 826 | sinon.assert.calledTwice(clock.clearTimeout); 827 | }); 828 | 829 | it("fires a stopped event only once when stopped multiple times", async () => { 830 | const handleStop = sandbox.stub().returns(null); 831 | 832 | consumer.on("stopped", handleStop); 833 | 834 | consumer.start(); 835 | consumer.stop(); 836 | consumer.stop(); 837 | consumer.stop(); 838 | await clock.runAllAsync(); 839 | 840 | sandbox.assert.calledOnce(handleStop); 841 | }); 842 | 843 | it("fires a stopped event a second time if started and stopped twice", async () => { 844 | const handleStop = sandbox.stub().returns(null); 845 | 846 | consumer.on("stopped", handleStop); 847 | 848 | consumer.start(); 849 | consumer.stop(); 850 | consumer.start(); 851 | consumer.stop(); 852 | await clock.runAllAsync(); 853 | 854 | sandbox.assert.calledTwice(handleStop); 855 | }); 856 | 857 | it("aborts requests when the abort param is true", async () => { 858 | const handleStop = sandbox.stub().returns(null); 859 | const handleAbort = sandbox.stub().returns(null); 860 | 861 | consumer.on("stopped", handleStop); 862 | consumer.on("aborted", handleAbort); 863 | 864 | consumer.start(); 865 | consumer.stop({ abort: true }); 866 | 867 | await clock.runAllAsync(); 868 | 869 | assert.isTrue(consumer.abortController.signal.aborted); 870 | sandbox.assert.calledOnce(handleMessage); 871 | sandbox.assert.calledOnce(handleAbort); 872 | sandbox.assert.calledOnce(handleStop); 873 | }); 874 | }); 875 | 876 | describe(".status", () => { 877 | beforeEach(() => { 878 | mockPullRequest({}); 879 | mockAckRequest({}); 880 | }); 881 | 882 | it("returns the defaults before the consumer is started", () => { 883 | assert.isFalse(consumer.status.isRunning); 884 | assert.isFalse(consumer.status.isPolling); 885 | }); 886 | 887 | it("returns true for `isRunning` if the consumer has not been stopped", async () => { 888 | consumer.start(); 889 | assert.isTrue(consumer.status.isRunning); 890 | await pEvent(consumer, "message_processed"); 891 | consumer.stop(); 892 | }); 893 | 894 | it("returns false for `isRunning` if the consumer has been stopped", async () => { 895 | consumer.start(); 896 | await pEvent(consumer, "message_processed"); 897 | consumer.stop(); 898 | assert.isFalse(consumer.status.isRunning); 899 | }); 900 | 901 | it.skip("returns true for `isPolling` if the consumer is polling for messages", async () => { 902 | consumer = new Consumer({ 903 | accountId: ACCOUNT_ID, 904 | queueId: QUEUE_ID, 905 | handleMessage: () => new Promise((resolve) => setTimeout(resolve, 20)), 906 | }); 907 | 908 | consumer.start(); 909 | await Promise.all([clock.tickAsync(1)]); 910 | assert.isTrue(consumer.status.isPolling); 911 | consumer.stop(); 912 | assert.isTrue(consumer.status.isPolling); 913 | await Promise.all([clock.tickAsync(21)]); 914 | assert.isFalse(consumer.status.isPolling); 915 | }); 916 | }); 917 | 918 | describe.skip(".updateOption", () => { 919 | beforeEach(() => { 920 | mockPullRequest({}); 921 | mockAckRequest({}); 922 | }); 923 | 924 | it("updates the visibilityTimeoutMs option and emits an event", () => { 925 | const optionUpdatedListener = sandbox.stub(); 926 | consumer.on("option_updated", optionUpdatedListener); 927 | 928 | consumer.updateOption("visibilityTimeoutMs", 45); 929 | 930 | assert.equal(consumer.visibilityTimeoutMs, 45); 931 | 932 | sandbox.assert.calledWithMatch( 933 | optionUpdatedListener, 934 | "visibilityTimeoutMs", 935 | 45, 936 | ); 937 | }); 938 | 939 | it("does not update the visibilityTimeoutMs if the value is more than 43200000", () => { 940 | const optionUpdatedListener = sandbox.stub(); 941 | consumer.on("option_updated", optionUpdatedListener); 942 | 943 | assert.throws(() => { 944 | consumer.updateOption("visibilityTimeoutMs", 43200000); 945 | }, "visibilityTimeoutMs must be less than 43200000"); 946 | 947 | assert.equal(consumer.visibilityTimeoutMs, 1); 948 | 949 | sandbox.assert.notCalled(optionUpdatedListener); 950 | }); 951 | 952 | it("updates the batchSize option and emits an event", () => { 953 | const optionUpdatedListener = sandbox.stub(); 954 | consumer.on("option_updated", optionUpdatedListener); 955 | 956 | consumer.updateOption("batchSize", 45); 957 | 958 | assert.equal(consumer.batchSize, 45); 959 | 960 | sandbox.assert.calledWithMatch(optionUpdatedListener, "batchSize", 45); 961 | }); 962 | 963 | it("does not update the batchSize if the value is less than 1", () => { 964 | const optionUpdatedListener = sandbox.stub(); 965 | consumer.on("option_updated", optionUpdatedListener); 966 | 967 | assert.throws(() => { 968 | consumer.updateOption("batchSize", 0); 969 | }, "batchSize must be between 1 and 100"); 970 | 971 | assert.equal(consumer.batchSize, 1); 972 | 973 | sandbox.assert.notCalled(optionUpdatedListener); 974 | }); 975 | 976 | it("does not update the batchSize if the value is more than 100", () => { 977 | const optionUpdatedListener = sandbox.stub(); 978 | consumer.on("option_updated", optionUpdatedListener); 979 | 980 | assert.throws(() => { 981 | consumer.updateOption("batchSize", 101); 982 | }, "batchSize must be between 1 and 100"); 983 | 984 | assert.equal(consumer.batchSize, 1); 985 | 986 | sandbox.assert.notCalled(optionUpdatedListener); 987 | }); 988 | 989 | it("updates the retryMessageDelay option and emits an event", () => { 990 | const optionUpdatedListener = sandbox.stub(); 991 | consumer.on("option_updated", optionUpdatedListener); 992 | 993 | consumer.updateOption("retryMessageDelay", 45); 994 | 995 | assert.equal(consumer.retryMessageDelay, 45); 996 | 997 | sandbox.assert.calledWithMatch( 998 | optionUpdatedListener, 999 | "retryMessageDelay", 1000 | 45, 1001 | ); 1002 | }); 1003 | 1004 | it("does not update the retryMessageDelay if the value is more than 42300", () => { 1005 | const optionUpdatedListener = sandbox.stub(); 1006 | consumer.on("option_updated", optionUpdatedListener); 1007 | 1008 | assert.throws(() => { 1009 | consumer.updateOption("retryMessageDelay", 42300); 1010 | }, "retryMessageDelay must be less than 42300"); 1011 | 1012 | assert.equal(consumer.retryMessageDelay, 1); 1013 | 1014 | sandbox.assert.notCalled(optionUpdatedListener); 1015 | }); 1016 | 1017 | it("updates the pollingWaitTimeMs option and emits an event", () => { 1018 | const optionUpdatedListener = sandbox.stub(); 1019 | consumer.on("option_updated", optionUpdatedListener); 1020 | 1021 | consumer.updateOption("pollingWaitTimeMs", 45); 1022 | 1023 | assert.equal(consumer.pollingWaitTimeMs, 45); 1024 | 1025 | sandbox.assert.calledWithMatch( 1026 | optionUpdatedListener, 1027 | "pollingWaitTimeMs", 1028 | 45, 1029 | ); 1030 | }); 1031 | 1032 | it("does not update the pollingWaitTimeMs if the value is less than 0", () => { 1033 | const optionUpdatedListener = sandbox.stub(); 1034 | consumer.on("option_updated", optionUpdatedListener); 1035 | 1036 | assert.throws(() => { 1037 | consumer.updateOption("pollingWaitTimeMs", -1); 1038 | }, "pollingWaitTimeMs must be greater than 0."); 1039 | 1040 | assert.equal(consumer.pollingWaitTimeMs, 1); 1041 | 1042 | sandbox.assert.notCalled(optionUpdatedListener); 1043 | }); 1044 | 1045 | it("throws an error for an unknown option", () => { 1046 | consumer = new Consumer({ 1047 | queueId: QUEUE_ID, 1048 | accountId: ACCOUNT_ID, 1049 | handleMessage, 1050 | visibilityTimeout: 60, 1051 | }); 1052 | 1053 | assert.throws(() => { 1054 | consumer.updateOption("unknown", "value"); 1055 | }, `The update unknown cannot be updated`); 1056 | }); 1057 | }); 1058 | 1059 | describe("Aborting", () => { 1060 | beforeEach(() => { 1061 | mockPullRequest({}); 1062 | mockAckRequest({}); 1063 | }); 1064 | 1065 | it.skip('aborts the request when the consumer is stopped with the "abort" option', async () => {}); 1066 | 1067 | it.skip("aborts the request with the correct handler", async () => {}); 1068 | }); 1069 | 1070 | describe("Event Listeners", () => { 1071 | beforeEach(() => { 1072 | mockPullRequest({}); 1073 | mockAckRequest({}); 1074 | }); 1075 | 1076 | it.skip("fires the event multiple times", async () => {}); 1077 | 1078 | it.skip("fires the events only once", async () => {}); 1079 | }); 1080 | 1081 | describe("Logger", () => { 1082 | beforeEach(() => { 1083 | mockPullRequest({}); 1084 | mockAckRequest({}); 1085 | }); 1086 | 1087 | it("logs a debug event when an event is emitted", async () => { 1088 | const loggerDebug = sandbox.stub(logger, "debug"); 1089 | 1090 | consumer.start(); 1091 | await pEvent(consumer, "message_processed"); 1092 | consumer.stop(); 1093 | 1094 | sandbox.assert.callCount(loggerDebug, 9); 1095 | sandbox.assert.calledWithMatch(loggerDebug, "starting"); 1096 | sandbox.assert.calledWithMatch(loggerDebug, "started"); 1097 | sandbox.assert.calledWithMatch(loggerDebug, "polling"); 1098 | sandbox.assert.calledWithMatch(loggerDebug, "stopping"); 1099 | sandbox.assert.calledWithMatch(loggerDebug, "stopped"); 1100 | }); 1101 | }); 1102 | }); 1103 | -------------------------------------------------------------------------------- /test/unit/lib/fetch.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it } from "node:test"; 2 | import { assert } from "chai"; 3 | 4 | import { throwErrorIfResponseNotOk } from "../../../src/lib/fetch"; 5 | import { ProviderError } from "../../../src/utils/errors"; 6 | 7 | describe("throwErrorIfResponseNotOk", () => { 8 | it("should not throw an error if the response is OK", () => { 9 | const mockResponse = { 10 | ok: true, 11 | status: 200, 12 | statusText: "OK", 13 | url: "http://example.com", 14 | }; 15 | 16 | // @ts-expect-error 17 | assert.doesNotThrow(() => throwErrorIfResponseNotOk(mockResponse)); 18 | }); 19 | 20 | it("should throw a ProviderError if the response is not OK and has a status", () => { 21 | const mockResponse = { 22 | ok: false, 23 | status: 404, 24 | statusText: "Not Found", 25 | url: "http://example.com", 26 | }; 27 | 28 | try { 29 | // @ts-expect-error 30 | throwErrorIfResponseNotOk(mockResponse); 31 | assert.fail("Expected error to be thrown"); 32 | } catch (error) { 33 | assert.instanceOf(error, ProviderError); 34 | assert.equal(error.message, "[404 - Not Found] http://example.com"); 35 | assert.equal(error.code, "ResponseNotOk"); 36 | } 37 | }); 38 | 39 | it("should throw a ProviderError if the response is not OK and does not have a status", () => { 40 | const mockResponse = { 41 | ok: false, 42 | status: undefined, 43 | statusText: "Unknown Error", 44 | url: "http://example.com", 45 | }; 46 | 47 | try { 48 | // @ts-expect-error 49 | throwErrorIfResponseNotOk(mockResponse); 50 | assert.fail("Expected error to be thrown"); 51 | } catch (error) { 52 | assert.instanceOf(error, ProviderError); 53 | assert.equal(error.message, "[Unknown Error] http://example.com"); 54 | assert.equal(error.code, "ResponseNotOk"); 55 | } 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /test/unit/utils/emitter.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, beforeEach, afterEach } from "node:test"; 2 | import { assert } from "chai"; 3 | import sinon from "sinon"; 4 | 5 | import { TypedEventEmitter } from "../../../src/utils/emitter"; 6 | import { logger } from "../../../src/utils/logger"; 7 | 8 | describe("TypedEventEmitter", () => { 9 | let emitter: TypedEventEmitter; 10 | let loggerStub: sinon.SinonStub; 11 | 12 | beforeEach(() => { 13 | emitter = new TypedEventEmitter(); 14 | loggerStub = sinon.stub(logger, "debug"); 15 | }); 16 | 17 | afterEach(() => { 18 | loggerStub.restore(); 19 | }); 20 | 21 | it("should log and emit an event with the emit method", () => { 22 | emitter.emit("empty"); 23 | 24 | assert.isTrue(loggerStub.calledOnce); 25 | assert.isTrue(loggerStub.calledWith("empty")); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /test/unit/utils/errors.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it } from "node:test"; 2 | import { assert } from "chai"; 3 | 4 | import { 5 | ProviderError, 6 | toProviderError, 7 | StandardError, 8 | toStandardError, 9 | TimeoutError, 10 | toTimeoutError, 11 | } from "../../../src/utils/errors"; 12 | 13 | describe("Errors", () => { 14 | describe("ProviderError", () => { 15 | it("should create a ProviderError with the correct properties", () => { 16 | const message = "Provider error occurred"; 17 | const error = new ProviderError(message); 18 | 19 | assert.instanceOf(error, ProviderError); 20 | assert.equal(error.message, message); 21 | assert.equal(error.name, "ProviderError"); 22 | }); 23 | }); 24 | 25 | describe("toProviderError", () => { 26 | it("should format an error to ProviderError with response", () => { 27 | const message = "Formatted provider error"; 28 | const mockResponse = { 29 | status: 404, 30 | statusText: "Not Found", 31 | url: "http://example.com", 32 | }; 33 | const err = { 34 | code: "ERR_CODE", 35 | response: mockResponse, 36 | stack: "Error stack", 37 | }; 38 | 39 | // @ts-expect-error 40 | const error = toProviderError(err, message); 41 | 42 | assert.instanceOf(error, ProviderError); 43 | assert.equal(error.message, message); 44 | assert.equal(error.code, err.code); 45 | assert.equal(error.stack, err.stack); 46 | assert.equal(error.status, mockResponse.status); 47 | assert.equal(error.statusText, mockResponse.statusText); 48 | assert.equal(error.url, mockResponse.url); 49 | }); 50 | 51 | it("should format an error to ProviderError without response", () => { 52 | const message = "Formatted provider error"; 53 | const err = { 54 | code: "ERR_CODE", 55 | stack: "Error stack", 56 | }; 57 | 58 | // @ts-expect-error 59 | const error = toProviderError(err, message); 60 | 61 | assert.instanceOf(error, ProviderError); 62 | assert.equal(error.message, message); 63 | assert.equal(error.code, err.code); 64 | assert.equal(error.stack, err.stack); 65 | assert.isUndefined(error.status); 66 | assert.isUndefined(error.statusText); 67 | assert.isUndefined(error.url); 68 | }); 69 | }); 70 | 71 | describe("StandardError", () => { 72 | it("should create a StandardError with the correct properties", () => { 73 | const message = "Standard error occurred"; 74 | const error = new StandardError(message); 75 | 76 | assert.instanceOf(error, StandardError); 77 | assert.equal(error.message, message); 78 | assert.equal(error.name, "StandardError"); 79 | }); 80 | }); 81 | 82 | describe("toStandardError", () => { 83 | it("should format an error to StandardError", () => { 84 | const message = "Formatted standard error"; 85 | const err = new Error("Original error"); 86 | 87 | const error = toStandardError(err, message); 88 | 89 | assert.instanceOf(error, StandardError); 90 | assert.equal(error.message, message); 91 | assert.equal(error.cause, err); 92 | }); 93 | }); 94 | 95 | describe("TimeoutError", () => { 96 | it("should create a TimeoutError with the correct properties", () => { 97 | const message = "Timeout error occurred"; 98 | const error = new TimeoutError(message); 99 | 100 | assert.instanceOf(error, TimeoutError); 101 | assert.equal(error.message, message); 102 | assert.equal(error.name, "TimeoutError"); 103 | }); 104 | }); 105 | 106 | describe("toTimeoutError", () => { 107 | it("should format an error to TimeoutError", () => { 108 | const message = "Formatted timeout error"; 109 | const err = new TimeoutError("Original timeout error"); 110 | 111 | const error = toTimeoutError(err, message); 112 | 113 | assert.instanceOf(error, TimeoutError); 114 | assert.equal(error.message, message); 115 | assert.equal(error.cause, err); 116 | }); 117 | }); 118 | }); 119 | -------------------------------------------------------------------------------- /test/unit/utils/validation.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it } from "node:test"; 2 | import { assert } from "chai"; 3 | 4 | import { 5 | assertOptions, 6 | validateOption, 7 | hasMessages, 8 | } from "../../../src/utils/validation"; 9 | 10 | describe("Validation Functions", () => { 11 | describe("validateOption", () => { 12 | it("should throw an error if batchSize is not between 1 and 100", () => { 13 | assert.throws( 14 | () => validateOption("batchSize", 0), 15 | "batchSize must be between 1 and 100", 16 | ); 17 | assert.throws( 18 | () => validateOption("batchSize", 101), 19 | "batchSize must be between 1 and 100", 20 | ); 21 | }); 22 | 23 | it("should not throw an error if batchSize is between 1 and 100", () => { 24 | assert.doesNotThrow(() => validateOption("batchSize", 50)); 25 | }); 26 | 27 | it("should throw an error if visibilityTimeoutMs is greater than 43200000", () => { 28 | assert.throws( 29 | () => validateOption("visibilityTimeoutMs", 43200001), 30 | "visibilityTimeoutMs must be less than 43200000", 31 | ); 32 | }); 33 | 34 | it("should not throw an error if visibilityTimeoutMs is less than or equal to 43200000", () => { 35 | assert.doesNotThrow(() => 36 | validateOption("visibilityTimeoutMs", 43200000), 37 | ); 38 | }); 39 | 40 | it("should throw an error if retryMessageDelay is greater than 42300", () => { 41 | assert.throws( 42 | () => validateOption("retryMessageDelay", 42301), 43 | "retryMessageDelay must be less than 42300", 44 | ); 45 | }); 46 | 47 | it("should not throw an error if retryMessageDelay is less than or equal to 42300", () => { 48 | assert.doesNotThrow(() => validateOption("retryMessageDelay", 42300)); 49 | }); 50 | 51 | it("should throw an error if pollingWaitTimeMs is less than 0", () => { 52 | assert.throws( 53 | () => validateOption("pollingWaitTimeMs", -1), 54 | "pollingWaitTimeMs must be greater than 0.", 55 | ); 56 | }); 57 | 58 | it("should not throw an error if pollingWaitTimeMs is greater than or equal to 0", () => { 59 | assert.doesNotThrow(() => validateOption("pollingWaitTimeMs", 0)); 60 | }); 61 | 62 | it("should throw an error for unknown options if strict is true", () => { 63 | assert.throws( 64 | () => validateOption("unknownOption", 0, true), 65 | "The update unknownOption cannot be updated", 66 | ); 67 | }); 68 | 69 | it("should not throw an error for unknown options if strict is false", () => { 70 | assert.doesNotThrow(() => validateOption("unknownOption", 0, false)); 71 | }); 72 | }); 73 | 74 | describe("assertOptions", () => { 75 | it("should throw an error if required options are missing", () => { 76 | const options = { 77 | accountId: "accountId", 78 | queueId: "queueId", 79 | }; 80 | 81 | assert.throws( 82 | () => assertOptions(options), 83 | "Missing consumer option [ handleMessage or handleMessageBatch ].", 84 | ); 85 | }); 86 | 87 | it("should not throw an error if required options are present", () => { 88 | const options = { 89 | accountId: "accountId", 90 | queueId: "queueId", 91 | handleMessage: () => {}, 92 | }; 93 | 94 | // @ts-expect-error 95 | assert.doesNotThrow(() => assertOptions(options)); 96 | }); 97 | 98 | it("should call validateOption for batchSize, visibilityTimeoutMs, and retryMessageDelay", () => { 99 | const options = { 100 | accountId: "accountId", 101 | queueId: "queueId", 102 | handleMessage: () => {}, 103 | batchSize: 50, 104 | visibilityTimeoutMs: 43200000, 105 | retryMessageDelay: 42300, 106 | }; 107 | 108 | // @ts-expect-error 109 | assert.doesNotThrow(() => assertOptions(options)); 110 | }); 111 | }); 112 | 113 | describe("hasMessages", () => { 114 | it("should return true if response has messages", () => { 115 | const response = { 116 | result: { 117 | messages: [{ id: "1" }], 118 | }, 119 | }; 120 | 121 | // @ts-expect-error 122 | assert.isTrue(hasMessages(response)); 123 | }); 124 | 125 | it("should return false if response does not have messages", () => { 126 | const response = { 127 | result: { 128 | messages: [], 129 | }, 130 | }; 131 | 132 | // @ts-expect-error 133 | assert.isFalse(hasMessages(response)); 134 | }); 135 | 136 | it("should return false if response is undefined", () => { 137 | const response = undefined; 138 | 139 | // @ts-expect-error 140 | assert.isFalse(hasMessages(response)); 141 | }); 142 | }); 143 | }); 144 | -------------------------------------------------------------------------------- /tsconfig.cjs.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./dist/cjs", 5 | "module": "commonjs", 6 | "moduleResolution": "node", 7 | "noEmit": false 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.esm.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./dist/esm", 5 | "module": "Node16", 6 | "moduleResolution": "Node16", 7 | "noEmit": false 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ES2021"], 4 | "target": "ES2021", 5 | "module": "Node16", 6 | "moduleResolution": "Node16", 7 | "noEmit": true, 8 | "sourceMap": false, 9 | "allowJs": false, 10 | "noUnusedLocals": true, 11 | "declaration": true, 12 | "declarationDir": "dist/types" 13 | }, 14 | "include": [ 15 | "src/*", 16 | "src/utils/validation.ts", 17 | "src/utils/errors.ts", 18 | "src/utils/emitter.ts", 19 | "src/utils/logger.ts", 20 | "src/lib/consumer.ts" 21 | ], 22 | "exclude": ["node_modules", "dist"] 23 | } 24 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://typedoc.org/schema.json", 3 | "sort": ["kind", "instance-first", "required-first", "alphabetical"], 4 | "treatWarningsAsErrors": false, 5 | "searchInComments": true, 6 | "entryPoints": ["./src/index.ts"], 7 | "out": "public", 8 | "name": "Cloudflare Queue Consumer", 9 | "hideGenerator": true, 10 | "navigationLinks": { 11 | "GitHub": "https://github.com/bbc/cloudflare-queue-consumer" 12 | } 13 | } 14 | --------------------------------------------------------------------------------