├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ ├── config.yml │ ├── documentation.yml │ └── feature-request.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── ci.yml │ ├── closed-issue-message.yml │ ├── handle-stale-discussions.yml │ └── stale_issues.yml ├── .gitignore ├── .gitmodules ├── .rspec ├── .rubocop.yml ├── .yardopts ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE ├── NOTICE ├── README.md ├── Rakefile ├── VERSION ├── aws-activejob-sqs.gemspec ├── bin └── aws_active_job_sqs ├── doc-src └── templates │ └── default │ └── layout │ └── html │ ├── footer.erb │ └── layout.erb ├── lib ├── active_job │ └── queue_adapters │ │ ├── sqs_adapter.rb │ │ ├── sqs_adapter │ │ └── params.rb │ │ └── sqs_async_adapter.rb ├── aws-activejob-sqs.rb └── aws │ └── active_job │ └── sqs │ ├── cli_options.rb │ ├── configuration.rb │ ├── deduplication.rb │ ├── executor.rb │ ├── job_runner.rb │ ├── lambda_handler.rb │ └── poller.rb └── spec ├── active_job └── queue_adapters │ ├── sqs_adapter │ └── params_spec.rb │ ├── sqs_adapter_spec.rb │ └── sqs_async_adapter_spec.rb ├── aws-activejob-sqs_spec.rb ├── aws └── active_job │ └── sqs │ ├── configuration_spec.rb │ ├── deduplication_spec.rb │ ├── executor_spec.rb │ ├── job_runner_spec.rb │ └── poller_spec.rb ├── dummy ├── config.ru └── config │ ├── application.rb │ └── aws_active_job_sqs.yml └── spec_helper.rb /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "🐛 Bug Report" 3 | description: Report a bug 4 | title: "(short issue description)" 5 | labels: [bug, needs-triage] 6 | assignees: [] 7 | body: 8 | - type: textarea 9 | id: description 10 | attributes: 11 | label: Describe the bug 12 | description: What is the problem? A clear and concise description of the bug. 13 | validations: 14 | required: true 15 | - type: textarea 16 | id: expected 17 | attributes: 18 | label: Expected Behavior 19 | description: | 20 | What did you expect to happen? 21 | validations: 22 | required: true 23 | - type: textarea 24 | id: current 25 | attributes: 26 | label: Current Behavior 27 | description: | 28 | What actually happened? 29 | 30 | Please include full errors, uncaught exceptions, stack traces, and relevant logs. 31 | If service responses are relevant, please include wire logs. 32 | validations: 33 | required: true 34 | - type: textarea 35 | id: reproduction 36 | attributes: 37 | label: Reproduction Steps 38 | description: | 39 | Provide a self-contained, concise snippet of code that can be used to reproduce the issue. 40 | For more complex issues provide a repo with the smallest sample that reproduces the bug. 41 | 42 | Avoid including business logic or unrelated code, it makes diagnosis more difficult. 43 | The code sample should be an SSCCE. See http://sscce.org/ for details. In short, please provide a code sample that we can copy/paste, run and reproduce. 44 | validations: 45 | required: true 46 | - type: textarea 47 | id: solution 48 | attributes: 49 | label: Possible Solution 50 | description: | 51 | Suggest a fix/reason for the bug 52 | validations: 53 | required: false 54 | - type: textarea 55 | id: context 56 | attributes: 57 | label: Additional Information/Context 58 | description: | 59 | Anything else that might be relevant for troubleshooting this bug. Providing context helps us come up with a solution that is most useful in the real world. 60 | validations: 61 | required: false 62 | - type: input 63 | id: sdk-version 64 | attributes: 65 | label: Gem name ('aws-sdk', 'aws-sdk-resources' or service gems like 'aws-sdk-s3') and its version 66 | validations: 67 | required: true 68 | - type: input 69 | id: environment 70 | attributes: 71 | label: Environment details (Version of Ruby, OS environment) 72 | validations: 73 | required: true 74 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: false 3 | contact_links: 4 | - name: 💬 General Question 5 | url: https://github.com/aws/aws-sdk-ruby/discussions/categories/q-a 6 | about: Please ask and answer questions as a discussion thread -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "📕 Documentation Issue" 3 | description: Report an issue in the API Reference documentation or Developer Guide 4 | title: "(short issue description)" 5 | labels: [documentation, needs-triage] 6 | assignees: [] 7 | body: 8 | - type: textarea 9 | id: description 10 | attributes: 11 | label: Describe the issue 12 | description: A clear and concise description of the issue. 13 | validations: 14 | required: true 15 | 16 | - type: textarea 17 | id: links 18 | attributes: 19 | label: Links 20 | description: | 21 | Include links to affected documentation page(s). 22 | validations: 23 | required: true 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature Request 3 | description: Suggest an idea for this project 4 | title: "(short issue description)" 5 | labels: [feature-request, needs-triage] 6 | assignees: [] 7 | body: 8 | - type: textarea 9 | id: description 10 | attributes: 11 | label: Describe the feature 12 | description: A clear and concise description of the feature you are proposing. 13 | validations: 14 | required: true 15 | - type: textarea 16 | id: use-case 17 | attributes: 18 | label: Use Case 19 | description: | 20 | Why do you need this feature? For example: "I'm always frustrated when..." 21 | validations: 22 | required: true 23 | - type: textarea 24 | id: solution 25 | attributes: 26 | label: Proposed Solution 27 | description: | 28 | Suggest how to implement the addition or change. Please include prototype/workaround/sketch/reference implementation. 29 | validations: 30 | required: false 31 | - type: textarea 32 | id: other 33 | attributes: 34 | label: Other Information 35 | description: | 36 | Any alternative solutions or features you considered, a more detailed explanation, stack traces, related issues, links for context, etc. 37 | validations: 38 | required: false 39 | - type: checkboxes 40 | id: ack 41 | attributes: 42 | label: Acknowledgements 43 | options: 44 | - label: I may be able to implement this feature request 45 | required: false 46 | - label: This feature might incur a breaking change 47 | required: false 48 | - type: input 49 | id: sdk-version 50 | attributes: 51 | label: SDK version used 52 | validations: 53 | required: true 54 | - type: input 55 | id: environment 56 | attributes: 57 | label: Environment details (OS name and version, etc.) 58 | validations: 59 | required: true 60 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | By submitting this pull request, I confirm that my contribution is made under 6 | the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | env: 13 | ruby_version: 3.4 14 | 15 | jobs: 16 | test: 17 | runs-on: ubuntu-latest 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | # JRuby-10.0 is expected to fail since Rails 8 is not fully supported until summer 2025 22 | # source: https://blog.jruby.org/2025/04/jruby-10-part-1-whats-new 23 | ruby: [2.7, '3.0', 3.1, 3.2, 3.3, 3.4, jruby-9.4, jruby-10.0] 24 | rails: [7.1, 7.2, '8.0', main] 25 | 26 | exclude: 27 | # Rails 7.2 is Ruby >= 3.1 28 | - rails: 7.2 29 | ruby: 2.7 30 | - rails: 7.2 31 | ruby: 3.0 32 | - rails: 7.2 33 | ruby: jruby-10.0 34 | # Rails 8.0 is Ruby >= 3.2 35 | - rails: '8.0' 36 | ruby: 2.7 37 | - rails: '8.0' 38 | ruby: 3.0 39 | - rails: '8.0' 40 | ruby: 3.1 41 | - rails: '8.0' 42 | ruby: jruby-9.4 43 | # Rails main is Ruby >= 3.2 44 | - rails: main 45 | ruby: 2.7 46 | - rails: main 47 | ruby: 3.0 48 | - rails: main 49 | ruby: 3.1 50 | - rails: main 51 | ruby: jruby-9.4 52 | 53 | steps: 54 | - uses: actions/checkout@v4 55 | 56 | - name: Set RAILS_VERSION 57 | run: echo "RAILS_VERSION=${{ matrix.rails }}" >> $GITHUB_ENV 58 | 59 | - name: Setup Ruby 60 | uses: ruby/setup-ruby@v1 61 | with: 62 | ruby-version: ${{ matrix.ruby }} 63 | bundler-cache: true 64 | 65 | - name: Test 66 | run: bundle exec rake spec 67 | 68 | rubocop: 69 | runs-on: ubuntu-latest 70 | 71 | steps: 72 | - uses: actions/checkout@v4 73 | 74 | - name: Setup Ruby 75 | uses: ruby/setup-ruby@v1 76 | with: 77 | ruby-version: ${{ env.ruby_version }} 78 | 79 | - name: Install gems 80 | run: | 81 | bundle config set --local with 'development' 82 | bundle install 83 | 84 | - name: Rubocop 85 | run: bundle exec rake rubocop 86 | -------------------------------------------------------------------------------- /.github/workflows/closed-issue-message.yml: -------------------------------------------------------------------------------- 1 | name: Closed Issue Message 2 | on: 3 | issues: 4 | types: [closed] 5 | permissions: {} 6 | jobs: 7 | auto_comment: 8 | permissions: 9 | issues: write # to comment on issues 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: aws-actions/closed-issue-message@v1 13 | with: 14 | # These inputs are both required 15 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 16 | message: | 17 | This issue is now closed. Comments on closed issues are hard for our team to see. 18 | If you need more assistance, please open a new issue that references this one. 19 | -------------------------------------------------------------------------------- /.github/workflows/handle-stale-discussions.yml: -------------------------------------------------------------------------------- 1 | name: HandleStaleDiscussions 2 | on: 3 | schedule: 4 | - cron: '0 */4 * * *' 5 | discussion_comment: 6 | types: [created] 7 | 8 | jobs: 9 | handle-stale-discussions: 10 | name: Handle stale discussions 11 | runs-on: ubuntu-latest 12 | permissions: 13 | discussions: write 14 | steps: 15 | - name: Stale discussions action 16 | uses: aws-github-ops/handle-stale-discussions@v1 17 | env: 18 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} -------------------------------------------------------------------------------- /.github/workflows/stale_issues.yml: -------------------------------------------------------------------------------- 1 | name: "Close stale issues" 2 | 3 | # Controls when the action will run. 4 | on: 5 | schedule: 6 | - cron: "0 0 * * *" 7 | 8 | permissions: {} 9 | jobs: 10 | cleanup: 11 | permissions: 12 | issues: write # to label, comment and close issues 13 | pull-requests: write # to label, comment and close pull requests 14 | 15 | runs-on: ubuntu-latest 16 | name: Stale issue job 17 | steps: 18 | - uses: aws-actions/stale-issue-cleanup@v6 19 | with: 20 | # Setting messages to an empty string will cause the automation to skip 21 | # that category 22 | ancient-issue-message: Greetings! We’re closing this issue because it has been open a long time and hasn’t been updated in a while and may not be getting the attention it deserves. We encourage you to check if this is still an issue in the latest release and if you find that this is still a problem, please feel free to comment or open a new issue. 23 | stale-issue-message: This issue has not received a response in 1 week. If you still think there is a problem, please leave a comment to avoid the issue from automatically closing. 24 | stale-pr-message: This PR has not received a response in 1 week. If you still think there is a problem, please leave a comment to avoid the PR from automatically closing. 25 | # These labels are required 26 | stale-issue-label: closing-soon 27 | exempt-issue-label: no-autoclose 28 | stale-pr-label: closing-soon 29 | exempt-pr-label: pr/needs-review 30 | response-requested-label: response-requested 31 | 32 | # Don't set closed-for-staleness label to skip closing very old issues 33 | # regardless of label 34 | closed-for-staleness-label: closed-for-staleness 35 | 36 | # Issue timing 37 | days-before-stale: 10 38 | days-before-close: 4 39 | days-before-ancient: 36500 40 | 41 | # If you don't want to mark a issue as being ancient based on a 42 | # threshold of "upvotes", you can set this here. An "upvote" is 43 | # the total number of +1, heart, hooray, and rocket reactions 44 | # on an issue. 45 | minimum-upvotes-to-exempt: 10 46 | 47 | repo-token: ${{ secrets.GITHUB_TOKEN }} 48 | # loglevel: DEBUG 49 | # Set dry-run to true to not perform label or close actions. 50 | # dry-run: true 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .byebug_history 3 | 4 | Gemfile.lock 5 | *.gem 6 | 7 | coverage/ 8 | .yardoc/ 9 | doc/ 10 | docs.zip 11 | .release/ 12 | 13 | spec/dummy/log/ 14 | spec/dummy/tmp/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tasks/release"] 2 | path = tasks/release 3 | url = git@github.com:aws/aws-sdk-ruby-release-tools.git 4 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | --format documentation -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | NewCops: enable 3 | TargetRubyVersion: 2.7 4 | SuggestExtensions: false 5 | Exclude: 6 | - 'tasks/release/**/*' 7 | 8 | Gemspec/RequireMFA: 9 | Enabled: false 10 | 11 | Naming/FileName: 12 | Exclude: 13 | - 'lib/aws-activejob-sqs.rb' 14 | - 'spec/aws-activejob-sqs_spec.rb' 15 | 16 | Metrics/BlockLength: 17 | Exclude: 18 | - 'spec/**/*.rb' 19 | 20 | Metrics/ClassLength: 21 | Max: 150 22 | 23 | Metrics/MethodLength: 24 | Max: 15 25 | 26 | Metrics/ModuleLength: 27 | Exclude: 28 | - 'spec/**/*' 29 | 30 | Style/BlockComments: 31 | Exclude: 32 | - 'spec/spec_helper.rb' 33 | 34 | Style/Documentation: 35 | Exclude: 36 | - 'spec/**/*' 37 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --title 'ActiveJob SQS' 2 | --template-path doc-src/templates 3 | --plugin sitemap 4 | --hide-api private 5 | --markup markdown -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Unreleased Changes 2 | ------------------ 3 | 4 | 1.0.2 (2025-04-01) 5 | ------------------ 6 | 7 | * Issue - Remove `aws_sqs_active_job` executable. 8 | * Issue - Handle `Time` objects correctly when using `ActiveJob.perform_all_later`. 9 | 10 | 1.0.1 (2024-12-23) 11 | ------------------ 12 | 13 | * Issue - Add deprecated `aws_sqs_active_job` executable to aid in migration. 14 | * Issue - Support legacy `queue: 'url'` config in file to aid in migration. 15 | 16 | 1.0.0 (2024-12-13) 17 | ------------------ 18 | 19 | * Feature - Support polling on multiple queues. (#4) 20 | * Feature - Support running without Rails. (#5) 21 | * Feature - Replace `retry_standard_errors` with `poller_error_handler`. (#6) 22 | * Feature - Support per queue configuration. (#4) 23 | * Feature - Support loading global and queue specific configuration from ENV. (#3) 24 | 25 | 0.1.1 (2024-12-02) 26 | ------------------ 27 | 28 | * Feature - Add lifecycle hooks for Executor. 29 | 30 | 0.1.0 (2024-11-16) 31 | ------------------ 32 | 33 | * Feature - Initial version of this gem. 34 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | gemspec 6 | 7 | gem 'rake', require: false 8 | 9 | case ENV.fetch('RAILS_VERSION', nil) 10 | when '7.1' 11 | gem 'rails', '~> 7.1.0' 12 | when '7.2' 13 | gem 'rails', '~> 7.2.0' 14 | when '8.0' 15 | gem 'rails', '~> 8.0.0' 16 | else 17 | gem 'rails', github: 'rails/rails' 18 | end 19 | 20 | group :development do 21 | gem 'byebug', platforms: :ruby 22 | gem 'rubocop' 23 | end 24 | 25 | group :test do 26 | gem 'rspec' 27 | end 28 | 29 | group :docs do 30 | gem 'yard' 31 | gem 'yard-sitemap', '~> 1.0' 32 | end 33 | 34 | group :release do 35 | gem 'octokit' 36 | end 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ActiveJob with Amazon Simple Queue Service 2 | 3 | [![Gem Version](https://badge.fury.io/rb/aws-activejob-sqs.svg)](https://badge.fury.io/rb/aws-activejob-sqs) 4 | [![Build Status](https://github.com/aws/aws-activejob-sqs-ruby/workflows/CI/badge.svg)](https://github.com/aws/aws-activejob-sqs-ruby/actions) 5 | [![Github forks](https://img.shields.io/github/forks/aws/aws-activejob-sqs-ruby.svg)](https://github.com/aws/aws-activejob-sqs-ruby/network) 6 | [![Github stars](https://img.shields.io/github/stars/aws/aws-activejob-sqs-ruby.svg)](https://github.com/aws/aws-activejob-sqs-ruby/stargazers) 7 | 8 | This gem contains [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) 9 | adapters for Amazon Simple Queue Service (SQS). 10 | 11 | ## Installation 12 | 13 | Add this gem to your Rails project's Gemfile: 14 | 15 | ```ruby 16 | gem 'aws-activejob-sqs', '~> 1' 17 | gem 'aws-sdk-rails', '~> 5' 18 | ``` 19 | 20 | Then run `bundle install`. 21 | 22 | This gem also brings in the following AWS gems: 23 | 24 | * `aws-sdk-sqs` 25 | 26 | You will have to ensure that you provide credentials for the SDK to use. See the 27 | latest [AWS SDK for Ruby Docs](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/index.html#Configuration) 28 | for details. 29 | 30 | If you're running your Rails application on Amazon EC2, the AWS SDK will 31 | check Amazon EC2 instance metadata for credentials to load. Learn more: 32 | [IAM Roles for Amazon EC2](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) 33 | 34 | ## Configuration 35 | 36 | To use SQS as your queuing backend, simply set the `active_job.queue_adapter` 37 | to `:sqs`. 38 | 39 | ```ruby 40 | # config/environments/production.rb 41 | config.active_job.queue_adapter = :sqs 42 | ``` 43 | 44 | To use the non-blocking (async) adapter, set `active_job.queue_adapter` to 45 | `:sqs_async`. If you have a lot of jobs to queue or you need to avoid the extra 46 | latency from an SQS call in your request then consider using the async adapter. 47 | When using the Async adapter, you may want to configure a 48 | `async_queue_error_handler` to handle errors that may occur when queuing jobs. 49 | 50 | ```ruby 51 | # config/environments/production.rb 52 | config.active_job.queue_adapter = :sqs_async 53 | ``` 54 | 55 | You can also set the adapter for a single job: 56 | 57 | ```ruby 58 | class YourJob < ApplicationJob 59 | self.queue_adapter = :sqs 60 | #.... 61 | end 62 | ``` 63 | 64 | You also need to configure a mapping of ActiveJob queue names to SQS Queue URLs: 65 | 66 | ```yaml 67 | # config/aws_active_job_sqs.yml 68 | backpressure: 5 # configure global options for poller 69 | max_messages: 3 70 | queues: 71 | default: 72 | url: 'https://my-queue-url.amazon.aws' 73 | max_messages: 2 # queue specific values override global values 74 | ``` 75 | 76 | For a complete list of configuration options see the 77 | [Aws::ActiveJob::SQS::Configuration](https://docs.aws.amazon.com/sdk-for-ruby/aws-activejob-sqs/api/Aws/ActiveJob/SQS/Configuration.html) 78 | documentation. 79 | 80 | You can configure SQS Active Job either through the environment, yaml file or 81 | through code in your `config/.rb` or initializers. 82 | 83 | For file based configuration, you can use either 84 | `config/aws_active_job_sqs/.yml` or `config/aws_active_job_sqs.yml`. 85 | You may specify the file used through the `:config_file` option in code or the 86 | `AWS_ACTIVE_JOB_SQS_CONFIG_FILE` environment variable. 87 | The yaml files support ERB. 88 | 89 | To configure in code: 90 | 91 | ```ruby 92 | Aws::ActiveJob::SQS.configure do |config| 93 | config.logger = Rails.logger 94 | config.max_messages = 5 95 | config.client = Aws::SQS::Client.new(region: 'us-east-1') 96 | end 97 | ``` 98 | 99 | SQS Active Job loads global and queue specific values from your 100 | environment. Global keys take the form of: 101 | `AWS_ACTIVE_JOB_SQS_` and queue specific keys take the 102 | form of: `AWS_ACTIVE_JOB_SQS__`. 103 | is case-insensitive and is always down cased. Configuring 104 | non-snake case queues (containing upper case) through ENV is 105 | not supported. 106 | 107 | Example: 108 | 109 | ```shell 110 | export AWS_ACTIVE_JOB_SQS_MAX_MESSAGES = 5 111 | export AWS_ACTIVE_JOB_SQS_DEFAULT_URL = https://my-queue.aws 112 | ``` 113 | 114 | ## Usage 115 | 116 | To queue a job, you can just use standard ActiveJob methods: 117 | 118 | ```ruby 119 | # To queue for immediate processing 120 | YourJob.perform_later(args) 121 | 122 | # or to schedule a job for a future time: 123 | YourJob.set(wait: 1.minute).perform_later(args) 124 | ``` 125 | 126 | **Note**: Due to limitations in SQS, you cannot schedule jobs for 127 | later than 15 minutes in the future. 128 | 129 | ### Retry Behavior and Handling Errors 130 | 131 | See the Rails ActiveJob Guide on 132 | [Exceptions](https://guides.rubyonrails.org/active_job_basics.html#exceptions) 133 | for background on how ActiveJob handles exceptions and retries. 134 | 135 | In general - you should configure retries for your jobs using 136 | [retry_on](https://api.rubyonrails.org/classes/ActiveJob/Exceptions/ClassMethods.html#method-i-retry_on). 137 | When configured, ActiveJob will catch the exception and reschedule the job for 138 | re-execution after the configured delay. This will delete the original 139 | message from the SQS queue and requeue a new message. 140 | 141 | By default, any `StandardError` raised during job execution will leave the message 142 | on the queue (retrying it later) and initiate shutdown for the poller 143 | and it will attempt to finish executing any in progress jobs. 144 | 145 | You can configure an error handler block for the ActiveJob SQS poller with 146 | `poller_error_handler`. Jobs that raise a `StandardError` and that do 147 | not handle that error via `retry_on` or `discard_on` will call the configured 148 | `poller_error_handler` with `|exception, sqs_message|`. 149 | You may re-raise the exception to terminate the poller. You may also choose 150 | whether to delete the sqs_message or not. If the message is not explicitly 151 | deleted then the message will be left on the queue and will be retried 152 | following the SQS Queue's configured 153 | [retry and DLQ settings](https://docs.aws.amazon.com/lambda/latest/operatorguide/sqs-retries.html). 154 | Retries provided by this mechanism are after any retries configured on the job 155 | with `retry_on`. 156 | 157 | If you do not have a DLQ configured, the message will continue to be attempted 158 | until it reaches the queues retention period. In general, it is a best practice 159 | to configure a DLQ to store unprocessable jobs for troubleshooting and re-drive. 160 | 161 | Configuring retry/redrive for all `StandardErrors`: 162 | 163 | ```ruby 164 | Aws::ActiveJob::SQS.configure do |config| 165 | config.poller_error_handler do |_exception, _sqs_message| 166 | # no-op, don't delete the message or re-raise the exception 167 | end 168 | end 169 | ``` 170 | 171 | Configuring different behavior for different exceptions: 172 | 173 | ```ruby 174 | module MyErrorHandlers 175 | HANDLE_ERRORS = proc do |exception, sqs_message| 176 | case exception 177 | when MyRetryableException 178 | # no-op, don't delete message 179 | when MyTerminalException 180 | # delete the message, preventing retry 181 | sqs_message.delete 182 | else 183 | # unhandled exceptions, re-raise the exception to terminate the poller 184 | raise exception 185 | end 186 | end 187 | end 188 | 189 | Aws::ActiveJob::SQS.configure do |config| 190 | config.poller_error_handler = MyErrorHandlers::HANDLE_ERRORS 191 | end 192 | ``` 193 | 194 | 195 | When using the Async adapter, you may want to configure a 196 | `async_queue_error_handler` to handle errors that may occur when queuing jobs. 197 | See 198 | [Aws::ActiveJob::SQS::Configuration](https://docs.aws.amazon.com/sdk-for-ruby/aws-activejob-sqs/api/Aws/ActiveJob/SQS/Configuration.html) 199 | for documentation. 200 | 201 | ### Running workers - Polling for jobs 202 | 203 | To start processing jobs, you need to start a separate process 204 | (in additional to your Rails app) with the `aws_active_job_sqs` 205 | executable script provided with this gem, eg: 206 | `bundle exec aws_active_job_sqs --queue default`. 207 | You can poll for one or multiple queues using the `--queue` argument(s). 208 | 209 | You can specify multiple queues 210 | in the arguments by passing`--queue` multiple times 211 | (eg `--queue queue_1 --queue queue_2`) or for all configured queues by 212 | providing no queue arguments. When multiple queues are specified, 1 thread 213 | per queue is started to run the poller for that queue. All queues share a 214 | single, common thread pool for executing jobs. 215 | 216 | It is generally recommended to start one polling process per queue instead of 217 | running a single poller for all queues as this generally allows you to better 218 | manage the resources used. 219 | 220 | ```sh 221 | RAILS_ENV=development bundle exec aws_active_job_sqs --queue default 222 | ``` 223 | 224 | To see a complete list of arguments use `--help`. 225 | 226 | You can kill the process at any time with `CTRL+C` - the processor will attempt 227 | to shutdown cleanly and will wait up to `:shutdown_timeout` seconds for all 228 | actively running jobs to finish before killing them. 229 | 230 | **Note**: When running in production, its recommended that use a process 231 | supervisor such as [foreman](https://github.com/ddollar/foreman), systemd, 232 | upstart, daemontools, launchd, runit, etc. 233 | 234 | ### Running without Rails 235 | By default the `aws_active_job_sqs` script will require and boot rails 236 | using your `config/environment.rb`; however, you can start `aws_active_job_sqs` 237 | with `--no-rails` to run the poller without Rails. You can specify an 238 | additional file that includes required Job/application definitions with 239 | `--require`. Example: 240 | 241 | ```sh 242 | bundle exec aws_active_job_sqs --queue default --no-rails --require my_jobs.rb 243 | ``` 244 | 245 | ### Serverless workers: Processing jobs using AWS Lambda 246 | 247 | Rather than managing the worker processes yourself, you can use Lambda with an 248 | SQS Trigger. There are two main ways to process jobs with lambda: 249 | 250 | 1. [Running without Rails](#processing-jobs-on-lambda-without-rails) using a ZIP, SAM CLI or CDK 251 | 2. [Running with Rails](#processing-jobs-on-lambda-with-rails-container-image) using a Container Image. 252 | 253 | #### Processing Jobs on Lambda Without Rails 254 | In general, if you can separate out your job logic and dependencies from your 255 | main application its recommended that you run a light weight Lambda function 256 | to process jobs. A basic job can be executed using around 80 Mb of memory with 257 | a lower cold start time than a Rails container. 258 | 259 | There are a number of ways to do this, including 260 | [deploying with a .zip file](https://docs.aws.amazon.com/lambda/latest/dg/ruby-package.html), 261 | creating an application with the 262 | [AWS Serverless Application Model (SAM)](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/what-is-sam.html), 263 | , or defining and deploying the function in 264 | [AWS CDK](https://docs.aws.amazon.com/lambda/latest/dg/lambda-cdk-tutorial.html). 265 | 266 | The below are general instructions for manually creating and using a .zip 267 | to handle basic jobs in a lambda function. 268 | 269 | 1. Create a Gemfile. It must include `aws-activejob-sqs` and your other dependencies. 270 | 2. Use a local folder to install dependencies: `bundle config set --local path 'vendor/bundle' && bundle install`. 271 | 3. Create an `app.rb` file that requires `aws-activejob-sqs` and your local job files (see example below) with a `handle` method that calls `Aws::ActiveJob::SQS::LambdaHandler.job_handler(...)`. 272 | 4. Create a zip that includes your ruby files and the vendor folder: `zip -r test_activejob_lambda.zip *.rb vendor` 273 | 5. Create a lambda function using the latest Ruby runtime and upload the zip. 274 | 6. Specify the handler. Edit the runtime settings and change the Handler to `app.handle` (file name.function). 275 | 7. Add SQS permissions to the lambda's execution role. 276 | 8. Add an SQS Trigger with your job queue(s). 277 | 278 | #### Processing jobs on Lambda with Rails (Container Image) 279 | With 280 | [Lambda Container Image Support](https://aws.amazon.com/blogs/aws/new-for-aws-lambda-container-image-support/) 281 | and the lambda handler provided with this gem, you can use lambda to run 282 | ActiveJobs for your dockerized rails app (see below for some tips). 283 | 284 | All you need to do is: 285 | 1. include the [aws_lambda_ric gem](https://github.com/aws/aws-lambda-ruby-runtime-interface-client) 286 | 2. Push your image to ECR 287 | 3. Create a lambda function from your image (see the lambda docs for details). 288 | 4. Add an SQS Trigger for the queue(s) you want to process jobs from. 289 | 5. Set the ENTRYPOINT to `/usr/local/bundle/bin/aws_lambda_ric` and the CMD 290 | to `config/environment.Aws::ActiveJob::SQS.lambda_job_handler` - this will load 291 | Rails and then use the lambda handler. You can do this either as function config 292 | or in your Dockerfile. 293 | 294 | There are a few 295 | [limitations/requirements](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-reqs) 296 | for lambda container images: the default lambda user must be able 297 | to read all the files and the image must be able to run on a read only file system. 298 | You may need to disable bootsnap, set a HOME env variable and 299 | set the logger to STDOUT (which lambda will record to cloudwatch for you). 300 | 301 | You can use the RAILS_ENV to control environment. If you need to execute 302 | specific configuration in the lambda, you can create a ruby file and use it 303 | as your entrypoint: 304 | 305 | ```ruby 306 | # app.rb 307 | # some custom config 308 | 309 | require_relative 'config/environment' # load rails 310 | 311 | # Rails.config.custom.... 312 | # Aws::ActiveJob::SQS.configure.... 313 | 314 | # no need to write a handler yourself here, as long as 315 | # aws-sdk-rails is loaded, you can still use the 316 | # Aws::ActiveJob::SQS::LambdaHandler.job_handler 317 | 318 | # To use this file, set CMD: app.Aws::ActiveJob::SQS::LambdaHandler.job_handler 319 | ``` 320 | 321 | ### Using FIFO queues 322 | 323 | If the order in which your jobs executes is important, consider using a 324 | [FIFO Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html). 325 | A FIFO queue ensures that messages are processed in the order they were sent 326 | (First-In-First-Out) and exactly-once processing (ensuring duplicates are never 327 | introduced into the queue). To use a fifo queue, simply set the queue url 328 | (which will end in ".fifo") in your config. 329 | 330 | When using FIFO queues, jobs will NOT be processed concurrently by the poller 331 | to ensure the correct ordering. Additionally, all jobs on a FIFO queue will be queued 332 | synchronously, even if you have configured the `sqs_async` adapter. 333 | 334 | #### Message Deduplication ID 335 | 336 | FIFO queues support [Message deduplication ID](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagededuplicationid-property.html), 337 | which is the token used for deduplication of sent messages. If a message with a 338 | particular message deduplication ID is sent successfully, any messages sent with 339 | the same message deduplication ID are accepted successfully but aren't delivered 340 | during the 5-minute deduplication interval. 341 | 342 | If necessary, the deduplication key used to create the message deduplication ID 343 | can be customized: 344 | 345 | ```ruby 346 | Aws::ActiveJob::SQS.configure do |config| 347 | config.excluded_deduplication_keys = [:job_class, :arguments] 348 | end 349 | 350 | # Or to set deduplication keys to exclude for a single job: 351 | class YourJob < ApplicationJob 352 | include Aws::ActiveJob::SQS::Deduplication 353 | deduplicate_without :job_class, :arguments 354 | #... 355 | end 356 | ``` 357 | 358 | By default, the following keys are used for deduplication keys: 359 | 360 | ``` 361 | job_class, provider_job_id, queue_name, priority, arguments, 362 | executions, exception_executions, locale, timezone, enqueued_at 363 | ``` 364 | 365 | Note that `job_id` is NOT included in deduplication keys because it is unique 366 | for each initialization of the job, and the run-once behavior must be guaranteed 367 | for ActiveJob retries. Even without setting `job_id`, it is implicitly excluded 368 | from deduplication keys. 369 | 370 | #### Message Group IDs 371 | 372 | FIFO queues require a message group id to be provided for the job. It is determined by: 373 | 1. Calling `message_group_id` on the job if it is defined 374 | 2. If `message_group_id` is not defined or the result is `nil`, the default value will be used. 375 | You can optionally specify a custom value in your config as the default that will be used by all jobs. 376 | 377 | 378 | ## Job Iteration (interruptible and resumable jobs) 379 | AWS Activejob SQS is supported as an interrupt adaptor by [job-iteration](https://github.com/Shopify/job-iteration). 380 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec/core/rake_task' 4 | require 'rubocop/rake_task' 5 | 6 | Dir.glob('tasks/**/*.rake').each do |task_file| 7 | load task_file 8 | end 9 | 10 | RuboCop::RakeTask.new 11 | 12 | RSpec::Core::RakeTask.new 13 | 14 | task 'release:test' => :spec 15 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.2 2 | -------------------------------------------------------------------------------- /aws-activejob-sqs.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | version = File.read(File.expand_path('VERSION', __dir__)).strip 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = 'aws-activejob-sqs' 7 | spec.version = version 8 | spec.author = 'Amazon Web Services' 9 | spec.email = ['aws-dr-rubygems@amazon.com'] 10 | spec.summary = 'ActiveJob integration with SQS' 11 | spec.description = 'Amazon Simple Queue Service as an ActiveJob adapter' 12 | spec.homepage = 'https://github.com/aws/aws-activejob-sqs-ruby' 13 | spec.license = 'Apache-2.0' 14 | spec.files = Dir['LICENSE', 'CHANGELOG.md', 'VERSION', 'lib/**/*'] 15 | spec.executables = ['aws_active_job_sqs'] 16 | 17 | # Require this version for user_agent_framework configs 18 | spec.add_dependency('aws-sdk-sqs', '~> 1', '>= 1.56.0') 19 | 20 | spec.add_dependency('activejob', '>= 7.1.0') 21 | spec.add_dependency('concurrent-ruby', '~> 1') 22 | 23 | spec.required_ruby_version = '>= 2.7' 24 | end 25 | -------------------------------------------------------------------------------- /bin/aws_active_job_sqs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require_relative '../lib/aws-activejob-sqs' 5 | require_relative '../lib/aws/active_job/sqs/cli_options' 6 | require_relative '../lib/aws/active_job/sqs/poller' 7 | 8 | opts = Aws::ActiveJob::SQS::CliOptions.parse(ARGV) 9 | 10 | if opts[:boot_rails] 11 | require 'rails' 12 | require File.expand_path('config/environment.rb') 13 | end 14 | 15 | require File.join(Dir.pwd, opts[:require]) if opts[:require] 16 | 17 | Aws::ActiveJob::SQS::Poller.new(opts.to_h.compact).run 18 | -------------------------------------------------------------------------------- /doc-src/templates/default/layout/html/footer.erb: -------------------------------------------------------------------------------- 1 | 7 | -------------------------------------------------------------------------------- /doc-src/templates/default/layout/html/layout.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | <%= erb(:headers) %> 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 |
14 |
15 | 16 |
17 | 22 | 23 |
24 | 25 | <%= yieldall %> 26 |
27 | 28 | <%= erb(:footer) %> 29 |
30 | 31 | 32 | -------------------------------------------------------------------------------- /lib/active_job/queue_adapters/sqs_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'aws-sdk-sqs' 4 | 5 | module ActiveJob 6 | module QueueAdapters 7 | # Synchronous adapter for Amazon SQS ActiveJob. This adapter queues jobs synchronously (ie blocking). 8 | # 9 | # To use this adapter, set up as: 10 | # 11 | # config.active_job.queue_adapter = :sqs 12 | class SqsAdapter 13 | def enqueue_after_transaction_commit? 14 | # can be removed after Rails 8 15 | true 16 | end 17 | 18 | def enqueue(job) 19 | _enqueue(job) 20 | end 21 | 22 | def enqueue_at(job, timestamp) 23 | delay = Params.assured_delay_seconds(timestamp) 24 | _enqueue(job, nil, delay_seconds: delay) 25 | end 26 | 27 | def enqueue_all(jobs) 28 | enqueued_count = 0 29 | jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs| 30 | enqueued_count += enqueue_batches(queue_name, same_queue_jobs) 31 | end 32 | enqueued_count 33 | end 34 | 35 | private 36 | 37 | def enqueue_batches(queue_name, same_queue_jobs) 38 | enqueued_count = 0 39 | queue_url = Aws::ActiveJob::SQS.config.url_for(queue_name) 40 | 41 | same_queue_jobs.each_slice(10) do |chunk| 42 | enqueued_count += enqueue_batch(queue_url, chunk) 43 | end 44 | enqueued_count 45 | end 46 | 47 | def enqueue_batch(queue_url, chunk) 48 | entries = chunk.map do |job| 49 | entry = Params.new(job, nil).entry 50 | entry[:id] = job.job_id 51 | entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at 52 | entry 53 | end 54 | 55 | send_message_opts = { 56 | queue_url: queue_url, 57 | entries: entries 58 | } 59 | 60 | send_message_batch_result = Aws::ActiveJob::SQS.config.client.send_message_batch(send_message_opts) 61 | send_message_batch_result.successful.count 62 | end 63 | 64 | def _enqueue(job, body = nil, send_message_opts = {}) 65 | body ||= job.serialize 66 | params = Params.new(job, body) 67 | send_message_opts = send_message_opts.merge(params.entry) 68 | send_message_opts[:queue_url] = params.queue_url 69 | Aws::ActiveJob::SQS.config.client.send_message(send_message_opts) 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/active_job/queue_adapters/sqs_adapter/params.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ActiveJob 4 | module QueueAdapters 5 | class SqsAdapter 6 | # Build request parameter of Aws::SQS::Client 7 | # @api private 8 | class Params 9 | class << self 10 | def assured_delay_seconds(timestamp) 11 | delay = (timestamp.to_f - Time.now.to_f).floor 12 | delay = 0 if delay.negative? 13 | raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes 14 | 15 | delay 16 | end 17 | end 18 | 19 | def initialize(job, body) 20 | @job = job 21 | @body = body || job.serialize 22 | end 23 | 24 | def queue_url 25 | @queue_url ||= Aws::ActiveJob::SQS.config.url_for(@job.queue_name) 26 | end 27 | 28 | def entry 29 | if Aws::ActiveJob::SQS.fifo?(queue_url) 30 | default_entry.merge(options_for_fifo) 31 | else 32 | default_entry 33 | end 34 | end 35 | 36 | private 37 | 38 | def default_entry 39 | { 40 | message_body: ActiveSupport::JSON.dump(@body), 41 | message_attributes: message_attributes 42 | } 43 | end 44 | 45 | def message_attributes 46 | { 47 | 'aws_sqs_active_job_class' => { 48 | string_value: @job.class.to_s, 49 | data_type: 'String' 50 | }, 51 | 'aws_sqs_active_job_version' => { 52 | string_value: Aws::ActiveJob::SQS::VERSION, 53 | data_type: 'String' 54 | } 55 | } 56 | end 57 | 58 | def options_for_fifo 59 | options = {} 60 | options[:message_deduplication_id] = 61 | Digest::SHA256.hexdigest(ActiveSupport::JSON.dump(deduplication_body)) 62 | 63 | message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id) 64 | message_group_id ||= Aws::ActiveJob::SQS.config.message_group_id_for(@job.queue_name) 65 | 66 | options[:message_group_id] = message_group_id 67 | options 68 | end 69 | 70 | def deduplication_body 71 | ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys) 72 | ex_dedup_keys ||= Aws::ActiveJob::SQS.config.excluded_deduplication_keys_for(@job.queue_name) 73 | 74 | @body.except(*ex_dedup_keys) 75 | end 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/active_job/queue_adapters/sqs_async_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'aws-sdk-sqs' 4 | require 'concurrent' 5 | 6 | module ActiveJob 7 | module QueueAdapters 8 | # Asynchronous adapter for Amazon SQS ActiveJob This adapter queues jobs asynchronously (ie non-blocking). 9 | # 10 | # An error handler can be configured with: 11 | # 12 | # Aws::ActiveJob::SQS.config.async_queue_error_handler = ->(error, job, send_message_opts) { ... } 13 | # 14 | # To use this adapter, set up as: 15 | # 16 | # config.active_job.queue_adapter = :sqs_async 17 | class SqsAsyncAdapter < SqsAdapter 18 | private 19 | 20 | def _enqueue(job, body = nil, send_message_opts = {}) 21 | # FIFO jobs must be queued in order, so do not queue async 22 | queue_url = Aws::ActiveJob::SQS.config.url_for(job.queue_name) 23 | if Aws::ActiveJob::SQS.fifo?(queue_url) 24 | super 25 | else 26 | # Serialize is called here because the job’s locale needs to be 27 | # determined in this thread and not in some other thread. 28 | body = job.serialize 29 | Concurrent::Promises 30 | .future { super(job, body, send_message_opts) } 31 | .rescue do |e| 32 | Aws::ActiveJob::SQS.config.logger.error "Failed to queue job #{job}. Reason: #{e}" 33 | error_handler = Aws::ActiveJob::SQS.config.async_queue_error_handler 34 | error_handler&.call(e, job, send_message_opts) 35 | end 36 | end 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/aws-activejob-sqs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_job' 4 | require_relative 'active_job/queue_adapters/sqs_adapter' 5 | require_relative 'active_job/queue_adapters/sqs_adapter/params' 6 | require_relative 'active_job/queue_adapters/sqs_async_adapter' 7 | require_relative 'aws/active_job/sqs/configuration' 8 | require_relative 'aws/active_job/sqs/deduplication' 9 | require_relative 'aws/active_job/sqs/executor' 10 | require_relative 'aws/active_job/sqs/job_runner' 11 | require_relative 'aws/active_job/sqs/lambda_handler' 12 | 13 | module Aws 14 | module ActiveJob 15 | # ActiveJob Adapter and backend queueing using AWS SQS. 16 | module SQS 17 | VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip 18 | 19 | # @return [Configuration] the (singleton) Configuration 20 | def self.config 21 | @config ||= Configuration.new 22 | end 23 | 24 | # @yield [Configuration] the (singleton) Configuration 25 | def self.configure 26 | yield(config) 27 | end 28 | 29 | # @param queue_url [String] 30 | # @return [Boolean] true if the queue_url is a FIFO queue 31 | def self.fifo?(queue_url) 32 | queue_url.end_with?('.fifo') 33 | end 34 | 35 | def self.on_worker_stop(...) 36 | Executor.on_stop(...) 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/cli_options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'optparse' 4 | 5 | module Aws 6 | module ActiveJob 7 | module SQS 8 | # options for the aws_active_job_sqs CLI. 9 | # @api private 10 | module CliOptions 11 | def self.option_parser(out) 12 | ::OptionParser.new do |opts| 13 | queues_option(opts, out) 14 | threads_option(opts, out) 15 | backpressure_option(opts, out) 16 | max_messages_option(opts, out) 17 | visibility_timeout_option(opts, out) 18 | shutdown_timeout_option(opts, out) 19 | boot_rails_option(opts, out) 20 | require_option(opts, out) 21 | end 22 | end 23 | private_class_method :option_parser 24 | 25 | def self.parse(argv) 26 | out = { boot_rails: true, queues: [] } 27 | parser = option_parser(out) 28 | 29 | parser.banner = 'aws_active_job_sqs [options]' 30 | parser.on_tail '-h', '--help', 'Show help' do 31 | puts parser 32 | exit 1 33 | end 34 | 35 | parser.parse(argv) 36 | out 37 | end 38 | 39 | def self.require_option(opts, out) 40 | doc = 'Additional file to require before starting the poller. ' \ 41 | 'Can be used to define/load job classes with --no-rails.' 42 | opts.on('-r', '--require STRING', String, doc) do |a| 43 | out[:require] = a 44 | end 45 | end 46 | 47 | def self.boot_rails_option(opts, out) 48 | doc = 'When set boots rails before running the poller.' 49 | opts.on('--[no-]rails [FLAG]', TrueClass, doc) do |a| 50 | out[:boot_rails] = a.nil? || a 51 | end 52 | end 53 | 54 | def self.shutdown_timeout_option(opts, out) 55 | doc = 'The amount of time to wait for a clean shutdown. Jobs that ' \ 56 | 'are unable to complete in this time will not be deleted from ' \ 57 | 'the SQS queue and will be retryable after the visibility timeout.' 58 | opts.on('-s', '--shutdown_timeout INTEGER', Integer, doc) do |a| 59 | out[:shutdown_timeout] = a 60 | end 61 | end 62 | 63 | def self.visibility_timeout_option(opts, out) 64 | doc = 'The visibility timeout is the number of seconds that a ' \ 65 | 'message will not be processable by any other consumers. ' \ 66 | 'You should set this value to be longer than your expected ' \ 67 | 'job runtime to prevent other processes from picking up an ' \ 68 | 'running job. See the SQS Visibility Timeout Documentation ' \ 69 | 'at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/' \ 70 | 'SQSDeveloperGuide/sqs-visibility-timeout.html.' 71 | opts.on('-v', '--visibility_timeout INTEGER', Integer, doc) do |a| 72 | out[:visibility_timeout] = a 73 | end 74 | end 75 | 76 | def self.max_messages_option(opts, out) 77 | doc = 'Max number of messages to receive in a batch from SQS.' 78 | opts.on('-m', '--max_messages INTEGER', Integer, doc) do |a| 79 | out[:max_messages] = a 80 | end 81 | end 82 | 83 | def self.backpressure_option(opts, out) 84 | doc = 'The maximum number of messages to have waiting in ' \ 85 | 'the Executor queue. This should be a low, but non zero number. ' \ 86 | 'Messages in the Executor queue cannot be picked up by other ' \ 87 | 'processes and will slow down shutdown.' 88 | opts.on('-b', '--backpressure INTEGER', Integer, doc) do |a| 89 | out[:backpressure] = a 90 | end 91 | end 92 | 93 | def self.threads_option(opts, out) 94 | doc = 'The maximum number of worker threads to create. ' \ 95 | 'Defaults to 2x the number of processors available on this system.' 96 | opts.on('-t', '--threads INTEGER', Integer, doc) do |a| 97 | out[:threads] = a 98 | end 99 | end 100 | 101 | def self.queues_option(opts, out) 102 | doc = 'Queue(s) to poll. You may specify this argument multiple ' \ 103 | 'times to poll multiple queues. If not specified, will ' \ 104 | 'start pollers for all queues defined.' 105 | opts.on('-q', '--queue STRING', doc) do |a| 106 | out[:queues] << a.to_sym 107 | end 108 | end 109 | end 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/configuration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | # This class provides a Configuration object for AWS ActiveJob 7 | # by pulling configuration options from runtime code, the ENV, a YAML file, 8 | # and default settings, in that order. Values set on queues are used 9 | # preferentially to global values. 10 | # 11 | # Use {Aws::ActiveJob::SQS.config Aws::ActiveJob::SQS.config} 12 | # to access the singleton config instance and use 13 | # {Aws::ActiveJob::SQS.configure Aws::ActiveJob::SQS.configure} to 14 | # configure in code: 15 | # 16 | # Aws::ActiveJob::SQS.configure do |config| 17 | # config.logger = Rails.logger 18 | # config.max_messages = 5 19 | # end 20 | # 21 | # # Configuation YAML File 22 | # By default, this class will load configuration from the 23 | # `config/aws_active_job_sqs/` and queue specific keys take the 37 | # form of: `AWS_ACTIVE_JOB_SQS__`. 38 | # is case-insensitive and is always down cased. Configuring 39 | # non-snake case queues (containing upper case) through ENV is 40 | # not supported. 41 | # 42 | # Example: 43 | # 44 | # export AWS_ACTIVE_JOB_SQS_MAX_MESSAGES = 5 45 | # export AWS_ACTIVE_JOB_SQS_DEFAULT_URL = https://my-queue.aws 46 | # 47 | # For supported global ENV configurations see 48 | # {GLOBAL_ENV_CONFIGS}. For supported queue specific ENV configurations 49 | # see: {QUEUE_ENV_CONFIGS}. 50 | # 51 | class Configuration 52 | # Default configuration options 53 | # @api private 54 | DEFAULTS = { 55 | threads: 2 * Concurrent.processor_count, 56 | backpressure: 10, 57 | max_messages: 10, 58 | shutdown_timeout: 15, 59 | queues: Hash.new { |h, k| h[k] = {} }, 60 | message_group_id: 'ActiveJobSqsGroup', 61 | excluded_deduplication_keys: ['job_id'] 62 | }.freeze 63 | 64 | GLOBAL_ENV_CONFIGS = %i[ 65 | config_file 66 | threads 67 | backpressure 68 | max_messages 69 | shutdown_timeout 70 | visibility_timeout 71 | message_group_id 72 | ].freeze 73 | 74 | QUEUE_ENV_CONFIGS = %i[ 75 | url 76 | max_messages 77 | visibility_timeout 78 | message_group_id 79 | ].freeze 80 | 81 | QUEUE_CONFIGS = QUEUE_ENV_CONFIGS + %i[excluded_deduplication_keys] 82 | 83 | QUEUE_KEY_REGEX = 84 | /AWS_ACTIVE_JOB_SQS_([\w]+)_(#{QUEUE_ENV_CONFIGS.map(&:upcase).join('|')})/.freeze 85 | 86 | # Don't use this method directly: Configuration is a singleton class, 87 | # use {Aws::ActiveJob::SQS.config Aws::ActiveJob::SQS.config} 88 | # to access the singleton config instance and use 89 | # {Aws::ActiveJob::SQS.configure Aws::ActiveJob::SQS.configure} to 90 | # configure in code: 91 | # 92 | # @param [Hash] options 93 | # @option options [Hash] :queues A mapping between the 94 | # active job queue name and the queue properties. Values 95 | # configured on the queue are used preferentially to the global 96 | # values. See: {QUEUE_CONFIGS} for supported queue specific options. 97 | # Note: multiple active job queues can map to the same SQS Queue URL. 98 | # 99 | # @option options [Integer] :max_messages 100 | # The max number of messages to poll for in a batch. 101 | # 102 | # @option options [Integer] :visibility_timeout 103 | # If unset, the visibility timeout configured on the 104 | # SQS queue will be used. 105 | # The visibility timeout is the number of seconds 106 | # that a message will not be processable by any other consumers. 107 | # You should set this value to be longer than your expected job runtime 108 | # to prevent other processes from picking up an running job. 109 | # See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html] 110 | # 111 | # @option options [Integer] :shutdown_timeout 112 | # the amount of time to wait 113 | # for a clean shutdown. Jobs that are unable to complete in this time 114 | # will not be deleted from the SQS queue and will be retryable after 115 | # the visibility timeout. 116 | # 117 | # @option options [Callable] :poller_error_handler and error handler to 118 | # be called when the poller encounters an error running a job. Called 119 | # with exception, sqs_message. You may re-raise the exception to 120 | # terminate the poller. You may also choose whether to delete the 121 | # sqs_message or not. If the message is not explicitly deleted 122 | # then the message will be left on the queue and will be 123 | # retried (pending the SQS Queue's redrive/DLQ/maximum 124 | # receive settings). Retries provided by this mechanism are 125 | # after any retries configured on the job with `retry_on`. 126 | # 127 | # @option options [ActiveSupport::Logger] :logger Logger to use 128 | # for the poller. 129 | # 130 | # @option options [String] :config_file 131 | # Override file to load configuration from. If not specified will 132 | # attempt to load from config/aws_active_job_sqs.yml. 133 | # 134 | # @option options [String] :message_group_id (SqsActiveJobGroup) 135 | # The message_group_id to use for queueing messages on a fifo queues. 136 | # Applies only to jobs queued on FIFO queues. 137 | # See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html] 138 | # 139 | # @option options [Callable] :async_queue_error_handler An error handler 140 | # to be called when the async active job adapter experiences an error 141 | # queueing a job. Only applies when 142 | # +active_job.queue_adapter = :sqs_async+. Called with: 143 | # [error, job, job_options] 144 | # 145 | # @option options [SQS::Client] :client SQS Client to use. A default 146 | # client will be created if none is provided. 147 | # 148 | # @option options [Array] :excluded_deduplication_keys (['job_id']) 149 | # The type of keys stored in the array should be String or Symbol. 150 | # Using this option, job_id is implicitly added to the keys. 151 | 152 | def initialize(options = {}) 153 | opts = env_options.deep_merge(options) 154 | opts = file_options(opts).deep_merge(opts) 155 | opts = DEFAULTS.merge(logger: default_logger).deep_merge(opts) 156 | 157 | apply_attributes(opts) 158 | end 159 | 160 | # @api private 161 | attr_accessor :queues, :threads, :backpressure, 162 | :shutdown_timeout, :logger, 163 | :async_queue_error_handler 164 | 165 | # @api private 166 | attr_writer :max_messages, :message_group_id, :visibility_timeout, 167 | :poller_error_handler, :client 168 | 169 | def excluded_deduplication_keys=(keys) 170 | @excluded_deduplication_keys = keys.map(&:to_s) | ['job_id'] 171 | end 172 | 173 | def poller_error_handler(&block) 174 | @poller_error_handler = block if block_given? 175 | @poller_error_handler 176 | end 177 | 178 | def client 179 | @client ||= begin 180 | client = Aws::SQS::Client.new 181 | client.config.user_agent_frameworks << 'aws-activejob-sqs' 182 | client 183 | end 184 | end 185 | 186 | QUEUE_CONFIGS.each do |key| 187 | define_method(:"#{key}_for") do |job_queue| 188 | queue_attribute_for(key, job_queue) 189 | end 190 | end 191 | 192 | # @api private 193 | def to_s 194 | to_h.to_s 195 | end 196 | 197 | # @api private 198 | def to_h 199 | h = {} 200 | instance_variables.each do |v| 201 | v_sym = v.to_s.delete('@').to_sym 202 | val = instance_variable_get(v) 203 | h[v_sym] = val 204 | end 205 | h 206 | end 207 | 208 | private 209 | 210 | def queue_attribute_for(attribute, job_queue) 211 | job_queue = job_queue.to_sym 212 | raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue 213 | 214 | queues[job_queue][attribute] || instance_variable_get("@#{attribute}") 215 | end 216 | 217 | # Set accessible attributes after merged options. 218 | def apply_attributes(options) 219 | options.each_key do |opt_name| 220 | instance_variable_set("@#{opt_name}", options[opt_name]) 221 | client.config.user_agent_frameworks << 'aws-activejob-sqs' if opt_name == :client 222 | end 223 | end 224 | 225 | # resolve ENV for global and queue specific options 226 | def env_options 227 | resolved = { queues: {} } 228 | resolve_global_env_options(resolved) 229 | resolve_queue_env__options(resolved) 230 | resolved 231 | end 232 | 233 | def resolve_queue_env__options(resolved) 234 | ENV.each_key do |key| 235 | next unless (match = QUEUE_KEY_REGEX.match(key)) 236 | 237 | queue_name = match[1].downcase.to_sym 238 | resolved[:queues][queue_name] ||= {} 239 | resolved[:queues][queue_name][match[2].downcase.to_sym] = 240 | parse_env_value(key) 241 | end 242 | end 243 | 244 | def resolve_global_env_options(resolved) 245 | GLOBAL_ENV_CONFIGS.each do |cfg| 246 | env_name = "AWS_ACTIVE_JOB_SQS_#{cfg.to_s.upcase}" 247 | resolved[cfg] = parse_env_value(env_name) if ENV.key? env_name 248 | end 249 | end 250 | 251 | def parse_env_value(key) 252 | val = ENV.fetch(key, nil) 253 | Integer(val) 254 | rescue ArgumentError, TypeError 255 | %w[true false].include?(val) ? val == 'true' : val 256 | end 257 | 258 | def file_options(options = {}) 259 | file_path = options[:config_file] || default_config_file 260 | if file_path 261 | load_from_file(file_path) 262 | else 263 | options 264 | end 265 | end 266 | 267 | def default_config_file 268 | return unless defined?(::Rails) 269 | 270 | file = ::Rails.root.join("config/aws_active_job_sqs/#{::Rails.env}.yml") 271 | file = ::Rails.root.join('config/aws_active_job_sqs.yml') unless File.exist?(file) 272 | file 273 | end 274 | 275 | # Load options from YAML file 276 | def load_from_file(file_path) 277 | opts = load_yaml(file_path) || {} 278 | opts = opts.deep_symbolize_keys 279 | opts[:queues]&.each_key do |queue| 280 | opts[:queues][queue] = { url: opts[:queues][queue] } if opts[:queues][queue].is_a?(String) 281 | end 282 | opts 283 | end 284 | 285 | def load_yaml(file_path) 286 | return {} unless File.exist?(file_path) 287 | 288 | require 'erb' 289 | source = ERB.new(File.read(file_path)).result 290 | 291 | # Avoid incompatible changes with Psych 4.0.0 292 | # https://bugs.ruby-lang.org/issues/17866 293 | begin 294 | YAML.safe_load(source, aliases: true) || {} 295 | rescue ArgumentError 296 | YAML.safe_load(source) || {} 297 | end 298 | end 299 | 300 | def default_logger 301 | if defined?(::Rails) 302 | ::Rails.logger 303 | else 304 | ActiveSupport::Logger.new($stdout) 305 | end 306 | end 307 | end 308 | end 309 | end 310 | end 311 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/deduplication.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | # Mixin module to configure job level deduplication keys 7 | module Deduplication 8 | extend ActiveSupport::Concern 9 | 10 | included do 11 | class_attribute :excluded_deduplication_keys 12 | end 13 | 14 | # class methods for SQS ActiveJob. 15 | module ClassMethods 16 | def deduplicate_without(*keys) 17 | self.excluded_deduplication_keys = keys.map(&:to_s) | ['job_id'] 18 | end 19 | end 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/executor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'concurrent' 4 | 5 | module Aws 6 | module ActiveJob 7 | module SQS 8 | # CLI runner for polling for SQS ActiveJobs 9 | class Executor 10 | DEFAULTS = { 11 | min_threads: 0, 12 | max_threads: Integer(Concurrent.available_processor_count || Concurrent.processor_count), 13 | auto_terminate: true, 14 | idletime: 60, # 1 minute 15 | fallback_policy: :abort # Concurrent::RejectedExecutionError must be handled 16 | }.freeze 17 | 18 | class << self 19 | def on_stop(&block) 20 | lifecycle_hooks[:stop] << block 21 | end 22 | 23 | def lifecycle_hooks 24 | @lifecycle_hooks ||= Hash.new { |h, k| h[k] = [] } 25 | end 26 | 27 | def clear_hooks 28 | @lifecycle_hooks = nil 29 | end 30 | end 31 | 32 | def initialize(options = {}) 33 | @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options)) 34 | @logger = options[:logger] || ActiveSupport::Logger.new($stdout) 35 | @task_complete = Concurrent::Event.new 36 | @post_mutex = Mutex.new 37 | 38 | @error_handler = options[:error_handler] 39 | @error_queue = Thread::Queue.new 40 | @error_handler_thread = Thread.new(&method(:handle_errors)) 41 | @error_handler_thread.abort_on_exception = true 42 | @error_handler_thread.report_on_exception = false 43 | @shutting_down = Concurrent::AtomicBoolean.new(false) 44 | end 45 | 46 | def execute(message) 47 | @post_mutex.synchronize do 48 | _execute(message) 49 | end 50 | end 51 | 52 | def shutdown(timeout = nil) 53 | @shutting_down.make_true 54 | 55 | run_hooks_for(:stop) 56 | @executor.shutdown 57 | clean_shutdown = @executor.wait_for_termination(timeout) 58 | if clean_shutdown 59 | @logger.info 'Clean shutdown complete. All executing jobs finished.' 60 | else 61 | @logger.info "Timeout (#{timeout}) exceeded. Some jobs may not have " \ 62 | 'finished cleanly. Unfinished jobs will not be removed from ' \ 63 | 'the queue and can be ru-run once their visibility timeout ' \ 64 | 'passes.' 65 | end 66 | @error_queue.push(nil) # process any remaining errors and then terminate 67 | @error_handler_thread.join unless @error_handler_thread == Thread.current 68 | @shutting_down.make_false 69 | end 70 | 71 | private 72 | 73 | def _execute(message) 74 | post_task(message) 75 | rescue Concurrent::RejectedExecutionError 76 | # no capacity, wait for a task to complete 77 | @task_complete.reset 78 | @task_complete.wait 79 | retry 80 | end 81 | 82 | def post_task(message) 83 | @executor.post(message) do |msg| 84 | execute_task(msg) 85 | end 86 | end 87 | 88 | def execute_task(message) 89 | job = JobRunner.new(message) 90 | @logger.info("Running job: #{job.id}[#{job.class_name}]") 91 | job.run 92 | message.delete 93 | rescue JSON::ParserError => e 94 | @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}." 95 | rescue StandardError => e 96 | handle_standard_error(e, job, message) 97 | ensure 98 | @task_complete.set 99 | end 100 | 101 | def handle_standard_error(error, job, message) 102 | job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job' 103 | @logger.info "Error processing job #{job_msg}: #{error}" 104 | @logger.debug error.backtrace.join("\n") 105 | 106 | @error_queue.push([error, message]) 107 | end 108 | 109 | def run_hooks_for(event_name) 110 | return unless (hooks = self.class.lifecycle_hooks[event_name]) 111 | 112 | hooks.each(&:call) 113 | end 114 | 115 | # run in the @error_handler_thread 116 | def handle_errors 117 | # wait until errors are placed in the error queue 118 | while ((exception, message) = @error_queue.pop) 119 | raise exception unless @error_handler 120 | 121 | @error_handler.call(exception, message) 122 | 123 | end 124 | rescue StandardError => e 125 | @logger.info("Unhandled exception executing jobs in poller: #{e}.") 126 | @logger.info('Shutting down executor') 127 | shutdown unless @shutting_down.true? 128 | 129 | raise e # re-raise the error, terminating the application 130 | end 131 | end 132 | end 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/job_runner.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | # @api private 7 | class JobRunner 8 | attr_reader :id, :class_name 9 | 10 | def initialize(message) 11 | @job_data = ActiveSupport::JSON.load(message.data.body) 12 | @class_name = @job_data['job_class'].constantize 13 | @id = @job_data['job_id'] 14 | end 15 | 16 | def run 17 | ::ActiveJob::Base.execute @job_data 18 | end 19 | 20 | def exception_executions? 21 | @job_data['exception_executions'] && 22 | !@job_data['exception_executions'].empty? 23 | end 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/lambda_handler.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'aws-sdk-sqs' 4 | 5 | module Aws 6 | module ActiveJob 7 | module SQS 8 | # Lambda event handler to run jobs from an SQS queue trigger 9 | module LambdaHandler 10 | class << self 11 | # A lambda event handler to run jobs from an SQS queue trigger. 12 | # Configure the entrypoint to: +config/environment.Aws::ActiveJob::SQS::LambdaHandler.job_handler+ 13 | # This will load your Rails environment, and then use this method as the handler. 14 | def job_handler(event:, context:) 15 | return 'no records to process' unless event['Records'] 16 | 17 | puts "job_handler running for #{event} with context: #{context}" 18 | 19 | event['Records'].each do |record| 20 | sqs_msg = to_sqs_msg(record) 21 | job = Aws::ActiveJob::SQS::JobRunner.new(sqs_msg) 22 | puts "Running job: #{job.id}[#{job.class_name}]" 23 | job.run 24 | sqs_msg.delete 25 | end 26 | "Processed #{event['Records'].length} jobs." 27 | end 28 | 29 | private 30 | 31 | def to_sqs_msg(record) 32 | msg = Aws::SQS::Types::Message.new( 33 | body: record['body'], 34 | md5_of_body: record['md5OfBody'], 35 | message_attributes: to_message_attributes(record), 36 | message_id: record['messageId'], 37 | receipt_handle: record['receiptHandle'] 38 | ) 39 | Aws::SQS::Message.new( 40 | queue_url: to_queue_url(record), 41 | receipt_handle: msg.receipt_handle, 42 | data: msg, 43 | client: Aws::ActiveJob::SQS.config.client 44 | ) 45 | end 46 | 47 | def to_message_attributes(record) 48 | record['messageAttributes'].transform_values do |value| 49 | { 50 | string_value: value['stringValue'], 51 | binary_value: value['binaryValue'], 52 | string_list_values: ['stringListValues'], 53 | binary_list_values: value['binaryListValues'], 54 | data_type: value['dataType'] 55 | } 56 | end 57 | end 58 | 59 | def to_queue_url(record) 60 | source_arn = record['eventSourceARN'] 61 | raise ArgumentError, "Invalid queue arn: #{source_arn}" unless Aws::ARNParser.arn?(source_arn) 62 | 63 | arn = Aws::ARNParser.parse(source_arn) 64 | sfx = Aws::Partitions::EndpointProvider.dns_suffix_for(arn.region) 65 | "https://sqs.#{arn.region}.#{sfx}/#{arn.account_id}/#{arn.resource}" 66 | end 67 | end 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/aws/active_job/sqs/poller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'aws-sdk-sqs' 4 | require 'concurrent' 5 | 6 | module Aws 7 | module ActiveJob 8 | module SQS 9 | # CLI runner for polling for SQS ActiveJobs 10 | # Use `aws_active_job_sqs --help` for detailed usage 11 | class Poller 12 | class Interrupt < StandardError; end 13 | 14 | def initialize(options = {}) 15 | @queues = options.delete(:queues) 16 | @options = options 17 | end 18 | 19 | def run 20 | init_config 21 | 22 | config = Aws::ActiveJob::SQS.config 23 | 24 | Signal.trap('INT') { raise Interrupt } 25 | Signal.trap('TERM') { raise Interrupt } 26 | @executor = Executor.new( 27 | max_threads: config.threads, 28 | logger: @logger, 29 | max_queue: config.backpressure, 30 | error_handler: config.poller_error_handler 31 | ) 32 | 33 | poll 34 | rescue Interrupt 35 | @logger.info 'Process Interrupted or killed - attempting to shutdown cleanly.' 36 | shutdown(config.shutdown_timeout) 37 | exit 38 | end 39 | 40 | private 41 | 42 | def init_config 43 | Aws::ActiveJob::SQS.configure do |cfg| 44 | @options.each_pair do |key, value| 45 | cfg.send(:"#{key}=", value) if cfg.respond_to?(:"#{key}=") 46 | end 47 | end 48 | 49 | # ensure we have a logger configured 50 | config = Aws::ActiveJob::SQS.config 51 | @logger = config.logger || ActiveSupport::Logger.new($stdout) 52 | @logger.info("Starting Poller with config=#{config.to_h}") 53 | end 54 | 55 | def shutdown(timeout) 56 | @executor.shutdown(timeout) 57 | end 58 | 59 | def poll 60 | config = Aws::ActiveJob::SQS.config 61 | if @queues && !@queues.empty? 62 | if @queues.size == 1 63 | # single queue, use main thread 64 | poll_foreground(@queues.first) 65 | else 66 | poll_background(@queues) 67 | end 68 | else 69 | # poll on all configured queues 70 | @logger.info("No queues specified - polling on all configured queues: #{config.queues.keys}") 71 | poll_background(config.queues.keys) 72 | end 73 | end 74 | 75 | def poll_foreground(queue) 76 | config = Aws::ActiveJob::SQS.config 77 | validate_config(queue) 78 | queue_url = config.url_for(queue) 79 | 80 | poller_options = poller_options(queue) 81 | @logger.info "Foreground Polling on: #{queue} => #{queue_url} with options=#{poller_options}" 82 | 83 | _poll(poller_options, queue_url) 84 | end 85 | 86 | def poll_background(queues) 87 | config = Aws::ActiveJob::SQS.config 88 | queues.each { |q| validate_config(q) } 89 | poller_threads = queues.map do |queue| 90 | Thread.new do 91 | queue_url = config.url_for(queue) 92 | 93 | poller_options = poller_options(queue) 94 | @logger.info "Background Polling on: #{queue} => #{queue_url} with options=#{poller_options}" 95 | 96 | _poll(poller_options, queue_url) 97 | end 98 | end 99 | poller_threads.each(&:join) 100 | end 101 | 102 | def validate_config(queue) 103 | return if Aws::ActiveJob::SQS.config.queues[queue]&.fetch(:url, nil) 104 | 105 | raise ArgumentError, "No URL configured for queue #{queue}" 106 | end 107 | 108 | def poller_options(queue) 109 | config = Aws::ActiveJob::SQS.config 110 | queue_url = config.url_for(queue) 111 | poller_options = { 112 | skip_delete: true, 113 | max_number_of_messages: config.max_messages_for(queue), 114 | visibility_timeout: config.visibility_timeout_for(queue) 115 | } 116 | 117 | # Limit max_number_of_messages for FIFO queues to 1 118 | # this ensures jobs with the same message_group_id are processed 119 | # in order 120 | # Jobs with different message_group_id will be processed in 121 | # parallel and may be out of order. 122 | poller_options[:max_number_of_messages] = 1 if Aws::ActiveJob::SQS.fifo?(queue_url) 123 | poller_options 124 | end 125 | 126 | def _poll(poller_options, queue_url) 127 | poller = Aws::SQS::QueuePoller.new( 128 | queue_url, 129 | client: Aws::ActiveJob::SQS.config.client 130 | ) 131 | single_message = poller_options[:max_number_of_messages] == 1 132 | poller.poll(poller_options) do |msgs| 133 | msgs = [msgs] if single_message 134 | execute_messages(msgs, queue_url) 135 | end 136 | end 137 | 138 | def execute_messages(msgs, queue_url) 139 | @logger.info "Processing batch of #{msgs.length} messages" 140 | msgs.each do |msg| 141 | sqs_message = Aws::SQS::Message.new( 142 | queue_url: queue_url, 143 | receipt_handle: msg.receipt_handle, 144 | data: msg, 145 | client: Aws::ActiveJob::SQS.config.client 146 | ) 147 | @executor.execute(sqs_message) 148 | end 149 | end 150 | end 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /spec/active_job/queue_adapters/sqs_adapter/params_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ActiveJob 4 | module QueueAdapters 5 | class SqsAdapter 6 | describe Params do 7 | describe '.assured_delay_seconds' do 8 | let(:now) { Time.now } 9 | 10 | before { allow(Time).to receive(:now).and_return(now) } 11 | 12 | it 'returns seconds from present' do 13 | unix_time = (now + 15.minutes).to_f 14 | expect(Params.assured_delay_seconds(unix_time)).to eq 900 15 | end 16 | 17 | it 'rounds up to zero' do 18 | unix_time = (now - 1.second).to_f 19 | expect(Params.assured_delay_seconds(unix_time)).to eq 0 20 | end 21 | 22 | it 'raise error when 15 minutes after present' do 23 | unix_time = (now + 15.minutes + 1.second).to_f 24 | expect { Params.assured_delay_seconds(unix_time) }.to raise_error ArgumentError 25 | end 26 | end 27 | 28 | describe '#queue_url' do 29 | let(:params) { Params.new(job, nil) } 30 | let(:job) { TestJob.new('a1', 'a2') } 31 | 32 | it 'returns url of job queue' do 33 | expect(params.queue_url).to eq 'https://queue-url' 34 | end 35 | end 36 | 37 | describe '#entry' do 38 | let(:params) { Params.new(job, nil) } 39 | let(:job) { TestJob.new('a1', 'a2') } 40 | 41 | it 'returns hash of core attributes' do 42 | expect(params.entry).to include( 43 | { 44 | message_body: instance_of(String), 45 | message_attributes: instance_of(Hash) 46 | } 47 | ) 48 | end 49 | 50 | describe 'fifo queue' do 51 | before do 52 | allow(Aws::ActiveJob::SQS.config).to receive(:url_for).and_return('https://queue-url.fifo') 53 | end 54 | 55 | it 'includes message_group_id and message_deduplication_id' do 56 | expect(params.entry).to include( 57 | { 58 | message_body: instance_of(String), 59 | message_attributes: instance_of(Hash), 60 | message_group_id: String, 61 | message_deduplication_id: String 62 | } 63 | ) 64 | end 65 | end 66 | end 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/active_job/queue_adapters/sqs_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ActiveJob 4 | module QueueAdapters 5 | describe SqsAdapter do 6 | let(:client) { double('Client') } 7 | before do 8 | allow(Aws::ActiveJob::SQS.config).to receive(:client).and_return(client) 9 | end 10 | 11 | it 'enqueues jobs' do 12 | expect(client).to receive(:send_message) 13 | .with( 14 | { 15 | queue_url: 'https://queue-url', 16 | message_body: instance_of(String), 17 | message_attributes: instance_of(Hash) 18 | } 19 | ) 20 | TestJob.perform_later('test') 21 | sleep(0.2) 22 | end 23 | 24 | context 'fifo queues' do 25 | before do 26 | allow(Aws::ActiveJob::SQS.config).to receive(:url_for).and_return('https://queue-url.fifo') 27 | end 28 | 29 | it 'adds message_deduplication_id and default message_group_id if job does not override it' do 30 | expect(client).to receive(:send_message) 31 | .with( 32 | { 33 | queue_url: 'https://queue-url.fifo', 34 | message_body: instance_of(String), 35 | message_attributes: instance_of(Hash), 36 | message_group_id: Aws::ActiveJob::SQS.config.message_group_id_for(:default), 37 | message_deduplication_id: instance_of(String) 38 | } 39 | ) 40 | TestJob.perform_later('test') 41 | sleep(0.2) 42 | end 43 | 44 | context 'when job has excluded deduplication keys defined' do 45 | let(:ex_dedup_keys) { %w[job_class queue_name] } 46 | let(:ex_dudup_keys_with_job_id) { ex_dedup_keys << 'job_id' } 47 | let(:hashed_body) { 'hashed_body' } 48 | 49 | context 'through #deduplicate_without' do 50 | before do 51 | TestJobWithDedupKeys.deduplicate_without(*ex_dedup_keys) 52 | end 53 | 54 | it 'adds customized message_deduplication_id' do 55 | expect(Digest::SHA256).to receive(:hexdigest) do |body| 56 | ex_dudup_keys_with_job_id.each do |key| 57 | expect(body).not_to include(%("#{key}")) 58 | end 59 | end.and_return(hashed_body) 60 | expect(client).to receive(:send_message).with( 61 | { 62 | queue_url: 'https://queue-url.fifo', 63 | message_body: instance_of(String), 64 | message_attributes: instance_of(Hash), 65 | message_group_id: Aws::ActiveJob::SQS.config.message_group_id_for(:default), 66 | message_deduplication_id: hashed_body 67 | } 68 | ) 69 | 70 | TestJobWithDedupKeys.perform_later('test') 71 | sleep(0.2) 72 | end 73 | end 74 | 75 | context 'through Aws::ActiveJob::SQS config' do 76 | before do 77 | Aws::ActiveJob::SQS.configure do |config| 78 | config.excluded_deduplication_keys = ex_dedup_keys 79 | end 80 | end 81 | 82 | it 'adds customized message_deduplication_id' do 83 | expect(Digest::SHA256).to receive(:hexdigest) do |body| 84 | ex_dudup_keys_with_job_id.each do |key| 85 | expect(body).not_to include(%("#{key}")) 86 | end 87 | end.and_return(hashed_body) 88 | expect(client).to receive(:send_message).with( 89 | { 90 | queue_url: 'https://queue-url.fifo', 91 | message_body: instance_of(String), 92 | message_attributes: instance_of(Hash), 93 | message_group_id: Aws::ActiveJob::SQS.config.message_group_id_for(:default), 94 | message_deduplication_id: hashed_body 95 | } 96 | ) 97 | 98 | TestJob.perform_later('test') 99 | sleep(0.2) 100 | end 101 | end 102 | end 103 | 104 | context 'when job has #message_group_id defined' do 105 | it 'adds message_deduplication_id and default message_group_id if job does not return a value' do 106 | expect(client).to receive(:send_message).with( 107 | { 108 | queue_url: 'https://queue-url.fifo', 109 | message_body: instance_of(String), 110 | message_attributes: instance_of(Hash), 111 | message_group_id: Aws::ActiveJob::SQS.config.message_group_id_for(:default), 112 | message_deduplication_id: instance_of(String) 113 | } 114 | ) 115 | 116 | TestJobWithMessageGroupID.perform_later('test') 117 | sleep(0.2) 118 | end 119 | 120 | it 'adds message_deduplication_id and given message_group_id if job returns a value' do 121 | arg = 'test' 122 | dbl = TestJobWithMessageGroupID.new(arg) 123 | message_group_id = "mgi_#{rand(0..100)}" 124 | 125 | expect(client).to receive(:send_message).with( 126 | { 127 | queue_url: 'https://queue-url.fifo', 128 | message_body: instance_of(String), 129 | message_attributes: instance_of(Hash), 130 | message_group_id: message_group_id, 131 | message_deduplication_id: instance_of(String) 132 | } 133 | ) 134 | 135 | expect(TestJobWithMessageGroupID).to receive(:new).with(arg).and_return(dbl) 136 | expect(dbl).to receive(:message_group_id).and_return(message_group_id) 137 | 138 | TestJobWithMessageGroupID.perform_later(arg) 139 | sleep(0.2) 140 | end 141 | end 142 | end 143 | 144 | context 'with queue delay' do 145 | it 'enqueues jobs with proper delay' do 146 | t1 = Time.now 147 | allow(Time).to receive(:now).and_return t1 148 | 149 | expect(client).to receive(:send_message).with( 150 | { 151 | queue_url: 'https://queue-url', 152 | delay_seconds: 60, 153 | message_body: instance_of(String), 154 | message_attributes: instance_of(Hash) 155 | } 156 | ) 157 | 158 | TestJob.set(wait: 1.minute).perform_later('test') 159 | sleep(0.2) 160 | end 161 | 162 | it 'enqueues jobs with zero or negative delay' do 163 | t1 = Time.now 164 | allow(Time).to receive(:now).and_return t1 165 | 166 | expect(client).to receive(:send_message).with( 167 | { 168 | queue_url: 'https://queue-url', 169 | delay_seconds: 0, 170 | message_body: instance_of(String), 171 | message_attributes: instance_of(Hash) 172 | } 173 | ).twice 174 | 175 | TestJob.set(wait: 0).perform_later('test') 176 | TestJob.set(wait: -1).perform_later('test') 177 | sleep(0.2) 178 | end 179 | 180 | it 'raises an error when job delay is great than SQS support' do 181 | t1 = Time.now 182 | allow(Time).to receive(:now).and_return t1 183 | expect do 184 | TestJob.set(wait: 1.day).perform_later('test') 185 | end.to raise_error ArgumentError 186 | end 187 | end 188 | 189 | context 'with multiple jobs' do 190 | before do 191 | response = double('Response') 192 | allow(response).to receive(:successful).and_return([1, 2]) 193 | allow(client).to receive(:send_message_batch).and_return(response) 194 | end 195 | 196 | it do 197 | expect(client).to receive(:send_message_batch).with( 198 | { 199 | queue_url: 'https://queue-url', 200 | entries: [ 201 | { 202 | delay_seconds: instance_of(Integer), 203 | id: instance_of(String), 204 | message_body: instance_of(String), 205 | message_attributes: instance_of(Hash) 206 | }, 207 | { 208 | delay_seconds: instance_of(Integer), 209 | id: instance_of(String), 210 | message_body: instance_of(String), 211 | message_attributes: instance_of(Hash) 212 | } 213 | ] 214 | } 215 | ).once 216 | 217 | jobs = [ 218 | TestJob.new('test').set(wait: 1.minute), 219 | TestJob.new('test').set(wait: 1.minute) 220 | ] 221 | ActiveJob.perform_all_later(jobs) 222 | end 223 | end 224 | end 225 | end 226 | end 227 | -------------------------------------------------------------------------------- /spec/active_job/queue_adapters/sqs_async_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ActiveJob 4 | module QueueAdapters 5 | describe SqsAsyncAdapter do 6 | let(:client) { double('Client') } 7 | 8 | before do 9 | allow(Aws::ActiveJob::SQS.config).to receive(:client).and_return(client) 10 | end 11 | 12 | def mock_send_message 13 | expect(client).to receive(:send_message).with( 14 | { 15 | queue_url: 'https://queue-url', 16 | message_attributes: instance_of(Hash), 17 | message_body: include("\"locale\":\"#{I18n.locale}\"") 18 | } 19 | ) 20 | end 21 | 22 | def mock_async 23 | expect(Concurrent::Promises).to receive(:future).and_call_original 24 | end 25 | 26 | it 'enqueues jobs without blocking' do 27 | mock_send_message 28 | mock_async 29 | 30 | TestJobAsync.perform_later('test') 31 | sleep(0.2) 32 | end 33 | 34 | it 'calls the custom error handler when set' do 35 | expect(client).to receive(:send_message).and_raise('error') 36 | allow(Aws::ActiveJob::SQS.config) 37 | .to receive(:async_queue_error_handler) 38 | .and_return(proc { @error_handled = true }) 39 | 40 | TestJobAsync.perform_later('test') 41 | sleep(0.2) 42 | 43 | expect(@error_handled).to be true 44 | end 45 | 46 | it 'passes the serialized I18n locale to promises' do 47 | I18n.available_locales = %i[en de] # necessary, defaults empty 48 | 49 | I18n.with_locale(:de) do 50 | mock_async 51 | mock_send_message 52 | 53 | TestJobAsync.perform_later('test') 54 | sleep(0.2) 55 | end 56 | 57 | I18n.available_locales = [] 58 | end 59 | 60 | it 'queues jobs to fifo queues synchronously' do 61 | allow(Aws::ActiveJob::SQS.config).to receive(:url_for) 62 | .and_return('https://queue-url.fifo') 63 | expect(Concurrent::Promises).not_to receive(:future) 64 | expect(client).to receive(:send_message) 65 | 66 | TestJobAsync.perform_later('test') 67 | sleep(0.2) 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /spec/aws-activejob-sqs_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | describe SQS do 6 | describe '.config' do 7 | before { Aws::ActiveJob::SQS.instance_variable_set(:@config, nil) } 8 | 9 | it 'creates and returns configuration' do 10 | expect(Aws::ActiveJob::SQS::Configuration).to receive(:new).and_call_original 11 | expect(Aws::ActiveJob::SQS.config).to be_a Aws::ActiveJob::SQS::Configuration 12 | end 13 | 14 | it 'creates config only once' do 15 | expect(Aws::ActiveJob::SQS::Configuration).to receive(:new).once.and_call_original 16 | # call twice 17 | Aws::ActiveJob::SQS.config 18 | Aws::ActiveJob::SQS.config 19 | end 20 | end 21 | 22 | describe '.configure' do 23 | it 'allows configuration through a block' do 24 | Aws::ActiveJob::SQS.configure do |config| 25 | config.threads = 3 26 | config.backpressure = 5 27 | end 28 | 29 | expect(Aws::ActiveJob::SQS.config).to have_attributes( 30 | threads: 3, 31 | backpressure: 5 32 | ) 33 | end 34 | end 35 | 36 | describe '.fifo?' do 37 | it 'returns true if queue_url is fifo' do 38 | queue_url = 'https://sqs.us-west-2.amazonaws.com/012345678910/queue.fifo' 39 | expect(Aws::ActiveJob::SQS.fifo?(queue_url)).to be(true) 40 | end 41 | 42 | it 'returns false if queue_url is not fifo' do 43 | queue_url = 'https://sqs.us-west-2.amazonaws.com/012345678910/queue' 44 | expect(Aws::ActiveJob::SQS.fifo?(queue_url)).to be(false) 45 | end 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /spec/aws/active_job/sqs/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | describe Configuration do 7 | let(:expected_file_opts) do 8 | { 9 | max_messages: 5, 10 | queues: { default: { url: 'https://queue-url', max_messages: 2 } } 11 | } 12 | end 13 | 14 | it 'configures defaults without runtime or YAML options' do 15 | allow(File).to receive(:exist?).and_return(false) 16 | cfg = Aws::ActiveJob::SQS::Configuration.new 17 | expect(cfg.to_h).to include(Aws::ActiveJob::SQS::Configuration::DEFAULTS) 18 | end 19 | 20 | it 'merges runtime options with default options' do 21 | allow(File).to receive(:exist?).and_return(false) 22 | cfg = Aws::ActiveJob::SQS::Configuration.new(shutdown_timeout: 360) 23 | expect(cfg.shutdown_timeout).to eq 360 24 | end 25 | 26 | it 'merges YAML options with default options' do 27 | cfg = Aws::ActiveJob::SQS::Configuration.new 28 | expected = Aws::ActiveJob::SQS::Configuration::DEFAULTS.merge(expected_file_opts) 29 | expect(cfg.to_h).to include(expected) 30 | end 31 | 32 | it 'merges runtime options with YAML options' do 33 | cfg = Aws::ActiveJob::SQS::Configuration.new(shutdown_timeout: 360) 34 | expected = Aws::ActiveJob::SQS::Configuration::DEFAULTS 35 | .merge(expected_file_opts) 36 | .merge(shutdown_timeout: 360) 37 | expect(cfg.to_h).to include(expected) 38 | end 39 | 40 | # For Ruby 3.1+, Psych 4 will normally raise BadAlias error 41 | it 'accepts YAML config with alias' do 42 | allow_any_instance_of(ERB).to receive(:result).and_return(<<~YAML) 43 | common: &common 44 | default: 45 | url: 'https://queue-url' 46 | queues: 47 | <<: *common 48 | YAML 49 | expect { Aws::ActiveJob::SQS::Configuration.new }.to_not raise_error 50 | end 51 | 52 | context 'ENV set' do 53 | Configuration::GLOBAL_ENV_CONFIGS.each do |config_name| 54 | next if config_name == :config_file 55 | 56 | describe "ENV #{config_name}" do 57 | let(:env_name) { "AWS_ACTIVE_JOB_SQS_#{config_name.to_s.upcase}" } 58 | 59 | let(:cfg) do 60 | options = {} 61 | options[config_name] = 'file_value' 62 | Tempfile.create('aws_active_job_sqs.yml') do |f| 63 | f << options.transform_keys(&:to_s).to_yaml 64 | f.rewind 65 | Configuration.new( 66 | config_file: f.path, 67 | queues: { default: {} } 68 | ) 69 | end 70 | end 71 | 72 | before(:each) do 73 | ENV[env_name] = 'env_value' 74 | end 75 | 76 | after(:each) do 77 | ENV.delete(env_name) 78 | end 79 | 80 | it 'uses values from ENV over default and file' do 81 | if Configuration::QUEUE_CONFIGS.include?(config_name) 82 | expect(cfg.send(:"#{config_name}_for", :default)).to eq('env_value') 83 | else 84 | expect(cfg.send(config_name)).to eq('env_value') 85 | end 86 | end 87 | end 88 | end 89 | 90 | Configuration::QUEUE_ENV_CONFIGS.each do |config_name| 91 | describe "ENV queue #{config_name}" do 92 | let(:env_name) { "AWS_ACTIVE_JOB_SQS_DEFAULT_#{config_name.to_s.upcase}" } 93 | 94 | let(:cfg) do 95 | options = { queues: { default: {} } } 96 | options[:queues][:default][config_name] = 'file_value' 97 | Tempfile.create('aws_active_job_sqs.yml') do |f| 98 | f << options.deep_transform_keys(&:to_s).to_yaml 99 | f.rewind 100 | Configuration.new( 101 | config_file: f.path, 102 | queues: { default: {} } 103 | ) 104 | end 105 | end 106 | 107 | before(:each) do 108 | ENV[env_name] = 'env_value' 109 | end 110 | 111 | after(:each) do 112 | ENV.delete(env_name) 113 | end 114 | 115 | it 'uses values from ENV over default and file' do 116 | expect(cfg.send(:"#{config_name}_for", :default)).to eq('env_value') 117 | end 118 | 119 | it 'uses runtime configured values over ENV' do 120 | options = { queues: { default: {} } } 121 | options[:queues][:default][config_name] = 'runtime_value' 122 | cfg = Configuration.new(options) 123 | expect(cfg.send(:"#{config_name}_for", :default)).to eq('runtime_value') 124 | end 125 | end 126 | end 127 | end 128 | 129 | describe '#client' do 130 | it 'does not create client on initialize' do 131 | expect(Aws::SQS::Client).not_to receive(:new) 132 | Aws::ActiveJob::SQS::Configuration.new 133 | end 134 | 135 | it 'creates a client on #client' do 136 | client = Aws::SQS::Client.new(stub_responses: true) 137 | cfg = Aws::ActiveJob::SQS::Configuration.new 138 | expect(Aws::SQS::Client).to receive(:new).and_return(client) 139 | cfg.client 140 | end 141 | end 142 | 143 | describe '#poller_error_handler' do 144 | it 'allows configuration through a block' do 145 | cfg = Aws::ActiveJob::SQS::Configuration.new 146 | cfg.poller_error_handler do 147 | # pass 148 | end 149 | expect(cfg.poller_error_handler).to be_a(Proc) 150 | end 151 | end 152 | 153 | Configuration::QUEUE_CONFIGS.each do |config_name| 154 | describe "##{config_name}_for" do 155 | let(:cfg) do 156 | queues = { 157 | default: {}, 158 | override: {} 159 | } 160 | queues[:override][config_name] = 'queue_value' 161 | options = { queues: queues, config_file: 'nonexistant' } 162 | options[config_name] = 'global_value' 163 | Aws::ActiveJob::SQS::Configuration.new(**options) 164 | end 165 | 166 | it 'returns the queue value when set' do 167 | expect(cfg.send(:"#{config_name}_for", :override)).to eq('queue_value') 168 | end 169 | 170 | it 'returns the global value when unset' do 171 | expect(cfg.send(:"#{config_name}_for", :default)).to eq('global_value') 172 | end 173 | 174 | it 'raises an ArgumentError when the queue is not mapped' do 175 | expect { cfg.send(:"#{config_name}_for", :not_mapped) }.to raise_error(ArgumentError) 176 | end 177 | end 178 | end 179 | end 180 | end 181 | end 182 | end 183 | -------------------------------------------------------------------------------- /spec/aws/active_job/sqs/deduplication_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | describe Deduplication do 7 | describe 'ClassMethods' do 8 | describe '.deduplicate_without' do 9 | let(:keys) { %w[job_id job_class queue_name] } 10 | let(:expected_keys) { keys.map(&:to_s) | ['job_id'] } 11 | 12 | it 'excluded deduplication keys set successfully' do 13 | expect(TestJobWithDedupKeys.deduplicate_without(*keys)).to contain_exactly(*expected_keys) 14 | end 15 | 16 | it 'excluded deduplication keys set successfully and job_id is added' do 17 | keys.delete(:job_id) 18 | expect(TestJobWithDedupKeys.deduplicate_without(*keys)).to contain_exactly(*expected_keys) 19 | end 20 | end 21 | end 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/aws/active_job/sqs/executor_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | describe Executor do 7 | let(:logger) { double(info: nil, debug: nil) } 8 | 9 | before do 10 | allow(ActiveSupport::Logger).to receive(:new).and_return(logger) 11 | end 12 | 13 | it 'merges runtime options with defaults' do 14 | expected = Executor::DEFAULTS.merge(max_queue: 10) 15 | expect(Concurrent::ThreadPoolExecutor).to receive(:new).with(expected) 16 | Executor.new(max_queue: 10) 17 | end 18 | 19 | describe '#execute' do 20 | let(:body) { ActiveSupport::JSON.dump(TestJob.new('a1', 'a2').serialize) } 21 | # message is a reserved minitest name 22 | let(:msg) { double(data: double(body: body)) } 23 | let(:executor) { Executor.new } 24 | let(:runner) { double('runner', id: 'jobid', class_name: 'jobclass', exception_executions?: false) } 25 | 26 | it 'executes the job and deletes the message' do 27 | expect(JobRunner).to receive(:new).and_return(runner) 28 | expect(runner).to receive(:run) 29 | expect(msg).to receive(:delete) 30 | executor.execute(msg) 31 | executor.shutdown # give the job a chance to run 32 | end 33 | 34 | it 'raises the error and terminates poller' do 35 | expect(JobRunner).to receive(:new).and_return(runner) 36 | expect(runner).to receive(:run).and_raise StandardError 37 | expect do 38 | executor.execute(msg) 39 | executor.shutdown # give the job a chance to run 40 | end.to raise_exception(StandardError) 41 | end 42 | 43 | describe 'error_handler' do 44 | let(:error_handler) { double } 45 | let(:executor) { Executor.new(error_handler: error_handler) } 46 | let(:exception) { StandardError.new } 47 | 48 | it 'calls the error handler with exception and message' do 49 | expect(JobRunner).to receive(:new).and_return(runner) 50 | expect(runner).to receive(:run).and_raise exception 51 | expect(error_handler).to receive(:call).with(exception, msg) 52 | expect(executor).to receive(:shutdown).exactly(1).times.and_call_original 53 | 54 | executor.execute(msg) 55 | executor.shutdown # give the job a chance to run 56 | end 57 | end 58 | 59 | describe 'backpressure' do 60 | let(:executor) { Executor.new(max_threads: 1, max_queue: 1) } 61 | let(:trigger) { Concurrent::Event.new } 62 | 63 | it 'waits for a tasks to complete before attempting to post new tasks' do 64 | task_complete_event = executor.instance_variable_get(:@task_complete) 65 | expect(JobRunner).to receive(:new).at_least(:once).and_return(runner) 66 | allow(msg).to receive(:delete) 67 | allow(runner).to receive(:run) do 68 | trigger.wait 69 | end 70 | expect(task_complete_event).to receive(:wait).at_least(:once) do 71 | trigger.set # unblock the task 72 | end 73 | executor.execute(msg) # first message runs 74 | executor.execute(msg) # second message enters queue 75 | executor.execute(msg) # third message triggers wait 76 | end 77 | end 78 | end 79 | 80 | describe '#shutdown' do 81 | let(:tp) { double } 82 | 83 | it 'calls shutdown and waits for termination' do 84 | expect(Concurrent::ThreadPoolExecutor).to receive(:new).and_return(tp) 85 | executor = Executor.new 86 | expect(tp).to receive(:shutdown) 87 | expect(tp).to receive(:wait_for_termination).with(5).and_return true 88 | executor.shutdown(5) 89 | end 90 | 91 | context 'errors during shutdown' do 92 | let(:error_handler) { double } 93 | let(:body) { ActiveSupport::JSON.dump(TestJob.new('a1', 'a2').serialize) } 94 | let(:msg) { double(data: double(body: body)) } 95 | let(:executor) { Executor.new(error_handler: error_handler) } 96 | let(:runner) { double('runner', id: 'jobid', class_name: 'jobclass', exception_executions?: false) } 97 | 98 | it 'handles errors from jobs during shutdown' do 99 | expect(JobRunner).to receive(:new).and_return(runner) 100 | expect(runner).to receive(:run) do 101 | sleep(0.1) 102 | raise StandardError 103 | end 104 | expect(error_handler).to receive(:call) 105 | expect(executor).to receive(:shutdown).exactly(1).times.and_call_original 106 | 107 | executor.execute(msg) 108 | executor.shutdown 109 | end 110 | end 111 | 112 | context 'lifecycle hooks are registered' do 113 | let(:hook) { double } 114 | 115 | before do 116 | allow(hook).to receive(:call) 117 | end 118 | 119 | after do 120 | Executor.clear_hooks 121 | end 122 | 123 | it 'executes hook when shutdown' do 124 | Aws::ActiveJob::SQS.on_worker_stop do 125 | hook.call 126 | end 127 | executor = Executor.new 128 | 129 | executor.shutdown 130 | 131 | expect(hook).to have_received(:call) 132 | end 133 | end 134 | end 135 | end 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /spec/aws/active_job/sqs/job_runner_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Aws 4 | module ActiveJob 5 | module SQS 6 | describe JobRunner do 7 | let(:job_data) { TestJob.new('a1', 'a2').serialize } 8 | let(:body) { ActiveSupport::JSON.dump(job_data) } 9 | # message is a reserved minitest name 10 | let(:msg) { double(data: double(body: body)) } 11 | 12 | it 'parses the job data' do 13 | job_runner = JobRunner.new(msg) 14 | expect(job_runner.instance_variable_get(:@job_data)).to eq job_data 15 | end 16 | 17 | describe '#run' do 18 | it 'calls Base.execute with the job data' do 19 | expect(::ActiveJob::Base).to receive(:execute).with(job_data) 20 | JobRunner.new(msg).run 21 | end 22 | end 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/aws/active_job/sqs/poller_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'aws/active_job/sqs/poller' 4 | 5 | module Aws 6 | module ActiveJob 7 | module SQS 8 | describe Poller do 9 | let(:queue_poller) { double(Aws::SQS::QueuePoller) } 10 | let(:msg) { double('SQSMessage', receipt_handle: '1234') } 11 | let(:logger) { double(info: nil) } 12 | let(:sqs_client) { Aws::SQS::Client.new(stub_responses: true) } 13 | let(:executor) { double } 14 | 15 | before do 16 | allow(ActiveSupport::Logger).to receive(:new).and_return(logger) 17 | allow(Aws::ActiveJob::SQS.config).to receive(:client).and_return(sqs_client) 18 | allow(Executor).to receive(:new).and_return(executor) 19 | end 20 | 21 | describe '#initialize' do 22 | it 'initializes options' do 23 | poller = Poller.new(max_messages: 3, visibility_timeout: 360) 24 | parsed = poller.instance_variable_get(:@options) 25 | expect(parsed[:max_messages]).to eq 3 26 | expect(parsed[:visibility_timeout]).to eq 360 27 | end 28 | end 29 | 30 | describe '#run' do 31 | let(:poller) do 32 | Poller.new( 33 | queues: [:default], 34 | visibility_timeout: 360, 35 | shutdown_timeout: 42 36 | ) 37 | end 38 | 39 | it 'merges args with loaded config' do 40 | expect(Aws::SQS::QueuePoller).to receive(:new).and_return(queue_poller) 41 | 42 | expect(queue_poller).to receive(:poll).with( 43 | { 44 | skip_delete: true, 45 | max_number_of_messages: 2, # from test app config file 46 | visibility_timeout: 360 # from options 47 | } 48 | ) 49 | 50 | poller.run 51 | end 52 | 53 | it 'polls the single configured queue in the main thread' do 54 | expect(Aws::SQS::QueuePoller).to receive(:new).with( 55 | 'https://queue-url', 56 | { client: sqs_client } 57 | ).and_return(queue_poller) 58 | 59 | expect(queue_poller).to receive(:poll) 60 | expect(Thread).not_to receive(:new) 61 | poller.run 62 | end 63 | 64 | it 'runs the poller with the configured options' do 65 | expect(Aws::SQS::QueuePoller).to receive(:new).and_return(queue_poller) 66 | 67 | expect(queue_poller).to receive(:poll).with( 68 | { 69 | skip_delete: true, 70 | max_number_of_messages: 2, # from queue config in app config file 71 | visibility_timeout: 360 72 | } 73 | ) 74 | 75 | poller.run 76 | end 77 | 78 | it 'sets max_number_of_messages to 1 for fifo queues' do 79 | allow_any_instance_of(Configuration).to receive(:url_for).and_return('https://queue-url.fifo') 80 | expect(Aws::SQS::QueuePoller).to receive(:new).and_return(queue_poller) 81 | 82 | expect(queue_poller).to receive(:poll).with( 83 | { 84 | skip_delete: true, 85 | max_number_of_messages: 1, 86 | visibility_timeout: 360 87 | } 88 | ) 89 | 90 | poller.run 91 | end 92 | 93 | it 'polls for messages and executes them' do 94 | executor = double(Executor) 95 | expect(Executor).to receive(:new).and_return(executor) 96 | 97 | expect(Aws::SQS::QueuePoller).to receive(:new).and_return(queue_poller) 98 | expect(queue_poller).to receive(:poll) { |&block| block.call([msg, msg]) } 99 | 100 | expect(executor).to receive(:execute).twice.with(instance_of(Aws::SQS::Message)) 101 | 102 | poller.run 103 | end 104 | 105 | it 'calls shutdown when interrupted' do 106 | executor = double(Executor) 107 | expect(Executor).to receive(:new).and_return(executor) 108 | 109 | expect(Aws::SQS::QueuePoller).to receive(:new).and_return(queue_poller) 110 | expect(queue_poller).to receive(:poll).and_raise(Poller::Interrupt) 111 | 112 | expect(executor).to receive(:shutdown).with(42) # from options 113 | expect(poller).to receive(:exit) # no-op the exit 114 | 115 | poller.run 116 | end 117 | 118 | context 'queue without configuration' do 119 | let(:poller) { Poller.new(queues: [:not_defined]) } 120 | 121 | it 'raises an error' do 122 | expect { poller.run }.to raise_error(ArgumentError) 123 | end 124 | end 125 | 126 | context 'queue without url configuration' do 127 | let(:poller) { Poller.new(queues: [:no_url]) } 128 | 129 | it 'raises an error' do 130 | Aws::ActiveJob::SQS.configure do |config| 131 | config.queues[:no_url] = {} 132 | end 133 | expect { poller.run }.to raise_error(ArgumentError) 134 | end 135 | end 136 | 137 | context 'multiple queues' do 138 | let(:poller) do 139 | Poller.new( 140 | queues: %i[default other], 141 | visibility_timeout: 360, 142 | shutdown_timeout: 42 143 | ) 144 | end 145 | 146 | before(:each) do 147 | Aws::ActiveJob::SQS.configure do |config| 148 | config.queues[:other] = { url: 'https://other-queue-url' } 149 | end 150 | end 151 | 152 | it 'starts a thread and polls for each queue' do 153 | thread = double(join: nil) 154 | allow(Thread).to receive(:new).and_yield.and_return(thread) 155 | 156 | queue_poller_default = double 157 | expect(Aws::SQS::QueuePoller).to receive(:new).with( 158 | 'https://queue-url', 159 | { client: sqs_client } 160 | ).and_return(queue_poller_default) 161 | expect(queue_poller_default).to receive(:poll) 162 | 163 | queue_poller_other = double 164 | expect(Aws::SQS::QueuePoller).to receive(:new).with( 165 | 'https://other-queue-url', 166 | { client: sqs_client } 167 | ).and_return(queue_poller_other) 168 | expect(queue_poller_other).to receive(:poll) 169 | 170 | poller.run 171 | end 172 | end 173 | end 174 | end 175 | end 176 | end 177 | end 178 | -------------------------------------------------------------------------------- /spec/dummy/config.ru: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This file indicates the Rails root directory 4 | -------------------------------------------------------------------------------- /spec/dummy/config/application.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rails' 4 | require 'action_controller/railtie' 5 | require 'active_job/railtie' 6 | 7 | require 'aws-activejob-sqs' 8 | 9 | # @api private 10 | module Dummy 11 | class Application < Rails::Application 12 | config.load_defaults Rails::VERSION::STRING.to_f 13 | config.eager_load = true 14 | config.secret_key_base = 'secret' 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/dummy/config/aws_active_job_sqs.yml: -------------------------------------------------------------------------------- 1 | max_messages: 5 2 | queues: 3 | default: 4 | url: 'https://queue-url' 5 | max_messages: 2 6 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | ENV['RAILS_ENV'] = 'test' 4 | 5 | require_relative 'dummy/config/application' 6 | 7 | Rails.application.initialize! 8 | 9 | class TestJob < ActiveJob::Base 10 | self.queue_adapter = :sqs 11 | self.logger = ActiveSupport::Logger.new(IO::NULL) 12 | queue_as :default 13 | 14 | def perform(arg1, arg2); end 15 | end 16 | 17 | class TestJobAsync < ActiveJob::Base 18 | self.queue_adapter = :sqs_async 19 | self.logger = ActiveSupport::Logger.new(IO::NULL) 20 | queue_as :default 21 | 22 | def perform(arg1, arg2); end 23 | end 24 | 25 | class TestJobWithMessageGroupID < TestJob 26 | def message_group_id; end 27 | end 28 | 29 | class TestJobWithDedupKeys < TestJob 30 | include Aws::ActiveJob::SQS::Deduplication 31 | end 32 | 33 | # This file was generated by the `rspec --init` command. Conventionally, all 34 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 35 | # The generated `.rspec` file contains `--require spec_helper` which will cause 36 | # this file to always be loaded, without a need to explicitly require it in any 37 | # files. 38 | # 39 | # Given that it is always loaded, you are encouraged to keep this file as 40 | # light-weight as possible. Requiring heavyweight dependencies from this file 41 | # will add to the boot time of your test suite on EVERY test run, even for an 42 | # individual file that may not need all of that loaded. Instead, consider making 43 | # a separate helper file that requires the additional dependencies and performs 44 | # the additional setup, and require it from the spec files that actually need 45 | # it. 46 | # 47 | # See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 48 | RSpec.configure do |config| 49 | # rspec-expectations config goes here. You can use an alternate 50 | # assertion/expectation library such as wrong or the stdlib/minitest 51 | # assertions if you prefer. 52 | config.expect_with :rspec do |expectations| 53 | # This option will default to `true` in RSpec 4. It makes the `description` 54 | # and `failure_message` of custom matchers include text for helper methods 55 | # defined using `chain`, e.g.: 56 | # be_bigger_than(2).and_smaller_than(4).description 57 | # # => "be bigger than 2 and smaller than 4" 58 | # ...rather than: 59 | # # => "be bigger than 2" 60 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 61 | end 62 | 63 | # rspec-mocks config goes here. You can use an alternate test double 64 | # library (such as bogus or mocha) by changing the `mock_with` option here. 65 | config.mock_with :rspec do |mocks| 66 | # Prevents you from mocking or stubbing a method that does not exist on 67 | # a real object. This is generally recommended, and will default to 68 | # `true` in RSpec 4. 69 | mocks.verify_partial_doubles = true 70 | end 71 | 72 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 73 | # have no way to turn it off -- the option exists only for backwards 74 | # compatibility in RSpec 3). It causes shared context metadata to be 75 | # inherited by the metadata hash of host groups and examples, rather than 76 | # triggering implicit auto-inclusion in groups with matching metadata. 77 | config.shared_context_metadata_behavior = :apply_to_host_groups 78 | 79 | # Run specs in random order to surface order dependencies. If you find an 80 | # order dependency and want to debug it, you can fix the order by providing 81 | # the seed, which is printed after each run. 82 | # --seed 1234 83 | config.order = :random 84 | 85 | # The settings below are suggested to provide a good initial experience 86 | # with RSpec, but feel free to customize to your heart's content. 87 | =begin 88 | # This allows you to limit a spec run to individual examples or groups 89 | # you care about by tagging them with `:focus` metadata. When nothing 90 | # is tagged with `:focus`, all examples get run. RSpec also provides 91 | # aliases for `it`, `describe`, and `context` that include `:focus` 92 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 93 | config.filter_run_when_matching :focus 94 | 95 | # Allows RSpec to persist some state between runs in order to support 96 | # the `--only-failures` and `--next-failure` CLI options. We recommend 97 | # you configure your source control system to ignore this file. 98 | config.example_status_persistence_file_path = "spec/examples.txt" 99 | 100 | # Limits the available syntax to the non-monkey patched syntax that is 101 | # recommended. For more details, see: 102 | # https://rspec.info/features/3-12/rspec-core/configuration/zero-monkey-patching-mode/ 103 | config.disable_monkey_patching! 104 | 105 | # This setting enables warnings. It's recommended, but in some cases may 106 | # be too noisy due to issues in dependencies. 107 | config.warnings = true 108 | 109 | # Many RSpec users commonly either run the entire suite or an individual 110 | # file, and it's useful to allow more verbose output when running an 111 | # individual spec file. 112 | if config.files_to_run.one? 113 | # Use the documentation formatter for detailed output, 114 | # unless a formatter has already been configured 115 | # (e.g. via a command-line flag). 116 | config.default_formatter = "doc" 117 | end 118 | 119 | # Print the 10 slowest examples and example groups at the 120 | # end of the spec run, to help surface which specs are running 121 | # particularly slow. 122 | config.profile_examples = 10 123 | 124 | # Seed global randomization in this process using the `--seed` CLI option. 125 | # Setting this allows you to use `--seed` to deterministically reproduce 126 | # test failures related to randomization by passing the same `--seed` value 127 | # as the one that triggered the failure. 128 | Kernel.srand config.seed 129 | =end 130 | end 131 | --------------------------------------------------------------------------------