├── .github └── workflows │ ├── codeql-analysis.yml │ └── main.yaml ├── .gitignore ├── .rubocop.yml ├── .tool-versions ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── lib ├── generators │ └── queue_classic │ │ ├── install_generator.rb │ │ └── templates │ │ ├── add_queue_classic.rb │ │ ├── update_queue_classic_3_0_0.rb │ │ ├── update_queue_classic_3_0_2.rb │ │ ├── update_queue_classic_3_1_0.rb │ │ └── update_queue_classic_4_0_0.rb ├── queue_classic.rb └── queue_classic │ ├── config.rb │ ├── conn_adapter.rb │ ├── queue.rb │ ├── railtie.rb │ ├── setup.rb │ ├── tasks.rb │ ├── version.rb │ └── worker.rb ├── queue_classic.gemspec ├── sql ├── create_table.sql ├── ddl.sql ├── downgrade_from_3_0_0.sql ├── downgrade_from_3_1_0.sql ├── downgrade_from_4_0_0.sql ├── drop_ddl.sql ├── update_to_3_0_0.sql ├── update_to_3_1_0.sql └── update_to_4_0_0.sql └── test ├── benchmark_test.rb ├── config_test.rb ├── hard_coding_test.rb ├── helper.rb ├── helper.sql ├── lib ├── queue_classic_rails_connection_test.rb ├── queue_classic_test.rb └── queue_classic_test_with_activerecord_typecast.rb ├── queue_test.rb ├── rails-tests ├── .gitignore └── rails523.sh └── worker_test.rb /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '30 21 * * 0' 22 | 23 | concurrency: 24 | group: ${{ github.workflow }}-${{ github.ref }} 25 | cancel-in-progress: true 26 | 27 | jobs: 28 | analyze: 29 | name: Analyze 30 | runs-on: ubuntu-latest 31 | permissions: 32 | actions: read 33 | contents: read 34 | security-events: write 35 | 36 | strategy: 37 | fail-fast: false 38 | matrix: 39 | language: [ 'ruby' ] 40 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 41 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 42 | 43 | steps: 44 | - name: Checkout repository 45 | uses: actions/checkout@v3 46 | 47 | # Initializes the CodeQL tools for scanning. 48 | - name: Initialize CodeQL 49 | uses: github/codeql-action/init@v3 50 | with: 51 | languages: ${{ matrix.language }} 52 | # If you wish to specify custom queries, you can do so here or in a config file. 53 | # By default, queries listed here will override any specified in a config file. 54 | # Prefix the list here with "+" to use these queries and those in the config file. 55 | 56 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 57 | # queries: security-extended,security-and-quality 58 | 59 | 60 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 61 | # If this step fails, then you should remove it and run the build manually (see below) 62 | - name: Autobuild 63 | uses: github/codeql-action/autobuild@v3 64 | 65 | # ℹ️ Command-line programs to run using the OS shell. 66 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 67 | 68 | # If the Autobuild fails above, remove it and uncomment the following three lines. 69 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 70 | 71 | # - run: | 72 | # echo "Run, Build Application using script" 73 | # ./location_of_script_within_repo/buildscript.sh 74 | 75 | - name: Perform CodeQL Analysis 76 | uses: github/codeql-action/analyze@v3 77 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Lint & Test 2 | on: push 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }} 5 | cancel-in-progress: true 6 | jobs: 7 | # test_rails: 8 | # runs-on: ubuntu-latest 9 | # services: 10 | # postgres: 11 | # image: postgres 12 | # env: 13 | # POSTGRES_PASSWORD: postgres 14 | # options: >- 15 | # --health-cmd pg_isready 16 | # --health-interval 10s 17 | # --health-timeout 5s 18 | # --health-retries 5 19 | # ports: 20 | # - 5432:5432 21 | # steps: 22 | # - uses: actions/checkout@v3 23 | # - name: Install postgresql-client 24 | # run: | 25 | # sudo apt-get update 26 | # sudo apt-get install -y libpq-dev 27 | # - uses: ruby/setup-ruby@v1 28 | # with: 29 | # ruby-version: "2.6" 30 | # bundler-cache: true # runs 'bundle install' and caches installed gems automatically 31 | # - name: Test clean install in Rails 32 | # env: 33 | # DATABASE_URL: postgres://postgres:postgres@localhost/qctest523 34 | # run: | 35 | # cd test/rails-tests/ 36 | # sh rails523.sh 37 | lint: 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: actions/checkout@v4 41 | - uses: ruby/setup-ruby@v1 42 | with: 43 | ruby-version: '3.3' 44 | bundler-cache: true 45 | - name: Run rubocop 46 | run: bundle exec rubocop 47 | env: 48 | RUBY_YJIT_ENABLE: true 49 | test: 50 | runs-on: ubuntu-latest 51 | needs: lint 52 | strategy: 53 | fail-fast: false 54 | matrix: 55 | ruby_version: ['3.0', '3.1', '3.2', '3.3'] 56 | services: 57 | postgres: 58 | image: postgres 59 | env: 60 | POSTGRES_PASSWORD: postgres 61 | options: >- 62 | --health-cmd pg_isready 63 | --health-interval 10s 64 | --health-timeout 5s 65 | --health-retries 5 66 | ports: 67 | - 5432:5432 68 | steps: 69 | - uses: actions/checkout@v4 70 | - name: Install postgresql-client 71 | run: | 72 | sudo apt-get update 73 | sudo apt-get install -y libpq-dev 74 | - uses: ruby/setup-ruby@v1 75 | with: 76 | ruby-version: ${{ matrix.ruby_version }} 77 | bundler-cache: true 78 | - name: Run tests 79 | run: bundle exec rake 80 | env: 81 | RAILS_ENV: test 82 | RACK_ENV: test 83 | QC_DATABASE_URL: postgres://postgres:postgres@localhost/postgres 84 | DATABASE_URL: postgres://postgres:postgres@localhost/postgres 85 | QC_BENCHMARK: true 86 | QC_BENCHMARK_MAX_TIME_DEQUEUE: 60 87 | QC_BENCHMARK_MAX_TIME_ENQUEUE: 10 88 | RUBY_YJIT_ENABLE: true 89 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .db 2 | .ruby-version 3 | .bundle 4 | .rvmrc 5 | etc/ 6 | *.gem 7 | .env 8 | Gemfile.lock 9 | tips-and-tricks.md 10 | test/reports/ 11 | vendor/ -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | NewCops: enable 3 | TargetRubyVersion: 3.0 4 | Exclude: 5 | - 'vendor/**/*' 6 | Layout/LineLength: 7 | Enabled: false 8 | Metrics/ClassLength: 9 | Enabled: false 10 | Metrics/MethodLength: 11 | Enabled: false 12 | Metrics/BlockLength: 13 | Enabled: false 14 | Naming/MethodParameterName: 15 | Enabled: false 16 | Metrics/AbcSize: 17 | Enabled: false 18 | Naming/VariableNumber: 19 | Enabled: false 20 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | ruby 3.3.0 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [4.0.0] - 2022-05-17 6 | 7 | Updates: 8 | - [Allow overriding of job successes](https://github.com/QueueClassic/queue_classic/pull/338) 9 | - [Move to @github actions - @circleci ended up being slower. Test for Ruby 2.6, 2.7, 3.0 and 3.1. Require a more modern, but still old PG gem.](https://github.com/QueueClassic/queue_classic/pull/335) 10 | - [Setup some code analysis](https://github.com/QueueClassic/queue_classic/pull/337) 11 | 12 | ## [4.0.0-alpha1] - 2019-07-18 13 | 14 | Updates: 15 | - [Change to only support >= Postgres 9.6. We will be bringing in newer changes and testing on only 9.6+ going forward.](https://github.com/QueueClassic/queue_classic/pull/307) 16 | - [Change to only support currently supported Ruby versions: 2.4, 2.5 and 2.6.](https://github.com/QueueClassic/queue_classic/pull/305) 17 | - [Use skip-locked](https://github.com/QueueClassic/queue_classic/pull/303) 18 | - [Add abilty to count ready and scheduled jobs](https://github.com/QueueClassic/queue_classic/pull/255) 19 | 20 | Bug fixes: 21 | - [Switched project to use CircleCI, as it's way more consistent speed wise](https://github.com/QueueClassic/queue_classic/pull/304) 22 | - [Automatically retry after a connection reset #294](https://github.com/QueueClassic/queue_classic/pull/294) 23 | - [Add tests for installing fresh on rails 5.2.3 + running migrations](https://github.com/QueueClassic/queue_classic/pull/308) 24 | - [Don't recuse Time.now errors](https://github.com/QueueClassic/queue_classic/pull/310) 25 | - Use the jsonb type for the args column from now on. If not available, fall back to json or text. 26 | - `enqueue`, `enqueue_at`, `enqueue_in` return job hash with id. 27 | - Fixed a bug in the offset calculation of `.enqueue_at`. 28 | 29 | ## [3.0.0rc] - 2014-01-07 30 | 31 | - Improved signal handling 32 | 33 | ## [3.0.0beta] - 2014-01-06 34 | 35 | - Workers can process many queues. 36 | 37 | ## [2.2.3] - 2013-10-24 38 | 39 | - Update pg dependency to 0.17.0 40 | 41 | ## [2.3.0beta] - 2013-09-05 YANKED 42 | 43 | - Concurrent job processing. 44 | 45 | ## [2.2.2] - 2013-08-04 46 | 47 | - Update pg dependency to 0.16.0 48 | 49 | ## [2.2.1] - 2013-07-12 50 | 51 | - Force listen/notify on worker 52 | - Notifications happen inside PostgreSQL trigger 53 | - Add rake task for generating rails migrations 54 | - Fix bug related to listening worker 55 | 56 | ## [2.2.0] - 2013-07-02 57 | 58 | - Use json from the stdlib in place of MultiJson. 59 | - Use postgresql's json type for the args column if json type is available 60 | - QC::Worker#handle_failure logs the job and the error 61 | - QC.default_queue= to set your own default queue. (can be used 62 | in testing to configure a mock queue) 63 | - QC.log now reports time elapsed in milliseconds. 64 | 65 | ## [2.1.4] 66 | 67 | - Update pg dependency to 0.15.1 68 | - Document logging behaviour 69 | 70 | ## [2.1.3] 71 | 72 | - Use MultiJson (Ezekiel Templin: #106) 73 | 74 | ## [2.1.2] 75 | 76 | - Use 64bit ints as default data types in PostgreSQL 77 | - Add process method in worker 78 | - Allow percent-encoded socket paths in DATABASE_URL 79 | 80 | ## [2.1.1] 81 | 82 | - Update pg gem version 83 | 84 | ## [2.1.0] 85 | 86 | - Wrap connection execution in mutex making it thread safe 87 | - Cleanup logging 88 | - Refactor worker class making it more extensible 89 | - Added rdoc style docs for worker class 90 | 91 | ## [2.0.5] 92 | 93 | - Allow term signal to halt the lock_job function 94 | 95 | ## [2.0.4] 96 | 97 | - Provider a connection setter. 98 | 99 | ## [2.0.3] 100 | 101 | - Fix typo :( 102 | 103 | ## [2.0.2] 104 | 105 | - Remove scrolls dependency 106 | - Fix issue with notify not working on non-default queues 107 | 108 | ## [2.0.1] 109 | 110 | ## [2.0.0] 111 | 112 | - Simpler setup via QC::Setup.create (rake qc:create) & QC::Setup.drop (rake 113 | qc:drop) 114 | - Simpler abstractions in implementation 115 | - Better support for instrumentation via log_yield hook in QC module 116 | - Multiple queues use one table with a queue_name column 117 | 118 | ## [1.0.2] 119 | 120 | - Update to latest okjson as the current has bugs 121 | 122 | ## [1.0.1] 123 | 124 | - Using OkJson instead of any sort of rubygem 125 | - Remove html from docs 126 | - Use parameterised queries 127 | - Don't set application name by default 128 | - Injection attack bug fixed in lock_head() 129 | - Notificaiton get sent on seperate chans for disjoint queues 130 | 131 | ## [1.0.0rc1] - 2011-08-29 132 | 133 | - Removed json gem and relying on ruby 1.9.2's stdlib 134 | - Added better documentation 135 | 136 | ## [0.3.6pre] 137 | 138 | - Added listen/notify support configured by $QC_LISTENING_WORKER otherwise uses Kernel.sleep() 139 | 140 | ## [0.3.5pre] - 2011-08-27 141 | 142 | - Removed debug statement. Mistake! 143 | 144 | ## [0.3.4pre] 145 | 146 | - Added logging configured by $VERBOSE or $QC_VERBOSE. 147 | - Added a method setup_child that gets called right after a worker forks. 148 | - Removed database helper methods: create_table, drop_table, silence_warnings. 149 | - Removed queue connection helper methods. Status should be discoverd by psql or the likes. 150 | 151 | ## [0.3.3pre] 152 | 153 | - Removed PUB/SUB 154 | - Added GC after working a job 155 | - Added support for a database_url other than $DATABASE_URL. $QC_DATABASE_URL 156 | - Added exp backoff configured by $QC_MAX_LOCK_ATTEMPTS (default = 5) 157 | - Added option for forking worker configured by $QC_FORK_WORKER (default = false) 158 | 159 | ## [0.3.2] - 2011-08-03 160 | 161 | - Fixed bug which caused workers to consume 2 connections. Now they only consume 1 162 | - Added a rake file for tests 163 | - Added support for postgres:///db_name DATABASE_URLs 164 | 165 | ## [0.3.1] - 2011-04-27 166 | 167 | - Added query interface for introspection success 168 | - Moved the locking of jobs into the DB as a PG function. SELECT lock_head() 169 | - Added requirement for DB connection. MUST BE URI i.e. DATABASE_URL=postgres://user:pass@localhost/db_name 170 | - Added rake qc:create_queue. This task will add a new table. Use this for multiple queues. 171 | - Added a bit of randomness to the lock_head() function. Helps you scale to a hilarious number of workers. 172 | - Added support for trapping INT and TERM signals in the worker. ^C to stop after finished and ^C^C to kill. 173 | - Renamed the jobs table to queue_classic_jobs 174 | - Renamed the jobs channel to queue_classic_jobs 175 | - Added support for multiple queues 176 | 177 | ## [0.2.2] - 2011-02-26 178 | 179 | - Fixed problems with enqueueing a list of parameters. 180 | 181 | ## [0.2.1] - 2011-02-22 182 | 183 | - Added method for handling errors. 184 | - Added ability to enqueue a Job instance. Makes retrying jobs easier. 185 | - Added delete_all. 186 | - Fixed connection algorithm. 1 connection per process. 187 | - Fixed API for enqueue. Now accepting 1 arg or many args. 188 | 189 | ## [0.2.0] - 2011-02-17 190 | 191 | - Beta Release 192 | - Added method for handling failed jobs 193 | - Added Benchmarks 194 | - Removed logging 195 | - Moved the Job class into it's own file 196 | 197 | ## [0.1.6] - 2011-02-03 198 | 199 | - Early release 200 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at isitaboat+qc@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | queue_classic is a volunteer effort. We encourage you to pitch in. 2 | 3 | 1. Fork queue_classic 4 | 2. Create a topic branch - `git checkout -b my_branch` 5 | 3. Push to your branch - `git push origin my_branch` 6 | 4. Send us a pull-request for your topic branch 7 | 5. That's it! 8 | 9 | If you make code changes, please check that your patch: 10 | 11 | 1. has tests 12 | 2. works on Rails and non-Rails projects 13 | 3. updates documentation 14 | 15 | Thanks! :heart: 16 | 17 | queue_classic Team 18 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # this, is dumb, but stops errors 4 | source 'https://rubygems.org' 5 | 6 | source 'https://rubygems.org' do 7 | gem 'rake' 8 | 9 | gemspec 10 | 11 | group :development do 12 | gem 'rubocop' 13 | end 14 | 15 | group :development, :test do 16 | gem 'activerecord', '>= 5.0.0', '< 6.1' 17 | end 18 | 19 | group :test do 20 | gem 'minitest', '~> 5.8' 21 | gem 'minitest-reporters' 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # queue_classic 2 | A simple, efficient worker queue for Ruby & PostgreSQL. 3 | 4 | Why this over something like Resque. Two reasons: 5 | 6 | 1. Your jobs can be equeued in the same transaction as other modifications to the database, and will only be processed when everything is commited. This is a hard pattern to develop around for queues done outside your database 7 | 2. Less things to run, if you don't already have Redis or a dedicated queue in your stack. 8 | 9 | ![master](https://github.com/QueueClassic/queue_classic/actions/workflows/main.yaml/badge.svg?branch=master) 10 | 11 | [![Gem Version](http://img.shields.io/gem/v/queue_classic.svg?style=flat)](http://badge.fury.io/rb/queue_classic) 12 | 13 | **IMPORTANT NOTE: This README is representing the current work for queue_classic, which is generally the pending next version.** 14 | 15 | You can always find the latest and previous releases here: 16 | 17 | https://github.com/QueueClassic/queue_classic/releases 18 | 19 | ## Other related projects 20 | If you're interested in this project, you might also want to checkout: 21 | 22 | * [Que](https://github.com/que-rb/que) 23 | * [GoodJob](https://github.com/bensheldon/good_job) 24 | * [Delayed Job](https://github.com/collectiveidea/delayed_job) 25 | 26 | For a list of other queues (which may or may not be Postgres backed), checkout - https://edgeapi.rubyonrails.org/classes/ActiveJob/QueueAdapters.html 27 | 28 | ## What is queue_classic? 29 | queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable. 30 | 31 | A major benefit is the ability to enqueue inside transactions, ensuring things are done only when your changes are commited. 32 | 33 | ## Other related projects 34 | * [Queue Classic Plus](https://github.com/rainforestapp/queue_classic_plus) - adds support for retrying with specific exceptions, transaction processing of jobs, metric collection, etc 35 | * [Queue Classic Admin](https://github.com/QueueClassic/queue_classic_admin) - Admin interface for queue_classic 36 | * [Queue Classic Matchers](https://github.com/rainforestapp/queue_classic_matchers) - RSpec matchers for queue_classic 37 | 38 | ## Features 39 | * Leverage of PostgreSQL's listen/notify, skip locked, and row locking. 40 | * Support for multiple queues with heterogeneous workers. 41 | * JSON data format. 42 | * Workers can work multiple queues. 43 | 44 | ### Requirements 45 | For this version, the requirements are as follows: 46 | * Ruby 3.0, 3.1, 3.2, 3.3 - i.e. currently supported non-EOL Ruby versions 47 | * Postgres ~> 9.6 48 | * Rubygem: pg ~> 1.1 49 | 50 | ## Table of contents 51 | * [Documentation](https://www.rubydoc.info/gems/queue_classic/) 52 | * [Usage](#usage) 53 | * [Setup](#setup) 54 | * [Upgrade from earlier versions to V3.1](#upgrade-from-earlier-versions) 55 | * [Configuration](#configuration) 56 | * [JSON](#json) 57 | * [Logging](#logging) 58 | * [Support](#support) 59 | * [Hacking](#hacking-on-queue_classic) 60 | * [License](#license) 61 | 62 | ## Usage 63 | There are 2 ways to use queue_classic: 64 | 65 | * Producing Jobs 66 | * Working Jobs 67 | 68 | ### Producing Jobs 69 | #### Simple Enqueue 70 | The first argument is a string which represents a ruby object and a method name. The second argument(s) will be passed along as arguments to the method defined by the first argument. The set of arguments will be encoded as JSON and stored in the database. 71 | 72 | ```ruby 73 | # This method has no arguments. 74 | QC.enqueue("Time.now") 75 | 76 | # This method has 1 argument. 77 | QC.enqueue("Kernel.puts", "hello world") 78 | 79 | # This method has 2 arguments. 80 | QC.enqueue("Kernel.printf", "hello %s", "world") 81 | 82 | # This method has a hash argument. 83 | QC.enqueue("Kernel.puts", {"hello" => "world"}) 84 | 85 | # This method has an array argument. 86 | QC.enqueue("Kernel.puts", ["hello", "world"]) 87 | 88 | # This method uses a non-default queue. 89 | p_queue = QC::Queue.new("priority_queue") 90 | p_queue.enqueue("Kernel.puts", ["hello", "world"]) 91 | ``` 92 | 93 | #### Scheduling for later 94 | There is also the ability to schedule a job to run at a specified time in the future. The job will become processable after the specified time, and will be processed as-soon-as-possible. 95 | 96 | ```ruby 97 | # Specify the job execution time exactly 98 | QC.enqueue_at(Time.new(2024,01,02,10,00), "Kernel.puts", "hello future") 99 | 100 | # Specify the job execution time as an offset in seconds 101 | QC.enqueue_in(60, "Kernel.puts", "hello from 1 minute later") 102 | ``` 103 | 104 | ### Working Jobs 105 | There are two ways to work/process jobs. The first approach is to use the Rake task. The second approach is to use a custom executable. 106 | 107 | #### Rake Task 108 | Require queue_classic in your Rakefile: 109 | 110 | ```ruby 111 | require 'queue_classic' 112 | require 'queue_classic/tasks' 113 | ``` 114 | 115 | ##### Work all queues 116 | Start the worker via the Rakefile: 117 | 118 | ```bash 119 | bundle exec rake qc:work 120 | ``` 121 | 122 | ##### Work a single specific queue 123 | Setup a worker to work only a specific, non-default queue: 124 | 125 | ```bash 126 | QUEUE="priority_queue" bundle exec rake qc:work 127 | ``` 128 | 129 | ##### Work multiple queues 130 | In this scenario, on each iteration of the worker's loop, it will look for jobs in the first queue prior to looking at the second queue. This means that the first queue must be empty before the worker will look at the second queue. 131 | 132 | ```bash 133 | QUEUES="priority_queue,secondary_queue" bundle exec rake qc:work 134 | ``` 135 | 136 | #### Custom Worker 137 | This example is probably not production ready; however, it serves as an example of how to leverage the code in the Worker class to fit your non-default requirements. 138 | 139 | ```ruby 140 | require 'timeout' 141 | require 'queue_classic' 142 | 143 | FailedQueue = QC::Queue.new("failed_jobs") 144 | 145 | class MyWorker < QC::Worker 146 | # A job is a Hash containing these attributes: 147 | # :id Integer, the job id 148 | # :method String, containing the object and method 149 | # :args String, the arguments 150 | # :q_name String, the queue name 151 | # :scheduled_at Time, the scheduled time if the job was scheduled 152 | 153 | # Execute the job using the methods and arguments 154 | def call(job) 155 | # Do something with the job 156 | ... 157 | end 158 | 159 | # This method will be called when an exception 160 | # is raised during the execution of the job. 161 | # First argument is the job that failed. 162 | # Second argument is the exception. 163 | def handle_failure(job, e) 164 | FailedQueue.enqueue(job[:method], *job[:args]) 165 | end 166 | end 167 | 168 | worker = MyWorker.new 169 | 170 | trap('INT') { exit } 171 | trap('TERM') { worker.stop } 172 | 173 | loop do 174 | queue, job = worker.lock_job 175 | Timeout::timeout(5) { worker.process(queue, job) } 176 | end 177 | ``` 178 | 179 | The `qc:work` rake task uses `QC::Worker` by default. However, it's easy to inject your own worker class: 180 | 181 | ```ruby 182 | QC.default_worker_class = MyWorker 183 | ``` 184 | 185 | ## Setup 186 | In addition to installing the rubygem, you will need to prepare your database. Database preparation includes creating a table and loading PL/pgSQL functions. You can issue the database preparation commands using `psql` or use a database migration script. 187 | 188 | ### Quick Start 189 | ```bash{:copy} 190 | gem install queue_classic 191 | createdb queue_classic_test 192 | export QC_DATABASE_URL="postgres://username:password@localhost/queue_classic_test" 193 | ruby -r queue_classic -e "QC::Setup.create" 194 | ruby -r queue_classic -e "QC.enqueue('Kernel.puts', 'hello world')" 195 | ruby -r queue_classic -e "QC::Worker.new.work" 196 | ``` 197 | 198 | ### Ruby on Rails Setup 199 | Declare dependencies in Gemfile: 200 | 201 | ```ruby{:copy} 202 | source 'https://rubygems.org' do 203 | gem 'queue_classic' 204 | end 205 | ``` 206 | 207 | Install queue_classic, which adds the needed migrations for the database tables and stored procedures: 208 | 209 | ```bash 210 | rails generate queue_classic:install 211 | bundle exec rake db:migrate 212 | ``` 213 | 214 | #### Database connection 215 | Starting with with queue_classic 3.1, Rails is automatically detected and its connection is used. If you don't want to use the automatic database connection, set this environment variable to false: `export QC_RAILS_DATABASE=false`. 216 | 217 | > **Note:** If you do not share the connection, you cannot enqueue in the same transaction as whatever you're doing in Rails. 218 | 219 | **Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`. If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database. 220 | 221 | #### Active Job 222 | If you use Rails 4.2+ and want to use Active Job, all you need to do is to set `config.active_job.queue_adapter = :queue_classic` in your `application.rb`. Everything else will be taken care for you. You can now use the Active Job functionality from now. 223 | 224 | ### Plain Ruby Setup 225 | If you're not using Rails, you can use the Rake task to prepare your database: 226 | 227 | ```bash{:copy} 228 | # Creating the table and functions 229 | bundle exec rake qc:create 230 | 231 | # Dropping the table and functions 232 | bundle exec rake qc:drop 233 | ``` 234 | 235 | #### Database connection 236 | By default, queue_classic will use the `QC_DATABASE_URL` falling back on `DATABASE_URL`. The URL must be in the following format: `postgres://username:password@localhost/database_name`. If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database. 237 | 238 | ## Upgrading from earlier versions 239 | If you are upgrading from a previous version of queue_classic, you might need some new database columns and/or functions. Luckily enough for you, it is easy to do so. 240 | 241 | ### Ruby on Rails 242 | These two commands will add the newer migrations: 243 | 244 | ```bash{:copy} 245 | rails generate queue_classic:install 246 | bundle exec rake db:migrate 247 | ``` 248 | 249 | ### Rake Task 250 | This rake task will update you to the latest version: 251 | 252 | ```bash{:copy} 253 | # Updating the table and functions 254 | bundle exec rake qc:update 255 | ``` 256 | 257 | ## Configuration 258 | All configuration takes place in the form of environment vars. See [queue_classic.rb](https://github.com/QueueClassic/queue_classic/blob/master/lib/queue_classic.rb#L23-62) for a list of options. 259 | 260 | ## Logging 261 | By default queue_classic does not talk very much. If you find yourself in a situation where you need to know what's happening inside QC, there are two different kind of logging you can enable: `DEBUG` and `MEASURE`. 262 | 263 | ### Measure 264 | This will output the time to process and some more statistics. To enable it, set the `QC_MEASURE`: 265 | 266 | ```bash{:copy} 267 | export QC_MEASURE="true" 268 | ``` 269 | 270 | ### Debug 271 | You can enable the debug output by setting the `DEBUG` environment variable: 272 | 273 | ```bash{:copy} 274 | export DEBUG="true" 275 | ``` 276 | 277 | ## Support 278 | If you think you have found a bug, feel free to open an issue. Use the following template for the new issue: 279 | 280 | 1. List your versions: Ruby, PostgreSQL, queue_classic. 281 | 2. Define what you would have expected to happen. 282 | 3. List what actually happened. 283 | 4. Provide sample codes & commands which will reproduce the problem. 284 | 285 | ## Hacking on queue_classic 286 | ### Running Tests 287 | ```bash{:copy} 288 | bundle 289 | createdb queue_classic_test 290 | export QC_DATABASE_URL="postgres://username:pass@localhost/queue_classic_test" 291 | bundle exec rake # run all tests 292 | bundle exec ruby test/queue_test.rb # run a single test 293 | ``` 294 | 295 | ## License 296 | Copyright (C) 2010 Ryan Smith 297 | 298 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 299 | 300 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 301 | 302 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 303 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH.unshift('lib') 4 | 5 | require 'bundler/gem_tasks' 6 | require 'rake/testtask' 7 | require './lib/queue_classic' 8 | require './lib/queue_classic/tasks' 9 | 10 | task default: ['test'] 11 | Rake::TestTask.new do |t| 12 | t.libs << 'test' 13 | t.test_files = FileList['test/**/*_test.rb'] 14 | t.verbose = true 15 | t.warning = true 16 | end 17 | -------------------------------------------------------------------------------- /lib/generators/queue_classic/install_generator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rails/generators' 4 | require 'rails/generators/migration' 5 | require 'active_record' 6 | 7 | module QC 8 | # Install generator to create migration files for rails 9 | class InstallGenerator < Rails::Generators::Base 10 | include Rails::Generators::Migration 11 | 12 | namespace 'queue_classic:install' 13 | source_paths << File.join(File.dirname(__FILE__), 'templates') 14 | desc 'Generates (but does not run) a migration to add a queue_classic table.' 15 | 16 | def self.next_migration_number(dirname) 17 | next_migration_number = current_migration_number(dirname) + 1 18 | ActiveRecord::Migration.next_migration_number(next_migration_number) 19 | end 20 | 21 | def create_migration_file 22 | migration_template 'add_queue_classic.rb', 'db/migrate/add_queue_classic.rb' if self.class.migration_exists?('db/migrate', 'add_queue_classic').nil? 23 | migration_template 'update_queue_classic_3_0_0.rb', 'db/migrate/update_queue_classic_3_0_0.rb' if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_0_0').nil? 24 | migration_template 'update_queue_classic_3_0_2.rb', 'db/migrate/update_queue_classic_3_0_2.rb' if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_0_2').nil? 25 | migration_template 'update_queue_classic_3_1_0.rb', 'db/migrate/update_queue_classic_3_1_0.rb' if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_1_0').nil? 26 | migration_template 'update_queue_classic_4_0_0.rb', 'db/migrate/update_queue_classic_4_0_0.rb' if self.class.migration_exists?('db/migrate', 'update_queue_classic_4_0_0').nil? 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/generators/queue_classic/templates/add_queue_classic.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # add QC 4 | class AddQueueClassic < ActiveRecord::Migration[4.2] 5 | def self.up 6 | QC::Setup.create 7 | end 8 | 9 | def self.down 10 | QC::Setup.drop 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # update QC 4 | class UpdateQueueClassic300 < ActiveRecord::Migration[4.2] 5 | def self.up 6 | QC::Setup.update_to_3_0_0 7 | end 8 | 9 | def self.down 10 | QC::Setup.downgrade_from_3_0_0 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # update QC 4 | class UpdateQueueClassic302 < ActiveRecord::Migration[4.2] 5 | def self.up 6 | QC::Setup.update_to_3_0_0 7 | end 8 | 9 | def self.down 10 | # This migration is fixing a bug, so we don't want to do anything here. 11 | # I didn't want to make it irreversible either, as it could prevent 12 | # rolling back other, unrelated, stuff. 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # update QC 4 | class UpdateQueueClassic310 < ActiveRecord::Migration[4.2] 5 | def self.up 6 | QC::Setup.update_to_3_1_0 7 | end 8 | 9 | def self.down 10 | QC::Setup.downgrade_from_3_1_0 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/generators/queue_classic/templates/update_queue_classic_4_0_0.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # update QC 4 | class UpdateQueueClassic400 < ActiveRecord::Migration[4.2] 5 | def self.up 6 | QC::Setup.update_to_4_0_0 7 | end 8 | 9 | def self.down 10 | QC::Setup.downgrade_from_4_0_0 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/queue_classic.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'queue_classic/config' 4 | 5 | # QC 6 | module QC 7 | extend QC::Config 8 | 9 | # Assign constants for backwards compatibility. 10 | # They should no longer be used. Prefer the corresponding methods. 11 | # See +QC::Config+ for more details. 12 | DEPRECATED_CONSTANTS = { 13 | APP_NAME: :app_name, 14 | WAIT_TIME: :wait_time, 15 | TABLE_NAME: :table_name, 16 | QUEUE: :queue, 17 | QUEUES: :queues, 18 | TOP_BOUND: :top_bound, 19 | FORK_WORKER: :fork_worker? 20 | }.freeze 21 | 22 | def self.const_missing(const_name) 23 | if DEPRECATED_CONSTANTS.key? const_name 24 | config_method = DEPRECATED_CONSTANTS[const_name] 25 | warn <<~MSG 26 | The constant QC::#{const_name} is deprecated and will be removed in the future. 27 | Please use the method QC.#{config_method} instead. 28 | MSG 29 | QC.public_send config_method 30 | else 31 | super 32 | end 33 | end 34 | 35 | # Defer method calls on the QC module to the 36 | # default queue. This facilitates QC.enqueue() 37 | def self.method_missing(sym, ...) 38 | if default_queue.respond_to? sym 39 | default_queue.public_send(sym, ...) 40 | else 41 | super 42 | end 43 | end 44 | 45 | # Ensure QC.respond_to?(:enqueue) equals true (ruby 1.9 only) 46 | def self.respond_to_missing?(method_name, _include_private = false) 47 | default_queue.respond_to?(method_name) 48 | end 49 | 50 | def self.has_connection? # rubocop:disable Naming/PredicateName 51 | !default_conn_adapter.nil? 52 | end 53 | 54 | def self.default_conn_adapter 55 | Thread.current[:qc_conn_adapter] ||= ConnAdapter.new(active_record_connection_share: rails_connection_sharing_enabled?) 56 | end 57 | 58 | def self.default_conn_adapter=(conn) 59 | Thread.current[:qc_conn_adapter] = conn 60 | end 61 | 62 | def self.log_yield(data) 63 | t0 = Time.now 64 | begin 65 | yield 66 | rescue StandardError => e 67 | log({ at: 'error', error: e.inspect }.merge(data)) 68 | raise 69 | ensure 70 | t = Integer((Time.now - t0) * 1000) 71 | log(data.merge(elapsed: t)) unless e 72 | end 73 | end 74 | 75 | def self.log(data) 76 | result = nil 77 | data = { lib: 'queue-classic' }.merge(data) 78 | if block_given? 79 | result = yield 80 | data.merge(elapsed: Integer((Time.now - t0) * 1000)) 81 | end 82 | data.reduce(out = String.new) do |s, tup| 83 | s << [tup.first, tup.last].join('=') << ' ' 84 | end 85 | puts(out) if ENV['DEBUG'] 86 | result 87 | end 88 | 89 | def self.measure(data) 90 | return unless ENV['QC_MEASURE'] 91 | 92 | $stdout.puts("measure#qc.#{data}") 93 | end 94 | 95 | # This will unlock all jobs any postgres' PID that is not existing anymore 96 | # to prevent any infinitely locked jobs 97 | def self.unlock_jobs_of_dead_workers 98 | default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);") 99 | end 100 | 101 | # private class methods 102 | class << self 103 | private 104 | 105 | def rails_connection_sharing_enabled? 106 | enabled = ENV.fetch('QC_RAILS_DATABASE', 'true') != 'false' 107 | return false unless enabled 108 | 109 | Object.const_defined?('ActiveRecord') && ActiveRecord::Base.respond_to?('connection') 110 | end 111 | end 112 | end 113 | 114 | require_relative 'queue_classic/queue' 115 | require_relative 'queue_classic/worker' 116 | require_relative 'queue_classic/setup' 117 | require_relative 'queue_classic/railtie' if defined?(Rails) 118 | -------------------------------------------------------------------------------- /lib/queue_classic/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module QC 4 | # QC Configuration 5 | module Config 6 | # You can use the APP_NAME to query for 7 | # postgres related process information in the 8 | # pg_stat_activity table. 9 | def app_name 10 | @app_name ||= ENV['QC_APP_NAME'] || 'queue_classic' 11 | end 12 | 13 | # Number of seconds to block on the listen chanel for new jobs. 14 | def wait_time 15 | @wait_time ||= (ENV['QC_LISTEN_TIME'] || 5).to_i 16 | end 17 | 18 | # Why do you want to change the table name? 19 | # Just deal with the default OK? 20 | # If you do want to change this, you will 21 | # need to update the PL/pgSQL lock_head() function. 22 | # Come on. Don't do it.... Just stick with the default. 23 | def table_name 24 | @table_name ||= 'queue_classic_jobs' 25 | end 26 | 27 | def queue 28 | @queue = ENV['QUEUE'] || 'default' 29 | end 30 | 31 | # The default queue used by `QC.enqueue`. 32 | def default_queue 33 | @default_queue ||= Queue.new(QC.queue) 34 | end 35 | 36 | def default_queue=(queue) 37 | @default_queue = queue 38 | end 39 | 40 | # Each row in the table will have a column that 41 | # notes the queue. You can point your workers 42 | # at different queues. 43 | def queues 44 | @queues ||= (ENV.fetch('QUEUES', nil) && ENV['QUEUES'].split(',').map(&:strip)) || [] 45 | end 46 | 47 | # Set this to 1 for strict FIFO. 48 | # There is nothing special about 9.... 49 | def top_bound 50 | @top_bound ||= (ENV['QC_TOP_BOUND'] || 9).to_i 51 | end 52 | 53 | # Set this variable if you wish for 54 | # the worker to fork a UNIX process for 55 | # each locked job. Remember to re-establish 56 | # any database connections. See the worker 57 | # for more details. 58 | def fork_worker? 59 | @fork_worker ||= !ENV['QC_FORK_WORKER'].nil? 60 | end 61 | 62 | # The worker class instantiated by QC's rake tasks. 63 | def default_worker_class 64 | @default_worker_class ||= begin 65 | class_name = ENV.fetch('QC_DEFAULT_WORKER_CLASS', nil) 66 | class_name ? Kernel.const_get(class_name) : QC::Worker 67 | rescue NameError 68 | QC::Worker 69 | end 70 | end 71 | 72 | def default_worker_class=(worker_class) 73 | @default_worker_class = worker_class 74 | end 75 | 76 | # reset memoized configuration 77 | def reset_config 78 | # TODO: we might want to think about storing these in a Hash. 79 | @app_name = nil 80 | @wait_time = nil 81 | @table_name = nil 82 | @queue = nil 83 | @default_queue = nil 84 | @queues = nil 85 | @top_bound = nil 86 | @fork_worker = nil 87 | @worker_class = nil 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/queue_classic/conn_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'uri' 4 | require 'pg' 5 | 6 | module QC 7 | # This class is responsible for managing the connection to the database. 8 | class ConnAdapter 9 | def initialize(args = {}) 10 | @active_record_connection_share = args[:active_record_connection_share] 11 | @connection = args[:connection] 12 | @mutex = Mutex.new 13 | end 14 | 15 | def connection 16 | if @active_record_connection_share && Object.const_defined?('ActiveRecord') 17 | ActiveRecord::Base.connection.raw_connection 18 | else 19 | @connection ||= establish_new 20 | end 21 | end 22 | 23 | def execute(stmt, *params) 24 | @mutex.synchronize do 25 | QC.log(at: 'exec_sql', sql: stmt.inspect) 26 | begin 27 | params = nil if params.empty? 28 | r = connection.exec(stmt, params) 29 | result = [] 30 | r.each { |t| result << t } 31 | result.length > 1 ? result : result.pop 32 | rescue PG::Error => e 33 | QC.log(error: e.inspect) 34 | connection.reset 35 | raise 36 | end 37 | end 38 | end 39 | 40 | def wait(time, *channels) 41 | @mutex.synchronize do 42 | listen_cmds = channels.map { |c| "LISTEN \"#{c}\"" } 43 | connection.exec(listen_cmds.join(';')) 44 | wait_for_notify(time) 45 | unlisten_cmds = channels.map { |c| "UNLISTEN \"#{c}\"" } 46 | connection.exec(unlisten_cmds.join(';')) 47 | drain_notify 48 | end 49 | end 50 | 51 | def disconnect 52 | @mutex.synchronize do 53 | connection.close 54 | rescue StandardError => e 55 | QC.log(at: 'disconnect', error: e.message) 56 | end 57 | end 58 | 59 | def server_version 60 | @server_version ||= begin 61 | version = execute('SHOW server_version_num;')['server_version_num'] 62 | version&.to_i 63 | end 64 | end 65 | 66 | private 67 | 68 | def wait_for_notify(t) 69 | [].tap do |msgs| 70 | connection.wait_for_notify(t) { |_event, _pid, msg| msgs << msg } 71 | end 72 | end 73 | 74 | def drain_notify 75 | QC.log(at: 'drain_notifications') until connection.notifies.nil? 76 | end 77 | 78 | def validate!(c) 79 | return c if c.is_a?(PG::Connection) 80 | 81 | err = "connection must be an instance of PG::Connection, but was #{c.class}" 82 | raise(ArgumentError, err) 83 | end 84 | 85 | def establish_new 86 | QC.log(at: 'establish_conn') 87 | conn = PG.connect(*normalize_db_url(db_url)) 88 | QC.log(error: conn.error) if conn.status != PG::CONNECTION_OK 89 | 90 | raise "This version of Queue Classic does not support Postgres older than 9.6 (90600). This version is #{conn.server_version}. If you need that support, please use an older version." if conn.server_version < 90_600 91 | 92 | conn.exec("SET application_name = '#{QC.app_name}'") 93 | conn 94 | end 95 | 96 | def normalize_db_url(url) 97 | host = url.host 98 | host = host.gsub(/%2F/i, '/') if host 99 | 100 | [ 101 | host, # host or percent-encoded socket path 102 | url.port || 5432, 103 | nil, nil, # opts, tty 104 | url.path.gsub('/', ''), # database name 105 | url.user, 106 | url.password 107 | ] 108 | end 109 | 110 | def db_url 111 | return @db_url if defined?(@db_url) && @db_url 112 | 113 | url = ENV['QC_DATABASE_URL'] || 114 | ENV['DATABASE_URL'] || 115 | raise(ArgumentError, 'missing QC_DATABASE_URL or DATABASE_URL') 116 | @db_url = URI.parse(url) 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /lib/queue_classic/queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'conn_adapter' 4 | require 'json' 5 | require 'time' 6 | 7 | module QC 8 | # The queue class maps a queue abstraction onto a database table. 9 | class Queue 10 | attr_reader :name, :top_bound 11 | 12 | def initialize(name, top_bound = nil) 13 | @name = name 14 | @top_bound = top_bound || QC.top_bound 15 | end 16 | 17 | attr_writer :conn_adapter 18 | 19 | def conn_adapter 20 | @conn_adapter ||= QC.default_conn_adapter 21 | end 22 | 23 | # enqueue(m,a) inserts a row into the jobs table and trigger a notification. 24 | # The job's queue is represented by a name column in the row. 25 | # There is a trigger on the table which will send a NOTIFY event 26 | # on a channel which corresponds to the name of the queue. 27 | # The method argument is a string encoded ruby expression. The expression 28 | # will be separated by a `.` character and then `eval`d. 29 | # Examples of the method argument include: `puts`, `Kernel.puts`, 30 | # `MyObject.new.puts`. 31 | # The args argument will be encoded as JSON and stored as a JSON datatype 32 | # in the row. (If the version of PG does not support JSON, 33 | # then the args will be stored as text. 34 | # The args are stored as a collection and then splatted inside the worker. 35 | # Examples of args include: `'hello world'`, `['hello world']`, 36 | # `'hello', 'world'`. 37 | # This method returns a hash with the id of the enqueued job. 38 | def enqueue(method, *args) 39 | QC.log_yield(measure: 'queue.enqueue') do 40 | s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id" 41 | begin 42 | retries ||= 0 43 | conn_adapter.execute(s, name, method, JSON.dump(args)) 44 | rescue PG::Error 45 | raise unless (retries += 1) < 2 46 | 47 | retry 48 | end 49 | end 50 | end 51 | 52 | # enqueue_at(t,m,a) inserts a row into the jobs table representing a job 53 | # to be executed not before the specified time. 54 | # The time argument must be a Time object or a float timestamp. The method 55 | # and args argument must be in the form described in the documentation for 56 | # the #enqueue method. 57 | # This method returns a hash with the id of the enqueued job. 58 | def enqueue_at(timestamp, method, *args) 59 | offset = Time.at(timestamp).to_i - Time.now.to_i 60 | enqueue_in(offset, method, *args) 61 | end 62 | 63 | # enqueue_in(t,m,a) inserts a row into the jobs table representing a job 64 | # to be executed not before the specified time offset. 65 | # The seconds argument must be an integer. The method and args argument 66 | # must be in the form described in the documentation for the #enqueue 67 | # method. 68 | # This method returns a hash with the id of the enqueued job. 69 | def enqueue_in(seconds, method, *args) 70 | QC.log_yield(measure: 'queue.enqueue') do 71 | s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at) 72 | VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds') 73 | RETURNING id" 74 | begin 75 | retries ||= 0 76 | conn_adapter.execute(s, name, method, JSON.dump(args)) 77 | rescue PG::Error 78 | raise unless (retries += 1) < 2 79 | 80 | retry 81 | end 82 | end 83 | end 84 | 85 | def lock 86 | QC.log_yield(measure: 'queue.lock') do 87 | s = <<~SQL 88 | WITH selected_job AS ( 89 | SELECT id 90 | FROM #{QC.table_name} 91 | WHERE locked_at IS NULL 92 | AND q_name = $1 93 | AND scheduled_at <= now() 94 | LIMIT 1 95 | FOR NO KEY UPDATE SKIP LOCKED 96 | ) 97 | UPDATE #{QC.table_name} 98 | SET 99 | locked_at = now(), 100 | locked_by = pg_backend_pid() 101 | FROM selected_job 102 | WHERE #{QC.table_name}.id = selected_job.id 103 | RETURNING * 104 | SQL 105 | 106 | if (r = conn_adapter.execute(s, name)) 107 | {}.tap do |job| 108 | job[:id] = r['id'] 109 | job[:q_name] = r['q_name'] 110 | job[:method] = r['method'] 111 | job[:args] = JSON.parse(r['args']) 112 | if r['scheduled_at'] 113 | job[:scheduled_at] = r['scheduled_at'].is_a?(Time) ? r['scheduled_at'] : Time.parse(r['scheduled_at']) 114 | ttl = Integer((Time.now - job[:scheduled_at]) * 1000) 115 | QC.measure("time-to-lock=#{ttl}ms source=#{name}") 116 | end 117 | end 118 | end 119 | end 120 | end 121 | 122 | def unlock(id) 123 | QC.log_yield(measure: 'queue.unlock') do 124 | s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1" 125 | conn_adapter.execute(s, id) 126 | end 127 | end 128 | 129 | def delete(id) 130 | QC.log_yield(measure: 'queue.delete') do 131 | conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id) 132 | end 133 | end 134 | 135 | def delete_all 136 | QC.log_yield(measure: 'queue.delete_all') do 137 | s = "DELETE FROM #{QC.table_name} WHERE q_name = $1" 138 | conn_adapter.execute(s, name) 139 | end 140 | end 141 | 142 | # Count the number of jobs in a specific queue. This returns all 143 | # jobs, including ones that are scheduled in the future. 144 | def count 145 | _count('queue.count', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1") 146 | end 147 | 148 | # Count the number of jobs in a specific queue, except ones scheduled in the future 149 | def count_ready 150 | _count('queue.count_scheduled', 151 | "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at <= now()") 152 | end 153 | 154 | # Count the number of jobs in a specific queue scheduled in the future 155 | def count_scheduled 156 | _count('queue.count_scheduled', 157 | "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at > now()") 158 | end 159 | 160 | private 161 | 162 | def _count(metric_name, sql) 163 | QC.log_yield(measure: metric_name) do 164 | r = conn_adapter.execute(sql, name) 165 | r['count'].to_i 166 | end 167 | end 168 | end 169 | end 170 | -------------------------------------------------------------------------------- /lib/queue_classic/railtie.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rails/railtie' 4 | 5 | module QC 6 | # Railtie integrates queue_classic with Rails applications. 7 | class Railtie < ::Rails::Railtie 8 | rake_tasks do 9 | load 'queue_classic/tasks.rb' 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/queue_classic/setup.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module QC 4 | # Setup is a module that provides methods to create, update and drop the queue_classic tables 5 | module Setup 6 | Root = File.expand_path('../..', File.dirname(__FILE__)) 7 | SqlFunctions = File.join(Root, '/sql/ddl.sql') 8 | CreateTable = File.join(Root, '/sql/create_table.sql') 9 | DropSqlFunctions = File.join(Root, '/sql/drop_ddl.sql') 10 | UpgradeTo_3_0_0 = File.join(Root, '/sql/update_to_3_0_0.sql') 11 | DowngradeFrom_3_0_0 = File.join(Root, '/sql/downgrade_from_3_0_0.sql') 12 | UpgradeTo_3_1_0 = File.join(Root, '/sql/update_to_3_1_0.sql') 13 | DowngradeFrom_3_1_0 = File.join(Root, '/sql/downgrade_from_3_1_0.sql') 14 | UpgradeTo_4_0_0 = File.join(Root, '/sql/update_to_4_0_0.sql') 15 | DowngradeFrom_4_0_0 = File.join(Root, '/sql/downgrade_from_4_0_0.sql') 16 | 17 | def self.create(c = QC.default_conn_adapter.connection) 18 | conn = QC::ConnAdapter.new(connection: c) 19 | conn.execute(File.read(CreateTable)) 20 | conn.execute(File.read(SqlFunctions)) 21 | conn.disconnect if c.nil? # Don't close a conn we didn't create. 22 | end 23 | 24 | def self.drop(c = QC.default_conn_adapter.connection) 25 | conn = QC::ConnAdapter.new(connection: c) 26 | conn.execute('DROP TABLE IF EXISTS queue_classic_jobs CASCADE') 27 | conn.execute(File.read(DropSqlFunctions)) 28 | conn.disconnect if c.nil? # Don't close a conn we didn't create. 29 | end 30 | 31 | def self.update(c = QC.default_conn_adapter.connection) 32 | conn = QC::ConnAdapter.new(connection: c) 33 | conn.execute(File.read(UpgradeTo_3_0_0)) 34 | conn.execute(File.read(UpgradeTo_3_1_0)) 35 | conn.execute(File.read(UpgradeTo_4_0_0)) 36 | conn.execute(File.read(DropSqlFunctions)) 37 | conn.execute(File.read(SqlFunctions)) 38 | end 39 | 40 | def self.update_to_3_0_0(c = QC.default_conn_adapter.connection) 41 | conn = QC::ConnAdapter.new(connection: c) 42 | conn.execute(File.read(UpgradeTo_3_0_0)) 43 | conn.execute(File.read(DropSqlFunctions)) 44 | conn.execute(File.read(SqlFunctions)) 45 | end 46 | 47 | def self.downgrade_from_3_0_0(c = QC.default_conn_adapter.connection) 48 | conn = QC::ConnAdapter.new(connection: c) 49 | conn.execute(File.read(DowngradeFrom_3_0_0)) 50 | end 51 | 52 | def self.update_to_3_1_0(c = QC.default_conn_adapter.connection) 53 | conn = QC::ConnAdapter.new(connection: c) 54 | conn.execute(File.read(UpgradeTo_3_1_0)) 55 | conn.execute(File.read(DropSqlFunctions)) 56 | conn.execute(File.read(SqlFunctions)) 57 | end 58 | 59 | def self.downgrade_from_3_1_0(c = QC.default_conn_adapter.connection) 60 | conn = QC::ConnAdapter.new(connection: c) 61 | conn.execute(File.read(DowngradeFrom_3_1_0)) 62 | end 63 | 64 | def self.update_to_4_0_0(c = QC.default_conn_adapter.connection) 65 | conn = QC::ConnAdapter.new(connection: c) 66 | conn.execute(File.read(UpgradeTo_4_0_0)) 67 | conn.execute(File.read(DropSqlFunctions)) 68 | conn.execute(File.read(SqlFunctions)) 69 | end 70 | 71 | def self.downgrade_from_4_0_0(c = QC.default_conn_adapter.connection) 72 | conn = QC::ConnAdapter.new(connection: c) 73 | conn.execute(File.read(DowngradeFrom_4_0_0)) 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/queue_classic/tasks.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | task :environment 4 | 5 | namespace :jobs do 6 | desc 'Alias for qc:work' 7 | task work: 'qc:work' 8 | end 9 | 10 | namespace :qc do 11 | desc 'Start a new worker for the (default or $QUEUE / $QUEUES) queue' 12 | task work: :environment do 13 | @worker = QC.default_worker_class.new 14 | 15 | trap('INT') do 16 | warn('Received INT. Shutting down.') 17 | abort('Worker has stopped running. Exit.') unless @worker.running 18 | @worker.stop 19 | end 20 | 21 | trap('TERM') do 22 | warn('Received Term. Shutting down.') 23 | @worker.stop 24 | end 25 | 26 | @worker.start 27 | end 28 | 29 | desc 'Returns the number of jobs in the (default or $QUEUE / $QUEUES) queue' 30 | task count: :environment do 31 | puts QC.default_queue.count 32 | end 33 | 34 | desc 'Setup queue_classic tables and functions in database' 35 | task create: :environment do 36 | QC::Setup.create 37 | end 38 | 39 | desc 'Remove queue_classic tables and functions from database.' 40 | task drop: :environment do 41 | QC::Setup.drop 42 | end 43 | 44 | desc 'Update queue_classic tables and functions in database' 45 | task update: :environment do 46 | QC::Setup.update 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/queue_classic/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module QC 4 | VERSION = '4.0.0' 5 | end 6 | -------------------------------------------------------------------------------- /lib/queue_classic/worker.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'queue' 4 | require_relative 'conn_adapter' 5 | 6 | module QC 7 | # A Worker object can process jobs from one or many queues. 8 | class Worker 9 | attr_accessor :queues, :running 10 | 11 | # Creates a new worker but does not start the worker. See Worker#start. 12 | # This method takes a single hash argument. The following keys are read: 13 | # fork_worker:: Worker forks each job execution. 14 | # wait_interval:: Time to wait between failed lock attempts 15 | # connection:: PG::Connection object. 16 | # q_name:: Name of a single queue to process. 17 | # q_names:: Names of queues to process. Will process left to right. 18 | # top_bound:: Offset to the head of the queue. 1 == strict FIFO. 19 | def initialize(args = {}) 20 | @fork_worker = args[:fork_worker] || QC.fork_worker? 21 | @wait_interval = args[:wait_interval] || QC.wait_time 22 | 23 | @conn_adapter = if args[:connection] 24 | ConnAdapter.new(connection: args[:connection]) 25 | else 26 | QC.default_conn_adapter 27 | end 28 | 29 | @queues = setup_queues(@conn_adapter, 30 | (args[:q_name] || QC.queue), 31 | (args[:q_names] || QC.queues), 32 | (args[:top_bound] || QC.top_bound)) 33 | log(args.merge(at: 'worker_initialized')) 34 | @running = true 35 | end 36 | 37 | # Commences the working of jobs. 38 | # start() spins on @running –which is initialized as true. 39 | # This method is the primary entry point to starting the worker. 40 | # The canonical example of starting a worker is as follows: 41 | # QC::Worker.new.start 42 | def start 43 | QC.unlock_jobs_of_dead_workers 44 | 45 | @fork_worker ? fork_and_work : work while @running 46 | end 47 | 48 | # Signals the worker to stop taking new work. 49 | # This method has no immediate effect. However, there are 50 | # two loops in the worker (one in #start and another in #lock_job) 51 | # which check the @running variable to determine if further progress 52 | # is desirable. In the case that @running is false, the aforementioned 53 | # methods will short circuit and cause the blocking call to #start 54 | # to unblock. 55 | def stop 56 | @running = false 57 | end 58 | 59 | # Calls Worker#work but after the current process is forked. 60 | # The parent process will wait on the child process to exit. 61 | def fork_and_work 62 | cpid = fork do 63 | setup_child 64 | work 65 | end 66 | log(at: :fork, pid: cpid) 67 | Process.wait(cpid) 68 | end 69 | 70 | # Blocks on locking a job, and once a job is locked, 71 | # it will process the job. 72 | def work 73 | queue, job = lock_job 74 | return unless queue && job 75 | 76 | QC.log_yield(at: 'work', job: job[:id]) do 77 | process(queue, job) 78 | end 79 | end 80 | 81 | # Attempt to lock a job in the queue's table. 82 | # If a job can be locked, this method returns an array with 83 | # 2 elements. The first element is the queue from which the job was locked 84 | # and the second is a hash representation of the job. 85 | # If a job is returned, its locked_at column has been set in the 86 | # job's row. It is the caller's responsibility to delete the job row 87 | # from the table when the job is complete. 88 | def lock_job 89 | log(at: 'lock_job') 90 | job = nil 91 | while @running 92 | @queues.each do |queue| 93 | if (job = queue.lock) 94 | return [queue, job] 95 | end 96 | end 97 | @conn_adapter.wait(@wait_interval, *@queues.map(&:name)) 98 | end 99 | end 100 | 101 | # A job is processed by evaluating the target code. 102 | # if the job is evaluated with no exceptions 103 | # then it is deleted from the queue. 104 | # If the job has raised an exception the responsibility of what 105 | # to do with the job is delegated to Worker#handle_failure. 106 | # If the job is not finished and an INT signal is trapped, 107 | # this method will unlock the job in the queue. 108 | def process(queue, job) 109 | start = Time.now 110 | finished = false 111 | begin 112 | call(job).tap do 113 | handle_success(queue, job) 114 | finished = true 115 | end 116 | rescue StandardError, ScriptError, NoMemoryError => e 117 | # We really only want to unlock the job for signal and system exit 118 | # exceptions. If we encounter a ScriptError or a NoMemoryError any 119 | # future run will likely encounter the same error. 120 | handle_failure(job, e) 121 | finished = true 122 | ensure 123 | queue.unlock(job[:id]) unless finished 124 | ttp = Integer((Time.now - start) * 1000) 125 | QC.measure("time-to-process=#{ttp} source=#{queue.name}") 126 | end 127 | end 128 | 129 | # Each job includes a method column. We will use ruby's eval 130 | # to grab the ruby object from memory. We send the method to 131 | # the object and pass the args. 132 | def call(job) 133 | args = job[:args] 134 | receiver_str, _, message = job[:method].rpartition('.') 135 | receiver = eval(receiver_str) # rubocop:disable Security/Eval 136 | receiver.send(message, *args) 137 | end 138 | 139 | def handle_success(queue, job) 140 | queue.delete(job[:id]) 141 | end 142 | 143 | # This method will be called when a StandardError, ScriptError or 144 | # NoMemoryError is raised during the execution of the job. 145 | def handle_failure(job, e) 146 | warn("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}") 147 | end 148 | 149 | # This method should be overriden if 150 | # your worker is forking and you need to 151 | # re-establish database connections 152 | def setup_child 153 | log(at: 'setup_child') 154 | end 155 | 156 | def log(data) 157 | QC.log(data) 158 | end 159 | 160 | private 161 | 162 | def setup_queues(adapter, queue, queues, top_bound) 163 | names = queues.length.positive? ? queues : [queue] 164 | names.map do |name| 165 | QC::Queue.new(name, top_bound).tap do |q| 166 | q.conn_adapter = adapter 167 | end 168 | end 169 | end 170 | end 171 | end 172 | -------------------------------------------------------------------------------- /queue_classic.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path('lib', __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require 'queue_classic/version' 6 | 7 | Gem::Specification.new do |spec| 8 | spec.name = 'queue_classic' 9 | spec.email = 'r@32k.io' 10 | spec.version = QC::VERSION 11 | spec.description = 'queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg.' 12 | spec.summary = 'Simple, efficient worker queue for Ruby & PostgreSQL.' 13 | spec.authors = ['Ryan Smith (♠ ace hacker)'] 14 | spec.homepage = 'https://github.com/QueueClassic/queue_classic' 15 | spec.license = 'MIT' 16 | 17 | spec.files = `git ls-files -z`.split("\x0") 18 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 19 | spec.require_paths = %w[lib] 20 | 21 | spec.metadata = { 22 | 'bug_tracker_uri' => 'https://github.com/QueueClassic/queue_classic/issues', 23 | 'changelog_uri' => 'https://github.com/QueueClassic/queue_classic/blob/master/CHANGELOG.md', 24 | 'source_code_uri' => 'https://github.com/QueueClassic/queue_classic', 25 | 'rubygems_mfa_required' => 'true' 26 | } 27 | 28 | spec.required_ruby_version = '>= 3.0.0' 29 | spec.add_runtime_dependency 'pg', '>= 1.1', '< 2.0' 30 | end 31 | -------------------------------------------------------------------------------- /sql/create_table.sql: -------------------------------------------------------------------------------- 1 | DO $$ BEGIN 2 | 3 | CREATE TABLE queue_classic_jobs ( 4 | id bigserial PRIMARY KEY, 5 | q_name text NOT NULL CHECK (length(q_name) > 0), 6 | method text NOT NULL CHECK (length(method) > 0), 7 | args jsonb NOT NULL, 8 | locked_at timestamptz, 9 | locked_by integer, 10 | created_at timestamptz DEFAULT now(), 11 | scheduled_at timestamptz DEFAULT now() 12 | ); 13 | 14 | END $$ LANGUAGE plpgsql; 15 | 16 | CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL; 17 | CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL; 18 | -------------------------------------------------------------------------------- /sql/ddl.sql: -------------------------------------------------------------------------------- 1 | -- queue_classic_notify function and trigger 2 | CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN 3 | perform pg_notify(new.q_name, ''); RETURN NULL; 4 | END $$ LANGUAGE plpgsql; 5 | 6 | CREATE TRIGGER queue_classic_notify 7 | AFTER INSERT ON queue_classic_jobs FOR EACH ROW 8 | EXECUTE PROCEDURE queue_classic_notify(); 9 | -------------------------------------------------------------------------------- /sql/downgrade_from_3_0_0.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE queue_classic_jobs DROP COLUMN locked_by; 2 | ALTER TABLE queue_classic_jobs DROP COLUMN created_at; 3 | -------------------------------------------------------------------------------- /sql/downgrade_from_3_1_0.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE queue_classic_jobs DROP COLUMN scheduled_at; 2 | -------------------------------------------------------------------------------- /sql/downgrade_from_4_0_0.sql: -------------------------------------------------------------------------------- 1 | DO $$DECLARE r record; 2 | BEGIN 3 | -- If jsonb type is available, do nothing as we're downgrading from 4.0.0 4 | IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN 5 | -- do nothing - it should already be already jsonb 6 | -- Otherwise, use json type for the args column if available 7 | ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN 8 | -- this should only happen if someone downgrades QC and their database < pg 9.4 9 | ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json; 10 | END IF; 11 | 12 | 13 | END$$; 14 | 15 | 16 | -- 17 | -- Re install the lock_head function 18 | -- 19 | 20 | -- We are declaring the return type to be queue_classic_jobs. 21 | -- This is ok since I am assuming that all of the users added queues will 22 | -- have identical columns to queue_classic_jobs. 23 | -- When QC supports queues with columns other than the default, we will have to change this. 24 | 25 | CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer) 26 | RETURNS SETOF queue_classic_jobs AS $$ 27 | DECLARE 28 | unlocked bigint; 29 | relative_top integer; 30 | job_count integer; 31 | BEGIN 32 | -- The purpose is to release contention for the first spot in the table. 33 | -- The select count(*) is going to slow down dequeue performance but allow 34 | -- for more workers. Would love to see some optimization here... 35 | 36 | EXECUTE 'SELECT count(*) FROM ' 37 | || '(SELECT * FROM queue_classic_jobs ' 38 | || ' WHERE locked_at IS NULL' 39 | || ' AND q_name = ' 40 | || quote_literal(q_name) 41 | || ' AND scheduled_at <= ' 42 | || quote_literal(now()) 43 | || ' LIMIT ' 44 | || quote_literal(top_boundary) 45 | || ') limited' 46 | INTO job_count; 47 | 48 | SELECT TRUNC(random() * (top_boundary - 1)) 49 | INTO relative_top; 50 | 51 | IF job_count < top_boundary THEN 52 | relative_top = 0; 53 | END IF; 54 | 55 | LOOP 56 | BEGIN 57 | EXECUTE 'SELECT id FROM queue_classic_jobs ' 58 | || ' WHERE locked_at IS NULL' 59 | || ' AND q_name = ' 60 | || quote_literal(q_name) 61 | || ' AND scheduled_at <= ' 62 | || quote_literal(now()) 63 | || ' ORDER BY id ASC' 64 | || ' LIMIT 1' 65 | || ' OFFSET ' || quote_literal(relative_top) 66 | || ' FOR UPDATE NOWAIT' 67 | INTO unlocked; 68 | EXIT; 69 | EXCEPTION 70 | WHEN lock_not_available THEN 71 | -- do nothing. loop again and hope we get a lock 72 | END; 73 | END LOOP; 74 | 75 | RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs ' 76 | || ' SET locked_at = (CURRENT_TIMESTAMP),' 77 | || ' locked_by = (select pg_backend_pid())' 78 | || ' WHERE id = $1' 79 | || ' AND locked_at is NULL' 80 | || ' RETURNING *' 81 | USING unlocked; 82 | 83 | RETURN; 84 | END $$ LANGUAGE plpgsql; 85 | 86 | CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN 87 | RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname; 88 | END $$ LANGUAGE plpgsql; 89 | -------------------------------------------------------------------------------- /sql/drop_ddl.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS lock_head(tname varchar); 2 | DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer); 3 | DROP FUNCTION IF EXISTS queue_classic_notify() cascade; 4 | -------------------------------------------------------------------------------- /sql/update_to_3_0_0.sql: -------------------------------------------------------------------------------- 1 | DO $$DECLARE r record; 2 | BEGIN 3 | BEGIN 4 | ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now(); 5 | EXCEPTION 6 | WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.'; 7 | END; 8 | END$$; 9 | 10 | DO $$DECLARE r record; 11 | BEGIN 12 | BEGIN 13 | ALTER TABLE queue_classic_jobs ADD COLUMN locked_by integer; 14 | EXCEPTION 15 | WHEN duplicate_column THEN RAISE NOTICE 'column locked_by already exists in queue_classic_jobs.'; 16 | END; 17 | END$$; 18 | -------------------------------------------------------------------------------- /sql/update_to_3_1_0.sql: -------------------------------------------------------------------------------- 1 | DO $$DECLARE r record; 2 | BEGIN 3 | BEGIN 4 | ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now(); 5 | CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL; 6 | EXCEPTION 7 | WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.'; 8 | END; 9 | END$$; 10 | -------------------------------------------------------------------------------- /sql/update_to_4_0_0.sql: -------------------------------------------------------------------------------- 1 | DO $$DECLARE r record; 2 | BEGIN 3 | ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE jsonb USING args::jsonb; 4 | DROP FUNCTION IF EXISTS lock_head(tname varchar); 5 | DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer); 6 | END$$; 7 | -------------------------------------------------------------------------------- /test/benchmark_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'helper' 4 | 5 | if ENV['QC_BENCHMARK'] 6 | class BenchmarkTest < QCTest 7 | BENCHMARK_SIZE = Integer(ENV.fetch('QC_BENCHMARK_SIZE', 10_000)) 8 | BENCHMARK_MAX_TIME_DEQUEUE = Integer(ENV.fetch('QC_BENCHMARK_MAX_TIME_DEQUEUE', 30)) 9 | BENCHMARK_MAX_TIME_ENQUEUE = Integer(ENV.fetch('QC_BENCHMARK_MAX_TIME_ENQUEUE', 5)) 10 | 11 | def test_enqueue 12 | start = Time.now 13 | BENCHMARK_SIZE.times do 14 | QC.enqueue('1.odd?') 15 | end 16 | assert_equal(BENCHMARK_SIZE, QC.count) 17 | 18 | elapsed = Time.now - start 19 | assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_ENQUEUE) 20 | end 21 | 22 | def test_dequeue 23 | worker = QC::Worker.new 24 | worker.running = true 25 | BENCHMARK_SIZE.times do 26 | QC.enqueue('1.odd?') 27 | end 28 | assert_equal(BENCHMARK_SIZE, QC.count) 29 | 30 | start = Time.now 31 | BENCHMARK_SIZE.times do 32 | worker.work 33 | end 34 | elapsed = Time.now - start 35 | 36 | assert_equal(0, QC.count) 37 | assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_DEQUEUE) 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/config_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'helper' 4 | 5 | class ConfigTest < QCTest 6 | def setup 7 | QC.reset_config 8 | end 9 | 10 | def teardown 11 | QC.reset_config 12 | end 13 | 14 | def test_app_name_default 15 | assert_equal 'queue_classic', QC.app_name 16 | end 17 | 18 | def test_configure_app_name_with_env_var 19 | with_env 'QC_APP_NAME' => 'zomg_qc' do 20 | assert_equal 'zomg_qc', QC.app_name 21 | end 22 | end 23 | 24 | def test_wait_time_default 25 | assert_equal 5, QC.wait_time 26 | end 27 | 28 | def test_configure_wait_time_with_env_var 29 | with_env 'QC_LISTEN_TIME' => '7' do 30 | assert_equal 7, QC.wait_time 31 | end 32 | end 33 | 34 | def test_table_name_default 35 | assert_equal 'queue_classic_jobs', QC.table_name 36 | end 37 | 38 | def test_queue_default 39 | assert_equal 'default', QC.queue 40 | assert_equal 'default', QC.default_queue.name 41 | end 42 | 43 | def test_configure_queue_with_env_var 44 | with_env 'QUEUE' => 'priority' do 45 | assert_equal 'priority', QC.queue 46 | assert_equal 'priority', QC.default_queue.name 47 | end 48 | end 49 | 50 | def test_assign_default_queue 51 | QC.default_queue = QC::Queue.new 'dispensable' 52 | assert_equal 'default', QC.queue 53 | assert_equal 'dispensable', QC.default_queue.name 54 | end 55 | 56 | def test_queues_default 57 | assert_equal [], QC.queues 58 | end 59 | 60 | def test_configure_queues_with_env_var 61 | with_env 'QUEUES' => 'first,second,third' do 62 | assert_equal %w[first second third], QC.queues 63 | end 64 | end 65 | 66 | def test_configure_queues_with_whitespace 67 | with_env 'QUEUES' => ' one, two, three ' do 68 | assert_equal %w[one two three], QC.queues 69 | end 70 | end 71 | 72 | def test_top_bound_default 73 | assert_equal 9, QC.top_bound 74 | end 75 | 76 | def test_configure_top_bound_with_env_var 77 | with_env 'QC_TOP_BOUND' => '5' do 78 | assert_equal 5, QC.top_bound 79 | end 80 | end 81 | 82 | def test_fork_worker_default 83 | refute QC.fork_worker? 84 | end 85 | 86 | def test_configure_fork_worker_with_env_var 87 | with_env 'QC_FORK_WORKER' => 'yo' do 88 | assert QC.fork_worker? 89 | end 90 | end 91 | 92 | def test_configuration_constants_are_deprecated 93 | warning = capture_stderr_output do 94 | QC::FORK_WORKER 95 | end 96 | assert_match 'QC::FORK_WORKER is deprecated', warning 97 | assert_match 'QC.fork_worker? instead', warning 98 | end 99 | 100 | class TestWorker < QC::Worker; end 101 | 102 | def test_default_worker_class 103 | QC.default_worker_class = nil 104 | with_env 'QC_DEFAULT_WORKER_CLASS' => nil do 105 | assert_equal QC::Worker, QC.default_worker_class 106 | end 107 | 108 | QC.default_worker_class = nil 109 | assert_equal QC::Worker, QC.default_worker_class 110 | end 111 | 112 | def test_invite_worker_class 113 | QC.default_worker_class = nil 114 | with_env 'QC_DEFAULT_WORKER_CLASS' => 'Hopefully::Does::Not::Exist' do 115 | assert_equal QC::Worker, QC.default_worker_class 116 | end 117 | end 118 | 119 | def test_configure_default_worker_class_with_env_var 120 | QC.default_worker_class = nil 121 | with_env 'QC_DEFAULT_WORKER_CLASS' => 'ConfigTest::TestWorker' do 122 | assert_equal TestWorker, QC.default_worker_class 123 | end 124 | end 125 | 126 | def test_assign_default_worker_class 127 | original_worker = QC.default_worker_class 128 | QC.default_worker_class = TestWorker 129 | 130 | assert_equal TestWorker, QC.default_worker_class 131 | ensure 132 | QC.default_worker_class = original_worker 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /test/hard_coding_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'helper' 4 | 5 | class HardCodingTest < Minitest::Test 6 | def test_for_hard_coded_table_names 7 | # This is a simple way to do this, but prolly there could be a better way. 8 | # 9 | # TLDR: do not hard code the table name! It should (at the moment) only appear twice. Once for setup (all the upgrade SQL is currently hardcoded...), 10 | # and once for the config. If you change this test to add more hard coded table names, please reconsider. 11 | # 12 | # Ideally, you should use the config throughout the codebase; more context @ https://github.com/QueueClassic/queue_classic/issues/346 13 | # 14 | # 15 | # 16 | assert_equal `grep queue_classic_jobs lib -R`.split("\n").sort, 17 | ['lib/queue_classic/config.rb: @table_name ||= \'queue_classic_jobs\'', 18 | 'lib/queue_classic/setup.rb: conn.execute(\'DROP TABLE IF EXISTS queue_classic_jobs CASCADE\')'].sort 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler' 4 | require 'minitest/reporters' 5 | 6 | Bundler.setup :default, :test 7 | 8 | if ENV['CIRCLECI'] == 'true' 9 | Minitest::Reporters.use! Minitest::Reporters::JUnitReporter.new 10 | else 11 | Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new 12 | end 13 | 14 | ENV['DATABASE_URL'] ||= 'postgres:///queue_classic_test' 15 | 16 | require_relative '../lib/queue_classic' 17 | require 'stringio' 18 | require 'minitest/autorun' 19 | 20 | class QCTest < Minitest::Test 21 | def setup 22 | init_db 23 | end 24 | 25 | def teardown 26 | QC.delete_all 27 | end 28 | 29 | def init_db 30 | c = QC::ConnAdapter.new 31 | c.execute("SET client_min_messages TO 'warning'") 32 | QC::Setup.drop(c.connection) 33 | QC::Setup.create(c.connection) 34 | c.execute(File.read('./test/helper.sql')) 35 | c.disconnect 36 | end 37 | 38 | def capture_stderr_output 39 | original_stderr = $stderr 40 | $stderr = StringIO.new 41 | yield 42 | $stderr.string 43 | ensure 44 | $stderr = original_stderr 45 | end 46 | 47 | def capture_debug_output 48 | original_debug = ENV.fetch('DEBUG', nil) 49 | original_stdout = $stdout 50 | 51 | ENV['DEBUG'] = 'true' 52 | $stdout = StringIO.new 53 | yield 54 | $stdout.string 55 | ensure 56 | ENV['DEBUG'] = original_debug 57 | $stdout = original_stdout 58 | end 59 | 60 | def with_env(temporary_environment) 61 | original_environment = {} 62 | temporary_environment.each do |name, value| 63 | original_environment[name] = ENV.fetch(name, nil) 64 | ENV[name] = value 65 | end 66 | yield 67 | ensure 68 | original_environment.each { |name, value| ENV[name] = value } 69 | end 70 | 71 | def stub_any_instance(class_name, method_name, definition) 72 | new_method_name = "new_#{method_name}" 73 | original_method_name = "original_#{method_name}" 74 | 75 | method_present = class_name.instance_methods(false).include? method_name 76 | 77 | if method_present 78 | class_name.send(:alias_method, original_method_name, method_name) 79 | class_name.send(:define_method, new_method_name, definition) 80 | class_name.send(:alias_method, method_name, new_method_name) 81 | 82 | yield 83 | else 84 | message = "#{class_name} does not have method #{method_name}." 85 | message << "\nAvailable methods: #{class_name.instance_methods(false)}" 86 | raise ArgumentError, message 87 | end 88 | ensure 89 | if method_present 90 | class_name.send(:alias_method, method_name, original_method_name) 91 | class_name.send(:undef_method, new_method_name) 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /test/helper.sql: -------------------------------------------------------------------------------- 1 | DO $$ 2 | -- Set initial sequence to a large number to test the entire toolchain 3 | -- works on integers with higher bits set. 4 | DECLARE 5 | quoted_name text; 6 | quoted_size text; 7 | BEGIN 8 | -- Find the name of the relevant sequence. 9 | -- 10 | -- pg_get_serial_sequence quotes identifiers as part of its 11 | -- behavior. 12 | SELECT name 13 | INTO STRICT quoted_name 14 | FROM pg_get_serial_sequence('queue_classic_jobs', 'id') AS name; 15 | 16 | -- Don't quote, because ALTER SEQUENCE RESTART doesn't like 17 | -- general literals, only unquoted numeric literals. 18 | SELECT pow(2, 34)::text AS size 19 | INTO STRICT quoted_size; 20 | 21 | EXECUTE 'ALTER SEQUENCE ' || quoted_name || 22 | ' RESTART ' || quoted_size || ';'; 23 | END; 24 | $$; 25 | 26 | -------------------------------------------------------------------------------- /test/lib/queue_classic_rails_connection_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path('../helper.rb', __dir__) 4 | 5 | class QueueClassicRailsConnectionTest < QCTest 6 | def before_setup 7 | @original_conn_adapter = QC.default_conn_adapter 8 | QC.default_conn_adapter = nil 9 | end 10 | 11 | def before_teardown 12 | ActiveRecord.send :remove_const, :Base 13 | Object.send :remove_const, :ActiveRecord 14 | 15 | QC.default_conn_adapter = @original_conn_adapter 16 | end 17 | 18 | def test_uses_active_record_connection_if_exists 19 | connection = test_connection 20 | QC.default_conn_adapter.execute('SELECT 1;') 21 | connection.verify 22 | end 23 | 24 | def test_does_not_use_active_record_connection_if_env_var_set 25 | with_env 'QC_RAILS_DATABASE' => 'false' do 26 | connection = test_connection 27 | QC.default_conn_adapter.execute('SELECT 1;') 28 | assert_raises(MockExpectationError) { connection.verify } 29 | end 30 | end 31 | 32 | private 33 | 34 | def test_connection 35 | connection = Minitest::Mock.new 36 | connection.expect(:raw_connection, QC::ConnAdapter.new(active_record_connection_share: true).connection) 37 | 38 | Object.send :const_set, :ActiveRecord, Module.new 39 | ActiveRecord.const_set :Base, Module.new 40 | ActiveRecord::Base.define_singleton_method(:connection) do 41 | connection 42 | end 43 | 44 | QC.default_conn_adapter 45 | connection 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /test/lib/queue_classic_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path('../helper.rb', __dir__) 4 | 5 | class QueueClassicTest < QCTest 6 | def test_only_delegate_calls_to_queue_it_understands 7 | e = assert_raises(NoMethodError) do 8 | QC.probably_not 9 | end 10 | 11 | assert_match(/probably_not/, e.message) 12 | assert_match(/undefined method/, e.message) 13 | end 14 | 15 | def test_default_conn_adapter_default_value 16 | assert(QC.default_conn_adapter.is_a?(QC::ConnAdapter)) 17 | end 18 | 19 | def test_assigning_a_default_conn_adapter 20 | original_conn_adapter = QC.default_conn_adapter 21 | connection = QC::ConnAdapter.new 22 | QC.default_conn_adapter = connection 23 | assert_equal(QC.default_conn_adapter, connection) 24 | ensure 25 | QC.default_conn_adapter = original_conn_adapter 26 | end 27 | 28 | def test_unlock_jobs_of_dead_workers 29 | # Insert a locked job 30 | adapter = QC::ConnAdapter.new 31 | query = "INSERT INTO #{QC.table_name} (q_name, method, args, locked_by, locked_at) VALUES ('whatever', 'Kernel.puts', '[\"ok?\"]', 0, (CURRENT_TIMESTAMP))" 32 | adapter.execute(query) 33 | 34 | # We should have no unlocked jobs 35 | query_locked_jobs = "SELECT * FROM #{QC.table_name} WHERE locked_at IS NULL" 36 | res = adapter.connection.exec(query_locked_jobs) 37 | assert_equal(0, res.count) 38 | 39 | # Unlock the job 40 | QC.unlock_jobs_of_dead_workers 41 | 42 | # We should have an unlocked job now 43 | res = adapter.connection.exec(query_locked_jobs) 44 | assert_equal(1, res.count) 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/lib/queue_classic_test_with_activerecord_typecast.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path('../helper.rb', __dir__) 4 | 5 | class QueueClassicTest < QCTest 6 | def before_teardown 7 | ActiveRecord.send :remove_const, :Base 8 | Object.send :remove_const, :ActiveRecord 9 | 10 | QC.default_conn_adapter = @original_conn_adapter 11 | end 12 | 13 | def test_lock_with_active_record_timestamp_type_cast 14 | # Insert an unlocked job 15 | p_queue = QC::Queue.new('priority_queue') 16 | conn_adapter = Minitest::Mock.new 17 | conn_adapter.expect(:execute, 18 | { 'id' => '1', 'q_name' => 'test', 'method' => 'Kernel.puts', 'args' => '[]', 'scheduled_at' => Time.now }, [String, String]) 19 | QC.default_conn_adapter = conn_adapter 20 | assert_equal(p_queue.lock, {}) 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/queue_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'helper' 4 | 5 | class QueueTest < QCTest 6 | ResetError = Class.new(PG::Error) 7 | 8 | def test_enqueue 9 | QC.enqueue('Klass.method') 10 | end 11 | 12 | def test_respond_to 13 | assert_equal(true, QC.respond_to?(:enqueue)) 14 | end 15 | 16 | def test_lock 17 | queue = QC::Queue.new('queue_classic_jobs') 18 | queue.enqueue('Klass.method') 19 | job = queue.lock 20 | # See helper.rb for more information about the large initial id number. 21 | assert_equal((2**34).to_s, job[:id]) 22 | assert_equal('queue_classic_jobs', job[:q_name]) 23 | assert_equal('Klass.method', job[:method]) 24 | assert_equal([], job[:args]) 25 | end 26 | 27 | def test_lock_when_empty 28 | assert_nil(QC.lock) 29 | end 30 | 31 | def test_lock_with_future_job_with_enqueue_in 32 | now = Time.now 33 | QC.enqueue_in(2, 'Klass.method') 34 | assert_nil QC.lock 35 | sleep 2 36 | job = QC.lock 37 | assert_equal('Klass.method', job[:method]) 38 | assert_equal([], job[:args]) 39 | assert_equal((now + 2).to_i, job[:scheduled_at].to_i) 40 | end 41 | 42 | def test_lock_with_future_job_with_enqueue_at_with_a_time_object 43 | future = Time.now + 2 44 | QC.enqueue_at(future, 'Klass.method') 45 | assert_nil QC.lock 46 | sleep 0.1 until Time.now >= future 47 | job = QC.lock 48 | assert_equal('Klass.method', job[:method]) 49 | assert_equal([], job[:args]) 50 | assert_equal(future.to_i, job[:scheduled_at].to_i) 51 | end 52 | 53 | def test_lock_with_future_job_with_enqueue_at_with_a_float_timestamp 54 | offset = (Time.now + 2).to_f 55 | QC.enqueue_at(offset, 'Klass.method') 56 | assert_nil QC.lock 57 | sleep 2 58 | job = QC.lock 59 | assert_equal('Klass.method', job[:method]) 60 | assert_equal([], job[:args]) 61 | end 62 | 63 | def test_count 64 | QC.enqueue('Klass.method') 65 | assert_equal(1, QC.count) 66 | 67 | QC.enqueue('Klass.method') 68 | assert_equal(2, QC.count) 69 | assert_equal(2, QC.count_ready) 70 | assert_equal(0, QC.count_scheduled) 71 | 72 | QC.enqueue_in(60, 'Klass.method') 73 | assert_equal(3, QC.count) 74 | assert_equal(2, QC.count_ready) 75 | assert_equal(1, QC.count_scheduled) 76 | 77 | assert_raises(ArgumentError) do 78 | QC.count(:potatoes) 79 | end 80 | end 81 | 82 | def test_delete 83 | QC.enqueue('Klass.method') 84 | assert_equal(1, QC.count) 85 | QC.delete(QC.lock[:id]) 86 | assert_equal(0, QC.count) 87 | end 88 | 89 | def test_delete_all 90 | QC.enqueue('Klass.method') 91 | QC.enqueue('Klass.method') 92 | assert_equal(2, QC.count) 93 | QC.delete_all 94 | assert_equal(0, QC.count) 95 | end 96 | 97 | def test_delete_all_by_queue_name 98 | p_queue = QC::Queue.new('priority_queue') 99 | s_queue = QC::Queue.new('secondary_queue') 100 | p_queue.enqueue('Klass.method') 101 | s_queue.enqueue('Klass.method') 102 | assert_equal(1, p_queue.count) 103 | assert_equal(1, s_queue.count) 104 | p_queue.delete_all 105 | assert_equal(0, p_queue.count) 106 | assert_equal(1, s_queue.count) 107 | end 108 | 109 | def test_queue_instance 110 | queue = QC::Queue.new('queue_classic_jobs') 111 | queue.enqueue('Klass.method') 112 | assert_equal(1, queue.count) 113 | queue.delete(queue.lock[:id]) 114 | assert_equal(0, queue.count) 115 | end 116 | 117 | def test_repair_after_error 118 | queue = QC::Queue.new('queue_classic_jobs') 119 | queue.conn_adapter = QC::ConnAdapter.new 120 | queue.enqueue('Klass.method') 121 | assert_equal(1, queue.count) 122 | 123 | queue.conn_adapter.connection.stub :exec, ->(*_args) { raise(PG::Error) } do 124 | queue.conn_adapter.connection.stub :reset, ->(*_args) { raise(ResetError) } do 125 | assert_raises(PG::Error, ResetError) { queue.enqueue('Klass.other_method') } 126 | end 127 | end 128 | 129 | queue.conn_adapter.disconnect 130 | end 131 | 132 | def test_enqueue_retry 133 | queue = QC::Queue.new('queue_classic_jobs') 134 | queue.conn_adapter = QC::ConnAdapter.new 135 | conn = queue.conn_adapter.connection 136 | begin 137 | conn.exec('select pg_terminate_backend(pg_backend_pid())') 138 | rescue StandardError 139 | nil 140 | end 141 | queue.enqueue('Klass.method') 142 | assert_equal(1, queue.count) 143 | queue.conn_adapter.disconnect 144 | end 145 | 146 | def test_enqueue_stops_retrying_on_permanent_error 147 | queue = QC::Queue.new('queue_classic_jobs') 148 | queue.conn_adapter = QC::ConnAdapter.new 149 | 150 | # Simulate permanent connection error 151 | queue.conn_adapter.connection.stub :exec, ->(*_args) { raise(PG::Error) } do 152 | assert_raises(PG::Error) { queue.enqueue('Klass.other_method') } 153 | end 154 | 155 | queue.conn_adapter.disconnect 156 | end 157 | 158 | def test_enqueue_in_retry 159 | queue = QC::Queue.new('queue_classic_jobs') 160 | queue.conn_adapter = QC::ConnAdapter.new 161 | conn = queue.conn_adapter.connection 162 | begin 163 | conn.exec('select pg_terminate_backend(pg_backend_pid())') 164 | rescue StandardError 165 | nil 166 | end 167 | queue.enqueue_in(10, 'Klass.method') 168 | assert_equal(1, queue.count) 169 | queue.conn_adapter.disconnect 170 | end 171 | 172 | def test_enqueue_in_stops_retrying_on_permanent_error 173 | queue = QC::Queue.new('queue_classic_jobs') 174 | queue.conn_adapter = QC::ConnAdapter.new 175 | 176 | begin 177 | queue.conn_adapter.connection.exec('select pg_terminate_backend(pg_backend_pid())') 178 | rescue StandardError 179 | nil 180 | end 181 | 182 | # Simulate permanent connection error 183 | queue.conn_adapter.connection.stub :exec, ->(*_args) { raise(PG::Error) } do 184 | # Ensure that the error is reraised on second time 185 | assert_raises(PG::Error) { queue.enqueue_in(10, 'Klass.method') } 186 | end 187 | queue.conn_adapter.disconnect 188 | end 189 | 190 | def test_custom_default_queue 191 | queue_class = Class.new do 192 | attr_accessor :jobs 193 | 194 | def enqueue(method, *_args) 195 | @jobs ||= [] 196 | @jobs << method 197 | end 198 | end 199 | 200 | queue_instance = queue_class.new 201 | QC.default_queue = queue_instance 202 | 203 | QC.enqueue('Klass.method1') 204 | QC.enqueue('Klass.method2') 205 | 206 | assert_equal ['Klass.method1', 'Klass.method2'], queue_instance.jobs 207 | ensure 208 | QC.default_queue = nil 209 | end 210 | 211 | def test_multi_threaded_server_can_specified_connection 212 | adapter1 = QC::ConnAdapter.new 213 | adapter2 = QC::ConnAdapter.new 214 | q1 = q2 = nil 215 | 216 | QC.default_conn_adapter = adapter1 217 | 218 | t1 = Thread.new do 219 | QC.default_conn_adapter = adapter1 220 | q1 = QC::Queue.new('queue1').conn_adapter 221 | end 222 | 223 | t2 = Thread.new do 224 | QC.default_conn_adapter = adapter2 225 | q2 = QC::Queue.new('queue2').conn_adapter 226 | end 227 | 228 | t1.join 229 | t2.join 230 | 231 | assert_equal adapter1, q1 232 | assert_equal adapter2, q2 233 | end 234 | 235 | def test_multi_threaded_server_each_thread_acquires_unique_connection 236 | q1 = q2 = nil 237 | 238 | t1 = Thread.new do 239 | q1 = QC::Queue.new('queue1').conn_adapter 240 | end 241 | 242 | t2 = Thread.new do 243 | q2 = QC::Queue.new('queue2').conn_adapter 244 | end 245 | 246 | t1.join 247 | t2.join 248 | 249 | refute_equal q1, q2 250 | end 251 | 252 | def test_enqueue_triggers_notify 253 | adapter = QC.default_conn_adapter 254 | adapter.execute("LISTEN \"#{QC.queue}\"") 255 | adapter.send(:drain_notify) 256 | 257 | msgs = adapter.send(:wait_for_notify, 0.25) 258 | assert_equal(0, msgs.length) 259 | 260 | QC.enqueue('Klass.method') 261 | msgs = adapter.send(:wait_for_notify, 0.25) 262 | assert_equal(1, msgs.length) 263 | end 264 | 265 | def test_enqueue_returns_job_id 266 | enqueued_job = QC.enqueue('Klass.method') 267 | locked_job = QC.lock 268 | assert_equal enqueued_job, 'id' => locked_job[:id] 269 | end 270 | 271 | def test_enqueue_in_returns_job_id 272 | enqueued_job = QC.enqueue_in(1, 'Klass.method') 273 | sleep 1 274 | locked_job = QC.lock 275 | assert_equal enqueued_job, 'id' => locked_job[:id] 276 | end 277 | 278 | def test_enqueue_at_returns_job_id 279 | enqueued_job = QC.enqueue_at(Time.now + 1, 'Klass.method') 280 | sleep 1 281 | locked_job = QC.lock 282 | assert_equal enqueued_job, 'id' => locked_job[:id] 283 | end 284 | end 285 | -------------------------------------------------------------------------------- /test/rails-tests/.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | qctest*/ -------------------------------------------------------------------------------- /test/rails-tests/rails523.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # remove any old folder, should only matter locally 5 | rm -rf qctest523 6 | 7 | # install rails but not with much stuff 8 | gem install rails -v 5.2.3 9 | rails new qctest523 --api --database=postgresql --skip-test-unit --skip-keeps --skip-spring --skip-sprockets --skip-javascript --skip-turbolinks 10 | cd qctest523 11 | 12 | # get the db setup, run any default migrations 13 | bundle install 14 | bundle exec rails db:drop:all || true 15 | bundle exec rails db:create 16 | bundle exec rails db:migrate 17 | bundle exec rails db:setup 18 | 19 | # install qc --> gem file, bundle, add ourselves and migrate. 20 | echo "gem 'queue_classic', path: '../../../'" >> Gemfile 21 | bundle install 22 | bundle exec rails generate queue_classic:install 23 | bundle exec rails db:migrate 24 | -------------------------------------------------------------------------------- /test/worker_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'helper' 4 | 5 | module TestObject 6 | module_function 7 | 8 | def no_args = nil 9 | def one_arg(a) = a 10 | def two_args(a, b) = [a, b] 11 | def forty_two = Struct.new(:number).new(42) 12 | 13 | def fail_on_args(a = nil) 14 | raise "fail on args called with #{a}" unless a.nil? 15 | 16 | Class.new do 17 | def number(n) 18 | n * 2 19 | end 20 | end.new 21 | end 22 | end 23 | 24 | # This not only allows me to test what happens 25 | # when a failure occurs but it also demonstrates 26 | # how to override the worker to handle failures the way 27 | # you want. 28 | class TestWorker < QC::Worker 29 | attr_accessor :failed_count 30 | 31 | def initialize(args = {}) 32 | super(args.merge(connection: QC.default_conn_adapter.connection)) 33 | @failed_count = 0 34 | end 35 | 36 | def handle_failure(job, e) 37 | @failed_count += 1 38 | super 39 | end 40 | end 41 | 42 | class WorkerTest < QCTest 43 | def test_work 44 | QC.enqueue('TestObject.no_args') 45 | worker = TestWorker.new 46 | assert_equal(1, QC.count) 47 | worker.work 48 | assert_equal(0, QC.count) 49 | assert_equal(0, worker.failed_count) 50 | end 51 | 52 | def test_failed_job 53 | QC.enqueue('TestObject.not_a_method') 54 | worker = TestWorker.new 55 | capture_stderr_output { worker.work } 56 | assert_equal(1, worker.failed_count) 57 | end 58 | 59 | def test_failed_job_is_logged 60 | output = capture_stderr_output do 61 | QC.enqueue('TestObject.not_a_method') 62 | TestWorker.new.work 63 | end 64 | assert(output.include?("#