├── .github └── workflows │ ├── ci.yml │ ├── publish_gem.yml │ └── rubocop.yml ├── .gitignore ├── .rspec ├── .rubocop.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE.md ├── README.md ├── Rakefile ├── benchmarks.rb ├── contrib ├── delayed_job.monitrc ├── delayed_job_multiple.monitrc ├── delayed_job_rails_4.monitrc └── delayed_job_rails_4_multiple.monitrc ├── delayed_job.gemspec ├── lib ├── delayed │ ├── backend │ │ ├── base.rb │ │ ├── job_preparer.rb │ │ └── shared_spec.rb │ ├── command.rb │ ├── compatibility.rb │ ├── deserialization_error.rb │ ├── exceptions.rb │ ├── lifecycle.rb │ ├── message_sending.rb │ ├── performable_mailer.rb │ ├── performable_method.rb │ ├── plugin.rb │ ├── plugins │ │ └── clear_locks.rb │ ├── psych_ext.rb │ ├── railtie.rb │ ├── recipes.rb │ ├── serialization │ │ └── active_record.rb │ ├── syck_ext.rb │ ├── tasks.rb │ ├── worker.rb │ └── yaml_ext.rb ├── delayed_job.rb └── generators │ └── delayed_job │ ├── delayed_job_generator.rb │ └── templates │ └── script ├── recipes └── delayed_job.rb └── spec ├── autoloaded ├── clazz.rb ├── instance_clazz.rb ├── instance_struct.rb └── struct.rb ├── daemons.rb ├── delayed ├── backend │ └── test.rb ├── command_spec.rb └── serialization │ └── test.rb ├── helper.rb ├── lifecycle_spec.rb ├── message_sending_spec.rb ├── performable_mailer_spec.rb ├── performable_method_spec.rb ├── psych_ext_spec.rb ├── sample_jobs.rb ├── test_backend_spec.rb ├── worker_spec.rb └── yaml_ext_spec.rb /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | name: Test (Ruby ${{ matrix.ruby }}, Rails ${{ matrix.rails_version }}) 12 | runs-on: ubuntu-${{ matrix.ubuntu }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | ruby: ['3.2', '3.3', jruby-9.4, jruby-head, ruby-head] 17 | rails_version: 18 | - '7.0.0' 19 | - '7.1.0' 20 | - '7.2.0' 21 | - '8.0.0' 22 | - 'edge' 23 | ubuntu: [latest] 24 | include: 25 | # Ruby 2.6 26 | - ruby: 2.6 27 | rails_version: '6.0.0' 28 | ubuntu: '20.04' 29 | - ruby: 2.6 30 | rails_version: '6.1.0' 31 | ubuntu: '20.04' 32 | 33 | # ruby 3.1 (Dropped by Rails 8) 34 | - ruby: 3.1 35 | rails_version: '7.0.0' 36 | ubuntu: 'latest' 37 | - ruby: 3.1 38 | rails_version: '7.1.0' 39 | ubuntu: 'latest' 40 | - ruby: 3.1 41 | rails_version: '7.2.0' 42 | ubuntu: 'latest' 43 | 44 | # jruby-9.2 45 | - ruby: jruby-9.2 46 | rails_version: '6.0.0' 47 | ubuntu: '20.04' 48 | - ruby: jruby-9.2 49 | rails_version: '6.1.0' 50 | ubuntu: '20.04' 51 | 52 | # 53 | # The past 54 | # 55 | # EOL Active Record 56 | - ruby: 2.2 57 | rails_version: '3.2.0' 58 | ubuntu: '20.04' 59 | - ruby: 2.1 60 | rails_version: '4.1.0' 61 | ubuntu: '20.04' 62 | - ruby: 2.4 63 | rails_version: '4.2.0' 64 | ubuntu: '20.04' 65 | - ruby: 2.4 66 | rails_version: '5.0.0' 67 | ubuntu: '20.04' 68 | - ruby: 2.5 69 | rails_version: '5.1.0' 70 | ubuntu: '20.04' 71 | - ruby: 2.6 72 | rails_version: '5.2.0' 73 | ubuntu: '20.04' 74 | - ruby: 2.7 75 | rails_version: '5.2.0' 76 | ubuntu: '22.04' 77 | - ruby: jruby-9.2 78 | rails_version: '5.2.0' 79 | ubuntu: '22.04' 80 | - ruby: 2.7 81 | rails_version: '6.0.0' 82 | ubuntu: '22.04' 83 | - ruby: 3.0 84 | rails_version: '6.0.0' 85 | ubuntu: '22.04' 86 | - ruby: 3.2 87 | rails_version: '6.0.0' 88 | ubuntu: '22.04' 89 | - ruby: jruby-9.4 90 | rails_version: '6.0.0' 91 | ubuntu: '22.04' 92 | - ruby: 2.7 93 | rails_version: '6.1.0' 94 | ubuntu: '22.04' 95 | - ruby: 3.0 96 | rails_version: '6.1.0' 97 | ubuntu: '22.04' 98 | - ruby: 3.2 99 | rails_version: '6.1.0' 100 | ubuntu: '22.04' 101 | - ruby: jruby-9.4 102 | rails_version: '6.0.0' 103 | ubuntu: '22.04' 104 | - ruby: 2.7 105 | rails_version: '7.0.0' 106 | ubuntu: '22.04' 107 | - ruby: 3.0 108 | rails_version: '7.0.0' 109 | ubuntu: '22.04' 110 | - ruby: jruby-9.4 111 | rails_version: '6.0.0' 112 | ubuntu: '22.04' 113 | 114 | # EOL Ruby 115 | - ruby: 2.7 116 | rails_version: '7.1.0' 117 | ubuntu: '22.04' 118 | - ruby: 3.0 119 | rails_version: '7.1.0' 120 | ubuntu: '22.04' 121 | 122 | continue-on-error: ${{ matrix.rails_version == 'edge' || endsWith(matrix.ruby, 'head') }} 123 | 124 | steps: 125 | - uses: actions/checkout@v4 126 | - uses: ruby/setup-ruby@v1 127 | env: 128 | RAILS_VERSION: ${{ matrix.rails_version }} 129 | with: 130 | ruby-version: ${{ matrix.ruby }} 131 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 132 | - name: Run tests 133 | env: 134 | RAILS_VERSION: ${{ matrix.rails_version }} 135 | run: bundle exec rspec 136 | - name: Coveralls Parallel 137 | uses: coverallsapp/github-action@main 138 | with: 139 | github-token: ${{ secrets.github_token }} 140 | flag-name: run-${{ matrix.ruby }}-${{ matrix.rails_version }} 141 | parallel: true 142 | 143 | finish: 144 | needs: test 145 | runs-on: ubuntu-latest 146 | steps: 147 | - name: Coveralls Finished 148 | uses: coverallsapp/github-action@main 149 | with: 150 | github-token: ${{ secrets.github_token }} 151 | parallel-finished: true 152 | -------------------------------------------------------------------------------- /.github/workflows/publish_gem.yml: -------------------------------------------------------------------------------- 1 | name: Publish Gem 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | push: 10 | if: github.repository == 'collectiveidea/delayed_job' 11 | runs-on: ubuntu-latest 12 | environment: publishing 13 | 14 | permissions: 15 | contents: write 16 | id-token: write 17 | 18 | steps: 19 | # Set up 20 | - uses: actions/checkout@v4 21 | - name: Set up Ruby 22 | uses: ruby/setup-ruby@v1 23 | with: 24 | bundler-cache: true 25 | ruby-version: ruby 26 | 27 | # Release 28 | - uses: rubygems/release-gem@v1 29 | -------------------------------------------------------------------------------- /.github/workflows/rubocop.yml: -------------------------------------------------------------------------------- 1 | name: RuboCop 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v4 11 | - name: Set up Ruby 2.7 12 | uses: ruby/setup-ruby@v1 13 | with: 14 | ruby-version: 2.7 15 | - name: Generate lockfile for cache key 16 | run: bundle lock 17 | - name: Cache gems 18 | uses: actions/cache@v4 19 | with: 20 | path: vendor/bundle 21 | key: ${{ runner.os }}-rubocop-${{ hashFiles('**/Gemfile.lock') }} 22 | restore-keys: | 23 | ${{ runner.os }}-rubocop- 24 | - name: Install gems 25 | run: | 26 | bundle config path vendor/bundle 27 | bundle config set without 'default test' 28 | bundle install --jobs 4 --retry 3 29 | - name: Run RuboCop 30 | run: bundle exec rubocop 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.swp 3 | .bundle 4 | .DS_Store 5 | .rvmrc 6 | /coverage 7 | Gemfile.lock 8 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --order random 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AbcSize: 2 | Enabled: false 3 | 4 | # Enforce outdenting of access modifiers (i.e. public, private, protected) 5 | AccessModifierIndentation: 6 | EnforcedStyle: outdent 7 | 8 | Alias: 9 | EnforcedStyle: prefer_alias_method 10 | 11 | AllCops: 12 | Include: 13 | - 'Gemfile' 14 | - 'Rakefile' 15 | - 'delayed_job.gemspec' 16 | TargetRubyVersion: 2.1 17 | 18 | RedundantBlockCall: 19 | Enabled: false 20 | 21 | BlockLength: 22 | Enabled: false 23 | 24 | # Avoid more than `Max` levels of nesting. 25 | BlockNesting: 26 | Max: 2 27 | 28 | # Indentation of when/else 29 | CaseIndentation: 30 | EnforcedStyle: end 31 | IndentOneStep: false 32 | 33 | ClassLength: 34 | Max: 100 35 | 36 | # Align with the style guide. 37 | CollectionMethods: 38 | PreferredMethods: 39 | collect: 'map' 40 | collect!: 'map!' 41 | reduce: 'inject' 42 | find: 'detect' 43 | find_all: 'select' 44 | 45 | # Disable documentation checking until a class needs to be documented once 46 | Documentation: 47 | Enabled: false 48 | 49 | # Allow dots at the end of lines 50 | DotPosition: 51 | EnforcedStyle: trailing 52 | 53 | DoubleNegation: 54 | Enabled: false 55 | 56 | # Detects any duplication as issue including our conditional requires 57 | DuplicatedGem: 58 | Enabled: false 59 | 60 | EmptyLinesAroundAccessModifier: 61 | Enabled: true 62 | 63 | # Don't require magic comment at the top of every file 64 | Encoding: 65 | Enabled: false 66 | 67 | # Align ends correctly 68 | EndAlignment: 69 | EnforcedStyleAlignWith: variable 70 | 71 | Style/FrozenStringLiteralComment: 72 | Enabled: false 73 | 74 | # Enforce Ruby 1.8-compatible hash syntax 75 | HashSyntax: 76 | EnforcedStyle: hash_rockets 77 | 78 | Lambda: 79 | Enabled: false 80 | 81 | LineLength: 82 | Enabled: false 83 | 84 | MethodLength: 85 | CountComments: false 86 | Max: 53 87 | 88 | MultilineOperationIndentation: 89 | EnforcedStyle: indented 90 | 91 | Style/NumericPredicate: 92 | Enabled: false 93 | 94 | # Avoid long parameter lists 95 | ParameterLists: 96 | Max: 4 97 | CountKeywordArgs: true 98 | 99 | PercentLiteralDelimiters: 100 | PreferredDelimiters: 101 | '%': () 102 | '%i': () 103 | '%q': () 104 | '%Q': () 105 | '%r': '{}' 106 | '%s': () 107 | '%w': '[]' 108 | '%W': '[]' 109 | '%x': () 110 | 111 | RaiseArgs: 112 | EnforcedStyle: exploded 113 | 114 | RegexpLiteral: 115 | Enabled: false 116 | 117 | RescueModifier: 118 | Enabled: false 119 | 120 | Style/SafeNavigation: 121 | Enabled: false 122 | 123 | SignalException: 124 | EnforcedStyle: only_raise 125 | 126 | # No spaces inside hash literals 127 | SpaceInsideHashLiteralBraces: 128 | EnforcedStyle: no_space 129 | 130 | Style/SymbolArray: 131 | Enabled: false 132 | 133 | SymbolProc: 134 | Enabled: false 135 | 136 | TrailingCommaInLiteral: 137 | Enabled: false 138 | 139 | TrailingCommaInArguments: 140 | Enabled: false 141 | 142 | YAMLLoad: 143 | Enabled: false 144 | 145 | ZeroLengthPredicate: 146 | Enabled: false 147 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 4.1.13 - 2024-11-08 2 | ======================= 3 | * Enable Rails 8 4 | 5 | 4.1.12 - 2024-08-14 6 | ======================= 7 | * Add missing require for extract_options 8 | * Fix rails 7.2 ActiveSupport::ProxyObject deprecation 9 | * Multiple contributors on current and legacy test suite improvements 10 | 11 | 4.1.12.rc1 - 2024-08-13 12 | ======================= 13 | * Validating trusted publishing release 14 | * Add missing require for extract_options 15 | * Fix rails 7.2 ActiveSupport::ProxyObject deprecation 16 | * Multiple contributors on current and legacy test suite improvements 17 | 18 | 4.1.11 - 2022-09-28 19 | =================== 20 | * Fix missing require for Rails 7.0.3+ 21 | 22 | 4.1.10 - 2022-01-17 23 | =================== 24 | * Support for Rails 7.0. NOTE: If you are using Delayed Job independent of Rails, Active Support 7 has dropped classic dependency autoloading. You will need to add and setup zeitwerk for autoloading to continue working in ActiveSupport 7. 25 | 26 | 4.1.9 - 2020-12-09 27 | ================== 28 | * Support for Rails 6.1 29 | * Add support for parameterized mailers via delay call (#1121) 30 | 31 | 4.1.8 - 2019-08-16 32 | ================== 33 | * Support for Rails 6.0.0 34 | 35 | 4.1.7 - 2019-06-20 36 | ================== 37 | * Fix loading Delayed::PerformableMailer when ActionMailer isn't loaded yet 38 | 39 | 4.1.6 - 2019-06-19 40 | ================== 41 | * Properly initialize ActionMailer outside railties (#1077) 42 | * Fix Psych load_tags support (#1093) 43 | * Replace REMOVED with FAILED in log message (#1048) 44 | * Misc doc updates (#1052, #1074, #1064, #1063) 45 | 46 | 4.1.5 - 2018-04-13 47 | ================== 48 | * Allow Rails 5.2 49 | 50 | 4.1.4 - 2017-12-29 51 | ================== 52 | * Use `yaml_tag` instead of deprecated `yaml_as` (#996) 53 | * Support ruby 2.5.0 54 | 55 | 4.1.3 - 2017-05-26 56 | ================== 57 | * Don't mutate the options hash (#877) 58 | * Log an error message when a deserialization error occurs (#894) 59 | * Adding the queue name to the log output (#917) 60 | * Don't include ClassMethods with MessageSending (#924) 61 | * Fix YAML deserialization error if original object is soft-deleted (#947) 62 | * Add support for Rails 5.1 (#982) 63 | 64 | 4.1.2 - 2016-05-16 65 | ================== 66 | * Added Delayed::Worker.queue_attributes 67 | * Limit what we require in ActiveSupport 68 | * Fix pid file creation when there is no tmp directory 69 | * Rails 5 support 70 | 71 | 4.1.1 - 2015-09-24 72 | ================== 73 | * Fix shared specs for back-ends that reload objects 74 | 75 | 4.1.0 - 2015-09-22 76 | ================== 77 | * Alter `Delayed::Command` to work with or without Rails 78 | * Allow `Delayed::Worker.delay_jobs` configuration to be a proc 79 | * Add ability to set destroy failed jobs on a per job basis 80 | * Make `Delayed::Worker.new` idempotent 81 | * Set quiet from the environment 82 | * Rescue `Exception` instead of `StandardError` in worker 83 | * Fix worker crash on serialization error 84 | 85 | 4.0.6 - 2014-12-22 86 | ================== 87 | * Revert removing test files from the gem 88 | 89 | 4.0.5 - 2014-12-22 90 | ================== 91 | * Support for Rails 4.2 92 | * Allow user to override where DJ writes log output 93 | * First attempt at automatic code reloading 94 | * Clearer error message when ActiveRecord object no longer exists 95 | * Various improvements to the README 96 | 97 | 4.0.4 - 2014-09-24 98 | ================== 99 | * Fix using options passed into delayed_job command 100 | * Add the ability to set a default queue for a custom job 101 | * Add the ability to override the max_run_time on a custom job. MUST be lower than worker setting 102 | * Psych YAML overrides are now exclusively used only when loading a job payload 103 | * SLEEP_DELAY and READ_AHEAD can be set for the rake task 104 | * Some updates for Rails 4.2 support 105 | 106 | 4.0.3 - 2014-09-04 107 | ================== 108 | * Added --pools option to delayed_job command 109 | * Removed a bunch of the Psych hacks 110 | * Improved deserialization error reporting 111 | * Misc bug fixes 112 | 113 | 4.0.2 - 2014-06-24 114 | ================== 115 | * Add support for RSpec 3 116 | 117 | 4.0.1 - 2014-04-12 118 | ================== 119 | * Update gemspec for Rails 4.1 120 | * Make logger calls more universal 121 | * Check that records are persisted? instead of new_record? 122 | 123 | 4.0.0 - 2013-07-30 124 | ================== 125 | * Rails 4 compatibility 126 | * Reverted threaded startup due to daemons incompatibilities 127 | * Attempt to recover from job reservation errors 128 | 129 | 4.0.0.beta2 - 2013-05-28 130 | ======================== 131 | * Rails 4 compatibility 132 | * Threaded startup script for faster multi-worker startup 133 | * YAML compatibility changes 134 | * Added jobs:check rake task 135 | 136 | 4.0.0.beta1 - 2013-03-02 137 | ======================== 138 | * Rails 4 compatibility 139 | 140 | 3.0.5 - 2013-01-28 141 | ================== 142 | * Better job timeout error logging 143 | * psych support for delayed_job_data_mapper deserialization 144 | * User can configure the worker to raise a SignalException on TERM and/or INT 145 | * Add the ability to run all available jobs and exit when complete 146 | 147 | 3.0.4 - 2012-11-09 148 | ================== 149 | * Allow the app to specify a default queue name 150 | * Capistrano script now allows user to specify the DJ command, allowing the user to add "bundle exec" if necessary 151 | * Persisted record check is now more general 152 | 153 | 3.0.3 - 2012-05-25 154 | ================== 155 | * Fix a bug where the worker would not respect the exit condition 156 | * Properly handle sleep delay command line argument 157 | 158 | 3.0.2 - 2012-04-02 159 | ================== 160 | * Fix deprecation warnings 161 | * Raise ArgumentError if attempting to enqueue a performable method on an object that hasn't been persisted yet 162 | * Allow the number of jobs read at a time to be configured from the command line using --read-ahead 163 | * Allow custom logger to be configured through Delayed::Worker.logger 164 | * Various documentation improvements 165 | 166 | 3.0.1 - 2012-01-24 167 | ================== 168 | * Added RecordNotFound message to deserialization error 169 | * Direct JRuby's yecht parser to syck extensions 170 | * Updated psych extensions for better compatibility with ruby 1.9.2 171 | * Updated syck extension for increased compatibility with class methods 172 | * Test grooming 173 | 174 | 3.0.0 - 2011-12-30 175 | ================== 176 | * New: Named queues 177 | * New: Job/Worker lifecycle callbacks 178 | * Change: daemons is no longer a runtime dependency 179 | * Change: Active Record backend support is provided by a separate gem 180 | * Change: Enqueue hook is called before jobs are saved so that they may be modified 181 | * Fix problem deserializing models that use a custom primary key column 182 | * Fix deserializing AR models when the object isn't in the default scope 183 | * Fix hooks not getting called when delay_jobs is false 184 | 185 | 2.1.4 - 2011-02-11 186 | ================== 187 | * Working around issues when psych is loaded, fixes issues with bundler 1.0.10 and Rails 3.0.4 188 | * Added -p/--prefix option to help differentiate multiple delayed job workers on the same host. 189 | 190 | 2.1.3 - 2011-01-20 191 | ================== 192 | * Revert worker contention fix due to regressions 193 | * Added Delayed::Worker.delay_jobs flag to support running jobs immediately 194 | 195 | 2.1.2 - 2010-12-01 196 | ================== 197 | * Remove contention between multiple workers by performing an update to lock a job before fetching it 198 | * Job payloads may implement #max_attempts to control how many times it should be retried 199 | * Fix for loading ActionMailer extension 200 | * Added 'delayed_job_server_role' Capistrano variable to allow delayed_job to run on its own worker server 201 | set :delayed_job_server_role, :worker 202 | * Fix `rake jobs:work` so it outputs to the console 203 | 204 | 2.1.1 - 2010-11-14 205 | ================== 206 | * Fix issue with worker name not getting properly set when locking a job 207 | * Fixes for YAML serialization 208 | 209 | 2.1.0 - 2010-11-14 210 | ================== 211 | * Added enqueue, before, after, success, error, and failure. See the README 212 | * Remove Merb support 213 | * Remove all non Active Record backends into separate gems. See https://github.com/collectiveidea/delayed_job/wiki/Backends 214 | * remove rails 2 support. delayed_job 2.1 will only support Rails 3 215 | * New pure-YAML serialization 216 | * Added Rails 3 railtie and generator 217 | * Changed @@sleep_delay to self.class.sleep_delay to be consistent with other class variable usage 218 | * Added --sleep-delay command line option 219 | 220 | 2.0.8 - Unreleased 221 | ================== 222 | * Backport fix for deserialization errors that bring down the daemon 223 | 224 | 2.0.7 - 2011-02-10 225 | ================== 226 | * Fixed missing generators and recipes for Rails 2.x 227 | 228 | 2.0.6 - 2011-01-20 229 | ================== 230 | * Revert worker contention fix due to regressions 231 | 232 | 2.0.5 - 2010-12-01 233 | ================== 234 | * Added #reschedule_at hook on payload to determine when the job should be rescheduled [backported from 2.1] 235 | * Added --sleep-delay command line option [backported from 2.1] 236 | * Added 'delayed_job_server_role' Capistrano variable to allow delayed_job to run on its own worker server 237 | set :delayed_job_server_role, :worker 238 | * Changed AR backend to reserve jobs using an UPDATE query to reduce worker contention [backported from 2.1] 239 | 240 | 2.0.4 - 2010-11-14 241 | ================== 242 | * Fix issue where dirty tracking prevented job from being properly unlocked 243 | * Add delayed_job_args variable for Capistrano recipe to allow configuration of started workers (e.g. "-n 2 --max-priority 10") 244 | * Added options to handle_asynchronously 245 | * Added Delayed::Worker.default_priority 246 | * Allow private methods to be delayed 247 | * Fixes for Ruby 1.9 248 | * Added -m command line option to start a monitor process 249 | * normalize logging in worker 250 | * Deprecate #send_later and #send_at in favor of new #delay method 251 | * Added @#delay@ to Object that allows you to delay any method and pass options: 252 | options = {:priority => 19, :run_at => 5.minutes.from_now} 253 | UserMailer.delay(options).deliver_confirmation(@user) 254 | 255 | 2.0.3 - 2010-04-16 256 | ================== 257 | * Fix initialization for Rails 2.x 258 | 259 | 2.0.2 - 2010-04-08 260 | ================== 261 | * Fixes to Mongo Mapper backend [ "14be7a24":http://github.com/collectiveidea/delayed_job/commit/14be7a24, "dafd5f46":http://github.com/collectiveidea/delayed_job/commit/dafd5f46, "54d40913":http://github.com/collectiveidea/delayed_job/commit/54d40913 ] 262 | * DataMapper backend performance improvements [ "93833cce":http://github.com/collectiveidea/delayed_job/commit/93833cce, "e9b1573e":http://github.com/collectiveidea/delayed_job/commit/e9b1573e, "37a16d11":http://github.com/collectiveidea/delayed_job/commit/37a16d11, "803f2bfa":http://github.com/collectiveidea/delayed_job/commit/803f2bfa ] 263 | * Fixed Delayed::Command to create tmp/pids directory [ "8ec8ca41":http://github.com/collectiveidea/delayed_job/commit/8ec8ca41 ] 264 | * Railtie to perform Rails 3 initialization [ "3e0fc41f":http://github.com/collectiveidea/delayed_job/commit/3e0fc41f ] 265 | * Added on_permanent_failure hook [ "d2f14cd6":http://github.com/collectiveidea/delayed_job/commit/d2f14cd6 ] 266 | 267 | 2.0.1 - 2010-04-03 268 | ================== 269 | * Bug fix for using ActiveRecord backend with daemon [martinbtt] 270 | 271 | 2.0.0 - 2010-04-03 272 | ================== 273 | * Multiple backend support (See README for more details) 274 | * Added MongoMapper backend [zbelzer, moneypools] 275 | * Added DataMapper backend [lpetre] 276 | * Reverse priority so the jobs table can be indexed. Lower numbers have higher priority. The default priority is 0, so increase it for jobs that are not important. 277 | * Move most of the heavy lifting from Job to Worker (#work_off, #reschedule, #run, #min_priority, #max_priority, #max_run_time, #max_attempts, #worker_name) [albus522] 278 | * Remove EvaledJob. Implement your own if you need this functionality. 279 | * Only use Time.zone if it is set. Closes #20 280 | * Fix for last_error recording when destroy_failed_jobs = false, max_attempts = 1 281 | * Implemented worker name_prefix to maintain dynamic nature of pid detection 282 | * Some Rails 3 compatibility fixes [fredwu] 283 | 284 | 1.8.5 - 2010-03-15 285 | ================== 286 | * Set auto_flushing=true on Rails logger to fix logging in production 287 | * Fix error message when trying to send_later on a method that doesn't exist 288 | * Don't use rails_env in capistrano if it's not set. closes #22 289 | * Delayed job should append to delayed_job.log not overwrite 290 | * Version bump to 1.8.5 291 | * fixing Time.now to be Time.zone.now if set to honor the app set local TimeZone 292 | * Replaced @Worker::SLEEP@, @Job::MAX_ATTEMPTS@, and @Job::MAX_RUN_TIME@ with class methods that can be overridden. 293 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | How to contribute 2 | ================= 3 | 4 | If you find what looks like a bug: 5 | 6 | * Search the "mailing list":http://groups.google.com/group/delayed_job to see 7 | if anyone else had the same issue. 8 | * Check the "GitHub issue tracker":http://github.com/collectiveidea/delayed_job/issues/ 9 | to see if anyone else has reported issue. 10 | * Make sure you are using the latest version of delayed_job 11 | ![Gem Version](https://badge.fury.io/rb/delayed_job.png) 12 | * Make sure you are using the latest backend gem for delayed_job 13 | * Active Record ![Gem Version](https://badge.fury.io/rb/delayed_job_active_record.png) 14 | * Mongoid ![Gem Version](https://badge.fury.io/rb/delayed_job_mongoid.png) 15 | * If you are still having an issue, create an issue including: 16 | * Ruby version 17 | * Gemfile.lock contents or at least major gem versions, such as Rails version 18 | * Steps to reproduce the issue 19 | * Full backtrace for any errors encountered 20 | 21 | If you want to contribute an enhancement or a fix: 22 | 23 | * Fork the project on GitHub. 24 | * Make your changes with tests. 25 | * Commit the changes without making changes to the Rakefile or any other files 26 | that aren't related to your enhancement or fix. 27 | * Send a pull request. 28 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'rake' 4 | 5 | platforms :ruby do 6 | # Rails 5.1 is the first to work with sqlite 1.4 7 | # Rails 6 now requires sqlite 1.4 8 | if ENV['RAILS_VERSION'] && ENV['RAILS_VERSION'] < '5.1' 9 | gem 'sqlite3', '< 1.4' 10 | elsif ENV['RAILS_VERSION'] && ENV['RAILS_VERSION'] < '7.2' 11 | gem 'sqlite3', '~> 1.4' 12 | else 13 | gem 'sqlite3' 14 | end 15 | end 16 | 17 | platforms :jruby do 18 | if ENV['RAILS_VERSION'] == '4.2.0' 19 | gem 'activerecord-jdbcsqlite3-adapter', '< 50.0' 20 | elsif ENV['RAILS_VERSION'] == '5.0.0' 21 | gem 'activerecord-jdbcsqlite3-adapter', '~> 50.0' 22 | elsif ENV['RAILS_VERSION'] == '5.1.0' 23 | gem 'activerecord-jdbcsqlite3-adapter', '~> 51.0' 24 | elsif ENV['RAILS_VERSION'] == '5.2.0' 25 | gem 'activerecord-jdbcsqlite3-adapter', '~> 52.0' 26 | elsif ENV['RAILS_VERSION'] == '6.0.0' 27 | gem 'activerecord-jdbcsqlite3-adapter', '~> 60.0' 28 | elsif ENV['RAILS_VERSION'] == '6.1.0' 29 | gem 'activerecord-jdbcsqlite3-adapter', '~> 61.0' 30 | else 31 | gem 'activerecord-jdbcsqlite3-adapter' 32 | end 33 | gem 'jruby-openssl' 34 | gem 'mime-types', ['~> 2.6', '< 2.99'] 35 | 36 | if ENV['RAILS_VERSION'] == 'edge' 37 | gem 'railties', :github => 'rails/rails' 38 | elsif ENV['RAILS_VERSION'] 39 | gem 'railties', "~> #{ENV['RAILS_VERSION']}" 40 | else 41 | gem 'railties', ['>= 3.0', '< 9.0'] 42 | end 43 | end 44 | 45 | platforms :rbx do 46 | gem 'psych' 47 | end 48 | 49 | group :test do 50 | if ENV['RAILS_VERSION'] == 'edge' 51 | gem 'actionmailer', :github => 'rails/rails' 52 | gem 'activerecord', :github => 'rails/rails' 53 | elsif ENV['RAILS_VERSION'] 54 | gem 'actionmailer', "~> #{ENV['RAILS_VERSION']}" 55 | gem 'activerecord', "~> #{ENV['RAILS_VERSION']}" 56 | if ENV['RAILS_VERSION'] < '5.1' 57 | gem 'loofah', '2.3.1' 58 | gem 'nokogiri', '< 1.11.0' 59 | gem 'rails-html-sanitizer', '< 1.4.0' 60 | end 61 | else 62 | gem 'actionmailer', ['>= 3.0', '< 9.0'] 63 | gem 'activerecord', ['>= 3.0', '< 9.0'] 64 | end 65 | gem 'net-smtp' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.1.0') 66 | gem 'rspec', '>= 3' 67 | gem 'simplecov', :require => false 68 | if /\A2.[12]/ =~ RUBY_VERSION 69 | # 0.8.0 doesn't work with simplecov < 0.18.0 and older ruby can't run 0.18.0 70 | gem 'simplecov-lcov', '< 0.8.0', :require => false 71 | else 72 | gem 'simplecov-lcov', :require => false 73 | end 74 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3.0') 75 | # New dependencies with a deprecation notice in Ruby 3.3 and required in Ruby 3.4 76 | # Probably won't get released in rails 7.0 77 | gem 'base64' 78 | gem 'bigdecimal' 79 | gem 'mutex_m' 80 | end 81 | if ENV['RAILS_VERSION'].nil? || ENV['RAILS_VERSION'] >= '6.0.0' 82 | gem 'zeitwerk', :require => false 83 | end 84 | end 85 | 86 | group :rubocop do 87 | gem 'rubocop', '>= 0.25', '< 0.49' 88 | end 89 | 90 | gemspec 91 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2005 Tobias Lütke 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOa AND 17 | NONINFRINGEMENT. IN NO EVENT SaALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **If you're viewing this at https://github.com/collectiveidea/delayed_job, 2 | you're reading the documentation for the master branch. 3 | [View documentation for the latest release 4 | (4.1.13).](https://github.com/collectiveidea/delayed_job/tree/v4.1.13)** 5 | 6 | Delayed::Job 7 | ============ 8 | [![Gem Version](https://badge.fury.io/rb/delayed_job.svg)][gem] 9 | ![CI](https://github.com/collectiveidea/delayed_job/workflows/CI/badge.svg) 10 | [![Code Climate](https://codeclimate.com/github/collectiveidea/delayed_job.svg)][codeclimate] 11 | [![Coverage Status](https://coveralls.io/repos/collectiveidea/delayed_job/badge.svg?branch=master)][coveralls] 12 | 13 | [gem]: https://rubygems.org/gems/delayed_job 14 | [codeclimate]: https://codeclimate.com/github/collectiveidea/delayed_job 15 | [coveralls]: https://coveralls.io/r/collectiveidea/delayed_job 16 | 17 | Delayed::Job (or DJ) encapsulates the common pattern of asynchronously executing 18 | longer tasks in the background. 19 | 20 | It is a direct extraction from Shopify where the job table is responsible for a 21 | multitude of core tasks. Amongst those tasks are: 22 | 23 | * sending massive newsletters 24 | * image resizing 25 | * http downloads 26 | * updating smart collections 27 | * updating solr, our search server, after product changes 28 | * batch imports 29 | * spam checks 30 | 31 | [Follow us on Twitter][twitter] to get updates and notices about new releases. 32 | 33 | [twitter]: https://twitter.com/delayedjob 34 | 35 | Installation 36 | ============ 37 | delayed_job 3.0.0 only supports Rails 3.0+. 38 | 39 | delayed_job supports multiple backends for storing the job queue. [See the wiki 40 | for other backends](https://github.com/collectiveidea/delayed_job/wiki/Backends). 41 | 42 | If you plan to use delayed_job with Active Record, add `delayed_job_active_record` to your `Gemfile`. 43 | 44 | ```ruby 45 | gem 'delayed_job_active_record' 46 | ``` 47 | 48 | If you plan to use delayed_job with Mongoid, add `delayed_job_mongoid` to your `Gemfile`. 49 | 50 | ```ruby 51 | gem 'delayed_job_mongoid' 52 | ``` 53 | 54 | Run `bundle install` to install the backend and delayed_job gems. 55 | 56 | The Active Record backend requires a jobs table. You can create that table by 57 | running the following command: 58 | 59 | rails generate delayed_job:active_record 60 | rake db:migrate 61 | 62 | For Rails 4.2+, see [below](#active-job) 63 | 64 | Development 65 | =========== 66 | In development mode, if you are using Rails 3.1+, your application code will automatically reload every 100 jobs or when the queue finishes. 67 | You no longer need to restart Delayed Job every time you update your code in development. 68 | 69 | Active Job 70 | ========== 71 | In Rails 4.2+, set the queue_adapter in config/application.rb 72 | 73 | ```ruby 74 | config.active_job.queue_adapter = :delayed_job 75 | ``` 76 | 77 | See the [rails guide](http://guides.rubyonrails.org/active_job_basics.html#setting-the-backend) for more details. 78 | 79 | Rails 4.x 80 | ========= 81 | If you are using the protected_attributes gem, it must appear before delayed_job in your gemfile. If your jobs are failing with: 82 | 83 | ActiveRecord::StatementInvalid: PG::NotNullViolation: ERROR: null value in column "handler" violates not-null constraint 84 | 85 | then this is the fix you're looking for. 86 | 87 | Upgrading from 2.x to 3.0.0 on Active Record 88 | ============================================ 89 | Delayed Job 3.0.0 introduces a new column to the delayed_jobs table. 90 | 91 | If you're upgrading from Delayed Job 2.x, run the upgrade generator to create a migration to add the column. 92 | 93 | rails generate delayed_job:upgrade 94 | rake db:migrate 95 | 96 | Queuing Jobs 97 | ============ 98 | Call `.delay.method(params)` on any object and it will be processed in the background. 99 | 100 | ```ruby 101 | # without delayed_job 102 | @user.activate!(@device) 103 | 104 | # with delayed_job 105 | @user.delay.activate!(@device) 106 | ``` 107 | 108 | If a method should always be run in the background, you can call 109 | `#handle_asynchronously` after the method declaration: 110 | 111 | ```ruby 112 | class Device 113 | def deliver 114 | # long running method 115 | end 116 | handle_asynchronously :deliver 117 | end 118 | 119 | device = Device.new 120 | device.deliver 121 | ``` 122 | 123 | ## Parameters 124 | 125 | `#handle_asynchronously` and `#delay` take these parameters: 126 | 127 | - `:priority` (number): lower numbers run first; default is 0 but can be reconfigured (see below) 128 | - `:run_at` (Time): run the job after this time (probably in the future) 129 | - `:queue` (string): named queue to put this job in, an alternative to priorities (see below) 130 | 131 | These params can be Proc objects, allowing call-time evaluation of the value. 132 | 133 | For example: 134 | 135 | ```ruby 136 | class LongTasks 137 | def send_mailer 138 | # Some other code 139 | end 140 | handle_asynchronously :send_mailer, :priority => 20 141 | 142 | def in_the_future 143 | # Some other code 144 | end 145 | # 5.minutes.from_now will be evaluated when in_the_future is called 146 | handle_asynchronously :in_the_future, :run_at => Proc.new { 5.minutes.from_now } 147 | 148 | def self.when_to_run 149 | 2.hours.from_now 150 | end 151 | 152 | class << self 153 | def call_a_class_method 154 | # Some other code 155 | end 156 | handle_asynchronously :call_a_class_method, :run_at => Proc.new { when_to_run } 157 | end 158 | 159 | attr_reader :how_important 160 | 161 | def call_an_instance_method 162 | # Some other code 163 | end 164 | handle_asynchronously :call_an_instance_method, :priority => Proc.new {|i| i.how_important } 165 | end 166 | ``` 167 | 168 | If you ever want to call a `handle_asynchronously`'d method without Delayed Job, for instance while debugging something at the console, just add `_without_delay` to the method name. For instance, if your original method was `foo`, then call `foo_without_delay`. 169 | 170 | Rails Mailers 171 | ============= 172 | Delayed Job uses special syntax for Rails Mailers. 173 | Do not call the `.deliver` method when using `.delay`. 174 | 175 | ```ruby 176 | # without delayed_job 177 | Notifier.signup(@user).deliver 178 | 179 | # with delayed_job 180 | Notifier.delay.signup(@user) 181 | 182 | # delayed_job running at a specific time 183 | Notifier.delay(run_at: 5.minutes.from_now).signup(@user) 184 | 185 | # when using parameters, the .with method must be called before the .delay method 186 | Notifier.with(foo: 1, bar: 2).delay.signup(@user) 187 | ``` 188 | 189 | You may also wish to consider using 190 | [Active Job with Action Mailer](https://edgeguides.rubyonrails.org/active_job_basics.html#action-mailer) 191 | which provides convenient `.deliver_later` syntax that forwards to Delayed Job under-the-hood. 192 | 193 | Named Queues 194 | ============ 195 | DJ 3 introduces Resque-style named queues while still retaining DJ-style 196 | priority. The goal is to provide a system for grouping tasks to be worked by 197 | separate pools of workers, which may be scaled and controlled individually. 198 | 199 | Jobs can be assigned to a queue by setting the `queue` option: 200 | 201 | ```ruby 202 | object.delay(:queue => 'tracking').method 203 | 204 | Delayed::Job.enqueue job, :queue => 'tracking' 205 | 206 | handle_asynchronously :tweet_later, :queue => 'tweets' 207 | ``` 208 | 209 | You can configure default priorities for named queues: 210 | 211 | ```ruby 212 | Delayed::Worker.queue_attributes = { 213 | high_priority: { priority: -10 }, 214 | low_priority: { priority: 10 } 215 | } 216 | ``` 217 | 218 | Configured queue priorities can be overriden by passing priority to the delay method 219 | 220 | ```ruby 221 | object.delay(:queue => 'high_priority', priority: 0).method 222 | ``` 223 | 224 | You can start processes to only work certain queues with the `queue` and `queues` 225 | options defined below. Processes started without specifying a queue will run jobs 226 | from **any** queue. To effectively have a process that runs jobs where a queue is not 227 | specified, set a default queue name with `Delayed::Worker.default_queue_name` and 228 | have the processes run that queue. 229 | 230 | Running Jobs 231 | ============ 232 | `script/delayed_job` can be used to manage a background process which will 233 | start working off jobs. 234 | 235 | To do so, add `gem "daemons"` to your `Gemfile` and make sure you've run `rails 236 | generate delayed_job`. 237 | 238 | You can then do the following: 239 | 240 | RAILS_ENV=production script/delayed_job start 241 | RAILS_ENV=production script/delayed_job stop 242 | 243 | # Runs two workers in separate processes. 244 | RAILS_ENV=production script/delayed_job -n 2 start 245 | RAILS_ENV=production script/delayed_job stop 246 | 247 | # Set the --queue or --queues option to work from a particular queue. 248 | RAILS_ENV=production script/delayed_job --queue=tracking start 249 | RAILS_ENV=production script/delayed_job --queues=mailers,tasks start 250 | 251 | # Use the --pool option to specify a worker pool. You can use this option multiple times to start different numbers of workers for different queues. 252 | # The following command will start 1 worker for the tracking queue, 253 | # 2 workers for the mailers and tasks queues, and 2 workers for any jobs: 254 | RAILS_ENV=production script/delayed_job --pool=tracking --pool=mailers,tasks:2 --pool=*:2 start 255 | 256 | # Runs all available jobs and then exits 257 | RAILS_ENV=production script/delayed_job start --exit-on-complete 258 | # or to run in the foreground 259 | RAILS_ENV=production script/delayed_job run --exit-on-complete 260 | 261 | **Rails 4:** *replace script/delayed_job with bin/delayed_job* 262 | 263 | Workers can be running on any computer, as long as they have access to the 264 | database and their clock is in sync. Keep in mind that each worker will check 265 | the database at least every 5 seconds. 266 | 267 | You can also invoke `rake jobs:work` which will start working off jobs. You can 268 | cancel the rake task with `CTRL-C`. 269 | 270 | If you want to just run all available jobs and exit you can use `rake jobs:workoff` 271 | 272 | Work off queues by setting the `QUEUE` or `QUEUES` environment variable. 273 | 274 | QUEUE=tracking rake jobs:work 275 | QUEUES=mailers,tasks rake jobs:work 276 | 277 | Restarting delayed_job 278 | ====================== 279 | 280 | The following syntax will restart delayed jobs: 281 | 282 | RAILS_ENV=production script/delayed_job restart 283 | 284 | To restart multiple delayed_job workers: 285 | 286 | RAILS_ENV=production script/delayed_job -n2 restart 287 | 288 | **Rails 4:** *replace script/delayed_job with bin/delayed_job* 289 | 290 | 291 | 292 | Custom Jobs 293 | =========== 294 | Jobs are simple ruby objects with a method called perform. Any object which responds to perform can be stuffed into the jobs table. Job objects are serialized to yaml so that they can later be resurrected by the job runner. 295 | 296 | ```ruby 297 | NewsletterJob = Struct.new(:text, :emails) do 298 | def perform 299 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 300 | end 301 | end 302 | 303 | Delayed::Job.enqueue NewsletterJob.new('lorem ipsum...', Customers.pluck(:email)) 304 | ``` 305 | 306 | To set a per-job max attempts that overrides the Delayed::Worker.max_attempts you can define a max_attempts method on the job 307 | 308 | ```ruby 309 | NewsletterJob = Struct.new(:text, :emails) do 310 | def perform 311 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 312 | end 313 | 314 | def max_attempts 315 | 3 316 | end 317 | end 318 | ``` 319 | 320 | To set a per-job max run time that overrides the Delayed::Worker.max_run_time you can define a max_run_time method on the job 321 | 322 | NOTE: this can ONLY be used to set a max_run_time that is lower than Delayed::Worker.max_run_time. Otherwise the lock on the job would expire and another worker would start the working on the in progress job. 323 | 324 | ```ruby 325 | NewsletterJob = Struct.new(:text, :emails) do 326 | def perform 327 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 328 | end 329 | 330 | def max_run_time 331 | 120 # seconds 332 | end 333 | end 334 | ``` 335 | 336 | To set a per-job default for destroying failed jobs that overrides the Delayed::Worker.destroy_failed_jobs you can define a destroy_failed_jobs? method on the job 337 | 338 | ```ruby 339 | NewsletterJob = Struct.new(:text, :emails) do 340 | def perform 341 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 342 | end 343 | 344 | def destroy_failed_jobs? 345 | false 346 | end 347 | end 348 | ``` 349 | 350 | To set a default queue name for a custom job that overrides Delayed::Worker.default_queue_name, you can define a queue_name method on the job 351 | 352 | ```ruby 353 | NewsletterJob = Struct.new(:text, :emails) do 354 | def perform 355 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 356 | end 357 | 358 | def queue_name 359 | 'newsletter_queue' 360 | end 361 | end 362 | ``` 363 | 364 | On error, the job is scheduled again in 5 seconds + N ** 4, where N is the number of attempts. You can define your own `reschedule_at` method to override this default behavior. 365 | 366 | ```ruby 367 | NewsletterJob = Struct.new(:text, :emails) do 368 | def perform 369 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 370 | end 371 | 372 | def reschedule_at(current_time, attempts) 373 | current_time + 5.seconds 374 | end 375 | end 376 | ``` 377 | 378 | Hooks 379 | ===== 380 | You can define hooks on your job that will be called at different stages in the process: 381 | 382 | 383 | **NOTE:** If you are using ActiveJob these hooks are **not** available to your jobs. You will need to use ActiveJob's callbacks. You can find details here https://guides.rubyonrails.org/active_job_basics.html#callbacks 384 | 385 | ```ruby 386 | class ParanoidNewsletterJob < NewsletterJob 387 | def enqueue(job) 388 | record_stat 'newsletter_job/enqueue' 389 | end 390 | 391 | def perform 392 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) } 393 | end 394 | 395 | def before(job) 396 | record_stat 'newsletter_job/start' 397 | end 398 | 399 | def after(job) 400 | record_stat 'newsletter_job/after' 401 | end 402 | 403 | def success(job) 404 | record_stat 'newsletter_job/success' 405 | end 406 | 407 | def error(job, exception) 408 | Airbrake.notify(exception) 409 | end 410 | 411 | def failure(job) 412 | page_sysadmin_in_the_middle_of_the_night 413 | end 414 | end 415 | ``` 416 | 417 | Gory Details 418 | ============ 419 | The library revolves around a delayed_jobs table which looks as follows: 420 | 421 | ```ruby 422 | create_table :delayed_jobs, :force => true do |table| 423 | table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue 424 | table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually. 425 | table.text :handler # YAML-encoded string of the object that will do work 426 | table.text :last_error # reason for last failure (See Note below) 427 | table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future. 428 | table.datetime :locked_at # Set when a client is working on this object 429 | table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) 430 | table.string :locked_by # Who is working on this object (if locked) 431 | table.string :queue # The name of the queue this job is in 432 | table.timestamps 433 | end 434 | ``` 435 | 436 | On error, the job is scheduled again in 5 seconds + N ** 4, where N is the number of attempts or using the job's defined `reschedule_at` method. 437 | 438 | The default `Worker.max_attempts` is 25. After this, the job is either deleted (default), or left in the database with "failed_at" set. 439 | With the default of 25 attempts, the last retry will be 20 days later, with the last interval being almost 100 hours. 440 | 441 | The default `Worker.max_run_time` is 4.hours. If your job takes longer than that, another computer could pick it up. It's up to you to 442 | make sure your job doesn't exceed this time. You should set this to the longest time you think the job could take. 443 | 444 | By default, it will delete failed jobs (and it always deletes successful jobs). If you want to keep failed jobs, set 445 | `Delayed::Worker.destroy_failed_jobs = false`. The failed jobs will be marked with non-null failed_at. 446 | 447 | By default all jobs are scheduled with `priority = 0`, which is top priority. You can change this by setting `Delayed::Worker.default_priority` to something else. Lower numbers have higher priority. 448 | 449 | The default behavior is to read 5 jobs from the queue when finding an available job. You can configure this by setting `Delayed::Worker.read_ahead`. 450 | 451 | By default all jobs will be queued without a named queue. A default named queue can be specified by using `Delayed::Worker.default_queue_name`. 452 | 453 | If no jobs are found, the worker sleeps for the amount of time specified by the sleep delay option. Set `Delayed::Worker.sleep_delay = 60` for a 60 second sleep time. 454 | 455 | It is possible to disable delayed jobs for testing purposes. Set `Delayed::Worker.delay_jobs = false` to execute all jobs realtime. 456 | 457 | Or `Delayed::Worker.delay_jobs` can be a Proc that decides whether to execute jobs inline on a per-job basis: 458 | 459 | ```ruby 460 | Delayed::Worker.delay_jobs = ->(job) { 461 | job.queue != 'inline' 462 | } 463 | ``` 464 | 465 | You may need to raise exceptions on SIGTERM signals, `Delayed::Worker.raise_signal_exceptions = :term` will cause the worker to raise a `SignalException` causing the running job to abort and be unlocked, which makes the job available to other workers. The default for this option is false. 466 | 467 | Here is an example of changing job parameters in Rails: 468 | 469 | ```ruby 470 | # config/initializers/delayed_job_config.rb 471 | Delayed::Worker.destroy_failed_jobs = false 472 | Delayed::Worker.sleep_delay = 60 473 | Delayed::Worker.max_attempts = 3 474 | Delayed::Worker.max_run_time = 5.minutes 475 | Delayed::Worker.read_ahead = 10 476 | Delayed::Worker.default_queue_name = 'default' 477 | Delayed::Worker.delay_jobs = !Rails.env.test? 478 | Delayed::Worker.raise_signal_exceptions = :term 479 | Delayed::Worker.logger = Logger.new(File.join(Rails.root, 'log', 'delayed_job.log')) 480 | ``` 481 | 482 | Cleaning up 483 | =========== 484 | You can invoke `rake jobs:clear` to delete all jobs in the queue. 485 | 486 | Having problems? 487 | ================ 488 | Good places to get help are: 489 | * [Google Groups](http://groups.google.com/group/delayed_job) where you can join our mailing list. 490 | * [StackOverflow](http://stackoverflow.com/questions/tagged/delayed-job) 491 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | Bundler::GemHelper.install_tasks 3 | 4 | require 'rspec/core/rake_task' 5 | desc 'Run the specs' 6 | RSpec::Core::RakeTask.new do |r| 7 | r.verbose = false 8 | end 9 | 10 | task :test => :spec 11 | 12 | require 'rubocop/rake_task' 13 | RuboCop::RakeTask.new 14 | 15 | task :default => [:spec, :rubocop] 16 | -------------------------------------------------------------------------------- /benchmarks.rb: -------------------------------------------------------------------------------- 1 | require 'spec/helper' 2 | require 'logger' 3 | require 'benchmark' 4 | 5 | # Delayed::Worker.logger = Logger.new('/dev/null') 6 | 7 | Benchmark.bm(10) do |x| 8 | Delayed::Job.delete_all 9 | n = 10_000 10 | n.times { 'foo'.delay.length } 11 | 12 | x.report { Delayed::Worker.new(:quiet => true).work_off(n) } 13 | end 14 | -------------------------------------------------------------------------------- /contrib/delayed_job.monitrc: -------------------------------------------------------------------------------- 1 | # an example Monit configuration file for delayed_job 2 | # See: http://stackoverflow.com/questions/1226302/how-to-monitor-delayedjob-with-monit/1285611 3 | # 4 | # To use: 5 | # 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc 6 | # 2. replace {app_name} as appropriate 7 | # 3. add this to your /etc/monit/monitrc 8 | # 9 | # include /var/www/apps/{app_name}/shared/delayed_job.monitrc 10 | 11 | check process delayed_job 12 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.pid 13 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job start" 14 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job stop" 15 | -------------------------------------------------------------------------------- /contrib/delayed_job_multiple.monitrc: -------------------------------------------------------------------------------- 1 | # an example Monit configuration file for delayed_job running multiple processes 2 | # 3 | # To use: 4 | # 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc 5 | # 2. replace {app_name} as appropriate 6 | # you might also need to change the program strings to 7 | # "/bin/su - {username} -c '/usr/bin/env ...'" 8 | # to load your shell environment. 9 | # 10 | # 3. add this to your /etc/monit/monitrc 11 | # 12 | # include /var/www/apps/{app_name}/shared/delayed_job.monitrc 13 | # 14 | # The processes are grouped so that monit can act on them as a whole, e.g. 15 | # 16 | # monit -g delayed_job restart 17 | 18 | check process delayed_job_0 19 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.0.pid 20 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job start -i 0" 21 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job stop -i 0" 22 | group delayed_job 23 | 24 | check process delayed_job_1 25 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.1.pid 26 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job start -i 1" 27 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job stop -i 1" 28 | group delayed_job 29 | 30 | check process delayed_job_2 31 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.2.pid 32 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job start -i 2" 33 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job stop -i 2" 34 | group delayed_job 35 | -------------------------------------------------------------------------------- /contrib/delayed_job_rails_4.monitrc: -------------------------------------------------------------------------------- 1 | # an example Monit configuration file for delayed_job 2 | # See: http://stackoverflow.com/questions/1226302/how-to-monitor-delayedjob-with-monit/1285611 3 | # 4 | # To use: 5 | # 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc 6 | # 2. replace {app_name} as appropriate 7 | # 3. add this to your /etc/monit/monitrc 8 | # 9 | # include /var/www/apps/{app_name}/shared/delayed_job.monitrc 10 | 11 | check process delayed_job 12 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.pid 13 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start" 14 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop" 15 | -------------------------------------------------------------------------------- /contrib/delayed_job_rails_4_multiple.monitrc: -------------------------------------------------------------------------------- 1 | # an example Monit configuration file for delayed_job running multiple processes 2 | # 3 | # To use: 4 | # 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc 5 | # 2. replace {app_name} as appropriate 6 | # you might also need to change the program strings to 7 | # "/bin/su - {username} -c '/usr/bin/env ...'" 8 | # to load your shell environment. 9 | # 10 | # 3. add this to your /etc/monit/monitrc 11 | # 12 | # include /var/www/apps/{app_name}/shared/delayed_job.monitrc 13 | # 14 | # The processes are grouped so that monit can act on them as a whole, e.g. 15 | # 16 | # monit -g delayed_job restart 17 | 18 | check process delayed_job_0 19 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.0.pid 20 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start -i 0" 21 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop -i 0" 22 | group delayed_job 23 | 24 | check process delayed_job_1 25 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.1.pid 26 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start -i 1" 27 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop -i 1" 28 | group delayed_job 29 | 30 | check process delayed_job_2 31 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.2.pid 32 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start -i 2" 33 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop -i 2" 34 | group delayed_job 35 | -------------------------------------------------------------------------------- /delayed_job.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | Gem::Specification.new do |spec| 4 | spec.add_dependency 'activesupport', ['>= 3.0', '< 9.0'] 5 | spec.authors = ['Brandon Keepers', 'Brian Ryckbost', 'Chris Gaffney', 'David Genord II', 'Erik Michaels-Ober', 'Matt Griffin', 'Steve Richert', 'Tobias Lütke'] 6 | spec.description = 'Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks.' 7 | spec.email = ['brian@collectiveidea.com'] 8 | spec.files = %w[CHANGELOG.md CONTRIBUTING.md LICENSE.md README.md Rakefile delayed_job.gemspec] 9 | spec.files += Dir.glob('{contrib,lib,recipes,spec}/**/*') # rubocop:disable SpaceAroundOperators 10 | spec.homepage = 'http://github.com/collectiveidea/delayed_job' 11 | spec.licenses = ['MIT'] 12 | spec.name = 'delayed_job' 13 | spec.require_paths = ['lib'] 14 | spec.summary = 'Database-backed asynchronous priority queue system -- Extracted from Shopify' 15 | spec.test_files = Dir.glob('spec/**/*') 16 | spec.version = '4.1.13' 17 | spec.metadata = { 18 | 'changelog_uri' => 'https://github.com/collectiveidea/delayed_job/blob/master/CHANGELOG.md', 19 | 'bug_tracker_uri' => 'https://github.com/collectiveidea/delayed_job/issues', 20 | 'source_code_uri' => 'https://github.com/collectiveidea/delayed_job' 21 | } 22 | end 23 | -------------------------------------------------------------------------------- /lib/delayed/backend/base.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | module Backend 3 | module Base 4 | def self.included(base) 5 | base.extend ClassMethods 6 | end 7 | 8 | module ClassMethods 9 | # Add a job to the queue 10 | def enqueue(*args) 11 | job_options = Delayed::Backend::JobPreparer.new(*args).prepare 12 | enqueue_job(job_options) 13 | end 14 | 15 | def enqueue_job(options) 16 | new(options).tap do |job| 17 | Delayed::Worker.lifecycle.run_callbacks(:enqueue, job) do 18 | job.hook(:enqueue) 19 | Delayed::Worker.delay_job?(job) ? job.save : job.invoke_job 20 | end 21 | end 22 | end 23 | 24 | def reserve(worker, max_run_time = Worker.max_run_time) 25 | # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next. 26 | # this leads to a more even distribution of jobs across the worker processes 27 | find_available(worker.name, worker.read_ahead, max_run_time).detect do |job| 28 | job.lock_exclusively!(max_run_time, worker.name) 29 | end 30 | end 31 | 32 | # Allow the backend to attempt recovery from reserve errors 33 | def recover_from(_error); end 34 | 35 | # Hook method that is called before a new worker is forked 36 | def before_fork; end 37 | 38 | # Hook method that is called after a new worker is forked 39 | def after_fork; end 40 | 41 | def work_off(num = 100) 42 | warn '[DEPRECATION] `Delayed::Job.work_off` is deprecated. Use `Delayed::Worker.new.work_off instead.' 43 | Delayed::Worker.new.work_off(num) 44 | end 45 | end 46 | 47 | attr_reader :error 48 | def error=(error) 49 | @error = error 50 | self.last_error = "#{error.message}\n#{error.backtrace.join("\n")}" if respond_to?(:last_error=) 51 | end 52 | 53 | def failed? 54 | !!failed_at 55 | end 56 | alias_method :failed, :failed? 57 | 58 | ParseObjectFromYaml = %r{\!ruby/\w+\:([^\s]+)} # rubocop:disable ConstantName 59 | 60 | def name 61 | @name ||= payload_object.respond_to?(:display_name) ? payload_object.display_name : payload_object.class.name 62 | rescue DeserializationError 63 | ParseObjectFromYaml.match(handler)[1] 64 | end 65 | 66 | def payload_object=(object) 67 | @payload_object = object 68 | self.handler = object.to_yaml 69 | end 70 | 71 | def payload_object 72 | @payload_object ||= YAML.load_dj(handler) 73 | rescue TypeError, LoadError, NameError, ArgumentError, SyntaxError, Psych::SyntaxError => e 74 | raise DeserializationError, "Job failed to load: #{e.message}. Handler: #{handler.inspect}" 75 | end 76 | 77 | def invoke_job 78 | Delayed::Worker.lifecycle.run_callbacks(:invoke_job, self) do 79 | begin 80 | hook :before 81 | payload_object.perform 82 | hook :success 83 | rescue Exception => e # rubocop:disable RescueException 84 | hook :error, e 85 | raise e 86 | ensure 87 | hook :after 88 | end 89 | end 90 | end 91 | 92 | # Unlock this job (note: not saved to DB) 93 | def unlock 94 | self.locked_at = nil 95 | self.locked_by = nil 96 | end 97 | 98 | def hook(name, *args) 99 | if payload_object.respond_to?(name) 100 | method = payload_object.method(name) 101 | method.arity.zero? ? method.call : method.call(self, *args) 102 | end 103 | rescue DeserializationError # rubocop:disable HandleExceptions 104 | end 105 | 106 | def reschedule_at 107 | if payload_object.respond_to?(:reschedule_at) 108 | payload_object.reschedule_at(self.class.db_time_now, attempts) 109 | else 110 | self.class.db_time_now + (attempts**4) + 5 111 | end 112 | end 113 | 114 | def max_attempts 115 | payload_object.max_attempts if payload_object.respond_to?(:max_attempts) 116 | end 117 | 118 | def max_run_time 119 | return unless payload_object.respond_to?(:max_run_time) 120 | return unless (run_time = payload_object.max_run_time) 121 | 122 | if run_time > Delayed::Worker.max_run_time 123 | Delayed::Worker.max_run_time 124 | else 125 | run_time 126 | end 127 | end 128 | 129 | def destroy_failed_jobs? 130 | payload_object.respond_to?(:destroy_failed_jobs?) ? payload_object.destroy_failed_jobs? : Delayed::Worker.destroy_failed_jobs 131 | rescue DeserializationError 132 | Delayed::Worker.destroy_failed_jobs 133 | end 134 | 135 | def fail! 136 | self.failed_at = self.class.db_time_now 137 | save! 138 | end 139 | 140 | protected 141 | 142 | def set_default_run_at 143 | self.run_at ||= self.class.db_time_now 144 | end 145 | 146 | # Call during reload operation to clear out internal state 147 | def reset 148 | @payload_object = nil 149 | end 150 | end 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /lib/delayed/backend/job_preparer.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/core_ext/array/extract_options' 2 | 3 | module Delayed 4 | module Backend 5 | class JobPreparer 6 | attr_reader :options, :args 7 | 8 | def initialize(*args) 9 | @options = args.extract_options!.dup 10 | @args = args 11 | end 12 | 13 | def prepare 14 | set_payload 15 | set_queue_name 16 | set_priority 17 | handle_deprecation 18 | options 19 | end 20 | 21 | private 22 | 23 | def set_payload 24 | options[:payload_object] ||= args.shift 25 | end 26 | 27 | def set_queue_name 28 | if options[:queue].nil? && options[:payload_object].respond_to?(:queue_name) 29 | options[:queue] = options[:payload_object].queue_name 30 | else 31 | options[:queue] ||= Delayed::Worker.default_queue_name 32 | end 33 | end 34 | 35 | def set_priority 36 | queue_attribute = Delayed::Worker.queue_attributes[options[:queue]] 37 | options[:priority] ||= (queue_attribute && queue_attribute[:priority]) || Delayed::Worker.default_priority 38 | end 39 | 40 | def handle_deprecation 41 | if args.size > 0 42 | warn '[DEPRECATION] Passing multiple arguments to `#enqueue` is deprecated. Pass a hash with :priority and :run_at.' 43 | options[:priority] = args.first || options[:priority] 44 | options[:run_at] = args[1] 45 | end 46 | 47 | # rubocop:disable GuardClause 48 | unless options[:payload_object].respond_to?(:perform) 49 | raise ArgumentError, 'Cannot enqueue items which do not respond to perform' 50 | end 51 | # rubocop:enabled GuardClause 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/delayed/backend/shared_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../../../spec/sample_jobs', __FILE__) 2 | 3 | require 'active_support/core_ext/numeric/time' 4 | 5 | shared_examples_for 'a delayed_job backend' do 6 | let(:worker) { Delayed::Worker.new } 7 | 8 | def create_job(opts = {}) 9 | described_class.create(opts.merge(:payload_object => SimpleJob.new)) 10 | end 11 | 12 | before do 13 | Delayed::Worker.max_priority = nil 14 | Delayed::Worker.min_priority = nil 15 | Delayed::Worker.default_priority = 99 16 | Delayed::Worker.delay_jobs = true 17 | Delayed::Worker.default_queue_name = 'default_tracking' 18 | SimpleJob.runs = 0 19 | described_class.delete_all 20 | end 21 | 22 | after do 23 | Delayed::Worker.reset 24 | end 25 | 26 | it 'sets run_at automatically if not set' do 27 | expect(described_class.create(:payload_object => ErrorJob.new).run_at).not_to be_nil 28 | end 29 | 30 | it 'does not set run_at automatically if already set' do 31 | later = described_class.db_time_now + 5.minutes 32 | job = described_class.create(:payload_object => ErrorJob.new, :run_at => later) 33 | expect(job.run_at).to be_within(1).of(later) 34 | end 35 | 36 | describe '#reload' do 37 | it 'reloads the payload' do 38 | job = described_class.enqueue :payload_object => SimpleJob.new 39 | expect(job.payload_object.object_id).not_to eq(job.reload.payload_object.object_id) 40 | end 41 | end 42 | 43 | describe 'enqueue' do 44 | context 'with a hash' do 45 | it "raises ArgumentError when handler doesn't respond_to :perform" do 46 | expect { described_class.enqueue(:payload_object => Object.new) }.to raise_error(ArgumentError) 47 | end 48 | 49 | it 'is able to set priority' do 50 | job = described_class.enqueue :payload_object => SimpleJob.new, :priority => 5 51 | expect(job.priority).to eq(5) 52 | end 53 | 54 | it 'uses default priority' do 55 | job = described_class.enqueue :payload_object => SimpleJob.new 56 | expect(job.priority).to eq(99) 57 | end 58 | 59 | it 'is able to set run_at' do 60 | later = described_class.db_time_now + 5.minutes 61 | job = described_class.enqueue :payload_object => SimpleJob.new, :run_at => later 62 | expect(job.run_at).to be_within(1).of(later) 63 | end 64 | 65 | it 'is able to set queue' do 66 | job = described_class.enqueue :payload_object => NamedQueueJob.new, :queue => 'tracking' 67 | expect(job.queue).to eq('tracking') 68 | end 69 | 70 | it 'uses default queue' do 71 | job = described_class.enqueue :payload_object => SimpleJob.new 72 | expect(job.queue).to eq(Delayed::Worker.default_queue_name) 73 | end 74 | 75 | it "uses the payload object's queue" do 76 | job = described_class.enqueue :payload_object => NamedQueueJob.new 77 | expect(job.queue).to eq(NamedQueueJob.new.queue_name) 78 | end 79 | end 80 | 81 | context 'with multiple arguments' do 82 | it "raises ArgumentError when handler doesn't respond_to :perform" do 83 | expect { described_class.enqueue(Object.new) }.to raise_error(ArgumentError) 84 | end 85 | 86 | it 'increases count after enqueuing items' do 87 | described_class.enqueue SimpleJob.new 88 | expect(described_class.count).to eq(1) 89 | end 90 | 91 | it 'is able to set priority [DEPRECATED]' do 92 | silence_warnings do 93 | job = described_class.enqueue SimpleJob.new, 5 94 | expect(job.priority).to eq(5) 95 | end 96 | end 97 | 98 | it 'uses default priority when it is not set' do 99 | @job = described_class.enqueue SimpleJob.new 100 | expect(@job.priority).to eq(99) 101 | end 102 | 103 | it 'is able to set run_at [DEPRECATED]' do 104 | silence_warnings do 105 | later = described_class.db_time_now + 5.minutes 106 | @job = described_class.enqueue SimpleJob.new, 5, later 107 | expect(@job.run_at).to be_within(1).of(later) 108 | end 109 | end 110 | 111 | it 'works with jobs in modules' do 112 | M::ModuleJob.runs = 0 113 | job = described_class.enqueue M::ModuleJob.new 114 | expect { job.invoke_job }.to change { M::ModuleJob.runs }.from(0).to(1) 115 | end 116 | 117 | it 'does not mutate the options hash' do 118 | options = {:priority => 1} 119 | described_class.enqueue SimpleJob.new, options 120 | expect(options).to eq(:priority => 1) 121 | end 122 | end 123 | 124 | context 'with delay_jobs = false' do 125 | before(:each) do 126 | Delayed::Worker.delay_jobs = false 127 | end 128 | 129 | it 'does not increase count after enqueuing items' do 130 | described_class.enqueue SimpleJob.new 131 | expect(described_class.count).to eq(0) 132 | end 133 | 134 | it 'invokes the enqueued job' do 135 | job = SimpleJob.new 136 | expect(job).to receive(:perform) 137 | described_class.enqueue job 138 | end 139 | 140 | it 'returns a job, not the result of invocation' do 141 | expect(described_class.enqueue(SimpleJob.new)).to be_instance_of(described_class) 142 | end 143 | end 144 | end 145 | 146 | describe 'callbacks' do 147 | before(:each) do 148 | CallbackJob.messages = [] 149 | end 150 | 151 | %w[before success after].each do |callback| 152 | it "calls #{callback} with job" do 153 | job = described_class.enqueue(CallbackJob.new) 154 | expect(job.payload_object).to receive(callback).with(job) 155 | job.invoke_job 156 | end 157 | end 158 | 159 | it 'calls before and after callbacks' do 160 | job = described_class.enqueue(CallbackJob.new) 161 | expect(CallbackJob.messages).to eq(['enqueue']) 162 | job.invoke_job 163 | expect(CallbackJob.messages).to eq(%w[enqueue before perform success after]) 164 | end 165 | 166 | it 'calls the after callback with an error' do 167 | job = described_class.enqueue(CallbackJob.new) 168 | expect(job.payload_object).to receive(:perform).and_raise(RuntimeError.new('fail')) 169 | 170 | expect { job.invoke_job }.to raise_error(RuntimeError) 171 | expect(CallbackJob.messages).to eq(['enqueue', 'before', 'error: RuntimeError', 'after']) 172 | end 173 | 174 | it 'calls error when before raises an error' do 175 | job = described_class.enqueue(CallbackJob.new) 176 | expect(job.payload_object).to receive(:before).and_raise(RuntimeError.new('fail')) 177 | expect { job.invoke_job }.to raise_error(RuntimeError) 178 | expect(CallbackJob.messages).to eq(['enqueue', 'error: RuntimeError', 'after']) 179 | end 180 | end 181 | 182 | describe 'payload_object' do 183 | it 'raises a DeserializationError when the job class is totally unknown' do 184 | job = described_class.new :handler => '--- !ruby/object:JobThatDoesNotExist {}' 185 | expect { job.payload_object }.to raise_error(Delayed::DeserializationError) 186 | end 187 | 188 | it 'raises a DeserializationError when the job struct is totally unknown' do 189 | job = described_class.new :handler => '--- !ruby/struct:StructThatDoesNotExist {}' 190 | expect { job.payload_object }.to raise_error(Delayed::DeserializationError) 191 | end 192 | 193 | it 'raises a DeserializationError when the YAML.load raises argument error' do 194 | job = described_class.new :handler => '--- !ruby/struct:GoingToRaiseArgError {}' 195 | expect(YAML).to receive(:load_dj).and_raise(ArgumentError) 196 | expect { job.payload_object }.to raise_error(Delayed::DeserializationError) 197 | end 198 | 199 | it 'raises a DeserializationError when the YAML.load raises syntax error' do 200 | # only test with Psych since the other YAML parsers don't raise a SyntaxError 201 | if YAML.parser.class.name !~ /syck|yecht/i 202 | job = described_class.new :handler => 'message: "no ending quote' 203 | expect { job.payload_object }.to raise_error(Delayed::DeserializationError) 204 | end 205 | end 206 | end 207 | 208 | describe 'reserve' do 209 | before do 210 | Delayed::Worker.max_run_time = 2.minutes 211 | end 212 | 213 | after do 214 | Time.zone = nil 215 | end 216 | 217 | it 'does not reserve failed jobs' do 218 | create_job :attempts => 50, :failed_at => described_class.db_time_now 219 | expect(described_class.reserve(worker)).to be_nil 220 | end 221 | 222 | it 'does not reserve jobs scheduled for the future' do 223 | create_job :run_at => described_class.db_time_now + 1.minute 224 | expect(described_class.reserve(worker)).to be_nil 225 | end 226 | 227 | it 'reserves jobs scheduled for the past' do 228 | job = create_job :run_at => described_class.db_time_now - 1.minute 229 | expect(described_class.reserve(worker)).to eq(job) 230 | end 231 | 232 | it 'reserves jobs scheduled for the past when time zones are involved' do 233 | Time.zone = 'America/New_York' 234 | job = create_job :run_at => described_class.db_time_now - 1.minute 235 | expect(described_class.reserve(worker)).to eq(job) 236 | end 237 | 238 | it 'does not reserve jobs locked by other workers' do 239 | job = create_job 240 | other_worker = Delayed::Worker.new 241 | other_worker.name = 'other_worker' 242 | expect(described_class.reserve(other_worker)).to eq(job) 243 | expect(described_class.reserve(worker)).to be_nil 244 | end 245 | 246 | it 'reserves open jobs' do 247 | job = create_job 248 | expect(described_class.reserve(worker)).to eq(job) 249 | end 250 | 251 | it 'reserves expired jobs' do 252 | job = create_job(:locked_by => 'some other worker', :locked_at => described_class.db_time_now - Delayed::Worker.max_run_time - 1.minute) 253 | expect(described_class.reserve(worker)).to eq(job) 254 | end 255 | 256 | it 'reserves own jobs' do 257 | job = create_job(:locked_by => worker.name, :locked_at => (described_class.db_time_now - 1.minutes)) 258 | expect(described_class.reserve(worker)).to eq(job) 259 | end 260 | end 261 | 262 | context '#name' do 263 | it 'is the class name of the job that was enqueued' do 264 | expect(described_class.create(:payload_object => ErrorJob.new).name).to eq('ErrorJob') 265 | end 266 | 267 | it 'is the method that will be called if its a performable method object' do 268 | job = described_class.new(:payload_object => NamedJob.new) 269 | expect(job.name).to eq('named_job') 270 | end 271 | 272 | it 'is the instance method that will be called if its a performable method object' do 273 | job = Story.create(:text => '...').delay.save 274 | expect(job.name).to eq('Story#save') 275 | end 276 | 277 | it 'parses from handler on deserialization error' do 278 | job = Story.create(:text => '...').delay.text 279 | job.payload_object.object.destroy 280 | expect(job.reload.name).to eq('Delayed::PerformableMethod') 281 | end 282 | end 283 | 284 | context 'worker prioritization' do 285 | after do 286 | Delayed::Worker.max_priority = nil 287 | Delayed::Worker.min_priority = nil 288 | Delayed::Worker.queue_attributes = {} 289 | end 290 | 291 | it 'fetches jobs ordered by priority' do 292 | 10.times { described_class.enqueue SimpleJob.new, :priority => rand(10) } 293 | jobs = [] 294 | 10.times { jobs << described_class.reserve(worker) } 295 | expect(jobs.size).to eq(10) 296 | jobs.each_cons(2) do |a, b| 297 | expect(a.priority).to be <= b.priority 298 | end 299 | end 300 | 301 | it 'only finds jobs greater than or equal to min priority' do 302 | min = 5 303 | Delayed::Worker.min_priority = min 304 | [4, 5, 6].sort_by { |_i| rand }.each { |i| create_job :priority => i } 305 | 2.times do 306 | job = described_class.reserve(worker) 307 | expect(job.priority).to be >= min 308 | job.destroy 309 | end 310 | expect(described_class.reserve(worker)).to be_nil 311 | end 312 | 313 | it 'only finds jobs less than or equal to max priority' do 314 | max = 5 315 | Delayed::Worker.max_priority = max 316 | [4, 5, 6].sort_by { |_i| rand }.each { |i| create_job :priority => i } 317 | 2.times do 318 | job = described_class.reserve(worker) 319 | expect(job.priority).to be <= max 320 | job.destroy 321 | end 322 | expect(described_class.reserve(worker)).to be_nil 323 | end 324 | 325 | it 'sets job priority based on queue_attributes configuration' do 326 | Delayed::Worker.queue_attributes = {'job_tracking' => {:priority => 4}} 327 | job = described_class.enqueue :payload_object => NamedQueueJob.new 328 | expect(job.priority).to eq(4) 329 | end 330 | 331 | it 'sets job priority based on the passed in priority overrideing queue_attributes configuration' do 332 | Delayed::Worker.queue_attributes = {'job_tracking' => {:priority => 4}} 333 | job = described_class.enqueue :payload_object => NamedQueueJob.new, :priority => 10 334 | expect(job.priority).to eq(10) 335 | end 336 | end 337 | 338 | context 'clear_locks!' do 339 | before do 340 | @job = create_job(:locked_by => 'worker1', :locked_at => described_class.db_time_now) 341 | end 342 | 343 | it 'clears locks for the given worker' do 344 | described_class.clear_locks!('worker1') 345 | expect(described_class.reserve(worker)).to eq(@job) 346 | end 347 | 348 | it 'does not clear locks for other workers' do 349 | described_class.clear_locks!('different_worker') 350 | expect(described_class.reserve(worker)).not_to eq(@job) 351 | end 352 | end 353 | 354 | context 'unlock' do 355 | before do 356 | @job = create_job(:locked_by => 'worker', :locked_at => described_class.db_time_now) 357 | end 358 | 359 | it 'clears locks' do 360 | @job.unlock 361 | expect(@job.locked_by).to be_nil 362 | expect(@job.locked_at).to be_nil 363 | end 364 | end 365 | 366 | context 'large handler' do 367 | before do 368 | text = 'Lorem ipsum dolor sit amet. ' * 1000 369 | @job = described_class.enqueue Delayed::PerformableMethod.new(text, :length, {}) 370 | end 371 | 372 | it 'has an id' do 373 | expect(@job.id).not_to be_nil 374 | end 375 | end 376 | 377 | context 'named queues' do 378 | context 'when worker has one queue set' do 379 | before(:each) do 380 | worker.queues = ['large'] 381 | end 382 | 383 | it 'only works off jobs which are from its queue' do 384 | expect(SimpleJob.runs).to eq(0) 385 | 386 | create_job(:queue => 'large') 387 | create_job(:queue => 'small') 388 | worker.work_off 389 | 390 | expect(SimpleJob.runs).to eq(1) 391 | end 392 | end 393 | 394 | context 'when worker has two queue set' do 395 | before(:each) do 396 | worker.queues = %w[large small] 397 | end 398 | 399 | it 'only works off jobs which are from its queue' do 400 | expect(SimpleJob.runs).to eq(0) 401 | 402 | create_job(:queue => 'large') 403 | create_job(:queue => 'small') 404 | create_job(:queue => 'medium') 405 | create_job 406 | worker.work_off 407 | 408 | expect(SimpleJob.runs).to eq(2) 409 | end 410 | end 411 | 412 | context 'when worker does not have queue set' do 413 | before(:each) do 414 | worker.queues = [] 415 | end 416 | 417 | it 'works off all jobs' do 418 | expect(SimpleJob.runs).to eq(0) 419 | 420 | create_job(:queue => 'one') 421 | create_job(:queue => 'two') 422 | create_job 423 | worker.work_off 424 | 425 | expect(SimpleJob.runs).to eq(3) 426 | end 427 | end 428 | end 429 | 430 | context 'max_attempts' do 431 | before(:each) do 432 | @job = described_class.enqueue SimpleJob.new 433 | end 434 | 435 | it 'is not defined' do 436 | expect(@job.max_attempts).to be_nil 437 | end 438 | 439 | it 'uses the max_attempts value on the payload when defined' do 440 | expect(@job.payload_object).to receive(:max_attempts).and_return(99) 441 | expect(@job.max_attempts).to eq(99) 442 | end 443 | end 444 | 445 | describe '#max_run_time' do 446 | before(:each) { @job = described_class.enqueue SimpleJob.new } 447 | 448 | it 'is not defined' do 449 | expect(@job.max_run_time).to be_nil 450 | end 451 | 452 | it 'results in a default run time when not defined' do 453 | expect(worker.max_run_time(@job)).to eq(Delayed::Worker::DEFAULT_MAX_RUN_TIME) 454 | end 455 | 456 | it 'uses the max_run_time value on the payload when defined' do 457 | expect(@job.payload_object).to receive(:max_run_time).and_return(30.minutes) 458 | expect(@job.max_run_time).to eq(30.minutes) 459 | end 460 | 461 | it 'results in an overridden run time when defined' do 462 | expect(@job.payload_object).to receive(:max_run_time).and_return(45.minutes) 463 | expect(worker.max_run_time(@job)).to eq(45.minutes) 464 | end 465 | 466 | it 'job set max_run_time can not exceed default max run time' do 467 | expect(@job.payload_object).to receive(:max_run_time).and_return(Delayed::Worker::DEFAULT_MAX_RUN_TIME + 60) 468 | expect(worker.max_run_time(@job)).to eq(Delayed::Worker::DEFAULT_MAX_RUN_TIME) 469 | end 470 | end 471 | 472 | describe 'destroy_failed_jobs' do 473 | context 'with a SimpleJob' do 474 | before(:each) do 475 | @job = described_class.enqueue SimpleJob.new 476 | end 477 | 478 | it 'is not defined' do 479 | expect(@job.destroy_failed_jobs?).to be true 480 | end 481 | 482 | it 'uses the destroy failed jobs value on the payload when defined' do 483 | expect(@job.payload_object).to receive(:destroy_failed_jobs?).and_return(false) 484 | expect(@job.destroy_failed_jobs?).to be false 485 | end 486 | end 487 | 488 | context 'with a job that raises DserializationError' do 489 | before(:each) do 490 | @job = described_class.new :handler => '--- !ruby/struct:GoingToRaiseArgError {}' 491 | end 492 | 493 | it 'falls back reasonably' do 494 | expect(YAML).to receive(:load_dj).and_raise(ArgumentError) 495 | expect(@job.destroy_failed_jobs?).to be true 496 | end 497 | end 498 | end 499 | 500 | describe 'yaml serialization' do 501 | context 'when serializing jobs' do 502 | it 'raises error ArgumentError for new records' do 503 | story = Story.new(:text => 'hello') 504 | if story.respond_to?(:new_record?) 505 | expect { story.delay.tell }.to raise_error( 506 | ArgumentError, 507 | "job cannot be created for non-persisted record: #{story.inspect}" 508 | ) 509 | end 510 | end 511 | 512 | it 'raises error ArgumentError for destroyed records' do 513 | story = Story.create(:text => 'hello') 514 | story.destroy 515 | expect { story.delay.tell }.to raise_error( 516 | ArgumentError, 517 | "job cannot be created for non-persisted record: #{story.inspect}" 518 | ) 519 | end 520 | end 521 | 522 | context 'when reload jobs back' do 523 | it 'reloads changed attributes' do 524 | story = Story.create(:text => 'hello') 525 | job = story.delay.tell 526 | story.text = 'goodbye' 527 | story.save! 528 | expect(job.reload.payload_object.object.text).to eq('goodbye') 529 | end 530 | 531 | it 'raises deserialization error for destroyed records' do 532 | story = Story.create(:text => 'hello') 533 | job = story.delay.tell 534 | story.destroy 535 | expect { job.reload.payload_object }.to raise_error(Delayed::DeserializationError) 536 | end 537 | end 538 | end 539 | 540 | describe 'worker integration' do 541 | before do 542 | Delayed::Job.delete_all 543 | SimpleJob.runs = 0 544 | end 545 | 546 | describe 'running a job' do 547 | it 'fails after Worker.max_run_time' do 548 | Delayed::Worker.max_run_time = 1.second 549 | job = Delayed::Job.create :payload_object => LongRunningJob.new 550 | worker.run(job) 551 | expect(job.error).to_not be_nil 552 | expect(job.reload.last_error).to match(/expired/) 553 | expect(job.reload.last_error).to match(/Delayed::Worker\.max_run_time is only 1 second/) 554 | expect(job.attempts).to eq(1) 555 | end 556 | 557 | context 'when the job raises a deserialization error' do 558 | after do 559 | Delayed::Worker.destroy_failed_jobs = true 560 | end 561 | 562 | it 'marks the job as failed' do 563 | Delayed::Worker.destroy_failed_jobs = false 564 | job = described_class.create! :handler => '--- !ruby/object:JobThatDoesNotExist {}' 565 | expect_any_instance_of(described_class).to receive(:destroy_failed_jobs?).and_return(false) 566 | worker.work_off 567 | job.reload 568 | expect(job).to be_failed 569 | end 570 | end 571 | end 572 | 573 | describe 'failed jobs' do 574 | before do 575 | @job = Delayed::Job.enqueue(ErrorJob.new, :run_at => described_class.db_time_now - 1) 576 | end 577 | 578 | after do 579 | # reset default 580 | Delayed::Worker.destroy_failed_jobs = true 581 | end 582 | 583 | it 'records last_error when destroy_failed_jobs = false, max_attempts = 1' do 584 | Delayed::Worker.destroy_failed_jobs = false 585 | Delayed::Worker.max_attempts = 1 586 | worker.run(@job) 587 | @job.reload 588 | expect(@job.error).to_not be_nil 589 | expect(@job.last_error).to match(/did not work/) 590 | expect(@job.attempts).to eq(1) 591 | expect(@job).to be_failed 592 | end 593 | 594 | it 're-schedules jobs after failing' do 595 | worker.work_off 596 | @job.reload 597 | expect(@job.last_error).to match(/did not work/) 598 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.4.0') 599 | # Ruby 3.4 produces a more verbose message 600 | expect(@job.last_error).to match(/sample_jobs.rb:\d+:in 'ErrorJob#perform'/) 601 | else 602 | expect(@job.last_error).to match(/sample_jobs.rb:\d+:in `perform'/) 603 | end 604 | expect(@job.attempts).to eq(1) 605 | expect(@job.run_at).to be > Delayed::Job.db_time_now - 10.minutes 606 | expect(@job.run_at).to be < Delayed::Job.db_time_now + 10.minutes 607 | expect(@job.locked_by).to be_nil 608 | expect(@job.locked_at).to be_nil 609 | end 610 | 611 | it 're-schedules jobs with handler provided time if present' do 612 | job = Delayed::Job.enqueue(CustomRescheduleJob.new(99.minutes)) 613 | worker.run(job) 614 | job.reload 615 | 616 | expect((Delayed::Job.db_time_now + 99.minutes - job.run_at).abs).to be < 1 617 | end 618 | 619 | it "does not fail when the triggered error doesn't have a message" do 620 | error_with_nil_message = StandardError.new 621 | expect(error_with_nil_message).to receive(:message).twice.and_return(nil) 622 | expect(@job).to receive(:invoke_job).and_raise error_with_nil_message 623 | expect { worker.run(@job) }.not_to raise_error 624 | end 625 | end 626 | 627 | context 'reschedule' do 628 | before do 629 | @job = Delayed::Job.create :payload_object => SimpleJob.new 630 | end 631 | 632 | shared_examples_for 'any failure more than Worker.max_attempts times' do 633 | context "when the job's payload has a #failure hook" do 634 | before do 635 | @job = Delayed::Job.create :payload_object => OnPermanentFailureJob.new 636 | expect(@job.payload_object).to respond_to(:failure) 637 | end 638 | 639 | it 'runs that hook' do 640 | expect(@job.payload_object).to receive(:failure) 641 | worker.reschedule(@job) 642 | end 643 | 644 | it 'handles error in hook' do 645 | Delayed::Worker.destroy_failed_jobs = false 646 | @job.payload_object.raise_error = true 647 | expect { worker.reschedule(@job) }.not_to raise_error 648 | expect(@job.failed_at).to_not be_nil 649 | end 650 | end 651 | 652 | context "when the job's payload has no #failure hook" do 653 | # It's a little tricky to test this in a straightforward way, 654 | # because putting a not_to receive expectation on 655 | # @job.payload_object.failure makes that object incorrectly return 656 | # true to payload_object.respond_to? :failure, which is what 657 | # reschedule uses to decide whether to call failure. So instead, we 658 | # just make sure that the payload_object as it already stands doesn't 659 | # respond_to? failure, then shove it through the iterated reschedule 660 | # loop and make sure we don't get a NoMethodError (caused by calling 661 | # that nonexistent failure method). 662 | 663 | before do 664 | expect(@job.payload_object).not_to respond_to(:failure) 665 | end 666 | 667 | it 'does not try to run that hook' do 668 | expect do 669 | Delayed::Worker.max_attempts.times { worker.reschedule(@job) } 670 | end.not_to raise_exception 671 | end 672 | end 673 | end 674 | 675 | context 'and we want to destroy jobs' do 676 | after do 677 | Delayed::Worker.destroy_failed_jobs = true 678 | end 679 | 680 | it_behaves_like 'any failure more than Worker.max_attempts times' 681 | 682 | it 'is destroyed if it failed more than Worker.max_attempts times' do 683 | expect(@job).to receive(:destroy) 684 | Delayed::Worker.max_attempts.times { worker.reschedule(@job) } 685 | end 686 | 687 | it 'is destroyed if the job has destroy failed jobs set' do 688 | Delayed::Worker.destroy_failed_jobs = false 689 | expect(@job).to receive(:destroy_failed_jobs?).and_return(true) 690 | expect(@job).to receive(:destroy) 691 | Delayed::Worker.max_attempts.times { worker.reschedule(@job) } 692 | end 693 | 694 | it 'is not destroyed if failed fewer than Worker.max_attempts times' do 695 | expect(@job).not_to receive(:destroy) 696 | (Delayed::Worker.max_attempts - 1).times { worker.reschedule(@job) } 697 | end 698 | end 699 | 700 | context "and we don't want to destroy jobs" do 701 | before do 702 | Delayed::Worker.destroy_failed_jobs = false 703 | end 704 | 705 | after do 706 | Delayed::Worker.destroy_failed_jobs = true 707 | end 708 | 709 | it_behaves_like 'any failure more than Worker.max_attempts times' 710 | 711 | context 'and destroy failed jobs is false' do 712 | it 'is failed if it failed more than Worker.max_attempts times' do 713 | expect(@job.reload).not_to be_failed 714 | Delayed::Worker.max_attempts.times { worker.reschedule(@job) } 715 | expect(@job.reload).to be_failed 716 | end 717 | 718 | it 'is not failed if it failed fewer than Worker.max_attempts times' do 719 | (Delayed::Worker.max_attempts - 1).times { worker.reschedule(@job) } 720 | expect(@job.reload).not_to be_failed 721 | end 722 | end 723 | 724 | context 'and destroy failed jobs for job is false' do 725 | before do 726 | Delayed::Worker.destroy_failed_jobs = true 727 | end 728 | 729 | it 'is failed if it failed more than Worker.max_attempts times' do 730 | expect(@job).to receive(:destroy_failed_jobs?).and_return(false) 731 | expect(@job.reload).not_to be_failed 732 | Delayed::Worker.max_attempts.times { worker.reschedule(@job) } 733 | expect(@job.reload).to be_failed 734 | end 735 | 736 | it 'is not failed if it failed fewer than Worker.max_attempts times' do 737 | (Delayed::Worker.max_attempts - 1).times { worker.reschedule(@job) } 738 | expect(@job.reload).not_to be_failed 739 | end 740 | end 741 | end 742 | end 743 | end 744 | end 745 | -------------------------------------------------------------------------------- /lib/delayed/command.rb: -------------------------------------------------------------------------------- 1 | unless ENV['RAILS_ENV'] == 'test' 2 | begin 3 | require 'daemons' 4 | rescue LoadError 5 | raise "You need to add gem 'daemons' to your Gemfile if you wish to use it." 6 | end 7 | end 8 | require 'fileutils' 9 | require 'optparse' 10 | require 'pathname' 11 | 12 | module Delayed 13 | class Command # rubocop:disable ClassLength 14 | attr_accessor :worker_count, :worker_pools 15 | 16 | DIR_PWD = Pathname.new Dir.pwd 17 | 18 | def initialize(args) # rubocop:disable MethodLength 19 | @options = { 20 | :quiet => true, 21 | :pid_dir => "#{root}/tmp/pids", 22 | :log_dir => "#{root}/log" 23 | } 24 | 25 | @worker_count = 1 26 | @monitor = false 27 | 28 | opts = OptionParser.new do |opt| 29 | opt.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options] start|stop|restart|run" 30 | 31 | opt.on('-h', '--help', 'Show this message') do 32 | puts opt 33 | exit 1 34 | end 35 | opt.on('-e', '--environment=NAME', 'Specifies the environment to run this delayed jobs under (test/development/production).') do |_e| 36 | STDERR.puts 'The -e/--environment option has been deprecated and has no effect. Use RAILS_ENV and see http://github.com/collectiveidea/delayed_job/issues/7' 37 | end 38 | opt.on('--min-priority N', 'Minimum priority of jobs to run.') do |n| 39 | @options[:min_priority] = n 40 | end 41 | opt.on('--max-priority N', 'Maximum priority of jobs to run.') do |n| 42 | @options[:max_priority] = n 43 | end 44 | opt.on('-n', '--number_of_workers=workers', 'Number of unique workers to spawn') do |worker_count| 45 | @worker_count = worker_count.to_i rescue 1 46 | end 47 | opt.on('--pid-dir=DIR', 'Specifies an alternate directory in which to store the process ids.') do |dir| 48 | @options[:pid_dir] = dir 49 | end 50 | opt.on('--log-dir=DIR', 'Specifies an alternate directory in which to store the delayed_job log.') do |dir| 51 | @options[:log_dir] = dir 52 | end 53 | opt.on('-i', '--identifier=n', 'A numeric identifier for the worker.') do |n| 54 | @options[:identifier] = n 55 | end 56 | opt.on('-m', '--monitor', 'Start monitor process.') do 57 | @monitor = true 58 | end 59 | opt.on('--sleep-delay N', 'Amount of time to sleep when no jobs are found') do |n| 60 | @options[:sleep_delay] = n.to_i 61 | end 62 | opt.on('--read-ahead N', 'Number of jobs from the queue to consider') do |n| 63 | @options[:read_ahead] = n 64 | end 65 | opt.on('-p', '--prefix NAME', 'String to be prefixed to worker process names') do |prefix| 66 | @options[:prefix] = prefix 67 | end 68 | opt.on('--queues=queues', 'Specify which queue DJ must look up for jobs') do |queues| 69 | @options[:queues] = queues.split(',') 70 | end 71 | opt.on('--queue=queue', 'Specify which queue DJ must look up for jobs') do |queue| 72 | @options[:queues] = queue.split(',') 73 | end 74 | opt.on('--pool=queue1[,queue2][:worker_count]', 'Specify queues and number of workers for a worker pool') do |pool| 75 | parse_worker_pool(pool) 76 | end 77 | opt.on('--exit-on-complete', 'Exit when no more jobs are available to run. This will exit if all jobs are scheduled to run in the future.') do 78 | @options[:exit_on_complete] = true 79 | end 80 | opt.on('--daemon-options a, b, c', Array, 'options to be passed through to daemons gem') do |daemon_options| 81 | @daemon_options = daemon_options 82 | end 83 | end 84 | @args = opts.parse!(args) + (@daemon_options || []) 85 | end 86 | 87 | def daemonize # rubocop:disable PerceivedComplexity 88 | dir = @options[:pid_dir] 89 | FileUtils.mkdir_p(dir) unless File.exist?(dir) 90 | 91 | if worker_pools 92 | setup_pools 93 | elsif @options[:identifier] 94 | # rubocop:disable GuardClause 95 | if worker_count > 1 96 | raise ArgumentError, 'Cannot specify both --number-of-workers and --identifier' 97 | else 98 | run_process("delayed_job.#{@options[:identifier]}", @options) 99 | end 100 | # rubocop:enable GuardClause 101 | else 102 | worker_count.times do |worker_index| 103 | process_name = worker_count == 1 ? 'delayed_job' : "delayed_job.#{worker_index}" 104 | run_process(process_name, @options) 105 | end 106 | end 107 | end 108 | 109 | def setup_pools 110 | worker_index = 0 111 | @worker_pools.each do |queues, worker_count| 112 | options = @options.merge(:queues => queues) 113 | worker_count.times do 114 | process_name = "delayed_job.#{worker_index}" 115 | run_process(process_name, options) 116 | worker_index += 1 117 | end 118 | end 119 | end 120 | 121 | def run_process(process_name, options = {}) 122 | Delayed::Worker.before_fork 123 | Daemons.run_proc(process_name, :dir => options[:pid_dir], :dir_mode => :normal, :monitor => @monitor, :ARGV => @args) do |*_args| 124 | $0 = File.join(options[:prefix], process_name) if @options[:prefix] 125 | run process_name, options 126 | end 127 | end 128 | 129 | def run(worker_name = nil, options = {}) 130 | Dir.chdir(root) 131 | 132 | Delayed::Worker.after_fork 133 | Delayed::Worker.logger ||= Logger.new(File.join(@options[:log_dir], 'delayed_job.log')) 134 | 135 | worker = Delayed::Worker.new(options) 136 | worker.name_prefix = "#{worker_name} " 137 | worker.start 138 | rescue => e 139 | STDERR.puts e.message 140 | STDERR.puts e.backtrace 141 | ::Rails.logger.fatal(e) if rails_logger_defined? 142 | exit_with_error_status 143 | end 144 | 145 | private 146 | 147 | def parse_worker_pool(pool) 148 | @worker_pools ||= [] 149 | 150 | queues, worker_count = pool.split(':') 151 | queues = ['*', '', nil].include?(queues) ? [] : queues.split(',') 152 | worker_count = (worker_count || 1).to_i rescue 1 153 | @worker_pools << [queues, worker_count] 154 | end 155 | 156 | def root 157 | @root ||= rails_root_defined? ? ::Rails.root : DIR_PWD 158 | end 159 | 160 | def rails_root_defined? 161 | defined?(::Rails.root) 162 | end 163 | 164 | def rails_logger_defined? 165 | defined?(::Rails.logger) 166 | end 167 | 168 | def exit_with_error_status 169 | exit 1 170 | end 171 | end 172 | end 173 | -------------------------------------------------------------------------------- /lib/delayed/compatibility.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/version' 2 | 3 | module Delayed 4 | module Compatibility 5 | if ActiveSupport::VERSION::MAJOR >= 4 6 | def self.executable_prefix 7 | 'bin' 8 | end 9 | else 10 | def self.executable_prefix 11 | 'script' 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/delayed/deserialization_error.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | class DeserializationError < StandardError 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /lib/delayed/exceptions.rb: -------------------------------------------------------------------------------- 1 | require 'timeout' 2 | 3 | module Delayed 4 | class WorkerTimeout < Timeout::Error 5 | def message 6 | seconds = Delayed::Worker.max_run_time.to_i 7 | "#{super} (Delayed::Worker.max_run_time is only #{seconds} second#{seconds == 1 ? '' : 's'})" 8 | end 9 | end 10 | 11 | class FatalBackendError < RuntimeError; end 12 | end 13 | -------------------------------------------------------------------------------- /lib/delayed/lifecycle.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | class InvalidCallback < RuntimeError; end 3 | 4 | class Lifecycle 5 | EVENTS = { 6 | :enqueue => [:job], 7 | :execute => [:worker], 8 | :loop => [:worker], 9 | :perform => [:worker, :job], 10 | :error => [:worker, :job], 11 | :failure => [:worker, :job], 12 | :invoke_job => [:job] 13 | }.freeze 14 | 15 | def initialize 16 | @callbacks = EVENTS.keys.each_with_object({}) do |e, hash| 17 | hash[e] = Callback.new 18 | end 19 | end 20 | 21 | def before(event, &block) 22 | add(:before, event, &block) 23 | end 24 | 25 | def after(event, &block) 26 | add(:after, event, &block) 27 | end 28 | 29 | def around(event, &block) 30 | add(:around, event, &block) 31 | end 32 | 33 | def run_callbacks(event, *args, &block) 34 | missing_callback(event) unless @callbacks.key?(event) 35 | 36 | unless EVENTS[event].size == args.size 37 | raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}" 38 | end 39 | 40 | @callbacks[event].execute(*args, &block) 41 | end 42 | 43 | private 44 | 45 | def add(type, event, &block) 46 | missing_callback(event) unless @callbacks.key?(event) 47 | @callbacks[event].add(type, &block) 48 | end 49 | 50 | def missing_callback(event) 51 | raise InvalidCallback, "Unknown callback event: #{event}" 52 | end 53 | end 54 | 55 | class Callback 56 | def initialize 57 | @before = [] 58 | @after = [] 59 | 60 | # Identity proc. Avoids special cases when there is no existing around chain. 61 | @around = lambda { |*args, &block| block.call(*args) } 62 | end 63 | 64 | def execute(*args, &block) 65 | @before.each { |c| c.call(*args) } 66 | result = @around.call(*args, &block) 67 | @after.each { |c| c.call(*args) } 68 | result 69 | end 70 | 71 | def add(type, &callback) 72 | case type 73 | when :before 74 | @before << callback 75 | when :after 76 | @after << callback 77 | when :around 78 | chain = @around # use a local variable so that the current chain is closed over in the following lambda 79 | @around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } } 80 | else 81 | raise InvalidCallback, "Invalid callback type: #{type}" 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/delayed/message_sending.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | class DelayProxy < BasicObject 3 | # What additional methods exist on BasicObject has changed over time 4 | (::BasicObject.instance_methods - [:__id__, :__send__, :instance_eval, :instance_exec]).each do |method| 5 | undef_method method 6 | end 7 | 8 | # Let DelayProxy raise exceptions. 9 | def raise(*args) 10 | ::Object.send(:raise, *args) 11 | end 12 | 13 | def initialize(payload_class, target, options) 14 | @payload_class = payload_class 15 | @target = target 16 | @options = options 17 | end 18 | 19 | # rubocop:disable MethodMissing 20 | def method_missing(method, *args) 21 | Job.enqueue({:payload_object => @payload_class.new(@target, method.to_sym, args)}.merge(@options)) 22 | end 23 | # rubocop:enable MethodMissing 24 | end 25 | 26 | module MessageSending 27 | def delay(options = {}) 28 | DelayProxy.new(PerformableMethod, self, options) 29 | end 30 | alias_method :__delay__, :delay 31 | 32 | def send_later(method, *args) 33 | warn '[DEPRECATION] `object.send_later(:method)` is deprecated. Use `object.delay.method' 34 | __delay__.__send__(method, *args) 35 | end 36 | 37 | def send_at(time, method, *args) 38 | warn '[DEPRECATION] `object.send_at(time, :method)` is deprecated. Use `object.delay(:run_at => time).method' 39 | __delay__(:run_at => time).__send__(method, *args) 40 | end 41 | end 42 | 43 | module MessageSendingClassMethods 44 | def handle_asynchronously(method, opts = {}) # rubocop:disable PerceivedComplexity 45 | aliased_method = method.to_s.sub(/([?!=])$/, '') 46 | punctuation = $1 # rubocop:disable PerlBackrefs 47 | with_method = "#{aliased_method}_with_delay#{punctuation}" 48 | without_method = "#{aliased_method}_without_delay#{punctuation}" 49 | define_method(with_method) do |*args| 50 | curr_opts = opts.clone 51 | curr_opts.each_key do |key| 52 | next unless (val = curr_opts[key]).is_a?(Proc) 53 | curr_opts[key] = if val.arity == 1 54 | val.call(self) 55 | else 56 | val.call 57 | end 58 | end 59 | delay(curr_opts).__send__(without_method, *args) 60 | end 61 | 62 | alias_method without_method, method 63 | alias_method method, with_method 64 | 65 | if public_method_defined?(without_method) 66 | public method 67 | elsif protected_method_defined?(without_method) 68 | protected method 69 | elsif private_method_defined?(without_method) 70 | private method 71 | end 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/delayed/performable_mailer.rb: -------------------------------------------------------------------------------- 1 | require 'mail' 2 | 3 | module Delayed 4 | class PerformableMailer < PerformableMethod 5 | def perform 6 | mailer = object.send(method_name, *args) 7 | mailer.respond_to?(:deliver_now) ? mailer.deliver_now : mailer.deliver 8 | end 9 | end 10 | 11 | module DelayMail 12 | def delay(options = {}) 13 | DelayProxy.new(PerformableMailer, self, options) 14 | end 15 | end 16 | end 17 | 18 | Mail::Message.class_eval do 19 | def delay(*_args) 20 | raise 'Use MyMailer.delay.mailer_action(args) to delay sending of emails.' 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/delayed/performable_method.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | class PerformableMethod 3 | attr_accessor :object, :method_name, :args 4 | 5 | def initialize(object, method_name, args) 6 | raise NoMethodError, "undefined method `#{method_name}' for #{object.inspect}" unless object.respond_to?(method_name, true) 7 | 8 | if object.respond_to?(:persisted?) && !object.persisted? 9 | raise(ArgumentError, "job cannot be created for non-persisted record: #{object.inspect}") 10 | end 11 | 12 | self.object = object 13 | self.args = args 14 | self.method_name = method_name.to_sym 15 | end 16 | 17 | def display_name 18 | if object.is_a?(Class) 19 | "#{object}.#{method_name}" 20 | else 21 | "#{object.class}##{method_name}" 22 | end 23 | end 24 | 25 | def perform 26 | object.send(method_name, *args) if object 27 | end 28 | 29 | def method(sym) 30 | object.method(sym) 31 | end 32 | 33 | # rubocop:disable MethodMissing 34 | def method_missing(symbol, *args) 35 | object.send(symbol, *args) 36 | end 37 | # rubocop:enable MethodMissing 38 | 39 | def respond_to?(symbol, include_private = false) 40 | super || object.respond_to?(symbol, include_private) 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/delayed/plugin.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/core_ext/class/attribute' 2 | 3 | module Delayed 4 | class Plugin 5 | class_attribute :callback_block 6 | 7 | def self.callbacks(&block) 8 | self.callback_block = block 9 | end 10 | 11 | def initialize 12 | self.class.callback_block.call(Delayed::Worker.lifecycle) if self.class.callback_block 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/delayed/plugins/clear_locks.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | module Plugins 3 | class ClearLocks < Plugin 4 | callbacks do |lifecycle| 5 | lifecycle.around(:execute) do |worker, &block| 6 | begin 7 | block.call(worker) 8 | ensure 9 | Delayed::Job.clear_locks!(worker.name) 10 | end 11 | end 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/delayed/psych_ext.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | class PerformableMethod 3 | # serialize to YAML 4 | def encode_with(coder) 5 | coder.map = { 6 | 'object' => object, 7 | 'method_name' => method_name, 8 | 'args' => args 9 | } 10 | end 11 | end 12 | end 13 | 14 | module Psych 15 | def self.load_dj(yaml) 16 | result = parse(yaml) 17 | result ? Delayed::PsychExt::ToRuby.create.accept(result) : result 18 | end 19 | end 20 | 21 | module Delayed 22 | module PsychExt 23 | class ToRuby < Psych::Visitors::ToRuby 24 | unless respond_to?(:create) 25 | def self.create 26 | new 27 | end 28 | end 29 | 30 | def visit_Psych_Nodes_Mapping(object) # rubocop:disable CyclomaticComplexity, MethodName, PerceivedComplexity 31 | klass = Psych.load_tags[object.tag] 32 | if klass 33 | # Implementation changed here https://github.com/ruby/psych/commit/2c644e184192975b261a81f486a04defa3172b3f 34 | # load_tags used to have class values, now the values are strings 35 | klass = resolve_class(klass) if klass.is_a?(String) 36 | return revive(klass, object) 37 | end 38 | 39 | case object.tag 40 | when %r{^!ruby/object} 41 | result = super 42 | if jruby_is_seriously_borked && result.is_a?(ActiveRecord::Base) 43 | klass = result.class 44 | id = result[klass.primary_key] 45 | begin 46 | klass.unscoped.find(id) 47 | rescue ActiveRecord::RecordNotFound => error # rubocop:disable BlockNesting 48 | raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{error.message})" 49 | end 50 | else 51 | result 52 | end 53 | when %r{^!ruby/ActiveRecord:(.+)$} 54 | klass = resolve_class(Regexp.last_match[1]) 55 | payload = Hash[*object.children.map { |c| accept c }] 56 | id = payload['attributes'][klass.primary_key] 57 | id = id.value if defined?(ActiveRecord::Attribute) && id.is_a?(ActiveRecord::Attribute) 58 | begin 59 | klass.unscoped.find(id) 60 | rescue ActiveRecord::RecordNotFound => error 61 | raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{error.message})" 62 | end 63 | when %r{^!ruby/Mongoid:(.+)$} 64 | klass = resolve_class(Regexp.last_match[1]) 65 | payload = Hash[*object.children.map { |c| accept c }] 66 | id = payload['attributes']['_id'] 67 | begin 68 | klass.find(id) 69 | rescue Mongoid::Errors::DocumentNotFound => error 70 | raise Delayed::DeserializationError, "Mongoid::Errors::DocumentNotFound, class: #{klass}, primary key: #{id} (#{error.message})" 71 | end 72 | when %r{^!ruby/DataMapper:(.+)$} 73 | klass = resolve_class(Regexp.last_match[1]) 74 | payload = Hash[*object.children.map { |c| accept c }] 75 | begin 76 | primary_keys = klass.properties.select(&:key?) 77 | key_names = primary_keys.map { |p| p.name.to_s } 78 | klass.get!(*key_names.map { |k| payload['attributes'][k] }) 79 | rescue DataMapper::ObjectNotFoundError => error 80 | raise Delayed::DeserializationError, "DataMapper::ObjectNotFoundError, class: #{klass} (#{error.message})" 81 | end 82 | else 83 | super 84 | end 85 | end 86 | 87 | # defined? is triggering something really messed up in 88 | # jruby causing both the if AND else clauses to execute, 89 | # however if the check is run here, everything is fine 90 | def jruby_is_seriously_borked 91 | defined?(ActiveRecord::Base) 92 | end 93 | 94 | def resolve_class(klass_name) 95 | return nil if !klass_name || klass_name.empty? 96 | klass_name.constantize 97 | rescue 98 | super 99 | end 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/delayed/railtie.rb: -------------------------------------------------------------------------------- 1 | require 'delayed_job' 2 | require 'rails' 3 | 4 | module Delayed 5 | class Railtie < Rails::Railtie 6 | initializer :after_initialize do 7 | Delayed::Worker.logger ||= if defined?(Rails) 8 | Rails.logger 9 | elsif defined?(RAILS_DEFAULT_LOGGER) 10 | RAILS_DEFAULT_LOGGER 11 | end 12 | end 13 | 14 | rake_tasks do 15 | load 'delayed/tasks.rb' 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/delayed/recipes.rb: -------------------------------------------------------------------------------- 1 | # Capistrano Recipes for managing delayed_job 2 | # 3 | # Add these callbacks to have the delayed_job process restart when the server 4 | # is restarted: 5 | # 6 | # after "deploy:stop", "delayed_job:stop" 7 | # after "deploy:start", "delayed_job:start" 8 | # after "deploy:restart", "delayed_job:restart" 9 | # 10 | # If you want to use command line options, for example to start multiple workers, 11 | # define a Capistrano variable delayed_job_args: 12 | # 13 | # set :delayed_job_args, "-n 2" 14 | # 15 | # If you've got delayed_job workers running on a servers, you can also specify 16 | # which servers have delayed_job running and should be restarted after deploy. 17 | # 18 | # set :delayed_job_server_role, :worker 19 | # 20 | 21 | Capistrano::Configuration.instance.load do 22 | namespace :delayed_job do 23 | def rails_env 24 | fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : '' 25 | end 26 | 27 | def args 28 | fetch(:delayed_job_args, '') 29 | end 30 | 31 | def roles 32 | fetch(:delayed_job_server_role, :app) 33 | end 34 | 35 | def delayed_job_command 36 | fetch(:delayed_job_command, 'script/delayed_job') 37 | end 38 | 39 | desc 'Stop the delayed_job process' 40 | task :stop, :roles => lambda { roles } do 41 | run "cd #{current_path} && #{rails_env} #{delayed_job_command} stop #{args}" 42 | end 43 | 44 | desc 'Start the delayed_job process' 45 | task :start, :roles => lambda { roles } do 46 | run "cd #{current_path} && #{rails_env} #{delayed_job_command} start #{args}" 47 | end 48 | 49 | desc 'Restart the delayed_job process' 50 | task :restart, :roles => lambda { roles } do 51 | run "cd #{current_path} && #{rails_env} #{delayed_job_command} restart #{args}" 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/delayed/serialization/active_record.rb: -------------------------------------------------------------------------------- 1 | if defined?(ActiveRecord) 2 | module ActiveRecord 3 | class Base 4 | yaml_tag 'tag:ruby.yaml.org,2002:ActiveRecord' 5 | 6 | def self.yaml_new(klass, _tag, val) 7 | klass.unscoped.find(val['attributes'][klass.primary_key]) 8 | rescue ActiveRecord::RecordNotFound 9 | raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass} , primary key: #{val['attributes'][klass.primary_key]}" 10 | end 11 | 12 | def to_yaml_properties 13 | ['@attributes'] 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/delayed/syck_ext.rb: -------------------------------------------------------------------------------- 1 | class Module 2 | yaml_tag 'tag:ruby.yaml.org,2002:module' 3 | 4 | def self.yaml_new(_klass, _tag, val) 5 | val.constantize 6 | end 7 | 8 | def to_yaml(options = {}) 9 | YAML.quick_emit(nil, options) do |out| 10 | out.scalar(taguri, name, :plain) 11 | end 12 | end 13 | 14 | def yaml_tag_read_class(name) 15 | # Constantize the object so that ActiveSupport can attempt 16 | # its auto loading magic. Will raise LoadError if not successful. 17 | name.constantize 18 | name 19 | end 20 | end 21 | 22 | class Class 23 | yaml_tag 'tag:ruby.yaml.org,2002:class' 24 | remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml 25 | end 26 | 27 | class Struct 28 | def self.yaml_tag_read_class(name) 29 | # Constantize the object so that ActiveSupport can attempt 30 | # its auto loading magic. Will raise LoadError if not successful. 31 | name.constantize 32 | "Struct::#{name}" 33 | end 34 | end 35 | 36 | module YAML 37 | def load_dj(yaml) 38 | # See https://github.com/dtao/safe_yaml 39 | # When the method is there, we need to load our YAML like this... 40 | respond_to?(:unsafe_load) ? load(yaml, :safe => false) : load(yaml) 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/delayed/tasks.rb: -------------------------------------------------------------------------------- 1 | namespace :jobs do 2 | desc 'Clear the delayed_job queue.' 3 | task :clear => :environment do 4 | Delayed::Job.delete_all 5 | end 6 | 7 | desc 'Start a delayed_job worker.' 8 | task :work => :environment_options do 9 | Delayed::Worker.new(@worker_options).start 10 | end 11 | 12 | desc 'Start a delayed_job worker and exit when all available jobs are complete.' 13 | task :workoff => :environment_options do 14 | Delayed::Worker.new(@worker_options.merge(:exit_on_complete => true)).start 15 | end 16 | 17 | task :environment_options => :environment do 18 | @worker_options = { 19 | :min_priority => ENV['MIN_PRIORITY'], 20 | :max_priority => ENV['MAX_PRIORITY'], 21 | :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','), 22 | :quiet => ENV['QUIET'] 23 | } 24 | 25 | @worker_options[:sleep_delay] = ENV['SLEEP_DELAY'].to_i if ENV['SLEEP_DELAY'] 26 | @worker_options[:read_ahead] = ENV['READ_AHEAD'].to_i if ENV['READ_AHEAD'] 27 | end 28 | 29 | desc "Exit with error status if any jobs older than max_age seconds haven't been attempted yet." 30 | task :check, [:max_age] => :environment do |_, args| 31 | args.with_defaults(:max_age => 300) 32 | 33 | unprocessed_jobs = Delayed::Job.where('attempts = 0 AND created_at < ?', Time.now - args[:max_age].to_i).count 34 | 35 | if unprocessed_jobs > 0 36 | raise "#{unprocessed_jobs} jobs older than #{args[:max_age]} seconds have not been processed yet" 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/delayed/worker.rb: -------------------------------------------------------------------------------- 1 | require 'timeout' 2 | require 'active_support/dependencies' 3 | require 'active_support/core_ext/kernel/reporting' 4 | require 'active_support/core_ext/numeric/time' 5 | require 'active_support/core_ext/class/attribute_accessors' 6 | require 'active_support/hash_with_indifferent_access' 7 | require 'active_support/core_ext/hash/indifferent_access' 8 | require 'logger' 9 | require 'benchmark' 10 | 11 | module Delayed 12 | class Worker # rubocop:disable ClassLength 13 | DEFAULT_LOG_LEVEL = 'info'.freeze 14 | DEFAULT_SLEEP_DELAY = 5 15 | DEFAULT_MAX_ATTEMPTS = 25 16 | DEFAULT_MAX_RUN_TIME = 4.hours 17 | DEFAULT_DEFAULT_PRIORITY = 0 18 | DEFAULT_DELAY_JOBS = true 19 | DEFAULT_QUEUES = [].freeze 20 | DEFAULT_QUEUE_ATTRIBUTES = HashWithIndifferentAccess.new.freeze 21 | DEFAULT_READ_AHEAD = 5 22 | 23 | cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, 24 | :default_priority, :sleep_delay, :logger, :delay_jobs, :queues, 25 | :read_ahead, :plugins, :destroy_failed_jobs, :exit_on_complete, 26 | :default_log_level 27 | 28 | # Named queue into which jobs are enqueued by default 29 | cattr_accessor :default_queue_name 30 | 31 | cattr_reader :backend, :queue_attributes 32 | 33 | # name_prefix is ignored if name is set directly 34 | attr_accessor :name_prefix 35 | 36 | def self.reset 37 | self.default_log_level = DEFAULT_LOG_LEVEL 38 | self.sleep_delay = DEFAULT_SLEEP_DELAY 39 | self.max_attempts = DEFAULT_MAX_ATTEMPTS 40 | self.max_run_time = DEFAULT_MAX_RUN_TIME 41 | self.default_priority = DEFAULT_DEFAULT_PRIORITY 42 | self.delay_jobs = DEFAULT_DELAY_JOBS 43 | self.queues = DEFAULT_QUEUES 44 | self.queue_attributes = DEFAULT_QUEUE_ATTRIBUTES 45 | self.read_ahead = DEFAULT_READ_AHEAD 46 | @lifecycle = nil 47 | end 48 | 49 | # Add or remove plugins in this list before the worker is instantiated 50 | self.plugins = [Delayed::Plugins::ClearLocks] 51 | 52 | # By default failed jobs are destroyed after too many attempts. If you want to keep them around 53 | # (perhaps to inspect the reason for the failure), set this to false. 54 | self.destroy_failed_jobs = true 55 | 56 | # By default, Signals INT and TERM set @exit, and the worker exits upon completion of the current job. 57 | # If you would prefer to raise a SignalException and exit immediately you can use this. 58 | # Be aware daemons uses TERM to stop and restart 59 | # false - No exceptions will be raised 60 | # :term - Will only raise an exception on TERM signals but INT will wait for the current job to finish 61 | # true - Will raise an exception on TERM and INT 62 | cattr_accessor :raise_signal_exceptions 63 | self.raise_signal_exceptions = false 64 | 65 | def self.backend=(backend) 66 | if backend.is_a? Symbol 67 | require "delayed/serialization/#{backend}" 68 | require "delayed/backend/#{backend}" 69 | backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize 70 | end 71 | @@backend = backend # rubocop:disable ClassVars 72 | silence_warnings { ::Delayed.const_set(:Job, backend) } 73 | end 74 | 75 | # rubocop:disable ClassVars 76 | def self.queue_attributes=(val) 77 | @@queue_attributes = val.with_indifferent_access 78 | end 79 | 80 | def self.guess_backend 81 | warn '[DEPRECATION] guess_backend is deprecated. Please remove it from your code.' 82 | end 83 | 84 | def self.before_fork 85 | unless @files_to_reopen 86 | @files_to_reopen = [] 87 | ObjectSpace.each_object(File) do |file| 88 | @files_to_reopen << file unless file.closed? 89 | end 90 | end 91 | 92 | backend.before_fork 93 | end 94 | 95 | def self.after_fork 96 | # Re-open file handles 97 | @files_to_reopen.each do |file| 98 | begin 99 | file.reopen file.path, 'a+' 100 | file.sync = true 101 | rescue ::Exception # rubocop:disable HandleExceptions, RescueException 102 | end 103 | end 104 | backend.after_fork 105 | end 106 | 107 | def self.lifecycle 108 | # In case a worker has not been set up, job enqueueing needs a lifecycle. 109 | setup_lifecycle unless @lifecycle 110 | 111 | @lifecycle 112 | end 113 | 114 | def self.setup_lifecycle 115 | @lifecycle = Delayed::Lifecycle.new 116 | plugins.each { |klass| klass.new } 117 | end 118 | 119 | def self.reload_app? 120 | defined?(ActionDispatch::Reloader) && Rails.application.config.cache_classes == false 121 | end 122 | 123 | def self.delay_job?(job) 124 | if delay_jobs.is_a?(Proc) 125 | delay_jobs.arity == 1 ? delay_jobs.call(job) : delay_jobs.call 126 | else 127 | delay_jobs 128 | end 129 | end 130 | 131 | def initialize(options = {}) 132 | @quiet = options.key?(:quiet) ? options[:quiet] : true 133 | @failed_reserve_count = 0 134 | 135 | [:min_priority, :max_priority, :sleep_delay, :read_ahead, :queues, :exit_on_complete].each do |option| 136 | self.class.send("#{option}=", options[option]) if options.key?(option) 137 | end 138 | 139 | # Reset lifecycle on the offhand chance that something lazily 140 | # triggered its creation before all plugins had been registered. 141 | self.class.setup_lifecycle 142 | end 143 | 144 | # Every worker has a unique name which by default is the pid of the process. There are some 145 | # advantages to overriding this with something which survives worker restarts: Workers can 146 | # safely resume working on tasks which are locked by themselves. The worker will assume that 147 | # it crashed before. 148 | def name 149 | return @name unless @name.nil? 150 | "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}" 151 | end 152 | 153 | # Sets the name of the worker. 154 | # Setting the name to nil will reset the default worker name 155 | attr_writer :name 156 | 157 | def start # rubocop:disable CyclomaticComplexity, PerceivedComplexity 158 | trap('TERM') do 159 | Thread.new { say 'Exiting...' } 160 | stop 161 | raise SignalException, 'TERM' if self.class.raise_signal_exceptions 162 | end 163 | 164 | trap('INT') do 165 | Thread.new { say 'Exiting...' } 166 | stop 167 | raise SignalException, 'INT' if self.class.raise_signal_exceptions && self.class.raise_signal_exceptions != :term 168 | end 169 | 170 | say 'Starting job worker' 171 | 172 | self.class.lifecycle.run_callbacks(:execute, self) do 173 | loop do 174 | self.class.lifecycle.run_callbacks(:loop, self) do 175 | @realtime = Benchmark.realtime do 176 | @result = work_off 177 | end 178 | end 179 | 180 | count = @result[0] + @result[1] 181 | 182 | if count.zero? 183 | if self.class.exit_on_complete 184 | say 'No more jobs available. Exiting' 185 | break 186 | elsif !stop? 187 | sleep(self.class.sleep_delay) 188 | reload! 189 | end 190 | else 191 | say format("#{count} jobs processed at %.4f j/s, %d failed", count / @realtime, @result.last) 192 | end 193 | 194 | break if stop? 195 | end 196 | end 197 | end 198 | 199 | def stop 200 | @exit = true 201 | end 202 | 203 | def stop? 204 | !!@exit 205 | end 206 | 207 | # Do num jobs and return stats on success/failure. 208 | # Exit early if interrupted. 209 | def work_off(num = 100) 210 | success = 0 211 | failure = 0 212 | 213 | num.times do 214 | case reserve_and_run_one_job 215 | when true 216 | success += 1 217 | when false 218 | failure += 1 219 | else 220 | break # leave if no work could be done 221 | end 222 | break if stop? # leave if we're exiting 223 | end 224 | 225 | [success, failure] 226 | end 227 | 228 | def run(job) 229 | job_say job, 'RUNNING' 230 | runtime = Benchmark.realtime do 231 | Timeout.timeout(max_run_time(job).to_i, WorkerTimeout) { job.invoke_job } 232 | job.destroy 233 | end 234 | job_say job, format('COMPLETED after %.4f', runtime) 235 | return true # did work 236 | rescue DeserializationError => error 237 | job_say job, "FAILED permanently with #{error.class.name}: #{error.message}", 'error' 238 | 239 | job.error = error 240 | failed(job) 241 | rescue Exception => error # rubocop:disable RescueException 242 | self.class.lifecycle.run_callbacks(:error, self, job) { handle_failed_job(job, error) } 243 | return false # work failed 244 | end 245 | 246 | # Reschedule the job in the future (when a job fails). 247 | # Uses an exponential scale depending on the number of failed attempts. 248 | def reschedule(job, time = nil) 249 | if (job.attempts += 1) < max_attempts(job) 250 | time ||= job.reschedule_at 251 | job.run_at = time 252 | job.unlock 253 | job.save! 254 | else 255 | job_say job, "FAILED permanently because of #{job.attempts} consecutive failures", 'error' 256 | failed(job) 257 | end 258 | end 259 | 260 | def failed(job) 261 | self.class.lifecycle.run_callbacks(:failure, self, job) do 262 | begin 263 | job.hook(:failure) 264 | rescue => error 265 | say "Error when running failure callback: #{error}", 'error' 266 | say error.backtrace.join("\n"), 'error' 267 | ensure 268 | job.destroy_failed_jobs? ? job.destroy : job.fail! 269 | end 270 | end 271 | end 272 | 273 | def job_say(job, text, level = default_log_level) 274 | text = "Job #{job.name} (id=#{job.id})#{say_queue(job.queue)} #{text}" 275 | say text, level 276 | end 277 | 278 | def say(text, level = default_log_level) 279 | text = "[Worker(#{name})] #{text}" 280 | puts text unless @quiet 281 | return unless logger 282 | # TODO: Deprecate use of Fixnum log levels 283 | unless level.is_a?(String) 284 | level = Logger::Severity.constants.detect { |i| Logger::Severity.const_get(i) == level }.to_s.downcase 285 | end 286 | logger.send(level, "#{Time.now.strftime('%FT%T%z')}: #{text}") 287 | end 288 | 289 | def max_attempts(job) 290 | job.max_attempts || self.class.max_attempts 291 | end 292 | 293 | def max_run_time(job) 294 | job.max_run_time || self.class.max_run_time 295 | end 296 | 297 | protected 298 | 299 | def say_queue(queue) 300 | " (queue=#{queue})" if queue 301 | end 302 | 303 | def handle_failed_job(job, error) 304 | job.error = error 305 | job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", 'error' 306 | reschedule(job) 307 | end 308 | 309 | # Run the next job we can get an exclusive lock on. 310 | # If no jobs are left we return nil 311 | def reserve_and_run_one_job 312 | job = reserve_job 313 | self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) } if job 314 | end 315 | 316 | def reserve_job 317 | job = Delayed::Job.reserve(self) 318 | @failed_reserve_count = 0 319 | job 320 | rescue ::Exception => error # rubocop:disable RescueException 321 | say "Error while reserving job: #{error}" 322 | Delayed::Job.recover_from(error) 323 | @failed_reserve_count += 1 324 | raise FatalBackendError if @failed_reserve_count >= 10 325 | nil 326 | end 327 | 328 | def reload! 329 | return unless self.class.reload_app? 330 | if defined?(ActiveSupport::Reloader) 331 | Rails.application.reloader.reload! 332 | else 333 | ActionDispatch::Reloader.cleanup! 334 | ActionDispatch::Reloader.prepare! 335 | end 336 | end 337 | end 338 | end 339 | 340 | Delayed::Worker.reset 341 | -------------------------------------------------------------------------------- /lib/delayed/yaml_ext.rb: -------------------------------------------------------------------------------- 1 | # These extensions allow properly serializing and autoloading of 2 | # Classes, Modules and Structs 3 | 4 | require 'yaml' 5 | if YAML.parser.class.name =~ /syck|yecht/i 6 | require File.expand_path('../syck_ext', __FILE__) 7 | require File.expand_path('../serialization/active_record', __FILE__) 8 | else 9 | require File.expand_path('../psych_ext', __FILE__) 10 | end 11 | -------------------------------------------------------------------------------- /lib/delayed_job.rb: -------------------------------------------------------------------------------- 1 | require 'active_support' 2 | require 'delayed/compatibility' 3 | require 'delayed/exceptions' 4 | require 'delayed/message_sending' 5 | require 'delayed/performable_method' 6 | require 'delayed/yaml_ext' 7 | require 'delayed/lifecycle' 8 | require 'delayed/plugin' 9 | require 'delayed/plugins/clear_locks' 10 | require 'delayed/backend/base' 11 | require 'delayed/backend/job_preparer' 12 | require 'delayed/worker' 13 | require 'delayed/deserialization_error' 14 | require 'delayed/railtie' if defined?(Rails::Railtie) 15 | 16 | ActiveSupport.on_load(:action_mailer) do 17 | require 'delayed/performable_mailer' 18 | ActionMailer::Base.extend(Delayed::DelayMail) 19 | ActionMailer::Parameterized::Mailer.include(Delayed::DelayMail) if defined?(ActionMailer::Parameterized::Mailer) 20 | end 21 | 22 | module Delayed 23 | autoload :PerformableMailer, 'delayed/performable_mailer' 24 | end 25 | 26 | Object.send(:include, Delayed::MessageSending) 27 | Module.send(:include, Delayed::MessageSendingClassMethods) 28 | -------------------------------------------------------------------------------- /lib/generators/delayed_job/delayed_job_generator.rb: -------------------------------------------------------------------------------- 1 | require 'rails/generators/base' 2 | require 'delayed/compatibility' 3 | 4 | class DelayedJobGenerator < Rails::Generators::Base 5 | source_paths << File.join(File.dirname(__FILE__), 'templates') 6 | 7 | def create_executable_file 8 | template 'script', "#{Delayed::Compatibility.executable_prefix}/delayed_job" 9 | chmod "#{Delayed::Compatibility.executable_prefix}/delayed_job", 0o755 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/generators/delayed_job/templates/script: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) 4 | require 'delayed/command' 5 | Delayed::Command.new(ARGV).daemonize 6 | -------------------------------------------------------------------------------- /recipes/delayed_job.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'delayed', 'recipes')) 2 | -------------------------------------------------------------------------------- /spec/autoloaded/clazz.rb: -------------------------------------------------------------------------------- 1 | # Make sure this file does not get required manually 2 | module Autoloaded 3 | class Clazz 4 | def perform; end 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /spec/autoloaded/instance_clazz.rb: -------------------------------------------------------------------------------- 1 | module Autoloaded 2 | class InstanceClazz 3 | def perform; end 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /spec/autoloaded/instance_struct.rb: -------------------------------------------------------------------------------- 1 | module Autoloaded 2 | InstanceStruct = ::Struct.new(nil) 3 | class InstanceStruct 4 | def perform; end 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /spec/autoloaded/struct.rb: -------------------------------------------------------------------------------- 1 | # Make sure this file does not get required manually 2 | module Autoloaded 3 | Struct = ::Struct.new(nil) 4 | class Struct 5 | def perform; end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/daemons.rb: -------------------------------------------------------------------------------- 1 | # Fake "daemons" file on the spec load path to allow spec/delayed/command_spec.rb 2 | # to test the Delayed::Command class without actually adding daemons as a dependency. 3 | -------------------------------------------------------------------------------- /spec/delayed/backend/test.rb: -------------------------------------------------------------------------------- 1 | require 'ostruct' 2 | 3 | # An in-memory backend suitable only for testing. Tries to behave as if it were an ORM. 4 | module Delayed 5 | module Backend 6 | module Test 7 | class Job 8 | attr_accessor :id 9 | attr_accessor :priority 10 | attr_accessor :attempts 11 | attr_accessor :handler 12 | attr_accessor :last_error 13 | attr_accessor :run_at 14 | attr_accessor :locked_at 15 | attr_accessor :locked_by 16 | attr_accessor :failed_at 17 | attr_accessor :queue 18 | 19 | include Delayed::Backend::Base 20 | 21 | cattr_accessor :id 22 | self.id = 0 23 | 24 | def initialize(hash = {}) 25 | self.attempts = 0 26 | self.priority = 0 27 | self.id = (self.class.id += 1) 28 | hash.each { |k, v| send(:"#{k}=", v) } 29 | end 30 | 31 | def self.all 32 | @jobs ||= [] 33 | end 34 | 35 | def self.count 36 | all.size 37 | end 38 | 39 | def self.delete_all 40 | all.clear 41 | end 42 | 43 | def self.create(attrs = {}) 44 | new(attrs).tap do |o| 45 | o.save 46 | end 47 | end 48 | 49 | def self.create!(*args) 50 | create(*args) 51 | end 52 | 53 | def self.clear_locks!(worker_name) 54 | all.select { |j| j.locked_by == worker_name }.each do |j| 55 | j.locked_by = nil 56 | j.locked_at = nil 57 | end 58 | end 59 | 60 | # Find a few candidate jobs to run (in case some immediately get locked by others). 61 | def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time) # rubocop:disable CyclomaticComplexity, PerceivedComplexity 62 | jobs = all.select do |j| 63 | j.run_at <= db_time_now && 64 | (j.locked_at.nil? || j.locked_at < db_time_now - max_run_time || j.locked_by == worker_name) && 65 | !j.failed? 66 | end 67 | jobs.select! { |j| j.priority <= Worker.max_priority } if Worker.max_priority 68 | jobs.select! { |j| j.priority >= Worker.min_priority } if Worker.min_priority 69 | jobs.select! { |j| Worker.queues.include?(j.queue) } if Worker.queues.any? 70 | jobs.sort_by! { |j| [j.priority, j.run_at] }[0..limit - 1] 71 | end 72 | 73 | # Lock this job for this worker. 74 | # Returns true if we have the lock, false otherwise. 75 | def lock_exclusively!(_max_run_time, worker) 76 | now = self.class.db_time_now 77 | if locked_by != worker 78 | # We don't own this job so we will update the locked_by name and the locked_at 79 | self.locked_at = now 80 | self.locked_by = worker 81 | end 82 | 83 | true 84 | end 85 | 86 | def self.db_time_now 87 | Time.current 88 | end 89 | 90 | def destroy 91 | self.class.all.delete(self) 92 | end 93 | 94 | def save 95 | self.run_at ||= Time.current 96 | 97 | self.class.all << self unless self.class.all.include?(self) 98 | true 99 | end 100 | 101 | def save! 102 | save 103 | end 104 | 105 | def reload 106 | reset 107 | self 108 | end 109 | end 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /spec/delayed/command_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require 'delayed/command' 3 | 4 | describe Delayed::Command do 5 | let(:options) { [] } 6 | let(:logger) { double('Logger') } 7 | 8 | subject { Delayed::Command.new options } 9 | 10 | before do 11 | allow(Delayed::Worker).to receive(:after_fork) 12 | allow(Dir).to receive(:chdir) 13 | allow(Logger).to receive(:new).and_return(logger) 14 | allow_any_instance_of(Delayed::Worker).to receive(:start) 15 | allow(Delayed::Worker).to receive(:logger=) 16 | allow(Delayed::Worker).to receive(:logger).and_return(nil, logger) 17 | end 18 | 19 | shared_examples_for 'uses --log-dir option' do 20 | context 'when --log-dir is specified' do 21 | let(:options) { ['--log-dir=/custom/log/dir'] } 22 | 23 | it 'creates the delayed_job.log in the specified directory' do 24 | expect(Logger).to receive(:new).with('/custom/log/dir/delayed_job.log') 25 | subject.run 26 | end 27 | end 28 | end 29 | 30 | describe 'run' do 31 | it 'sets the Delayed::Worker logger' do 32 | expect(Delayed::Worker).to receive(:logger=).with(logger) 33 | subject.run 34 | end 35 | 36 | context 'when Rails root is defined' do 37 | let(:rails_root) { Pathname.new '/rails/root' } 38 | let(:rails) { double('Rails', :root => rails_root) } 39 | 40 | before do 41 | stub_const('Rails', rails) 42 | end 43 | 44 | it 'runs the Delayed::Worker process in Rails.root' do 45 | expect(Dir).to receive(:chdir).with(rails_root) 46 | subject.run 47 | end 48 | 49 | context 'when --log-dir is not specified' do 50 | it 'creates the delayed_job.log in Rails.root/log' do 51 | expect(Logger).to receive(:new).with('/rails/root/log/delayed_job.log') 52 | subject.run 53 | end 54 | end 55 | 56 | include_examples 'uses --log-dir option' 57 | end 58 | 59 | context 'when Rails root is not defined' do 60 | let(:rails_without_root) { double('Rails') } 61 | 62 | before do 63 | stub_const('Rails', rails_without_root) 64 | end 65 | 66 | it 'runs the Delayed::Worker process in $PWD' do 67 | expect(Dir).to receive(:chdir).with(Delayed::Command::DIR_PWD) 68 | subject.run 69 | end 70 | 71 | context 'when --log-dir is not specified' do 72 | it 'creates the delayed_job.log in $PWD/log' do 73 | expect(Logger).to receive(:new).with("#{Delayed::Command::DIR_PWD}/log/delayed_job.log") 74 | subject.run 75 | end 76 | end 77 | 78 | include_examples 'uses --log-dir option' 79 | end 80 | 81 | context 'when an error is raised' do 82 | let(:test_error) { Class.new(StandardError) } 83 | 84 | before do 85 | allow(Delayed::Worker).to receive(:new).and_raise(test_error.new('An error')) 86 | allow(subject).to receive(:exit_with_error_status) 87 | allow(STDERR).to receive(:puts) 88 | end 89 | 90 | it 'prints the error message to STDERR' do 91 | expect(STDERR).to receive(:puts).with('An error') 92 | subject.run 93 | end 94 | 95 | it 'exits with an error status' do 96 | expect(subject).to receive(:exit_with_error_status) 97 | subject.run 98 | end 99 | 100 | context 'when Rails logger is not defined' do 101 | let(:rails) { double('Rails') } 102 | 103 | before do 104 | stub_const('Rails', rails) 105 | end 106 | 107 | it 'does not attempt to use the Rails logger' do 108 | subject.run 109 | end 110 | end 111 | 112 | context 'when Rails logger is defined' do 113 | let(:rails_logger) { double('Rails logger') } 114 | let(:rails) { double('Rails', :logger => rails_logger) } 115 | 116 | before do 117 | stub_const('Rails', rails) 118 | end 119 | 120 | it 'logs the error to the Rails logger' do 121 | expect(rails_logger).to receive(:fatal).with(test_error) 122 | subject.run 123 | end 124 | end 125 | end 126 | end 127 | 128 | describe 'parsing --pool argument' do 129 | it 'should parse --pool correctly' do 130 | command = Delayed::Command.new(['--pool=*:1', '--pool=test_queue:4', '--pool=mailers,misc:2']) 131 | 132 | expect(command.worker_pools).to eq [ 133 | [[], 1], 134 | [['test_queue'], 4], 135 | [%w[mailers misc], 2] 136 | ] 137 | end 138 | 139 | it 'should allow * or blank to specify any pools' do 140 | command = Delayed::Command.new(['--pool=*:4']) 141 | expect(command.worker_pools).to eq [ 142 | [[], 4], 143 | ] 144 | 145 | command = Delayed::Command.new(['--pool=:4']) 146 | expect(command.worker_pools).to eq [ 147 | [[], 4], 148 | ] 149 | end 150 | 151 | it 'should default to one worker if not specified' do 152 | command = Delayed::Command.new(['--pool=mailers']) 153 | expect(command.worker_pools).to eq [ 154 | [['mailers'], 1], 155 | ] 156 | end 157 | end 158 | 159 | describe 'running worker pools defined by multiple --pool arguments' do 160 | it 'should run the correct worker processes' do 161 | command = Delayed::Command.new(['--pool=*:1', '--pool=test_queue:4', '--pool=mailers,misc:2']) 162 | expect(FileUtils).to receive(:mkdir_p).with('./tmp/pids').once 163 | 164 | [ 165 | ['delayed_job.0', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => []}], 166 | ['delayed_job.1', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => ['test_queue']}], 167 | ['delayed_job.2', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => ['test_queue']}], 168 | ['delayed_job.3', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => ['test_queue']}], 169 | ['delayed_job.4', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => ['test_queue']}], 170 | ['delayed_job.5', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => %w[mailers misc]}], 171 | ['delayed_job.6', {:quiet => true, :pid_dir => './tmp/pids', :log_dir => './log', :queues => %w[mailers misc]}] 172 | ].each do |args| 173 | expect(command).to receive(:run_process).with(*args).once 174 | end 175 | 176 | command.daemonize 177 | end 178 | end 179 | end 180 | -------------------------------------------------------------------------------- /spec/delayed/serialization/test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectiveidea/delayed_job/ea4879dd3c2f3f5aa7f616a6f0fd58f2bbc2a481/spec/delayed/serialization/test.rb -------------------------------------------------------------------------------- /spec/helper.rb: -------------------------------------------------------------------------------- 1 | require 'simplecov' 2 | require 'simplecov-lcov' 3 | 4 | SimpleCov::Formatter::LcovFormatter.config do |c| 5 | c.report_with_single_file = true 6 | c.single_report_path = 'coverage/lcov.info' 7 | end 8 | SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new( 9 | [ 10 | SimpleCov::Formatter::HTMLFormatter, 11 | SimpleCov::Formatter::LcovFormatter 12 | ] 13 | ) 14 | 15 | SimpleCov.start do 16 | add_filter '/spec/' 17 | end 18 | 19 | require 'logger' 20 | require 'rspec' 21 | 22 | require 'action_mailer' 23 | require 'active_record' 24 | 25 | require 'delayed_job' 26 | require 'delayed/backend/shared_spec' 27 | 28 | if ENV['DEBUG_LOGS'] 29 | Delayed::Worker.logger = Logger.new(STDOUT) 30 | else 31 | require 'tempfile' 32 | 33 | tf = Tempfile.new('dj.log') 34 | Delayed::Worker.logger = Logger.new(tf.path) 35 | tf.unlink 36 | end 37 | ENV['RAILS_ENV'] = 'test' 38 | 39 | # Trigger AR to initialize 40 | ActiveRecord::Base # rubocop:disable Void 41 | 42 | module Rails 43 | def self.root 44 | '.' 45 | end 46 | end 47 | 48 | Delayed::Worker.backend = :test 49 | 50 | if ActiveSupport::VERSION::MAJOR < 7 51 | require 'active_support/dependencies' 52 | 53 | # Add this directory so the ActiveSupport autoloading works 54 | ActiveSupport::Dependencies.autoload_paths << File.dirname(__FILE__) 55 | else 56 | # Rails 7 dropped classic dependency auto-loading. This does a basic 57 | # zeitwerk setup to test against zeitwerk directly as the Rails zeitwerk 58 | # setup is intertwined in the application boot process. 59 | require 'zeitwerk' 60 | 61 | loader = Zeitwerk::Loader.new 62 | loader.push_dir File.dirname(__FILE__) 63 | loader.setup 64 | end 65 | 66 | # Used to test interactions between DJ and an ORM 67 | ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:' 68 | ActiveRecord::Base.logger = Delayed::Worker.logger 69 | ActiveRecord::Migration.verbose = false 70 | 71 | ActiveRecord::Schema.define do 72 | create_table :stories, :primary_key => :story_id, :force => true do |table| 73 | table.string :text 74 | table.boolean :scoped, :default => true 75 | end 76 | end 77 | 78 | class Story < ActiveRecord::Base 79 | self.primary_key = 'story_id' 80 | def tell 81 | text 82 | end 83 | 84 | def whatever(n, _) 85 | tell * n 86 | end 87 | default_scope { where(:scoped => true) } 88 | 89 | handle_asynchronously :whatever 90 | end 91 | 92 | RSpec.configure do |config| 93 | config.after(:each) do 94 | Delayed::Worker.reset 95 | end 96 | 97 | config.expect_with :rspec do |c| 98 | c.syntax = :expect 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /spec/lifecycle_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe Delayed::Lifecycle do 4 | let(:lifecycle) { Delayed::Lifecycle.new } 5 | let(:callback) { lambda { |*_args| } } 6 | let(:arguments) { [1] } 7 | let(:behavior) { double(Object, :before! => nil, :after! => nil, :inside! => nil) } 8 | let(:wrapped_block) { proc { behavior.inside! } } 9 | 10 | describe 'before callbacks' do 11 | before(:each) do 12 | lifecycle.before(:execute, &callback) 13 | end 14 | 15 | it 'executes before wrapped block' do 16 | expect(callback).to receive(:call).with(*arguments).ordered 17 | expect(behavior).to receive(:inside!).ordered 18 | lifecycle.run_callbacks :execute, *arguments, &wrapped_block 19 | end 20 | end 21 | 22 | describe 'after callbacks' do 23 | before(:each) do 24 | lifecycle.after(:execute, &callback) 25 | end 26 | 27 | it 'executes after wrapped block' do 28 | expect(behavior).to receive(:inside!).ordered 29 | expect(callback).to receive(:call).with(*arguments).ordered 30 | lifecycle.run_callbacks :execute, *arguments, &wrapped_block 31 | end 32 | end 33 | 34 | describe 'around callbacks' do 35 | before(:each) do 36 | lifecycle.around(:execute) do |*args, &block| 37 | behavior.before! 38 | block.call(*args) 39 | behavior.after! 40 | end 41 | end 42 | 43 | it 'wraps a block' do 44 | expect(behavior).to receive(:before!).ordered 45 | expect(behavior).to receive(:inside!).ordered 46 | expect(behavior).to receive(:after!).ordered 47 | lifecycle.run_callbacks :execute, *arguments, &wrapped_block 48 | end 49 | 50 | it 'executes multiple callbacks in order' do 51 | expect(behavior).to receive(:one).ordered 52 | expect(behavior).to receive(:two).ordered 53 | expect(behavior).to receive(:three).ordered 54 | 55 | lifecycle.around(:execute) do |*args, &block| 56 | behavior.one 57 | block.call(*args) 58 | end 59 | lifecycle.around(:execute) do |*args, &block| 60 | behavior.two 61 | block.call(*args) 62 | end 63 | lifecycle.around(:execute) do |*args, &block| 64 | behavior.three 65 | block.call(*args) 66 | end 67 | lifecycle.run_callbacks(:execute, *arguments, &wrapped_block) 68 | end 69 | end 70 | 71 | it 'raises if callback is executed with wrong number of parameters' do 72 | lifecycle.before(:execute, &callback) 73 | expect { lifecycle.run_callbacks(:execute, 1, 2, 3) {} }.to raise_error(ArgumentError, /1 parameter/) 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /spec/message_sending_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe Delayed::MessageSending do 4 | it 'does not include ClassMethods along with MessageSending' do 5 | expect { ClassMethods }.to raise_error(NameError) 6 | expect(defined?(String::ClassMethods)).to eq(nil) 7 | end 8 | 9 | describe 'handle_asynchronously' do 10 | class Story 11 | def tell!(_arg); end 12 | handle_asynchronously :tell! 13 | end 14 | 15 | it 'aliases original method' do 16 | expect(Story.new).to respond_to(:tell_without_delay!) 17 | expect(Story.new).to respond_to(:tell_with_delay!) 18 | end 19 | 20 | it 'creates a PerformableMethod' do 21 | story = Story.create 22 | expect do 23 | job = story.tell!(1) 24 | expect(job.payload_object.class).to eq(Delayed::PerformableMethod) 25 | expect(job.payload_object.method_name).to eq(:tell_without_delay!) 26 | expect(job.payload_object.args).to eq([1]) 27 | end.to(change { Delayed::Job.count }) 28 | end 29 | 30 | describe 'with options' do 31 | class Fable 32 | cattr_accessor :importance 33 | def tell; end 34 | handle_asynchronously :tell, :priority => proc { importance } 35 | end 36 | 37 | it 'sets the priority based on the Fable importance' do 38 | Fable.importance = 10 39 | job = Fable.new.tell 40 | expect(job.priority).to eq(10) 41 | 42 | Fable.importance = 20 43 | job = Fable.new.tell 44 | expect(job.priority).to eq(20) 45 | end 46 | 47 | describe 'using a proc with parameters' do 48 | class Yarn 49 | attr_accessor :importance 50 | def spin; end 51 | handle_asynchronously :spin, :priority => proc { |y| y.importance } 52 | end 53 | 54 | it 'sets the priority based on the Fable importance' do 55 | job = Yarn.new.tap { |y| y.importance = 10 }.spin 56 | expect(job.priority).to eq(10) 57 | 58 | job = Yarn.new.tap { |y| y.importance = 20 }.spin 59 | expect(job.priority).to eq(20) 60 | end 61 | end 62 | end 63 | end 64 | 65 | context 'delay' do 66 | class FairyTail 67 | attr_accessor :happy_ending 68 | def self.princesses; end 69 | 70 | def tell 71 | @happy_ending = true 72 | end 73 | end 74 | 75 | after do 76 | Delayed::Worker.default_queue_name = nil 77 | end 78 | 79 | it 'creates a new PerformableMethod job' do 80 | expect do 81 | job = 'hello'.delay.count('l') 82 | expect(job.payload_object.class).to eq(Delayed::PerformableMethod) 83 | expect(job.payload_object.method_name).to eq(:count) 84 | expect(job.payload_object.args).to eq(['l']) 85 | end.to change { Delayed::Job.count }.by(1) 86 | end 87 | 88 | it 'sets default priority' do 89 | Delayed::Worker.default_priority = 99 90 | job = FairyTail.delay.to_s 91 | expect(job.priority).to eq(99) 92 | end 93 | 94 | it 'sets default queue name' do 95 | Delayed::Worker.default_queue_name = 'abbazabba' 96 | job = FairyTail.delay.to_s 97 | expect(job.queue).to eq('abbazabba') 98 | end 99 | 100 | it 'sets job options' do 101 | run_at = Time.parse('2010-05-03 12:55 AM') 102 | job = FairyTail.delay(:priority => 20, :run_at => run_at).to_s 103 | expect(job.run_at).to eq(run_at) 104 | expect(job.priority).to eq(20) 105 | end 106 | 107 | it 'does not delay the job when delay_jobs is false' do 108 | Delayed::Worker.delay_jobs = false 109 | fairy_tail = FairyTail.new 110 | expect do 111 | expect do 112 | fairy_tail.delay.tell 113 | end.to change(fairy_tail, :happy_ending).from(nil).to(true) 114 | end.not_to(change { Delayed::Job.count }) 115 | end 116 | 117 | it 'does delay the job when delay_jobs is true' do 118 | Delayed::Worker.delay_jobs = true 119 | fairy_tail = FairyTail.new 120 | expect do 121 | expect do 122 | fairy_tail.delay.tell 123 | end.not_to change(fairy_tail, :happy_ending) 124 | end.to change { Delayed::Job.count }.by(1) 125 | end 126 | 127 | it 'does delay when delay_jobs is a proc returning true' do 128 | Delayed::Worker.delay_jobs = ->(_job) { true } 129 | fairy_tail = FairyTail.new 130 | expect do 131 | expect do 132 | fairy_tail.delay.tell 133 | end.not_to change(fairy_tail, :happy_ending) 134 | end.to change { Delayed::Job.count }.by(1) 135 | end 136 | 137 | it 'does not delay the job when delay_jobs is a proc returning false' do 138 | Delayed::Worker.delay_jobs = ->(_job) { false } 139 | fairy_tail = FairyTail.new 140 | expect do 141 | expect do 142 | fairy_tail.delay.tell 143 | end.to change(fairy_tail, :happy_ending).from(nil).to(true) 144 | end.not_to(change { Delayed::Job.count }) 145 | end 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /spec/performable_mailer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class MyMailer < ActionMailer::Base 4 | def signup(email) 5 | mail :to => email, :subject => 'Delaying Emails', :from => 'delayedjob@example.com', :body => 'Delaying Emails Body' 6 | end 7 | end 8 | 9 | describe ActionMailer::Base do 10 | describe 'delay' do 11 | it 'enqueues a PerformableEmail job' do 12 | expect do 13 | job = MyMailer.delay.signup('john@example.com') 14 | expect(job.payload_object.class).to eq(Delayed::PerformableMailer) 15 | expect(job.payload_object.method_name).to eq(:signup) 16 | expect(job.payload_object.args).to eq(['john@example.com']) 17 | end.to change { Delayed::Job.count }.by(1) 18 | end 19 | end 20 | 21 | describe 'delay on a mail object' do 22 | it 'raises an exception' do 23 | expect do 24 | MyMailer.signup('john@example.com').delay 25 | end.to raise_error(RuntimeError) 26 | end 27 | end 28 | 29 | describe Delayed::PerformableMailer do 30 | describe 'perform' do 31 | it 'calls the method and #deliver on the mailer' do 32 | email = double('email', :deliver => true) 33 | mailer_class = double('MailerClass', :signup => email) 34 | mailer = Delayed::PerformableMailer.new(mailer_class, :signup, ['john@example.com']) 35 | 36 | expect(mailer_class).to receive(:signup).with('john@example.com') 37 | expect(email).to receive(:deliver) 38 | mailer.perform 39 | end 40 | end 41 | end 42 | end 43 | 44 | if defined?(ActionMailer::Parameterized::Mailer) 45 | describe ActionMailer::Parameterized::Mailer do 46 | describe 'delay' do 47 | it 'enqueues a PerformableEmail job' do 48 | expect do 49 | job = MyMailer.with(:foo => 1, :bar => 2).delay.signup('john@example.com') 50 | expect(job.payload_object.class).to eq(Delayed::PerformableMailer) 51 | expect(job.payload_object.object.class).to eq(ActionMailer::Parameterized::Mailer) 52 | expect(job.payload_object.object.instance_variable_get('@mailer')).to eq(MyMailer) 53 | expect(job.payload_object.object.instance_variable_get('@params')).to eq(:foo => 1, :bar => 2) 54 | expect(job.payload_object.method_name).to eq(:signup) 55 | expect(job.payload_object.args).to eq(['john@example.com']) 56 | end.to change { Delayed::Job.count }.by(1) 57 | end 58 | end 59 | 60 | describe 'delay on a mail object' do 61 | it 'raises an exception' do 62 | expect do 63 | MyMailer.with(:foo => 1, :bar => 2).signup('john@example.com').delay 64 | end.to raise_error(RuntimeError) 65 | end 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/performable_method_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe Delayed::PerformableMethod do 4 | describe 'perform' do 5 | before do 6 | @method = Delayed::PerformableMethod.new('foo', :count, ['o']) 7 | end 8 | 9 | context 'with the persisted record cannot be found' do 10 | before do 11 | @method.object = nil 12 | end 13 | 14 | it 'does nothing if object is nil' do 15 | expect { @method.perform }.not_to raise_error 16 | end 17 | end 18 | 19 | it 'calls the method on the object' do 20 | expect(@method.object).to receive(:count).with('o') 21 | @method.perform 22 | end 23 | end 24 | 25 | it "raises a NoMethodError if target method doesn't exist" do 26 | expect do 27 | Delayed::PerformableMethod.new(Object, :method_that_does_not_exist, []) 28 | end.to raise_error(NoMethodError) 29 | end 30 | 31 | it 'does not raise NoMethodError if target method is private' do 32 | clazz = Class.new do 33 | def private_method; end 34 | private :private_method 35 | end 36 | expect { Delayed::PerformableMethod.new(clazz.new, :private_method, []) }.not_to raise_error 37 | end 38 | 39 | describe 'display_name' do 40 | it 'returns class_name#method_name for instance methods' do 41 | expect(Delayed::PerformableMethod.new('foo', :count, ['o']).display_name).to eq('String#count') 42 | end 43 | 44 | it 'returns class_name.method_name for class methods' do 45 | expect(Delayed::PerformableMethod.new(Class, :inspect, []).display_name).to eq('Class.inspect') 46 | end 47 | end 48 | 49 | describe 'hooks' do 50 | %w[before after success].each do |hook| 51 | it "delegates #{hook} hook to object" do 52 | story = Story.create 53 | job = story.delay.tell 54 | 55 | expect(story).to receive(hook).with(job) 56 | job.invoke_job 57 | end 58 | end 59 | 60 | it 'delegates enqueue hook to object' do 61 | story = Story.create 62 | expect(story).to receive(:enqueue).with(an_instance_of(Delayed::Job)) 63 | story.delay.tell 64 | end 65 | 66 | it 'delegates error hook to object' do 67 | story = Story.create 68 | expect(story).to receive(:error).with(an_instance_of(Delayed::Job), an_instance_of(RuntimeError)) 69 | expect(story).to receive(:tell).and_raise(RuntimeError) 70 | expect { story.delay.tell.invoke_job }.to raise_error(RuntimeError) 71 | end 72 | 73 | it 'delegates failure hook to object' do 74 | method = Delayed::PerformableMethod.new('object', :size, []) 75 | expect(method.object).to receive(:failure) 76 | method.failure 77 | end 78 | 79 | context 'with delay_job == false' do 80 | before do 81 | Delayed::Worker.delay_jobs = false 82 | end 83 | 84 | after do 85 | Delayed::Worker.delay_jobs = true 86 | end 87 | 88 | %w[before after success].each do |hook| 89 | it "delegates #{hook} hook to object" do 90 | story = Story.create 91 | expect(story).to receive(hook).with(an_instance_of(Delayed::Job)) 92 | story.delay.tell 93 | end 94 | end 95 | 96 | it 'delegates error hook to object' do 97 | story = Story.create 98 | expect(story).to receive(:error).with(an_instance_of(Delayed::Job), an_instance_of(RuntimeError)) 99 | expect(story).to receive(:tell).and_raise(RuntimeError) 100 | expect { story.delay.tell }.to raise_error(RuntimeError) 101 | end 102 | 103 | it 'delegates failure hook to object' do 104 | method = Delayed::PerformableMethod.new('object', :size, []) 105 | expect(method.object).to receive(:failure) 106 | method.failure 107 | end 108 | end 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /spec/psych_ext_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe 'Psych::Visitors::ToRuby', :if => defined?(Psych::Visitors::ToRuby) do 4 | context BigDecimal do 5 | it 'deserializes correctly' do 6 | deserialized = YAML.load_dj("--- !ruby/object:BigDecimal 18:0.1337E2\n...\n") 7 | 8 | expect(deserialized).to be_an_instance_of(BigDecimal) 9 | expect(deserialized).to eq(BigDecimal('13.37')) 10 | end 11 | end 12 | 13 | context 'load_tag handling' do 14 | # This only broadly works in ruby 2.0 but will cleanly work through load_dj 15 | # here because this class is so simple it only touches our extention 16 | YAML.load_tags['!ruby/object:RenamedClass'] = SimpleJob 17 | # This is how ruby 2.1 and newer works throughout the yaml handling 18 | YAML.load_tags['!ruby/object:RenamedString'] = 'SimpleJob' 19 | 20 | it 'deserializes class tag' do 21 | deserialized = YAML.load_dj("--- !ruby/object:RenamedClass\ncheck: 12\n") 22 | 23 | expect(deserialized).to be_an_instance_of(SimpleJob) 24 | expect(deserialized.instance_variable_get(:@check)).to eq(12) 25 | end 26 | 27 | it 'deserializes string tag' do 28 | deserialized = YAML.load_dj("--- !ruby/object:RenamedString\ncheck: 12\n") 29 | 30 | expect(deserialized).to be_an_instance_of(SimpleJob) 31 | expect(deserialized.instance_variable_get(:@check)).to eq(12) 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /spec/sample_jobs.rb: -------------------------------------------------------------------------------- 1 | NamedJob = Struct.new(:perform) 2 | class NamedJob 3 | def display_name 4 | 'named_job' 5 | end 6 | end 7 | 8 | class SimpleJob 9 | cattr_accessor :runs 10 | @runs = 0 11 | def perform 12 | self.class.runs += 1 13 | end 14 | end 15 | 16 | class NamedQueueJob < SimpleJob 17 | def queue_name 18 | 'job_tracking' 19 | end 20 | end 21 | 22 | class ErrorJob 23 | cattr_accessor :runs 24 | @runs = 0 25 | def perform 26 | raise Exception, 'did not work' 27 | end 28 | end 29 | 30 | CustomRescheduleJob = Struct.new(:offset) 31 | class CustomRescheduleJob 32 | cattr_accessor :runs 33 | @runs = 0 34 | def perform 35 | raise 'did not work' 36 | end 37 | 38 | def reschedule_at(time, _attempts) 39 | time + offset 40 | end 41 | end 42 | 43 | class LongRunningJob 44 | def perform 45 | sleep 250 46 | end 47 | end 48 | 49 | class OnPermanentFailureJob < SimpleJob 50 | attr_writer :raise_error 51 | 52 | def initialize 53 | @raise_error = false 54 | end 55 | 56 | def failure 57 | raise 'did not work' if @raise_error 58 | end 59 | 60 | def max_attempts 61 | 1 62 | end 63 | end 64 | 65 | module M 66 | class ModuleJob 67 | cattr_accessor :runs 68 | @runs = 0 69 | def perform 70 | self.class.runs += 1 71 | end 72 | end 73 | end 74 | 75 | class CallbackJob 76 | cattr_accessor :messages 77 | 78 | def enqueue(_job) 79 | self.class.messages << 'enqueue' 80 | end 81 | 82 | def before(_job) 83 | self.class.messages << 'before' 84 | end 85 | 86 | def perform 87 | self.class.messages << 'perform' 88 | end 89 | 90 | def after(_job) 91 | self.class.messages << 'after' 92 | end 93 | 94 | def success(_job) 95 | self.class.messages << 'success' 96 | end 97 | 98 | def error(_job, error) 99 | self.class.messages << "error: #{error.class}" 100 | end 101 | 102 | def failure(_job) 103 | self.class.messages << 'failure' 104 | end 105 | end 106 | 107 | class EnqueueJobMod < SimpleJob 108 | def enqueue(job) 109 | job.run_at = 20.minutes.from_now 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /spec/test_backend_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe Delayed::Backend::Test::Job do 4 | it_should_behave_like 'a delayed_job backend' 5 | 6 | describe '#reload' do 7 | it 'causes the payload object to be reloaded' do 8 | job = 'foo'.delay.length 9 | o = job.payload_object 10 | expect(o.object_id).not_to eq(job.reload.payload_object.object_id) 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe Delayed::Worker do 4 | describe 'backend=' do 5 | before do 6 | @clazz = Class.new 7 | Delayed::Worker.backend = @clazz 8 | end 9 | 10 | after do 11 | Delayed::Worker.backend = :test 12 | end 13 | 14 | it 'sets the Delayed::Job constant to the backend' do 15 | expect(Delayed::Job).to eq(@clazz) 16 | end 17 | 18 | it 'sets backend with a symbol' do 19 | Delayed::Worker.backend = :test 20 | expect(Delayed::Worker.backend).to eq(Delayed::Backend::Test::Job) 21 | end 22 | end 23 | 24 | describe 'job_say' do 25 | before do 26 | @worker = Delayed::Worker.new 27 | @job = double('job', :id => 123, :name => 'ExampleJob', :queue => nil) 28 | end 29 | 30 | it 'logs with job name and id' do 31 | expect(@job).to receive(:queue) 32 | expect(@worker).to receive(:say). 33 | with('Job ExampleJob (id=123) message', Delayed::Worker.default_log_level) 34 | @worker.job_say(@job, 'message') 35 | end 36 | 37 | it 'logs with job name, queue and id' do 38 | expect(@job).to receive(:queue).and_return('test') 39 | expect(@worker).to receive(:say). 40 | with('Job ExampleJob (id=123) (queue=test) message', Delayed::Worker.default_log_level) 41 | @worker.job_say(@job, 'message') 42 | end 43 | 44 | it 'has a configurable default log level' do 45 | Delayed::Worker.default_log_level = 'error' 46 | 47 | expect(@worker).to receive(:say). 48 | with('Job ExampleJob (id=123) message', 'error') 49 | @worker.job_say(@job, 'message') 50 | end 51 | end 52 | 53 | context 'worker read-ahead' do 54 | before do 55 | @read_ahead = Delayed::Worker.read_ahead 56 | end 57 | 58 | after do 59 | Delayed::Worker.read_ahead = @read_ahead 60 | end 61 | 62 | it 'reads five jobs' do 63 | expect(Delayed::Job).to receive(:find_available).with(anything, 5, anything).and_return([]) 64 | Delayed::Job.reserve(Delayed::Worker.new) 65 | end 66 | 67 | it 'reads a configurable number of jobs' do 68 | Delayed::Worker.read_ahead = 15 69 | expect(Delayed::Job).to receive(:find_available).with(anything, Delayed::Worker.read_ahead, anything).and_return([]) 70 | Delayed::Job.reserve(Delayed::Worker.new) 71 | end 72 | end 73 | 74 | context 'worker exit on complete' do 75 | before do 76 | Delayed::Worker.exit_on_complete = true 77 | end 78 | 79 | after do 80 | Delayed::Worker.exit_on_complete = false 81 | end 82 | 83 | it 'exits the loop when no jobs are available' do 84 | worker = Delayed::Worker.new 85 | Timeout.timeout(2) do 86 | worker.start 87 | end 88 | end 89 | end 90 | 91 | context 'worker job reservation' do 92 | before do 93 | Delayed::Worker.exit_on_complete = true 94 | end 95 | 96 | after do 97 | Delayed::Worker.exit_on_complete = false 98 | end 99 | 100 | it 'handles error during job reservation' do 101 | expect(Delayed::Job).to receive(:reserve).and_raise(Exception) 102 | Delayed::Worker.new.work_off 103 | end 104 | 105 | it 'gives up after 10 backend failures' do 106 | expect(Delayed::Job).to receive(:reserve).exactly(10).times.and_raise(Exception) 107 | worker = Delayed::Worker.new 108 | 9.times { worker.work_off } 109 | expect { worker.work_off }.to raise_exception Delayed::FatalBackendError 110 | end 111 | 112 | it 'allows the backend to attempt recovery from reservation errors' do 113 | expect(Delayed::Job).to receive(:reserve).and_raise(Exception) 114 | expect(Delayed::Job).to receive(:recover_from).with(instance_of(Exception)) 115 | Delayed::Worker.new.work_off 116 | end 117 | end 118 | 119 | context '#say' do 120 | before(:each) do 121 | @worker = Delayed::Worker.new 122 | @worker.name = 'ExampleJob' 123 | @worker.logger = double('job') 124 | time = Time.now 125 | allow(Time).to receive(:now).and_return(time) 126 | @text = 'Job executed' 127 | @worker_name = '[Worker(ExampleJob)]' 128 | @expected_time = time.strftime('%FT%T%z') 129 | end 130 | 131 | after(:each) do 132 | @worker.logger = nil 133 | end 134 | 135 | shared_examples_for 'a worker which logs on the correct severity' do |severity| 136 | it "logs a message on the #{severity[:level].upcase} level given a string" do 137 | expect(@worker.logger).to receive(:send). 138 | with(severity[:level], "#{@expected_time}: #{@worker_name} #{@text}") 139 | @worker.say(@text, severity[:level]) 140 | end 141 | 142 | it "logs a message on the #{severity[:level].upcase} level given a fixnum" do 143 | expect(@worker.logger).to receive(:send). 144 | with(severity[:level], "#{@expected_time}: #{@worker_name} #{@text}") 145 | @worker.say(@text, severity[:index]) 146 | end 147 | end 148 | 149 | severities = [{:index => 0, :level => 'debug'}, 150 | {:index => 1, :level => 'info'}, 151 | {:index => 2, :level => 'warn'}, 152 | {:index => 3, :level => 'error'}, 153 | {:index => 4, :level => 'fatal'}, 154 | {:index => 5, :level => 'unknown'}] 155 | severities.each do |severity| 156 | it_behaves_like 'a worker which logs on the correct severity', severity 157 | end 158 | 159 | it 'logs a message on the default log\'s level' do 160 | expect(@worker.logger).to receive(:send). 161 | with('info', "#{@expected_time}: #{@worker_name} #{@text}") 162 | @worker.say(@text, Delayed::Worker.default_log_level) 163 | end 164 | end 165 | 166 | describe 'plugin registration' do 167 | it 'does not double-register plugins on worker instantiation' do 168 | performances = 0 169 | plugin = Class.new(Delayed::Plugin) do 170 | callbacks do |lifecycle| 171 | lifecycle.before(:enqueue) { performances += 1 } 172 | end 173 | end 174 | Delayed::Worker.plugins << plugin 175 | 176 | Delayed::Worker.new 177 | Delayed::Worker.new 178 | Delayed::Worker.lifecycle.run_callbacks(:enqueue, nil) {} 179 | 180 | expect(performances).to eq(1) 181 | end 182 | end 183 | end 184 | -------------------------------------------------------------------------------- /spec/yaml_ext_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe 'YAML' do 4 | it 'autoloads classes' do 5 | expect do 6 | yaml = "--- !ruby/class Autoloaded::Clazz\n" 7 | expect(load_with_delayed_visitor(yaml)).to eq(Autoloaded::Clazz) 8 | end.not_to raise_error 9 | end 10 | 11 | it 'autoloads the class of a struct' do 12 | expect do 13 | yaml = "--- !ruby/class Autoloaded::Struct\n" 14 | expect(load_with_delayed_visitor(yaml)).to eq(Autoloaded::Struct) 15 | end.not_to raise_error 16 | end 17 | 18 | it 'autoloads the class for the instance of a struct' do 19 | expect do 20 | yaml = '--- !ruby/struct:Autoloaded::InstanceStruct {}' 21 | expect(load_with_delayed_visitor(yaml).class).to eq(Autoloaded::InstanceStruct) 22 | end.not_to raise_error 23 | end 24 | 25 | it 'autoloads the class of an anonymous struct' do 26 | expect do 27 | yaml = "--- !ruby/struct\nn: 1\n" 28 | object = load_with_delayed_visitor(yaml) 29 | expect(object).to be_kind_of(Struct) 30 | expect(object.n).to eq(1) 31 | end.not_to raise_error 32 | end 33 | 34 | it 'autoloads the class for the instance' do 35 | expect do 36 | yaml = "--- !ruby/object:Autoloaded::InstanceClazz {}\n" 37 | expect(load_with_delayed_visitor(yaml).class).to eq(Autoloaded::InstanceClazz) 38 | end.not_to raise_error 39 | end 40 | 41 | it 'does not throw an uninitialized constant Syck::Syck when using YAML.load with poorly formed yaml' do 42 | expect { YAML.load(YAML.dump('foo: *bar')) }.not_to raise_error 43 | end 44 | 45 | def load_with_delayed_visitor(yaml) 46 | YAML.load_dj(yaml) 47 | end 48 | end 49 | --------------------------------------------------------------------------------