├── VERSION
├── .gitignore
├── init.rb
├── tasks
└── jobs.rake
├── recipes
└── delayed_job.rb
├── generators
└── delayed_job
│ ├── templates
│ ├── script
│ └── migration.rb
│ └── delayed_job_generator.rb
├── spec
├── story_spec.rb
├── sample_jobs.rb
├── job_spec.rb
├── spec_helper.rb
├── performable_method_spec.rb
├── mongo_job_spec.rb
├── delayed_method_spec.rb
├── worker_spec.rb
└── shared_backend_spec.rb
├── lib
├── delayed_job.rb
└── delayed
│ ├── tasks.rb
│ ├── recipes.rb
│ ├── performable_method.rb
│ ├── message_sending.rb
│ ├── command.rb
│ ├── backend
│ ├── base.rb
│ ├── active_record.rb
│ └── mongo.rb
│ └── worker.rb
├── contrib
└── delayed_job.monitrc
├── MIT-LICENSE
├── Rakefile
├── delayed_job.gemspec
└── README.textile
/VERSION:
--------------------------------------------------------------------------------
1 | 1.8.4
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.gem
2 | .idea/*
3 |
--------------------------------------------------------------------------------
/init.rb:
--------------------------------------------------------------------------------
1 | require File.dirname(__FILE__) + '/lib/delayed_job'
2 |
--------------------------------------------------------------------------------
/tasks/jobs.rake:
--------------------------------------------------------------------------------
1 | require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'delayed', 'tasks'))
2 |
--------------------------------------------------------------------------------
/recipes/delayed_job.rb:
--------------------------------------------------------------------------------
1 | require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'delayed', 'recipes'))
2 |
--------------------------------------------------------------------------------
/generators/delayed_job/templates/script:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env ruby
2 |
3 | require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
4 | require 'delayed/command'
5 | Delayed::Command.new(ARGV).daemonize
6 |
--------------------------------------------------------------------------------
/spec/story_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe "A story" do
4 |
5 | before(:all) do
6 | @story = Story.create :text => "Once upon a time..."
7 | end
8 |
9 | it "should be shared" do
10 | @story.tell.should == 'Once upon a time...'
11 | end
12 |
13 | it "should not return its result if it storytelling is delayed" do
14 | @story.send_later(:tell).should_not == 'Once upon a time...'
15 | end
16 |
17 | end
--------------------------------------------------------------------------------
/spec/sample_jobs.rb:
--------------------------------------------------------------------------------
1 | class SimpleJob
2 | cattr_accessor :runs; self.runs = 0
3 | def perform; @@runs += 1; end
4 | end
5 |
6 | class ErrorJob
7 | cattr_accessor :runs; self.runs = 0
8 | def perform; raise 'did not work'; end
9 | end
10 |
11 | class LongRunningJob
12 | def perform; sleep 250; end
13 | end
14 |
15 | module M
16 | class ModuleJob
17 | cattr_accessor :runs; self.runs = 0
18 | def perform; @@runs += 1; end
19 | end
20 |
21 | end
--------------------------------------------------------------------------------
/lib/delayed_job.rb:
--------------------------------------------------------------------------------
1 | require File.dirname(__FILE__) + '/delayed/message_sending'
2 | require File.dirname(__FILE__) + '/delayed/performable_method'
3 | require File.dirname(__FILE__) + '/delayed/backend/base'
4 | require File.dirname(__FILE__) + '/delayed/worker'
5 |
6 | Object.send(:include, Delayed::MessageSending)
7 | Module.send(:include, Delayed::MessageSending::ClassMethods)
8 |
9 | Delayed::Worker.backend = :active_record
10 |
11 | if defined?(Merb::Plugins)
12 | Merb::Plugins.add_rakefiles File.dirname(__FILE__) / 'delayed' / 'tasks'
13 | end
14 |
--------------------------------------------------------------------------------
/generators/delayed_job/delayed_job_generator.rb:
--------------------------------------------------------------------------------
1 | class DelayedJobGenerator < Rails::Generator::Base
2 | default_options :skip_migration => false
3 |
4 | def manifest
5 | record do |m|
6 | m.template 'script', 'script/delayed_job', :chmod => 0755
7 | unless options[:skip_migration]
8 | m.migration_template "migration.rb", 'db/migrate',
9 | :migration_file_name => "create_delayed_jobs"
10 | end
11 | end
12 | end
13 |
14 | protected
15 |
16 | def add_options!(opt)
17 | opt.separator ''
18 | opt.separator 'Options:'
19 | opt.on("--skip-migration", "Don't generate a migration") { |v| options[:skip_migration] = v }
20 | end
21 |
22 | end
23 |
--------------------------------------------------------------------------------
/contrib/delayed_job.monitrc:
--------------------------------------------------------------------------------
1 | # an example Monit configuration file for delayed_job
2 | # See: http://stackoverflow.com/questions/1226302/how-to-monitor-delayedjob-with-monit/1285611
3 | #
4 | # To use:
5 | # 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc
6 | # 2. replace {app_name} as appropriate
7 | # 3. add this to your /etc/monit/monitrc
8 | #
9 | # include /var/www/apps/{app_name}/shared/delayed_job.monitrc
10 |
11 | check process delayed_job
12 | with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.pid
13 | start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job start"
14 | stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/script/delayed_job stop"
--------------------------------------------------------------------------------
/lib/delayed/tasks.rb:
--------------------------------------------------------------------------------
1 | # Re-definitions are appended to existing tasks
2 | task :environment
3 | task :merb_env
4 |
5 | namespace :jobs do
6 | desc "Clear the delayed_job queue. You can specify a queue with this: rake jobs:clear[queue_name] If no queue is specified, all jobs from all queues will be cleared."
7 | task :clear, [:queue] => [:merb_env, :environment] do |t, args|
8 | if args.queue
9 | Delayed::Job.delete_all :queue => args.queue
10 | else
11 | Delayed::Job.delete_all
12 | end
13 | end
14 |
15 | desc "Start a delayed_job worker. You can specify which queue to process from, for example: rake jobs:work[my_queue], or: QUEUE=my_queue rake jobs:work"
16 | task :work, [:queue] => [:merb_env, :environment] do |t,args|
17 | queue = args.queue || ENV['QUEUE']
18 | Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY'], :queue => queue).start
19 | end
20 | end
21 |
--------------------------------------------------------------------------------
/lib/delayed/recipes.rb:
--------------------------------------------------------------------------------
1 | # Capistrano Recipes for managing delayed_job
2 | #
3 | # Add these callbacks to have the delayed_job process restart when the server
4 | # is restarted:
5 | #
6 | # after "deploy:stop", "delayed_job:stop"
7 | # after "deploy:start", "delayed_job:start"
8 | # after "deploy:restart", "delayed_job:restart"
9 |
10 | Capistrano::Configuration.instance.load do
11 | namespace :delayed_job do
12 | def rails_env
13 | fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
14 | end
15 |
16 | desc "Stop the delayed_job process"
17 | task :stop, :roles => :app do
18 | run "cd #{current_path};#{rails_env} script/delayed_job stop"
19 | end
20 |
21 | desc "Start the delayed_job process"
22 | task :start, :roles => :app do
23 | run "cd #{current_path};#{rails_env} script/delayed_job start"
24 | end
25 |
26 | desc "Restart the delayed_job process"
27 | task :restart, :roles => :app do
28 | run "cd #{current_path};#{rails_env} script/delayed_job restart"
29 | end
30 | end
31 | end
--------------------------------------------------------------------------------
/MIT-LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2005 Tobias Luetke
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining
4 | a copy of this software and associated documentation files (the
5 | "Software"), to deal in the Software without restriction, including
6 | without limitation the rights to use, copy, modify, merge, publish,
7 | distribute, sublicense, and/or sell copies of the Software, and to
8 | permit persons to whom the Software is furnished to do so, subject to
9 | the following conditions:
10 |
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 |
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOa AND
17 | NONINFRINGEMENT. IN NO EVENT SaALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
/spec/job_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Delayed::Job do
4 | before(:all) do
5 | @backend = Delayed::Job
6 | end
7 |
8 | before(:each) do
9 | Delayed::Worker.max_priority = nil
10 | Delayed::Worker.min_priority = nil
11 | Delayed::Job.delete_all
12 | SimpleJob.runs = 0
13 | end
14 |
15 | after do
16 | Time.zone = nil
17 | end
18 |
19 | it_should_behave_like 'a backend'
20 |
21 | context "db_time_now" do
22 | it "should return time in current time zone if set" do
23 | Time.zone = 'Eastern Time (US & Canada)'
24 | Delayed::Job.db_time_now.zone.should == 'EST'
25 | end
26 |
27 | it "should return UTC time if that is the AR default" do
28 | Time.zone = nil
29 | ActiveRecord::Base.default_timezone = :utc
30 | Delayed::Job.db_time_now.zone.should == 'UTC'
31 | end
32 |
33 | it "should return local time if that is the AR default" do
34 | Time.zone = 'Central Time (US & Canada)'
35 | ActiveRecord::Base.default_timezone = :local
36 | Delayed::Job.db_time_now.zone.should == 'CST'
37 | end
38 | end
39 |
40 | end
41 |
--------------------------------------------------------------------------------
/generators/delayed_job/templates/migration.rb:
--------------------------------------------------------------------------------
1 | class CreateDelayedJobs < ActiveRecord::Migration
2 | def self.up
3 | create_table :delayed_jobs, :force => true do |table|
4 | table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
5 | table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
6 | table.text :handler # YAML-encoded string of the object that will do work
7 | table.text :last_error # reason for last failure (See Note below)
8 | table.string :queue, :default => nil # The queue that this job is in
9 | table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
10 | table.datetime :locked_at # Set when a client is working on this object
11 | table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
12 | table.string :locked_by # Who is working on this object (if locked)
13 | table.timestamps
14 | end
15 |
16 | add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
17 | end
18 |
19 | def self.down
20 | drop_table :delayed_jobs
21 | end
22 | end
--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | # -*- encoding: utf-8 -*-
2 | begin
3 | require 'jeweler'
4 | rescue LoadError
5 | puts "Jeweler not available. Install it with: sudo gem install jeweler"
6 | exit 1
7 | end
8 |
9 | Jeweler::Tasks.new do |s|
10 | s.name = "delayed_job"
11 | s.summary = "Database-backed asynchronous priority queue system -- Extracted from Shopify"
12 | s.email = "tobi@leetsoft.com"
13 | s.homepage = "http://github.com/collectiveidea/delayed_job"
14 | s.description = "Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks."
15 | s.authors = ["Brandon Keepers", "Tobias Lütke"]
16 |
17 | s.has_rdoc = true
18 | s.rdoc_options = ["--main", "README.textile", "--inline-source", "--line-numbers"]
19 | s.extra_rdoc_files = ["README.textile"]
20 |
21 | s.test_files = Dir['spec/**/*']
22 |
23 | s.add_dependency "daemons"
24 | s.add_development_dependency "rspec"
25 | s.add_development_dependency "sqlite3-ruby"
26 | end
27 |
28 | require 'spec/rake/spectask'
29 |
30 | task :default => :spec
31 |
32 | desc 'Run the specs'
33 | Spec::Rake::SpecTask.new(:spec) do |t|
34 | t.libs << 'lib'
35 | t.pattern = 'spec/**/*_spec.rb'
36 | t.verbose = true
37 | end
38 | task :spec => :check_dependencies
39 |
40 |
--------------------------------------------------------------------------------
/spec/spec_helper.rb:
--------------------------------------------------------------------------------
1 | $:.unshift(File.dirname(__FILE__) + '/../lib')
2 |
3 | require 'rubygems'
4 | require 'spec'
5 | require 'active_record'
6 | require 'delayed_job'
7 |
8 | logger = Logger.new('/tmp/dj.log')
9 | ActiveRecord::Base.logger = logger
10 | Delayed::Worker.logger = logger
11 | ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
12 | ActiveRecord::Migration.verbose = false
13 |
14 | ActiveRecord::Schema.define do
15 |
16 | create_table :delayed_jobs, :force => true do |table|
17 | table.integer :priority, :default => 0
18 | table.integer :attempts, :default => 0
19 | table.text :handler
20 | table.text :queue, :default => nil
21 | table.string :last_error
22 | table.datetime :run_at
23 | table.datetime :locked_at
24 | table.string :locked_by
25 | table.datetime :failed_at
26 | table.timestamps
27 | end
28 |
29 | create_table :stories, :force => true do |table|
30 | table.string :text
31 | end
32 |
33 | end
34 |
35 | # Purely useful for test cases...
36 | class Story < ActiveRecord::Base
37 | def tell; text; end
38 | def whatever(n, _); tell*n; end
39 | def whatever_else(n, _); tell*n; end
40 |
41 | handle_asynchronously :whatever
42 | handle_asynchronously_with_queue :whatever_else, "testqueue"
43 | end
44 |
45 | require 'sample_jobs'
46 | require 'shared_backend_spec'
--------------------------------------------------------------------------------
/spec/performable_method_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | class StoryReader
4 | def read(story)
5 | "Epilog: #{story.tell}"
6 | end
7 | end
8 |
9 | describe Delayed::PerformableMethod do
10 |
11 | it "should ignore ActiveRecord::RecordNotFound errors because they are permanent" do
12 | story = Story.create :text => 'Once upon...'
13 | p = Delayed::PerformableMethod.new(story, :tell, [])
14 | story.destroy
15 | lambda { p.perform }.should_not raise_error
16 | end
17 |
18 | it "should store the object as string if its an active record" do
19 | story = Story.create :text => 'Once upon...'
20 | p = Delayed::PerformableMethod.new(story, :tell, [])
21 | p.class.should == Delayed::PerformableMethod
22 | p.object.should == "LOAD;Story;#{story.id}"
23 | p.method.should == :tell
24 | p.args.should == []
25 | p.perform.should == 'Once upon...'
26 | end
27 |
28 | it "should allow class methods to be called on ActiveRecord models" do
29 | p = Delayed::PerformableMethod.new(Story, :count, [])
30 | lambda { p.send(:load, p.object) }.should_not raise_error
31 | end
32 |
33 | it "should store arguments as string if they are active record objects" do
34 | story = Story.create :text => 'Once upon...'
35 | reader = StoryReader.new
36 | p = Delayed::PerformableMethod.new(reader, :read, [story])
37 | p.class.should == Delayed::PerformableMethod
38 | p.method.should == :read
39 | p.args.should == ["LOAD;Story;#{story.id}"]
40 | p.perform.should == 'Epilog: Once upon...'
41 | end
42 |
43 |
44 |
45 | end
--------------------------------------------------------------------------------
/lib/delayed/performable_method.rb:
--------------------------------------------------------------------------------
1 | class Class
2 | def load_for_delayed_job(arg)
3 | self
4 | end
5 |
6 | def dump_for_delayed_job
7 | name
8 | end
9 | end
10 |
11 | module Delayed
12 | class PerformableMethod < Struct.new(:object, :method, :args)
13 | STRING_FORMAT = /^LOAD\;([A-Z][\w\:]+)(?:\;(\w+))?$/
14 |
15 | class LoadError < StandardError
16 | end
17 |
18 | def initialize(object, method, args)
19 | raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method)
20 |
21 | self.object = dump(object)
22 | self.args = args.map { |a| dump(a) }
23 | self.method = method.to_sym
24 | end
25 |
26 | def display_name
27 | if STRING_FORMAT === object
28 | "#{$1}#{$2 ? '#' : '.'}#{method}"
29 | else
30 | "#{object.class}##{method}"
31 | end
32 | end
33 |
34 | def perform
35 | load(object).send(method, *args.map{|a| load(a)})
36 | rescue PerformableMethod::LoadError
37 | # We cannot do anything about objects that can't be loaded
38 | true
39 | end
40 |
41 | private
42 |
43 | def load(obj)
44 | if STRING_FORMAT === obj
45 | $1.constantize.load_for_delayed_job($2)
46 | else
47 | obj
48 | end
49 | rescue => e
50 | Delayed::Worker.logger.warn "Could not load object for job: #{e.message}"
51 | raise PerformableMethod::LoadError
52 | end
53 |
54 | def dump(obj)
55 | if obj.respond_to?(:dump_for_delayed_job)
56 | "LOAD;#{obj.dump_for_delayed_job}"
57 | else
58 | obj
59 | end
60 | end
61 | end
62 | end
--------------------------------------------------------------------------------
/lib/delayed/message_sending.rb:
--------------------------------------------------------------------------------
1 | module Delayed
2 | module MessageSending
3 | def send_later(method, *args)
4 | Delayed::Job.enqueue Delayed::PerformableMethod.new(self, method.to_sym, args)
5 | end
6 |
7 | def send_later_with_queue(method, queue, *args)
8 | Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), :queue => queue)
9 | end
10 |
11 | def send_at(time, method, *args)
12 | Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), :priority => 0, :run_at => time)
13 | end
14 |
15 | def send_at_with_queue(time, method, queue, *args)
16 | Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), :priority => 0, :run_at => time, :queue => queue)
17 | end
18 |
19 | module ClassMethods
20 | def handle_asynchronously(method)
21 | aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
22 | with_method, without_method = "#{aliased_method}_with_send_later#{punctuation}", "#{aliased_method}_without_send_later#{punctuation}"
23 | define_method(with_method) do |*args|
24 | send_later(without_method, *args)
25 | end
26 | alias_method_chain method, :send_later
27 | end
28 |
29 | def handle_asynchronously_with_queue(method, queue)
30 | aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
31 | with_method, without_method = "#{aliased_method}_with_send_later_with_queue#{punctuation}", "#{aliased_method}_without_send_later_with_queue#{punctuation}"
32 | define_method(with_method) do |*args|
33 | send_later_with_queue(without_method, queue, *args)
34 | end
35 | alias_method_chain method, :send_later_with_queue
36 | end
37 | end
38 | end
39 | end
--------------------------------------------------------------------------------
/spec/mongo_job_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | require 'delayed/backend/mongo'
4 |
5 | MongoMapper.connection = Mongo::Connection.new nil, nil, :logger => ActiveRecord::Base.logger
6 | MongoMapper.database = 'delayed_job'
7 |
8 | describe Delayed::Backend::Mongo::Job do
9 | before(:all) do
10 | @backend = Delayed::Backend::Mongo::Job
11 | end
12 |
13 | before(:each) do
14 | MongoMapper.database.collections.each(&:remove)
15 | end
16 |
17 | it_should_behave_like 'a backend'
18 |
19 | describe "delayed method" do
20 | class MongoStoryReader
21 | def read(story)
22 | "Epilog: #{story.tell}"
23 | end
24 | end
25 |
26 | class MongoStory
27 | include MongoMapper::Document
28 | key :text, String
29 |
30 | def tell
31 | text
32 | end
33 | end
34 |
35 | it "should ignore not found errors because they are permanent" do
36 | story = MongoStory.create :text => 'Once upon a time…'
37 | job = story.send_later(:tell)
38 | story.destroy
39 | lambda { job.invoke_job }.should_not raise_error
40 | end
41 |
42 | it "should store the object as string" do
43 | story = MongoStory.create :text => 'Once upon a time…'
44 | job = story.send_later(:tell)
45 |
46 | job.payload_object.class.should == Delayed::PerformableMethod
47 | job.payload_object.object.should == "LOAD;MongoStory;#{story.id}"
48 | job.payload_object.method.should == :tell
49 | job.payload_object.args.should == []
50 | job.payload_object.perform.should == 'Once upon a time…'
51 | end
52 |
53 | it "should store arguments as string" do
54 | story = MongoStory.create :text => 'Once upon a time…'
55 | job = MongoStoryReader.new.send_later(:read, story)
56 | job.payload_object.class.should == Delayed::PerformableMethod
57 | job.payload_object.method.should == :read
58 | job.payload_object.args.should == ["LOAD;MongoStory;#{story.id}"]
59 | job.payload_object.perform.should == 'Epilog: Once upon a time…'
60 | end
61 | end
62 | end
--------------------------------------------------------------------------------
/delayed_job.gemspec:
--------------------------------------------------------------------------------
1 | # Generated by jeweler
2 | # DO NOT EDIT THIS FILE DIRECTLY
3 | # Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
4 | # -*- encoding: utf-8 -*-
5 |
6 | Gem::Specification.new do |s|
7 | s.name = %q{delayed_job}
8 | s.version = "1.8.5"
9 |
10 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11 | s.authors = ["Brandon Keepers", "Tobias L\303\274tke"]
12 | s.date = %q{2010-02-04}
13 | s.description = %q{Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks.}
14 | s.email = %q{tobi@leetsoft.com}
15 | s.extra_rdoc_files = [
16 | "README.textile"
17 | ]
18 | s.files = [
19 | ".gitignore",
20 | "MIT-LICENSE",
21 | "README.textile",
22 | "Rakefile",
23 | "VERSION",
24 | "contrib/delayed_job.monitrc",
25 | "delayed_job.gemspec",
26 | "generators/delayed_job/delayed_job_generator.rb",
27 | "generators/delayed_job/templates/migration.rb",
28 | "generators/delayed_job/templates/script",
29 | "init.rb",
30 | "lib/delayed/command.rb",
31 | "lib/delayed/message_sending.rb",
32 | "lib/delayed/performable_method.rb",
33 | "lib/delayed/recipes.rb",
34 | "lib/delayed/tasks.rb",
35 | "lib/delayed/worker.rb",
36 | "lib/delayed_job.rb",
37 | "recipes/delayed_job.rb",
38 | "spec/delayed_method_spec.rb",
39 | "spec/job_spec.rb",
40 | "spec/story_spec.rb",
41 | "tasks/jobs.rake"
42 | ]
43 | s.homepage = %q{http://github.com/collectiveidea/delayed_job}
44 | s.rdoc_options = ["--main", "README.textile", "--inline-source", "--line-numbers"]
45 | s.require_paths = ["lib"]
46 | s.rubygems_version = %q{1.3.5}
47 | s.summary = %q{Database-backed asynchronous priority queue system -- Extracted from Shopify}
48 | s.test_files = [
49 | "spec/database.rb",
50 | "spec/delayed_method_spec.rb",
51 | "spec/job_spec.rb",
52 | "spec/story_spec.rb"
53 | ]
54 |
55 | if s.respond_to? :specification_version then
56 | current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
57 | s.specification_version = 3
58 |
59 | if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
60 | else
61 | end
62 | else
63 | end
64 | end
65 |
66 |
--------------------------------------------------------------------------------
/lib/delayed/command.rb:
--------------------------------------------------------------------------------
1 | require 'rubygems'
2 | require 'daemons'
3 | require 'optparse'
4 |
5 | module Delayed
6 | class Command
7 | attr_accessor :worker_count, :process_name
8 |
9 | def initialize(args)
10 | @files_to_reopen = []
11 | @options = {:quiet => true}
12 |
13 | @worker_count = 1
14 |
15 | opts = OptionParser.new do |opts|
16 | opts.banner = "Usage: #{File.basename($0)} [options] start|stop|restart|run"
17 |
18 | opts.on('-h', '--help', 'Show this message') do
19 | puts opts
20 | exit 1
21 | end
22 | opts.on('-e', '--environment=NAME', 'Specifies the environment to run this delayed jobs under (test/development/production).') do |e|
23 | STDERR.puts "The -e/--environment option has been deprecated and has no effect. Use RAILS_ENV and see http://github.com/collectiveidea/delayed_job/issues/#issue/7"
24 | end
25 | opts.on('--min-priority N', 'Minimum priority of jobs to run.') do |n|
26 | @options[:min_priority] = n
27 | end
28 | opts.on('--max-priority N', 'Maximum priority of jobs to run.') do |n|
29 | @options[:max_priority] = n
30 | end
31 | opts.on('-n', '--number_of_workers=workers', "Number of unique workers to spawn") do |worker_count|
32 | @worker_count = worker_count.to_i rescue 1
33 | end
34 | opts.on('-p', '--process-name=NAME', "The name to append to the process name. eg. delayed_job_NAME") do |process_name|
35 | @process_name = process_name
36 | end
37 | opts.on('-q', '--queue=QUEUE_NAME', "The name of the queue for the workers to pull work from") do |queue|
38 | @options[:queue] = queue
39 | end
40 | end
41 | @args = opts.parse!(args)
42 | end
43 |
44 | def daemonize
45 | ObjectSpace.each_object(File) do |file|
46 | @files_to_reopen << file unless file.closed?
47 | end
48 |
49 | worker_count.times do |worker_index|
50 | base_name = @process_name ? "delayed_job_#{@process_name}" : "delayed_job"
51 | process_name = worker_count == 1 ? base_name : "#{base_name}.#{worker_index}"
52 | Daemons.run_proc(process_name, :dir => "#{RAILS_ROOT}/tmp/pids", :dir_mode => :normal, :ARGV => @args) do |*args|
53 | run process_name
54 | end
55 | end
56 | end
57 |
58 | def run(worker_name = nil)
59 | Dir.chdir(RAILS_ROOT)
60 |
61 | # Re-open file handles
62 | @files_to_reopen.each do |file|
63 | begin
64 | file.reopen File.join(RAILS_ROOT, 'log', 'delayed_job.log'), 'a+'
65 | file.sync = true
66 | rescue ::Exception
67 | end
68 | end
69 |
70 | ActiveRecord::Base.connection.reconnect!
71 |
72 | worker = Delayed::Worker.new(@options)
73 | worker.name_prefix = "#{worker_name} "
74 | worker.start
75 | rescue => e
76 | Rails.logger.fatal e
77 | STDERR.puts e.message
78 | exit 1
79 | end
80 |
81 | end
82 | end
83 |
--------------------------------------------------------------------------------
/lib/delayed/backend/base.rb:
--------------------------------------------------------------------------------
1 | module Delayed
2 | module Backend
3 | class DeserializationError < StandardError
4 | end
5 |
6 | module Base
7 | def self.included(base)
8 | base.extend ClassMethods
9 | end
10 |
11 | module ClassMethods
12 | # Add a job to the queue
13 | # The first argument should be an object that respond_to?(:perform)
14 | # The rest should be named arguments, these keys are expected:
15 | # :priority, :run_at, :queue
16 | # Example: Delayed::Job.enqueue(object, :priority => 0, :run_at => time, :queue => queue)
17 | def enqueue(*args)
18 | object = args.shift
19 | unless object.respond_to?(:perform)
20 | raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
21 | end
22 |
23 | options = args.first || {}
24 | options[:priority] ||= 0
25 | options[:payload_object] = object
26 | options[:queue] ||= Delayed::Worker.queue
27 | self.create(options)
28 | end
29 | end
30 |
31 | ParseObjectFromYaml = /\!ruby\/\w+\:([^\s]+)/
32 |
33 | def failed?
34 | failed_at
35 | end
36 | alias_method :failed, :failed?
37 |
38 | def payload_object
39 | @payload_object ||= deserialize(self['handler'])
40 | end
41 |
42 | def name
43 | @name ||= begin
44 | payload = payload_object
45 | if payload.respond_to?(:display_name)
46 | payload.display_name
47 | else
48 | payload.class.name
49 | end
50 | end
51 | end
52 |
53 | def payload_object=(object)
54 | self['handler'] = object.to_yaml
55 | end
56 |
57 | # Moved into its own method so that new_relic can trace it.
58 | def invoke_job
59 | payload_object.perform
60 | end
61 |
62 | # Unlock this job (note: not saved to DB)
63 | def unlock
64 | self.locked_at = nil
65 | self.locked_by = nil
66 | end
67 |
68 | private
69 |
70 | def deserialize(source)
71 | handler = YAML.load(source) rescue nil
72 |
73 | unless handler.respond_to?(:perform)
74 | if handler.nil? && source =~ ParseObjectFromYaml
75 | handler_class = $1
76 | end
77 | attempt_to_load(handler_class || handler.class)
78 | handler = YAML.load(source)
79 | end
80 |
81 | return handler if handler.respond_to?(:perform)
82 |
83 | raise DeserializationError,
84 | 'Job failed to load: Unknown handler. Try to manually require the appropriate file.'
85 | rescue TypeError, LoadError, NameError => e
86 | raise DeserializationError,
87 | "Job failed to load: #{e.message}. Try to manually require the required file."
88 | end
89 |
90 | # Constantize the object so that ActiveSupport can attempt
91 | # its auto loading magic. Will raise LoadError if not successful.
92 | def attempt_to_load(klass)
93 | klass.constantize
94 | end
95 |
96 | protected
97 |
98 | def before_save
99 | self.run_at ||= self.class.db_time_now
100 | end
101 |
102 | end
103 | end
104 | end
--------------------------------------------------------------------------------
/lib/delayed/backend/active_record.rb:
--------------------------------------------------------------------------------
1 | require 'active_record'
2 |
3 | class ActiveRecord::Base
4 | def self.load_for_delayed_job(id)
5 | if id
6 | find(id)
7 | else
8 | super
9 | end
10 | end
11 |
12 | def dump_for_delayed_job
13 | "#{self.class};#{id}"
14 | end
15 | end
16 |
17 | module Delayed
18 | module Backend
19 | module ActiveRecord
20 | # A job object that is persisted to the database.
21 | # Contains the work object as a YAML field.
22 | class Job < ::ActiveRecord::Base
23 | include Delayed::Backend::Base
24 | set_table_name :delayed_jobs
25 |
26 | named_scope :ready_to_run, lambda {|worker_name, max_run_time|
27 | {:conditions => ['(run_at <= ? AND (locked_at IS NULL OR locked_at < ?) OR locked_by = ?) AND failed_at IS NULL', db_time_now, db_time_now - max_run_time, worker_name]}
28 | }
29 | named_scope :by_priority, :order => 'priority ASC, run_at ASC'
30 |
31 | # When a worker is exiting, make sure we don't have any locked jobs.
32 | def self.clear_locks!(worker_name)
33 | update_all("locked_by = null, locked_at = null", ["locked_by = ?", worker_name])
34 | end
35 |
36 | # Find a few candidate jobs to run (in case some immediately get locked by others).
37 | def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time, queue=nil)
38 | scope = self.ready_to_run(worker_name, max_run_time)
39 | scope = scope.scoped(:conditions => ['priority >= ?', Worker.min_priority]) if Worker.min_priority
40 | scope = scope.scoped(:conditions => ['priority <= ?', Worker.max_priority]) if Worker.max_priority
41 | scope = scope.scoped(:conditions => ['queue = ?', queue]) if queue
42 | scope = scope.scoped(:conditions => ['queue is null']) unless queue
43 |
44 | ::ActiveRecord::Base.silence do
45 | scope.by_priority.all(:limit => limit)
46 | end
47 | end
48 |
49 | # Lock this job for this worker.
50 | # Returns true if we have the lock, false otherwise.
51 | def lock_exclusively!(max_run_time, worker)
52 | now = self.class.db_time_now
53 | affected_rows = if locked_by != worker
54 | # We don't own this job so we will update the locked_by name and the locked_at
55 | self.class.update_all(["locked_at = ?, locked_by = ?", now, worker], ["id = ? and (locked_at is null or locked_at < ?) and (run_at <= ?)", id, (now - max_run_time.to_i), now])
56 | else
57 | # We already own this job, this may happen if the job queue crashes.
58 | # Simply resume and update the locked_at
59 | self.class.update_all(["locked_at = ?", now], ["id = ? and locked_by = ?", id, worker])
60 | end
61 | if affected_rows == 1
62 | self.locked_at = now
63 | self.locked_by = worker
64 | return true
65 | else
66 | return false
67 | end
68 | end
69 |
70 | # Get the current time (GMT or local depending on DB)
71 | # Note: This does not ping the DB to get the time, so all your clients
72 | # must have syncronized clocks.
73 | def self.db_time_now
74 | if Time.zone
75 | Time.zone.now
76 | elsif ::ActiveRecord::Base.default_timezone == :utc
77 | Time.now.utc
78 | else
79 | Time.now
80 | end
81 | end
82 |
83 | end
84 | end
85 | end
86 | end
--------------------------------------------------------------------------------
/lib/delayed/backend/mongo.rb:
--------------------------------------------------------------------------------
1 | require 'mongo_mapper'
2 |
3 | module MongoMapper
4 | module Document
5 | module ClassMethods
6 | def load_for_delayed_job(id)
7 | find!(id)
8 | end
9 | end
10 |
11 | module InstanceMethods
12 | def dump_for_delayed_job
13 | "#{self.class};#{id}"
14 | end
15 | end
16 | end
17 | end
18 |
19 | module Delayed
20 | module Backend
21 | module Mongo
22 | class Job
23 | include MongoMapper::Document
24 | include Delayed::Backend::Base
25 | set_collection_name 'delayed_jobs'
26 |
27 | key :priority, Integer, :default => 0
28 | key :attempts, Integer, :default => 0
29 | key :handler, String
30 | key :run_at, Time
31 | key :locked_at, Time
32 | key :locked_by, String
33 | key :failed_at, Time
34 | key :last_error, String
35 | timestamps!
36 |
37 | before_save :set_default_run_at
38 |
39 | def self.db_time_now
40 | MongoMapper.time_class.now.utc
41 | end
42 |
43 | def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time, queue=nil)
44 | where = "this.run_at <= new Date(#{db_time_now.to_f * 1000}) && (this.locked_at == null || this.locked_at < new Date(#{(db_time_now - max_run_time).to_f * 1000})) || this.locked_by == #{worker_name.to_json}"
45 | # all(:limit => limit, :failed_at => nil, '$where' => where)
46 |
47 | conditions = {
48 | '$where' => where,
49 | :limit => limit,
50 | :failed_at => nil,
51 | :queue => queue,
52 | :sort => [['priority', 1], ['run_at', 1]]
53 | }
54 |
55 | # (conditions[:priority] ||= {})['$gte'] = Worker.min_priority if Worker.min_priority
56 | # (conditions[:priority] ||= {})['$lte'] = Worker.max_priority if Worker.max_priority
57 |
58 | all(conditions)
59 | end
60 |
61 | # When a worker is exiting, make sure we don't have any locked jobs.
62 | def self.clear_locks!(worker_name)
63 | collection.update({:locked_by => worker_name}, {"$set" => {:locked_at => nil, :locked_by => nil}}, :multi => true)
64 | end
65 |
66 | # Lock this job for this worker.
67 | # Returns true if we have the lock, false otherwise.
68 | def lock_exclusively!(max_run_time, worker = worker_name)
69 | now = self.class.db_time_now
70 | overtime = make_date(now - max_run_time.to_i)
71 |
72 | query = "this._id == #{id.to_json} && this.run_at <= #{make_date(now)} && (this.locked_at == null || this.locked_at < #{overtime} || this.locked_by == #{worker.to_json})"
73 |
74 | conditions = {"$where" => make_query(query)}
75 | collection.update(conditions, {"$set" => {:locked_at => now, :locked_by => worker}}, :multi => true)
76 | affected_rows = collection.find({:_id => id, :locked_by => worker}).count
77 | if affected_rows == 1
78 | self.locked_at = now
79 | self.locked_by = worker
80 | return true
81 | else
82 | return false
83 | end
84 | end
85 |
86 | private
87 |
88 | def self.make_date(date)
89 | "new Date(#{date.to_f * 1000})"
90 | end
91 |
92 | def make_date(date)
93 | self.class.make_date(date)
94 | end
95 |
96 | def self.make_query(string)
97 | "function() { return (#{string}); }"
98 | end
99 |
100 | def make_query(string)
101 | self.class.make_query(string)
102 | end
103 |
104 |
105 | def set_default_run_at
106 | self.run_at ||= self.class.db_time_now
107 | end
108 | end
109 | end
110 | end
111 | end
--------------------------------------------------------------------------------
/spec/delayed_method_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe 'random ruby objects' do
4 | before :each do
5 | Delayed::Worker.queue = nil
6 | Delayed::Job.delete_all
7 | end
8 |
9 | it "should respond_to :send_later method" do
10 | Object.new.respond_to?(:send_later)
11 | end
12 |
13 | it "should raise a ArgumentError if send_later is called but the target method doesn't exist" do
14 | lambda { Object.new.send_later(:method_that_deos_not_exist) }.should raise_error(NoMethodError)
15 | end
16 |
17 | it "should add a new entry to the job table when send_later is called on it" do
18 | lambda { Object.new.send_later(:to_s) }.should change { Delayed::Job.count }.by(1)
19 | end
20 |
21 | it "should add a new entry to the job table when send_later_with_queue is called on it" do
22 | lambda { Object.new.send_later_with_queue(:to_s, "testqueue") }.should change { Delayed::Job.count }.by(1)
23 | end
24 |
25 | it "should add a new entry to the job table when send_later is called on the class" do
26 | lambda { Object.send_later(:to_s) }.should change { Delayed::Job.count }.by(1)
27 | end
28 |
29 | it "should add a new entry to the job table when send_later_with_queue is called on the class" do
30 | lambda { Object.send_later_with_queue(:to_s, "testqueue") }.should change { Delayed::Job.count }.by(1)
31 | end
32 |
33 | it "should call send later on methods which are wrapped with handle_asynchronously" do
34 | story = Story.create :text => 'Once upon...'
35 |
36 | Delayed::Job.count.should == 0
37 |
38 | story.whatever(1, 5)
39 |
40 | Delayed::Job.count.should == 1
41 | job = Delayed::Job.find(:first)
42 | job.payload_object.class.should == Delayed::PerformableMethod
43 | job.payload_object.method.should == :whatever_without_send_later
44 | job.payload_object.args.should == [1, 5]
45 | job.payload_object.perform.should == 'Once upon...'
46 | end
47 |
48 | it "should call send later on methods which are wrapped with handle_asynchronously_with_queue" do
49 | story = Story.create :text => 'Once upon...'
50 |
51 | Delayed::Job.count.should == 0
52 |
53 | story.whatever_else(1, 5)
54 |
55 | Delayed::Job.count.should == 1
56 | job = Delayed::Job.find(:first)
57 | job.payload_object.class.should == Delayed::PerformableMethod
58 | job.payload_object.method.should == :whatever_else_without_send_later_with_queue
59 | job.payload_object.args.should == [1, 5]
60 | job.payload_object.perform.should == 'Once upon...'
61 | end
62 |
63 | context "send_later" do
64 | it "should use the default queue if there is one" do
65 | Delayed::Worker.queue = "testqueue"
66 | job = "string".send_later :reverse
67 | job.queue.should == "testqueue"
68 | end
69 |
70 | it "should have nil queue if there is not a default" do
71 | job = "string".send_later :reverse
72 | job.queue.should == nil
73 | end
74 | end
75 |
76 | context "send_at" do
77 | it "should queue a new job" do
78 | lambda do
79 | "string".send_at(1.hour.from_now, :length)
80 | end.should change { Delayed::Job.count }.by(1)
81 | end
82 |
83 | it "should schedule the job in the future" do
84 | time = 1.hour.from_now
85 | job = "string".send_at(time, :length)
86 | job.run_at.should == time
87 | end
88 |
89 | it "should store payload as PerformableMethod" do
90 | job = "string".send_at(1.hour.from_now, :count, 'r')
91 | job.payload_object.class.should == Delayed::PerformableMethod
92 | job.payload_object.method.should == :count
93 | job.payload_object.args.should == ['r']
94 | job.payload_object.perform.should == 1
95 | end
96 |
97 | it "should use the default queue if there is one" do
98 | Delayed::Worker.queue = "testqueue"
99 | job = "string".send_at 1.hour.from_now, :reverse
100 | job.queue.should == "testqueue"
101 | end
102 |
103 | it "should have nil queue if there is not a default" do
104 | job = "string".send_at 1.hour.from_now, :reverse
105 | job.queue.should == nil
106 | end
107 | end
108 |
109 | context "send_at_with_queue" do
110 | it "should queue a new job" do
111 | lambda do
112 | "string".send_at_with_queue(1.hour.from_now, :length, "testqueue")
113 | end.should change { Delayed::Job.count }.by(1)
114 | end
115 |
116 | it "should schedule the job in the future" do
117 | time = 1.hour.from_now
118 | job = "string".send_at_with_queue(time, :length, "testqueue")
119 | job.run_at.should == time
120 | end
121 |
122 | it "should override the default queue" do
123 | Delayed::Worker.queue = "default_queue"
124 | job = "string".send_at_with_queue(1.hour.from_now, :length, "testqueue")
125 | job.queue.should == "testqueue"
126 | end
127 |
128 | it "should store payload as PerformableMethod" do
129 | job = "string".send_at_with_queue(1.hour.from_now, :count, "testqueue", 'r')
130 | job.payload_object.class.should == Delayed::PerformableMethod
131 | job.payload_object.method.should == :count
132 | job.payload_object.args.should == ['r']
133 | job.payload_object.perform.should == 1
134 | end
135 | end
136 |
137 | end
138 |
--------------------------------------------------------------------------------
/lib/delayed/worker.rb:
--------------------------------------------------------------------------------
1 | require 'timeout'
2 |
3 | module Delayed
4 | class Worker
5 | cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :sleep_delay, :logger, :queue
6 | self.sleep_delay = 5
7 | self.max_attempts = 25
8 | self.max_run_time = 4.hours
9 | self.queue = nil
10 |
11 | # By default failed jobs are destroyed after too many attempts. If you want to keep them around
12 | # (perhaps to inspect the reason for the failure), set this to false.
13 | cattr_accessor :destroy_failed_jobs
14 | self.destroy_failed_jobs = true
15 |
16 | self.logger = if defined?(Merb::Logger)
17 | Merb.logger
18 | elsif defined?(RAILS_DEFAULT_LOGGER)
19 | RAILS_DEFAULT_LOGGER
20 | end
21 |
22 | # name_prefix is ignored if name is set directly
23 | attr_accessor :name_prefix, :queue
24 |
25 | cattr_reader :backend
26 |
27 | def self.backend=(backend)
28 | if backend.is_a? Symbol
29 | require "delayed/backend/#{backend}"
30 | backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
31 | end
32 | @@backend = backend
33 | silence_warnings { ::Delayed.const_set(:Job, backend) }
34 | end
35 |
36 | def initialize(options={})
37 | @quiet = options[:quiet]
38 | @queue = options[:queue] || self.class.queue
39 | self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
40 | self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
41 | @already_retried = false
42 | end
43 |
44 | # Every worker has a unique name which by default is the pid of the process. There are some
45 | # advantages to overriding this with something which survives worker retarts: Workers can#
46 | # safely resume working on tasks which are locked by themselves. The worker will assume that
47 | # it crashed before.
48 | def name
49 | return @name unless @name.nil?
50 | "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
51 | end
52 |
53 | # Sets the name of the worker.
54 | # Setting the name to nil will reset the default worker name
55 | def name=(val)
56 | @name = val
57 | end
58 |
59 | def start
60 | say "*** Starting job worker #{name}"
61 |
62 | trap('TERM') { say 'Exiting...'; $exit = true }
63 | trap('INT') { say 'Exiting...'; $exit = true }
64 |
65 | loop do
66 | result = nil
67 |
68 | realtime = Benchmark.realtime do
69 | result = work_off
70 | end
71 |
72 | count = result.sum
73 |
74 | break if $exit
75 |
76 | if count.zero?
77 | sleep(@@sleep_delay)
78 | else
79 | say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
80 | end
81 |
82 | break if $exit
83 | end
84 |
85 | ensure
86 | Delayed::Job.clear_locks!(name)
87 | end
88 |
89 | # Do num jobs and return stats on success/failure.
90 | # Exit early if interrupted.
91 | def work_off(num = 100)
92 | success, failure = 0, 0
93 |
94 | num.times do
95 | case reserve_and_run_one_job
96 | when true
97 | success += 1
98 | when false
99 | failure += 1
100 | else
101 | break # leave if no work could be done
102 | end
103 | break if $exit # leave if we're exiting
104 | end
105 |
106 | return [success, failure]
107 | end
108 |
109 | def run(job)
110 | self.ensure_db_connection
111 | runtime = Benchmark.realtime do
112 | Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
113 | job.destroy
114 | end
115 | # TODO: warn if runtime > max_run_time ?
116 | say "* [JOB] #{name} completed after %.4f" % runtime
117 | return true # did work
118 | rescue Exception => e
119 | handle_failed_job(job, e)
120 | return false # work failed
121 | end
122 |
123 | # Reschedule the job in the future (when a job fails).
124 | # Uses an exponential scale depending on the number of failed attempts.
125 | def reschedule(job, time = nil)
126 | if (job.attempts += 1) < self.class.max_attempts
127 | time ||= Job.db_time_now + (job.attempts ** 4) + 5
128 | job.run_at = time
129 | job.unlock
130 | job.save!
131 | else
132 | say "* [JOB] PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
133 | self.class.destroy_failed_jobs ? job.destroy : job.update_attribute(:failed_at, Delayed::Job.db_time_now)
134 | end
135 | end
136 |
137 | def say(text, level = Logger::INFO)
138 | puts text unless @quiet
139 | logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
140 | end
141 |
142 | protected
143 |
144 | def handle_failed_job(job, error)
145 | job.last_error = error.message + "\n" + error.backtrace.join("\n")
146 | say "* [JOB] #{name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
147 | reschedule(job)
148 | end
149 |
150 | # Run the next job we can get an exclusive lock on.
151 | # If no jobs are left we return nil
152 | def reserve_and_run_one_job
153 |
154 | # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
155 | # this leads to a more even distribution of jobs across the worker processes
156 | job = Delayed::Job.find_available(name, 5, self.class.max_run_time, @queue).detect do |job|
157 | if job.lock_exclusively!(self.class.max_run_time, name)
158 | say "* [Worker(#{name})] acquired lock on #{job.name}"
159 | true
160 | else
161 | say "* [Worker(#{name})] failed to acquire exclusive lock for #{job.name}", Logger::WARN
162 | false
163 | end
164 | end
165 |
166 | run(job) if job
167 | end
168 |
169 | # Makes a dummy call to the database to make sure we're still connected
170 | def ensure_db_connection
171 | begin
172 | ActiveRecord::Base.connection.execute("select 'I am alive'")
173 | rescue ActiveRecord::StatementInvalid
174 | ActiveRecord::Base.connection.reconnect!
175 | unless @already_retried
176 | @already_retried = true
177 | retry
178 | end
179 | raise
180 | else
181 | @already_retried = false
182 | end
183 | end
184 |
185 | end
186 | end
187 |
--------------------------------------------------------------------------------
/spec/worker_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Delayed::Worker do
4 | def job_create(opts = {})
5 | Delayed::Job.create(opts.merge(:payload_object => SimpleJob.new))
6 | end
7 | def worker_create(opts = {})
8 | Delayed::Worker.new(opts.merge(:max_priority => nil, :min_priority => nil, :quiet => true))
9 | end
10 |
11 | before(:all) do
12 | Delayed::Worker.send :public, :work_off
13 | end
14 |
15 | before(:each) do
16 | # Make sure backend is set to active record
17 | Delayed::Worker.backend = :active_record
18 |
19 | @worker = worker_create
20 |
21 | Delayed::Job.delete_all
22 |
23 | SimpleJob.runs = 0
24 | end
25 |
26 | describe "backend=" do
27 | it "should set the Delayed::Job constant to the backend" do
28 | @clazz = Class.new
29 | Delayed::Worker.backend = @clazz
30 | Delayed::Job.should == @clazz
31 | end
32 |
33 | it "should set backend with a symbol" do
34 | Delayed::Worker.backend = Class.new
35 | Delayed::Worker.backend = :active_record
36 | Delayed::Worker.backend.should == Delayed::Backend::ActiveRecord::Job
37 | end
38 | end
39 |
40 | describe "running a job" do
41 | it "should fail after Worker.max_run_time" do
42 | begin
43 | old_max_run_time = Delayed::Worker.max_run_time
44 | Delayed::Worker.max_run_time = 1.second
45 | @job = Delayed::Job.create :payload_object => LongRunningJob.new
46 | @worker.run(@job)
47 | @job.reload.last_error.should =~ /expired/
48 | @job.attempts.should == 1
49 | ensure
50 | Delayed::Worker.max_run_time = old_max_run_time
51 | end
52 | end
53 | end
54 |
55 | context "worker prioritization" do
56 | before(:each) do
57 | @worker = Delayed::Worker.new(:max_priority => 5, :min_priority => -5, :quiet => true)
58 | end
59 |
60 | it "should only work_off jobs that are >= min_priority" do
61 | SimpleJob.runs.should == 0
62 |
63 | job_create(:priority => -10)
64 | job_create(:priority => 0)
65 | @worker.work_off
66 |
67 | SimpleJob.runs.should == 1
68 | end
69 |
70 | it "should only work_off jobs that are <= max_priority" do
71 | SimpleJob.runs.should == 0
72 |
73 | job_create(:priority => 10)
74 | job_create(:priority => 0)
75 |
76 | @worker.work_off
77 |
78 | SimpleJob.runs.should == 1
79 | end
80 | end
81 |
82 | context "while running with locked and expired jobs" do
83 | before(:each) do
84 | @worker.name = 'worker1'
85 | end
86 |
87 | it "should not run jobs locked by another worker" do
88 | job_create(:locked_by => 'other_worker', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
89 | lambda { @worker.work_off }.should_not change { SimpleJob.runs }
90 | end
91 |
92 | it "should run open jobs" do
93 | job_create
94 | lambda { @worker.work_off }.should change { SimpleJob.runs }.from(0).to(1)
95 | end
96 |
97 | it "should run expired jobs" do
98 | expired_time = Delayed::Job.db_time_now - (1.minutes + Delayed::Worker.max_run_time)
99 | job_create(:locked_by => 'other_worker', :locked_at => expired_time)
100 | lambda { @worker.work_off }.should change { SimpleJob.runs }.from(0).to(1)
101 | end
102 |
103 | it "should run own jobs" do
104 | job_create(:locked_by => @worker.name, :locked_at => (Delayed::Job.db_time_now - 1.minutes))
105 | lambda { @worker.work_off }.should change { SimpleJob.runs }.from(0).to(1)
106 | end
107 | end
108 |
109 | describe "failed jobs" do
110 | before do
111 | # reset defaults
112 | Delayed::Worker.destroy_failed_jobs = true
113 | Delayed::Worker.max_attempts = 25
114 |
115 | @job = Delayed::Job.enqueue ErrorJob.new
116 | end
117 |
118 | it "should record last_error when destroy_failed_jobs = false, max_attempts = 1" do
119 | Delayed::Worker.destroy_failed_jobs = false
120 | Delayed::Worker.max_attempts = 1
121 | @worker.run(@job)
122 | @job.reload
123 | @job.last_error.should =~ /did not work/
124 | @job.last_error.should =~ /worker_spec.rb/
125 | @job.attempts.should == 1
126 | @job.failed_at.should_not be_nil
127 | end
128 |
129 | it "should re-schedule jobs after failing" do
130 | @worker.run(@job)
131 | @job.reload
132 | @job.last_error.should =~ /did not work/
133 | @job.last_error.should =~ /sample_jobs.rb:8:in `perform'/
134 | @job.attempts.should == 1
135 | @job.run_at.should > Delayed::Job.db_time_now - 10.minutes
136 | @job.run_at.should < Delayed::Job.db_time_now + 10.minutes
137 | end
138 | end
139 |
140 | context "reschedule" do
141 | before do
142 | @job = Delayed::Job.create :payload_object => SimpleJob.new
143 | end
144 |
145 | context "and we want to destroy jobs" do
146 | before do
147 | Delayed::Worker.destroy_failed_jobs = true
148 | end
149 |
150 | it "should be destroyed if it failed more than Worker.max_attempts times" do
151 | @job.should_receive(:destroy)
152 | Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
153 | end
154 |
155 | it "should not be destroyed if failed fewer than Worker.max_attempts times" do
156 | @job.should_not_receive(:destroy)
157 | (Delayed::Worker.max_attempts - 1).times { @worker.reschedule(@job) }
158 | end
159 | end
160 |
161 | context "and we don't want to destroy jobs" do
162 | before do
163 | Delayed::Worker.destroy_failed_jobs = false
164 | end
165 |
166 | it "should be failed if it failed more than Worker.max_attempts times" do
167 | @job.reload.failed_at.should == nil
168 | Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
169 | @job.reload.failed_at.should_not == nil
170 | end
171 |
172 | it "should not be failed if it failed fewer than Worker.max_attempts times" do
173 | (Delayed::Worker.max_attempts - 1).times { @worker.reschedule(@job) }
174 | @job.reload.failed_at.should == nil
175 | end
176 |
177 | end
178 | end
179 |
180 |
181 | context "Queue workers" do
182 | before :each do
183 | Delayed::Worker.queue = nil
184 | job_create(:queue => 'queue1')
185 | job_create(:queue => 'queue2')
186 | job_create(:queue => nil)
187 | end
188 |
189 | it "should only work off jobs assigned to themselves" do
190 | worker = worker_create(:queue=>'queue1')
191 | SimpleJob.runs.should == 0
192 | worker.work_off
193 | SimpleJob.runs.should == 1
194 |
195 | SimpleJob.runs = 0
196 |
197 | worker = worker_create(:queue=>'queue2')
198 | SimpleJob.runs.should == 0
199 | worker.work_off
200 | SimpleJob.runs.should == 1
201 | end
202 |
203 | it "should not work off jobs not assigned to themselves" do
204 | worker = worker_create(:queue=>'queue3')
205 |
206 | SimpleJob.runs.should == 0
207 | worker.work_off
208 | SimpleJob.runs.should == 0
209 | end
210 |
211 | it "should run non-named queue jobs when the queue has no name set" do
212 | worker = worker_create(:queue=>nil)
213 | SimpleJob.runs.should == 0
214 | worker.work_off
215 | SimpleJob.runs.should == 1
216 | end
217 |
218 | it "should get the default queue if none is set" do
219 | queue_name = "default_queue"
220 | Delayed::Worker.queue = queue_name
221 | worker = worker_create(:queue=>nil)
222 | worker.queue.should == queue_name
223 | end
224 |
225 | it "should override default queue name if specified in initialize" do
226 | queue_name = "my_queue"
227 | Delayed::Worker.queue = "default_queue"
228 | worker = worker_create(:queue=>queue_name)
229 | worker.queue.should == queue_name
230 | end
231 |
232 | it "shouldn't have a queue if none is set" do
233 | worker = worker_create(:queue=>nil)
234 | worker.queue.should == nil
235 | end
236 | end
237 | end
238 |
--------------------------------------------------------------------------------
/README.textile:
--------------------------------------------------------------------------------
1 | h1. Delayed::Job
2 |
3 | Delated_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background.
4 |
5 | This fork adds a 'queue' column for each job, and the ability to specify which queue a worker should pull from.
6 |
7 | It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks. Amongst those tasks are:
8 |
9 | * sending massive newsletters
10 | * image resizing
11 | * http downloads
12 | * updating smart collections
13 | * updating solr, our search server, after product changes
14 | * batch imports
15 | * spam checks
16 |
17 | h2. Installation
18 |
19 | To install as a gem, add the following to @config/environment.rb@:
20 |
21 |
22 | config.gem 'delayed_job'
23 |
24 |
25 | Rake tasks are not automatically loaded from gems, so you'll need to add the following to your Rakefile:
26 |
27 |
28 | begin
29 | require 'delayed/tasks'
30 | rescue LoadError
31 | STDERR.puts "Run `rake gems:install` to install delayed_job"
32 | end
33 |
34 |
35 | To install as a plugin: (main branch)
36 |
37 |
38 | script/plugin install git://github.com/collectiveidea/delayed_job.git
39 |
40 |
41 | To install as a plugin: (this fork)
42 |
43 |
44 | script/plugin install git://github.com/bracken/delayed_job.git
45 |
46 |
47 | After delayed_job is installed, you will need to setup the backend.
48 |
49 | h2. Backends
50 |
51 | delayed_job supports multiple backends for storing the job queue. The default is Active Record, which requires a jobs table.
52 |
53 |
54 | script/generate delayed_job
55 | rake db:migrate
56 |
57 |
58 | You can change the backend in an initializer:
59 |
60 |
61 | # config/initializers/delayed_job.rb
62 | Delayed::Worker.backend = :mongo
63 |
64 |
65 | h2. Upgrading to 1.8
66 |
67 | If you are upgrading from a previous release, you will need to generate the new @script/delayed_job@:
68 |
69 |
70 | script/generate delayed_job --skip-migration
71 |
72 |
73 | Known Issues: script/delayed_job does not work properly with anything besides the Active Record backend. That will be resolved before the next gem release.
74 |
75 | h2. Queuing Jobs
76 |
77 | Call @#send_later(method, params)@ on any object and it will be processed in the background.
78 |
79 |
80 | # without delayed_job
81 | Notifier.deliver_signup(@user)
82 |
83 | # with delayed_job
84 | Notifier.send_later :deliver_signup, @user
85 |
86 | # putting it in a specific queue
87 | Notifier.send_later_with_queue :deliver_signup, "my_queue", @user
88 |
89 |
90 | If a method should always be run in the background, you can call @#handle_asynchronously@ after the method declaration:
91 |
92 |
93 | class Device
94 | def deliver
95 | # long running method
96 | end
97 | handle_asynchronously :deliver
98 | #handle_asynchronously_with_queue :deliver, "my_queue"
99 | end
100 |
101 | device = Device.new
102 | device.deliver
103 |
104 |
105 | h2. Running Jobs
106 |
107 | @script/delayed_job@ can be used to manage a background process which will start working off jobs.
108 |
109 |
110 | $ RAILS_ENV=production script/delayed_job start
111 | $ RAILS_ENV=production script/delayed_job stop
112 |
113 | # Runs two workers in separate processes.
114 | $ RAILS_ENV=production script/delayed_job -n 2 start
115 | $ RAILS_ENV=production script/delayed_job stop
116 |
117 | # Specify the queue the worker should work out of with -q/--queue=QUEUE_NAME
118 | $ RAILS_ENV=production script/delayed_job start --queue=my_queue
119 | $ RAILS_ENV=production script/delayed_job stop
120 |
121 | # Specify a name to append to the process name -p/--process-name=NAME
122 | # the process name will be "delayed_job_#{NAME}"
123 | $ RAILS_ENV=production script/delayed_job start -p custom_name
124 | $ RAILS_ENV=production script/delayed_job stop
125 |
126 |
127 | Workers can be running on any computer, as long as they have access to the database and their clock is in sync. Keep in mind that each worker will check the database at least every 5 seconds.
128 |
129 | You can also invoke @RAILS_ENV=development rake jobs:work@ which will start working off jobs. You can cancel the rake task with @CTRL-C@.
130 | You can specify the queue for the rake task: @RAILS_ENV=development rake jobs:work[my_queue]@, or: @RAILS_ENV=development QUEUE=my_queue rake jobs:work@
131 |
132 | h2. Custom Jobs
133 |
134 | Jobs are simple ruby objects with a method called perform. Any object which responds to perform can be stuffed into the jobs table. Job objects are serialized to yaml so that they can later be resurrected by the job runner.
135 |
136 |
137 | class NewsletterJob < Struct.new(:text, :emails)
138 | def perform
139 | emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) }
140 | end
141 | end
142 |
143 | Delayed::Job.enqueue(NewsletterJob.new('lorem ipsum...', Customers.find(:all).collect(&:email)), :queue=>"my_queue")
144 |
145 |
146 | h2. Gory Details
147 |
148 | The library evolves around a delayed_jobs table which looks as follows:
149 |
150 |
151 | create_table :delayed_jobs, :force => true do |table|
152 | table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
153 | table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
154 | table.text :handler # YAML-encoded string of the object that will do work
155 | table.text :last_error # reason for last failure (See Note below)
156 | table.text :queue, :default => nil # The queue that this job is in
157 | table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
158 | table.datetime :locked_at # Set when a client is working on this object
159 | table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
160 | table.string :locked_by # Who is working on this object (if locked)
161 | table.timestamps
162 | end
163 |
164 |
165 | On failure, the job is scheduled again in 5 seconds + N ** 4, where N is the number of retries.
166 |
167 | The default Worker.max_attempts is 25. After this, the job either deleted (default), or left in the database with "failed_at" set.
168 | With the default of 25 attempts, the last retry will be 20 days later, with the last interval being almost 100 hours.
169 |
170 | The default Worker.max_run_time is 4.hours. If your job takes longer than that, another computer could pick it up. It's up to you to
171 | make sure your job doesn't exceed this time. You should set this to the longest time you think the job could take.
172 |
173 | By default, it will delete failed jobs (and it always deletes successful jobs). If you want to keep failed jobs, set
174 | Delayed::Worker.destroy_failed_jobs = false. The failed jobs will be marked with non-null failed_at.
175 |
176 | Here is an example of changing job parameters in Rails:
177 |
178 |
179 | # config/initializers/delayed_job_config.rb
180 | Delayed::Worker.destroy_failed_jobs = false
181 | Delayed::Worker.sleep_delay = 60
182 | Delayed::Worker.max_attempts = 3
183 | Delayed::Worker.max_run_time = 5.minutes
184 | Delayed::Worker.queue = "my_queue"
185 |
186 |
187 | h3. Cleaning up
188 |
189 | You can invoke @rake jobs:clear@ to delete all jobs in the queue.
190 | To only delete jobs for a specific queue run @rake jobs:clear[queue_name]@.
191 |
192 | h2. Mailing List
193 |
194 | Join us on the mailing list at http://groups.google.com/group/delayed_job
195 |
196 | h2. How to contribute
197 |
198 | If you find what looks like a bug:
199 |
200 | # Check the GitHub issue tracker to see if anyone else has had the same issue.
201 | http://github.com/collectiveidea/delayed_job/issues/
202 | # If you don't see anything, create an issue with information on how to reproduce it.
203 |
204 | If you want to contribute an enhancement or a fix:
205 |
206 | # Fork the project on github.
207 | http://github.com/collectiveidea/delayed_job/
208 | # Make your changes with tests.
209 | # Commit the changes without making changes to the Rakefile, VERSION, or any other files that aren't related to your enhancement or fix
210 | # Send a pull request.
211 |
212 | h3. Changes
213 |
214 | * 1.7.0: Added failed_at column which can optionally be set after a certain amount of failed job attempts. By default failed job attempts are destroyed after about a month.
215 |
216 | * 1.6.0: Renamed locked_until to locked_at. We now store when we start a given job instead of how long it will be locked by the worker. This allows us to get a reading on how long a job took to execute.
217 |
218 | * 1.5.0: Job runners can now be run in parallel. Two new database columns are needed: locked_until and locked_by. This allows us to use pessimistic locking instead of relying on row level locks. This enables us to run as many worker processes as we need to speed up queue processing.
219 |
220 | * 1.2.0: Added #send_later to Object for simpler job creation
221 |
222 | * 1.0.0: Initial release
223 |
--------------------------------------------------------------------------------
/spec/shared_backend_spec.rb:
--------------------------------------------------------------------------------
1 | shared_examples_for 'a backend' do
2 | def create_job(opts = {})
3 | @backend.create(opts.merge(:payload_object => SimpleJob.new))
4 | end
5 |
6 | before do
7 | SimpleJob.runs = 0
8 | end
9 |
10 | it "should set run_at automatically if not set" do
11 | @backend.create(:payload_object => ErrorJob.new ).run_at.should_not be_nil
12 | end
13 |
14 | it "should not set run_at automatically if already set" do
15 | later = @backend.db_time_now + 5.minutes
16 | @backend.create(:payload_object => ErrorJob.new, :run_at => later).run_at.should be_close(later, 1)
17 | end
18 |
19 | it "should raise ArgumentError when handler doesn't respond_to :perform" do
20 | lambda { @backend.enqueue(Object.new) }.should raise_error(ArgumentError)
21 | end
22 |
23 | it "should increase count after enqueuing items" do
24 | @backend.enqueue SimpleJob.new
25 | @backend.count.should == 1
26 | end
27 |
28 | it "should be able to set priority when enqueuing items" do
29 | @job = @backend.enqueue SimpleJob.new, :priority => 5
30 | @job.priority.should == 5
31 | end
32 |
33 | it "should be able to set run_at when enqueuing items" do
34 | later = @backend.db_time_now + 5.minutes
35 | @job = @backend.enqueue SimpleJob.new, :priority => 5, :run_at => later
36 | @job.run_at.should be_close(later, 1)
37 | end
38 |
39 | it "should work with jobs in modules" do
40 | M::ModuleJob.runs = 0
41 | job = @backend.enqueue M::ModuleJob.new
42 | lambda { job.invoke_job }.should change { M::ModuleJob.runs }.from(0).to(1)
43 | end
44 |
45 | it "should raise an DeserializationError when the job class is totally unknown" do
46 | job = @backend.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
47 | lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
48 | end
49 |
50 | it "should try to load the class when it is unknown at the time of the deserialization" do
51 | job = @backend.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
52 | job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
53 | lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
54 | end
55 |
56 | it "should try include the namespace when loading unknown objects" do
57 | job = @backend.new :handler => "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
58 | job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
59 | lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
60 | end
61 |
62 | it "should also try to load structs when they are unknown (raises TypeError)" do
63 | job = @backend.new :handler => "--- !ruby/struct:JobThatDoesNotExist {}"
64 | job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
65 | lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
66 | end
67 |
68 | it "should try include the namespace when loading unknown structs" do
69 | job = @backend.new :handler => "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
70 | job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
71 | lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
72 | end
73 |
74 | describe "find_available" do
75 | it "should not find failed jobs" do
76 | @job = create_job :attempts => 50, :failed_at => @backend.db_time_now
77 | @backend.find_available('worker', 5, 1.second).should_not include(@job)
78 | end
79 |
80 | it "should not find jobs scheduled for the future" do
81 | @job = create_job :run_at => (@backend.db_time_now + 1.minute)
82 | @backend.find_available('worker', 5, 4.hours).should_not include(@job)
83 | end
84 |
85 | it "should not find jobs locked by another worker" do
86 | @job = create_job(:locked_by => 'other_worker', :locked_at => @backend.db_time_now - 1.minute)
87 | @backend.find_available('worker', 5, 4.hours).should_not include(@job)
88 | end
89 |
90 | it "should find open jobs" do
91 | @job = create_job
92 | @backend.find_available('worker', 5, 4.hours).should include(@job)
93 | end
94 |
95 | it "should find expired jobs" do
96 | @job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now - 2.minutes)
97 | @backend.find_available('worker', 5, 1.minute).should include(@job)
98 | end
99 |
100 | it "should find own jobs" do
101 | @job = create_job(:locked_by => 'worker', :locked_at => (@backend.db_time_now - 1.minutes))
102 | @backend.find_available('worker', 5, 4.hours).should include(@job)
103 | end
104 | end
105 |
106 | context "when another worker is already performing an task, it" do
107 |
108 | before :each do
109 | @job = @backend.create :payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => @backend.db_time_now - 5.minutes
110 | end
111 |
112 | it "should not allow a second worker to get exclusive access" do
113 | @job.lock_exclusively!(4.hours, 'worker2').should == false
114 | end
115 |
116 | it "should allow a second worker to get exclusive access if the timeout has passed" do
117 | @job.lock_exclusively!(1.minute, 'worker2').should == true
118 | end
119 |
120 | it "should be able to get access to the task if it was started more then max_age ago" do
121 | @job.locked_at = 5.hours.ago
122 | @job.save
123 |
124 | @job.lock_exclusively! 4.hours, 'worker2'
125 | @job.reload
126 | @job.locked_by.should == 'worker2'
127 | @job.locked_at.should > 1.minute.ago
128 | end
129 |
130 | it "should not be found by another worker" do
131 | @backend.find_available('worker2', 1, 6.minutes).length.should == 0
132 | end
133 |
134 | it "should be found by another worker if the time has expired" do
135 | @backend.find_available('worker2', 1, 4.minutes).length.should == 1
136 | end
137 |
138 | it "should be able to get exclusive access again when the worker name is the same" do
139 | @job.lock_exclusively!(5.minutes, 'worker1').should be_true
140 | @job.lock_exclusively!(5.minutes, 'worker1').should be_true
141 | @job.lock_exclusively!(5.minutes, 'worker1').should be_true
142 | end
143 | end
144 |
145 | context "when another worker has worked on a task since the job was found to be available, it" do
146 |
147 | before :each do
148 | @job = @backend.create :payload_object => SimpleJob.new
149 | @job_copy_for_worker_2 = @backend.find(@job.id)
150 | end
151 |
152 | it "should not allow a second worker to get exclusive access if already successfully processed by worker1" do
153 | @job.destroy
154 | @job_copy_for_worker_2.lock_exclusively!(4.hours, 'worker2').should == false
155 | end
156 |
157 | it "should not allow a second worker to get exclusive access if failed to be processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
158 | @job.update_attributes(:attempts => 1, :run_at => 1.day.from_now)
159 | @job_copy_for_worker_2.lock_exclusively!(4.hours, 'worker2').should == false
160 | end
161 | end
162 |
163 | context "#name" do
164 | it "should be the class name of the job that was enqueued" do
165 | @backend.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
166 | end
167 |
168 | it "should be the method that will be called if its a performable method object" do
169 | @job = Story.send_later(:create)
170 | @job.name.should == "Story.create"
171 | end
172 |
173 | it "should be the instance method that will be called if its a performable method object" do
174 | @job = Story.create(:text => "...").send_later(:save)
175 | @job.name.should == 'Story#save'
176 | end
177 | end
178 |
179 | context "worker prioritization" do
180 | before(:each) do
181 | Delayed::Worker.max_priority = nil
182 | Delayed::Worker.min_priority = nil
183 | end
184 |
185 | it "should fetch jobs ordered by priority" do
186 | 10.times { @backend.enqueue SimpleJob.new, :priority => rand(10) }
187 | jobs = @backend.find_available('worker', 10)
188 | jobs.size.should == 10
189 | jobs.each_cons(2) do |a, b|
190 | a.priority.should <= b.priority
191 | end
192 | end
193 | end
194 |
195 | context "clear_locks!" do
196 | before do
197 | @job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now)
198 | end
199 |
200 | it "should clear locks for the given worker" do
201 | @backend.clear_locks!('worker')
202 | @backend.find_available('worker2', 5, 1.minute).should include(@job)
203 | end
204 |
205 | it "should not clear locks for other workers" do
206 | @backend.clear_locks!('worker1')
207 | @backend.find_available('worker1', 5, 1.minute).should_not include(@job)
208 | end
209 | end
210 |
211 | context "unlock" do
212 | before do
213 | @job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now)
214 | end
215 |
216 | it "should clear locks" do
217 | @job.unlock
218 | @job.locked_by.should be_nil
219 | @job.locked_at.should be_nil
220 | end
221 | end
222 |
223 | end
--------------------------------------------------------------------------------