├── Gemfile ├── .document ├── .gitignore ├── Rakefile ├── Gemfile.lock ├── resque-multi-job-forks.gemspec ├── LICENSE ├── README.rdoc ├── test ├── helper.rb └── test_resque-multi-job-forks.rb └── lib └── resque-multi-job-forks.rb /Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gemspec -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | README.rdoc 2 | lib/**/*.rb 3 | bin/* 4 | features/**/*.feature 5 | LICENSE 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## MAC OS 2 | .DS_Store 3 | 4 | ## TEXTMATE 5 | *.tmproj 6 | tmtags 7 | 8 | ## EMACS 9 | *~ 10 | \#* 11 | .\#* 12 | 13 | ## VIM 14 | *.swp 15 | 16 | ## PROJECT::GENERAL 17 | coverage 18 | rdoc 19 | pkg 20 | .bundle 21 | 22 | ## PROJECT::SPECIFIC 23 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler::GemHelper.install_tasks 3 | 4 | require 'rake/testtask' 5 | Rake::TestTask.new(:test) do |test| 6 | test.libs << 'lib' << 'test' 7 | test.pattern = 'test/**/test_*.rb' 8 | test.verbose = true 9 | end 10 | 11 | task :default => :test 12 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | resque-multi-job-forks (0.3.1) 5 | json 6 | resque (~> 1.10.0) 7 | 8 | GEM 9 | remote: http://rubygems.org/ 10 | specs: 11 | json (1.4.6) 12 | rack (1.2.1) 13 | rake (0.8.7) 14 | redis (2.1.1) 15 | redis-namespace (0.8.0) 16 | redis (< 3.0.0) 17 | resque (1.10.0) 18 | json (~> 1.4.6) 19 | redis-namespace (~> 0.8.0) 20 | sinatra (>= 0.9.2) 21 | vegas (~> 0.1.2) 22 | sinatra (1.1.0) 23 | rack (~> 1.1) 24 | tilt (~> 1.1) 25 | tilt (1.1) 26 | vegas (0.1.8) 27 | rack (>= 1.0.0) 28 | 29 | PLATFORMS 30 | ruby 31 | 32 | DEPENDENCIES 33 | bundler 34 | json 35 | rake 36 | resque (~> 1.10.0) 37 | resque-multi-job-forks! 38 | -------------------------------------------------------------------------------- /resque-multi-job-forks.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "resque-multi-job-forks" 6 | s.version = "0.3.1" 7 | s.authors = ["Mick Staugaard", "Luke Antins"] 8 | s.email = ["mick@zendesk.com", "luke@lividpenguin.com"] 9 | s.homepage = "http://github.com/staugaard/resque-multi-job-forks" 10 | s.summary = "Have your resque workers process more that one job" 11 | s.description = "When your resque jobs are frequent and fast, the overhead of forking and running your after_fork might get too big." 12 | 13 | s.add_runtime_dependency("resque", "~> 1.10.0") 14 | s.add_runtime_dependency("json") 15 | 16 | s.add_development_dependency("rake") 17 | s.add_development_dependency("bundler") 18 | 19 | s.files = Dir["lib/**/*"] 20 | s.test_files = Dir["test/**/*"] 21 | s.require_paths = ["lib"] 22 | end 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 Mick Staugaard 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = resque-multi-job-forks 2 | 3 | If you have very frequent and fast resque jobs, the overhead of forking and running your after_fork hook, might get too big. Using this resque plugin, you can have your workers perform more than one job, before terminating. 4 | 5 | You simply specify the number of minutes you want each fork to run using the MINUTES_PER_FORK environment variable: 6 | 7 | QUEUE=* MINUTES_PER_FORK=5 rake resque:work 8 | 9 | This will have each fork process jobs for 5 minutes, before terminating. 10 | 11 | This plugin also defines a new hook, that gets called right before the fork terminates: 12 | 13 | Resque.before_child_exit do |worker| 14 | worker.log("#{worker.jobs_processed} were processed in this fork") 15 | end 16 | 17 | 18 | == Note on Patches/Pull Requests 19 | 20 | * Fork the project. 21 | * Make your feature addition or bug fix. 22 | * Add tests for it. This is important so I don't break it in a 23 | future version unintentionally. 24 | * Commit, do not mess with rakefile, version, or history. 25 | (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull) 26 | * Send me a pull request. Bonus points for topic branches. 27 | 28 | == Copyright 29 | 30 | Copyright (c) 2010 Mick Staugaard. See LICENSE for details. 31 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 2 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 3 | 4 | $TESTING = true 5 | 6 | require 'rubygems' 7 | require 'bundler' 8 | Bundler.setup 9 | Bundler.require 10 | require 'test/unit' 11 | require 'resque-multi-job-forks' 12 | 13 | # setup redis & resque. 14 | redis = Redis.new(:db => 1) 15 | Resque.redis = redis 16 | 17 | # adds simple STDOUT logging to test workers. 18 | # set `VERBOSE=true` when running the tests to view resques log output. 19 | module Resque 20 | class Worker 21 | def log(msg) 22 | puts "*** #{msg}" unless ENV['VERBOSE'].nil? 23 | end 24 | alias_method :log!, :log 25 | end 26 | end 27 | 28 | # stores a record of the job processing sequence. 29 | # you may wish to reset this in the test `setup` method. 30 | $SEQUENCE = [] 31 | 32 | # test job, tracks sequence. 33 | class SequenceJob 34 | @queue = :jobs 35 | def self.perform(i) 36 | $SEQUENCE << "work_#{i}".to_sym 37 | sleep(2) 38 | end 39 | end 40 | 41 | class QuickSequenceJob 42 | @queue = :jobs 43 | def self.perform(i) 44 | $SEQUENCE << "work_#{i}".to_sym 45 | end 46 | end 47 | 48 | 49 | # test hooks, tracks sequence. 50 | Resque.after_fork do 51 | $SEQUENCE << :after_fork 52 | end 53 | 54 | Resque.before_fork do 55 | $SEQUENCE << :before_fork 56 | end 57 | 58 | Resque.before_child_exit do |worker| 59 | $SEQUENCE << "before_child_exit_#{worker.jobs_processed}".to_sym 60 | end 61 | -------------------------------------------------------------------------------- /test/test_resque-multi-job-forks.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/helper' 2 | 3 | class TestResqueMultiJobForks < Test::Unit::TestCase 4 | def setup 5 | $SEQUENCE = [] 6 | Resque.redis.flushdb 7 | @worker = Resque::Worker.new(:jobs) 8 | end 9 | 10 | def test_timeout_limit_sequence_of_events 11 | # only allow enough time for 3 jobs to process. 12 | @worker.seconds_per_fork = 3 13 | 14 | Resque.enqueue(SequenceJob, 1) 15 | Resque.enqueue(SequenceJob, 2) 16 | Resque.enqueue(SequenceJob, 3) 17 | Resque.enqueue(SequenceJob, 4) 18 | 19 | # make sure we don't take longer then 15 seconds. 20 | begin 21 | Timeout::timeout(15) { @worker.work(1) } 22 | rescue Timeout::Error 23 | end 24 | 25 | # test the sequence is correct. 26 | assert_equal([:before_fork, :after_fork, :work_1, :work_2, :work_3, 27 | :before_child_exit_3, :before_fork, :after_fork, :work_4, 28 | :before_child_exit_1], $SEQUENCE, 'correct sequence') 29 | end 30 | 31 | # test we can also limit fork job process by a job limit. 32 | def test_job_limit_sequence_of_events 33 | # only allow enough time for 3 jobs to process. 34 | ENV['JOBS_PER_FORK'] = '20' 35 | 36 | # queue 40 jobs. 37 | (1..40).each { |i| Resque.enqueue(QuickSequenceJob, i) } 38 | 39 | begin 40 | Timeout::timeout(3) { @worker.work(1) } 41 | rescue Timeout::Error 42 | end 43 | 44 | assert_equal :before_fork, $SEQUENCE[0], 'first before_fork call.' 45 | assert_equal :after_fork, $SEQUENCE[1], 'first after_fork call.' 46 | assert_equal :work_20, $SEQUENCE[21], '20th chunk of work.' 47 | assert_equal :before_child_exit_20, $SEQUENCE[22], 'first before_child_exit call.' 48 | assert_equal :before_fork, $SEQUENCE[23], 'final before_fork call.' 49 | assert_equal :after_fork, $SEQUENCE[24], 'final after_fork call.' 50 | assert_equal :work_40, $SEQUENCE[44], '40th chunk of work.' 51 | assert_equal :before_child_exit_20, $SEQUENCE[45], 'final before_child_exit call.' 52 | end 53 | 54 | def teardown 55 | # make sure we don't clobber any other tests. 56 | ENV['JOBS_PER_FORK'] = nil 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /lib/resque-multi-job-forks.rb: -------------------------------------------------------------------------------- 1 | require 'resque' 2 | require 'resque/worker' 3 | 4 | module Resque 5 | class Worker 6 | attr_accessor :seconds_per_fork 7 | attr_accessor :jobs_per_fork 8 | attr_reader :jobs_processed 9 | 10 | unless method_defined?(:shutdown_without_multi_job_forks) 11 | def perform_with_multi_job_forks(job = nil) 12 | perform_without_multi_job_forks(job) 13 | hijack_fork unless fork_hijacked? 14 | @jobs_processed += 1 15 | end 16 | alias_method :perform_without_multi_job_forks, :perform 17 | alias_method :perform, :perform_with_multi_job_forks 18 | 19 | def shutdown_with_multi_job_forks 20 | release_fork if fork_hijacked? && fork_job_limit_reached? 21 | shutdown_without_multi_job_forks 22 | end 23 | alias_method :shutdown_without_multi_job_forks, :shutdown? 24 | alias_method :shutdown?, :shutdown_with_multi_job_forks 25 | 26 | def working_on_with_worker_registration(job) 27 | register_worker 28 | working_on_without_worker_registration(job) 29 | end 30 | alias_method :working_on_without_worker_registration, :working_on 31 | alias_method :working_on, :working_on_with_worker_registration 32 | end 33 | 34 | def fork_hijacked? 35 | @release_fork_limit 36 | end 37 | 38 | def hijack_fork 39 | log 'hijack fork.' 40 | @suppressed_fork_hooks = [Resque.after_fork, Resque.before_fork] 41 | Resque.after_fork = Resque.before_fork = nil 42 | @release_fork_limit = fork_job_limit 43 | @jobs_processed = 0 44 | @cant_fork = true 45 | end 46 | 47 | def release_fork 48 | log "jobs processed by child: #{jobs_processed}" 49 | run_hook :before_child_exit, self 50 | Resque.after_fork, Resque.before_fork = *@suppressed_fork_hooks 51 | @release_fork_limit = @jobs_processed = @cant_fork = nil 52 | log 'hijack over, counter terrorists win.' 53 | @shutdown = true unless $TESTING 54 | end 55 | 56 | def fork_job_limit 57 | jobs_per_fork.nil? ? Time.now.to_i + seconds_per_fork : jobs_per_fork 58 | end 59 | 60 | def fork_job_limit_reached? 61 | fork_job_limit_remaining <= 0 ? true : false 62 | end 63 | 64 | def fork_job_limit_remaining 65 | jobs_per_fork.nil? ? @release_fork_limit - Time.now.to_i : jobs_per_fork - @jobs_processed 66 | end 67 | 68 | def seconds_per_fork 69 | @seconds_per_fork ||= minutes_per_fork * 60 70 | end 71 | 72 | def minutes_per_fork 73 | ENV['MINUTES_PER_FORK'].nil? ? 1 : ENV['MINUTES_PER_FORK'].to_i 74 | end 75 | 76 | def jobs_per_fork 77 | @jobs_per_fork ||= ENV['JOBS_PER_FORK'].nil? ? nil : ENV['JOBS_PER_FORK'].to_i 78 | end 79 | end 80 | 81 | # the `before_child_exit` hook will run in the child process 82 | # right before the child process terminates 83 | # 84 | # Call with a block to set the hook. 85 | # Call with no arguments to return the hook. 86 | def self.before_child_exit(&block) 87 | block ? (@before_child_exit = block) : @before_child_exit 88 | end 89 | 90 | # Set the before_child_exit proc. 91 | def self.before_child_exit=(before_child_exit) 92 | @before_child_exit = before_child_exit 93 | end 94 | 95 | end 96 | --------------------------------------------------------------------------------