├── .gitignore ├── .travis.yml ├── ChangeLog ├── Gemfile ├── Gemfile-rails4 ├── MIT-LICENSE ├── README.md ├── Rakefile ├── examples ├── example_setup.rb ├── graceful_shutdown_disabled.rb ├── graceful_shutdown_enabled.rb ├── hooks_perform.rb ├── hooks_push.rb ├── instrumentation.rb ├── logging.rb ├── queues_custom_names.rb └── queues_default_name.rb ├── lib ├── qu-immediate.rb ├── qu-mongo.rb ├── qu-rails.rb ├── qu-redis.rb ├── qu-sqs.rb ├── qu.rb └── qu │ ├── backend │ ├── base.rb │ ├── immediate.rb │ ├── instrumented.rb │ ├── memory.rb │ ├── mongo.rb │ ├── redis.rb │ ├── spec.rb │ └── sqs.rb │ ├── failure.rb │ ├── failure │ └── log.rb │ ├── hooks.rb │ ├── instrumentation │ ├── log_subscriber.rb │ ├── statsd.rb │ ├── statsd_subscriber.rb │ └── subscriber.rb │ ├── instrumenter.rb │ ├── instrumenters │ ├── memory.rb │ └── noop.rb │ ├── job.rb │ ├── logger.rb │ ├── payload.rb │ ├── runner │ ├── base.rb │ ├── direct.rb │ ├── forking.rb │ └── spec.rb │ ├── tasks.rb │ ├── util │ ├── process_wrapper.rb │ ├── procline.rb │ ├── signal_handler.rb │ └── thread_safe_hash.rb │ ├── version.rb │ └── worker.rb ├── qu-mongo.gemspec ├── qu-rails.gemspec ├── qu-redis.gemspec ├── qu-sqs.gemspec ├── qu.gemspec ├── script ├── bootstrap └── test └── spec ├── qu ├── backend │ ├── immediate_spec.rb │ ├── instrumented_spec.rb │ ├── memory_spec.rb │ ├── mongo_spec.rb │ ├── redis_spec.rb │ └── sqs_spec.rb ├── failure_spec.rb ├── hooks_spec.rb ├── instrumentation │ ├── log_subscriber_spec.rb │ └── statsd_subscriber_spec.rb ├── instrumenters │ ├── memory_spec.rb │ └── noop_spec.rb ├── job_spec.rb ├── payload_spec.rb ├── runner │ ├── direct_spec.rb │ └── forking_spec.rb ├── util │ └── thread_safe_hash_spec.rb └── worker_spec.rb ├── qu_spec.rb ├── spec_helper.rb └── support └── fake_udp_socket.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | log/*.log 6 | script/config/kestrel/target 7 | tmp 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3 4 | - 2.0.0 5 | - 2.1.0 6 | services: 7 | - mongodb 8 | - redis 9 | script: bundle exec rake unattended_spec 10 | cache: bundler 11 | gemfile: 12 | - Gemfile 13 | - Gemfile-rails4 14 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 0.3.0 2 | 3 | * Remove qu-airbrake failure backend 4 | 5 | 0.2.0 - 2012-06-13 6 | 7 | Backwards-incompatible Changes: 8 | 9 | * Remove #requeue from all backends. It was not used internally. 10 | * Move implicit Rails dependency to it's own Gem, which you must declare now in your Gemfile. 11 | 12 | gem 'qu-rails' 13 | 14 | Enhancements 15 | 16 | * qu-mongo: Add mongolab environment variable for automatic configuration on Heroku. 17 | 18 | Bug Fixes: 19 | 20 | * Fixing Immediate backend to satisfy the backend interface 21 | 22 | https://github.com/bkeepers/qu/compare/v0.1.4...v0.2.0 23 | 24 | 25 | 0.1.4 - 2012-01-07 26 | 27 | Enhancements: 28 | 29 | * Add `poll_frequency` config for mongo backend 30 | * Add qu-airbrake failure backend 31 | * Add immediate backend for performing jobs immediately 32 | 33 | require 'qu-immediate' 34 | 35 | https://github.com/bkeepers/qu/compare/v0.1.3...v0.1.4 36 | 37 | 38 | 0.1.3 - 2011-10-10 39 | 40 | Bug Fixes: 41 | 42 | * Delete jobs when clearing the queue in the Redis backend 43 | 44 | Enhancements: 45 | 46 | * Added support for Rails 2 47 | * Retry connection failures in Mongo backend and added two config options: 48 | 49 | Qu.configure do |c| 50 | c.backend.max_retries = 10 # default: 5 51 | c.backend.retry_frequency = 5 # default: 1 52 | end 53 | 54 | https://github.com/bkeepers/qu/compare/v0.1.2..v0.1.3 55 | 56 | 57 | 0.1.2 - 2011-10-05 58 | 59 | Bug Fixes: 60 | 61 | * Build gem with Ruby 1.8 to avoid YAML errors (http://blog.rubygems.org/2011/08/31/shaving-the-yaml-yacc.html) 62 | 63 | https://github.com/bkeepers/qu/compare/v0.1.1...v0.1.2 64 | 65 | 66 | 0.1.1 - 2011-10-02 67 | 68 | Bug Fixes: 69 | 70 | * Fixes for Ruby 1.9 71 | * Fix Mongo backend against MongoDB 2.0 72 | * Fix #unregister_worker 73 | * Add no-op environment task (so rake tasks will work in non-rails environment) 74 | 75 | Enhancements: 76 | 77 | * Forward rake tasks for jobs:work and resque:work to qu:work 78 | * Added logging 79 | 80 | https://github.com/bkeepers/qu/compare/v0.1.0...v0.1.1 81 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | gemspec :name => 'qu' 3 | 4 | Dir['qu-*.gemspec'].each do |gemspec| 5 | plugin = gemspec.scan(/qu-(.*)\.gemspec/).flatten.first 6 | gemspec(:name => "qu-#{plugin}", :development_group => plugin) 7 | end 8 | 9 | group :test do 10 | gem 'activesupport', :require => false 11 | gem 'statsd-ruby', :require => false 12 | gem 'rake' 13 | gem "rspec", "~> 2.14.1" 14 | end 15 | -------------------------------------------------------------------------------- /Gemfile-rails4: -------------------------------------------------------------------------------- 1 | original_gemfile = File.join(File.dirname(__FILE__), 'Gemfile') 2 | 3 | eval IO.read(original_gemfile) 4 | 5 | gem 'railties', '~> 4.0' 6 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Brandon Keepers 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOa AND 17 | NONINFRINGEMENT. IN NO EVENT SaALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Qu 2 | 3 | Qu is a Ruby library for queuing and processing background jobs. It is heavily inspired by delayed_job and Resque. 4 | 5 | Qu was created to overcome some shortcomings in the existing queuing libraries that we experienced at [Ordered List](http://orderedlist.com) while building [SpeakerDeck](http://speakerdeck.com), [Gaug.es](http://get.gaug.es) and [Harmony](http://get.harmonyapp.com). The advantages of Qu are: 6 | 7 | * Multiple backends (redis, mongo) 8 | * Jobs are requeued when worker is killed 9 | * Resque-like API 10 | 11 | ## Information & Help 12 | 13 | * Find more information on the [Wiki](https://github.com/bkeepers/qu/wiki). 14 | * Post to the [Google Group](http://groups.google.com/group/qu-users) for help or questions. 15 | * See the [issue tracker](https://github.com/bkeepers/qu/issues) for known issues or to report an issue. 16 | 17 | ## Installation 18 | 19 | ### Rails 3 and 4 20 | 21 | Decide which backend you want to use and add the gem to your `Gemfile`. 22 | 23 | ``` ruby 24 | gem 'qu-rails' 25 | gem 'qu-redis' 26 | ``` 27 | 28 | That's all you need to do! 29 | 30 | ### Rails 2 31 | 32 | Decide which backend you want to use and add the gem to `config.gems` in `environment.rb`: 33 | 34 | ``` ruby 35 | config.gem 'qu-redis' 36 | ```` 37 | 38 | To load the rake tasks, add the following to your `Rakefile`: 39 | 40 | ``` ruby 41 | require 'qu/tasks' 42 | ``` 43 | 44 | ## Usage 45 | 46 | Jobs are defined by extending the `Qu::Job` class: 47 | 48 | ``` ruby 49 | class ProcessPresentation < Qu::Job 50 | def initialize(presentation_id) 51 | @presentation_id = presentation_id 52 | end 53 | 54 | def perform 55 | Presentation.find(@presentation_id).process! 56 | end 57 | end 58 | ``` 59 | 60 | You can add a job to the queue by calling `create` on your job: 61 | 62 | ``` ruby 63 | job = ProcessPresentation.create(@presentation.id) 64 | puts "Created job #{job.id}" 65 | ``` 66 | 67 | The job will be initialized with any parameters that are passed to it when it is performed. These parameters will be stored in the backend, so they must be simple types that can easily be serialized and unserialized. Don't try to pass in an ActiveRecord object. 68 | 69 | Processing the jobs on the queue can be done with a Rake task: 70 | 71 | ``` sh 72 | $ bundle exec rake qu:work 73 | ``` 74 | 75 | You can easily inspect the queue or clear it: 76 | 77 | ``` ruby 78 | puts "Jobs on the queue:", Qu.size 79 | Qu.clear 80 | ``` 81 | 82 | ### Queues 83 | 84 | The `default` queue is used, um…by default. Jobs that don't specify a queue will be placed in that queue, and workers that don't specify a queue will work on that queue. 85 | 86 | However, if you have some background jobs that are more or less important, or some that take longer than others, you may want to consider using multiple queues. You can have workers dedicated to specific queues, or simply tell all your workers to work on the most important queue first. 87 | 88 | Jobs can be placed in a specific queue by setting the queue: 89 | 90 | ``` ruby 91 | class CallThePresident < Qu::Job 92 | queue :urgent 93 | 94 | def initialize(message) 95 | @message = message 96 | end 97 | 98 | def perform 99 | # … 100 | end 101 | end 102 | ``` 103 | 104 | You can then tell workers to work on this queue by passing an environment variable 105 | 106 | ``` sh 107 | $ bundle exec rake qu:work QUEUES=urgent,default 108 | ``` 109 | 110 | Note that if you still want your worker to process the default queue, you must specify it. Queues will be process in the order they are specified. 111 | 112 | You can also get the size or clear a specific queue: 113 | 114 | ``` ruby 115 | Qu.size(:urgent) 116 | Qu.clear(:urgent) 117 | ``` 118 | 119 | ## Configuration 120 | 121 | Most of the configuration for Qu should be automatic. It will also automatically detect ENV variables from Heroku for backend connections, so you shouldn't need to do anything to configure the backend. 122 | 123 | However, if you do need to customize it, you can by calling the `Qu.configure`: 124 | 125 | ``` ruby 126 | Qu.configure do |c| 127 | c.logger = Logger.new('log/qu.log') 128 | c.graceful_shutdown = true 129 | end 130 | ``` 131 | 132 | ## Tests 133 | 134 | If you prefer to have jobs processed immediatly in your tests, there is an `Immediate` backend that will perform the job instead of enqueuing it. In your test helper, require qu-immediate: 135 | 136 | ``` ruby 137 | require 'qu-immediate' 138 | ``` 139 | 140 | ## Why another queuing library? 141 | 142 | [Resque](https://github.com/resque/resque) and [delayed_job](https://github.com/collectiveidea/delayed_job) are both great, but both of them have shortcomings that can be frustrating in production applications. 143 | 144 | delayed_job was a brilliantly simple pioneer in the world of database-backed queues. While most asynchronous queuing systems were tending toward overly complex, it made use of your existing database and just worked. But there are a few flaws: 145 | 146 | * Occasionally fails silently. 147 | * Use of priority instead of separate named queues. 148 | * Contention in the ActiveRecord backend with multiple workers. Occasionally the same job gets performed by multiple workers. 149 | 150 | Resque, the wiser relative of delayed_job, fixes most of those issues. But in doing so, it forces some of its beliefs on you, and sometimes those beliefs just don't make sense for your environment. Here are some of the flaws of Resque: 151 | 152 | * Redis is a great queue backend, but it doesn't make sense for every environment. 153 | * Forking before each job prevents memory leaks, but it is terribly inefficient in environments with a lot of fast jobs (the resque-jobs-per-fork plugin alleviates this) 154 | 155 | Those shortcomings lead us to write Qu. It is not perfect, but we hope to overcome the issues we faced with other queuing libraries. 156 | 157 | ## Contributing 158 | 159 | If you find what looks like a bug: 160 | 161 | 1. Search the [issues on GitHub](http://github.com/bkeepers/qu/issues/) to see if anyone else has reported issue. 162 | 2. If you don't see anything, [create an issue](http://github.com/bkeepers/qu/issues/new) with information on how to reproduce it. 163 | 164 | If you want to contribute an enhancement or a fix: 165 | 166 | 1. [Fork the project](https://github.com/bkeepers/qu/fork) on GitHub. 167 | 2. Make your changes with tests. 168 | 3. Commit the changes without making changes to the Rakefile, Gemfile, gemspec, or any other files that aren't related to your enhancement or fix 169 | 4. Send a pull request. 170 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "qu/version" 4 | 5 | desc 'Build gem into the pkg directory' 6 | task :build do 7 | FileUtils.rm_rf('pkg') 8 | Dir['*.gemspec'].each do |gemspec| 9 | system "gem build #{gemspec}" 10 | end 11 | FileUtils.mkdir_p('pkg') 12 | FileUtils.mv(Dir['*.gem'], 'pkg') 13 | end 14 | 15 | desc 'Tags version, pushes to remote, and pushes gem' 16 | task :release => :build do 17 | sh 'git', 'tag', '-m', changelog, "v#{Qu::VERSION}" 18 | sh "git push origin master" 19 | sh "git push origin v#{Qu::VERSION}" 20 | sh "ls pkg/*.gem | xargs -n 1 gem push" 21 | end 22 | 23 | require 'rspec/core/rake_task' 24 | 25 | desc "Run all specs" 26 | RSpec::Core::RakeTask.new(:spec) do |t| 27 | t.rspec_opts = %w[--color] 28 | t.verbose = false 29 | end 30 | 31 | namespace :spec do 32 | Backends = %w(mongo redis) 33 | 34 | Backends.each do |backend| 35 | desc "Run specs for #{backend} backend" 36 | RSpec::Core::RakeTask.new(backend) do |t| 37 | t.rspec_opts = %w[--color] 38 | t.verbose = false 39 | t.pattern = "spec/qu/backend/#{backend}_spec.rb" 40 | end 41 | end 42 | 43 | task :backends => Backends 44 | end 45 | 46 | def changelog 47 | File.read('ChangeLog').split("\n\n\n", 2).first 48 | end 49 | 50 | desc "Start fake services, run tests, cleanup" 51 | task :unattended_spec do |t| 52 | require 'tmpdir' 53 | require 'socket' 54 | 55 | dir = Dir.mktmpdir 56 | data_file = File.join(dir, "data.fdb") 57 | 58 | sqs_pid = Process.spawn 'fake_sqs', '-p', '5111', err: '/dev/null', out: '/dev/null' 59 | 60 | at_exit { 61 | Process.kill('TERM', sqs_pid) 62 | FileUtils.rmtree(dir) 63 | } 64 | 65 | 40.downto(0) do |count| 66 | begin 67 | s = TCPSocket.new 'localhost', 5111 68 | s.close 69 | break 70 | rescue Errno::ECONNREFUSED 71 | raise if count == 0 72 | sleep 0.1 73 | end 74 | end 75 | 76 | Rake::Task["spec"].invoke 77 | end 78 | 79 | task :default => :unattended_spec 80 | -------------------------------------------------------------------------------- /examples/example_setup.rb: -------------------------------------------------------------------------------- 1 | require 'pp' 2 | require 'pathname' 3 | root_path = Pathname(__FILE__).dirname.join('..').expand_path 4 | lib_path = root_path.join('lib') 5 | $:.unshift(lib_path) 6 | 7 | require 'qu-redis' 8 | 9 | backend = Qu::Backend::Redis.new 10 | backend.connection.flushdb 11 | 12 | Qu.configure do |config| 13 | config.logger = Logger.new(STDOUT) 14 | config.logger.level = Logger::DEBUG 15 | config.backend = backend 16 | end 17 | 18 | def work_and_die(die_after_seconds = 1, *queues) 19 | # tell qu worker to terminate after N seconds by sending terminate signal 20 | Thread.new { 21 | sleep die_after_seconds 22 | Process.kill 'SIGTERM', $$ 23 | } 24 | 25 | worker = Qu::Worker.new(queues) 26 | 27 | begin 28 | worker.start 29 | rescue Qu::Worker::Stop 30 | puts 'worker stopped' 31 | rescue Qu::Worker::Abort 32 | puts 'worker aborted' 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /examples/graceful_shutdown_disabled.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned off. 2 | require_relative './example_setup' 3 | 4 | Qu.configure do |config| 5 | config.graceful_shutdown = false 6 | end 7 | 8 | class SleepJob < Qu::Job 9 | def initialize(sleep_for = 3) 10 | @sleep_for = sleep_for 11 | end 12 | 13 | def perform 14 | logger.debug "Performing job, sleeping for #{@sleep_for}" 15 | sleep @sleep_for 16 | logger.debug 'Job complete' 17 | end 18 | end 19 | 20 | # job created 21 | SleepJob.create 3 22 | Qu.logger.info "# of jobs: #{Qu.size}" 23 | 24 | # die before job is performed 25 | work_and_die 0.1 26 | 27 | # job is aborted and pushed back on queue 28 | Qu.logger.info "# of jobs: #{Qu.size}" 29 | -------------------------------------------------------------------------------- /examples/graceful_shutdown_enabled.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require_relative './example_setup' 3 | 4 | Qu.configure do |config| 5 | config.graceful_shutdown = true 6 | end 7 | 8 | class SleepJob < Qu::Job 9 | def initialize(sleep_for = 3) 10 | @sleep_for = sleep_for 11 | end 12 | 13 | def perform 14 | logger.debug "Performing job, sleeping for #{@sleep_for}" 15 | sleep @sleep_for 16 | logger.debug 'Job complete' 17 | end 18 | end 19 | 20 | SleepJob.create 3 21 | 22 | work_and_die 1 23 | -------------------------------------------------------------------------------- /examples/hooks_perform.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require_relative './example_setup' 3 | 4 | class CookJob < Qu::Job 5 | around_perform :instrument 6 | 7 | before_perform :purchase 8 | after_perform :eat 9 | 10 | def perform 11 | sleep rand 12 | logger.info "cooking" 13 | end 14 | 15 | private 16 | 17 | def instrument 18 | start = Time.now 19 | yield 20 | duration = ((Time.now - start) * 1_000).round 21 | logger.info "job time: #{duration}ms" 22 | end 23 | 24 | def purchase 25 | logger.info "purchasing" 26 | end 27 | 28 | def eat 29 | logger.info "eating" 30 | end 31 | end 32 | 33 | CookJob.create 34 | 35 | work_and_die 36 | -------------------------------------------------------------------------------- /examples/hooks_push.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require_relative './example_setup' 3 | 4 | class MaybeJob < Qu::Job 5 | before_push :determine_for_real 6 | after_push :log_push 7 | 8 | def initialize(actually_push = true) 9 | @actually_push = actually_push 10 | end 11 | 12 | def perform 13 | logger.info "performing job" 14 | end 15 | 16 | private 17 | 18 | def determine_for_real 19 | halt unless @actually_push 20 | end 21 | 22 | def log_push 23 | logger.info "pushed job" 24 | end 25 | end 26 | 27 | MaybeJob.create(false) 28 | MaybeJob.create(false) 29 | MaybeJob.create(true) 30 | MaybeJob.create(false) 31 | MaybeJob.create(false) 32 | 33 | Qu.logger.info "Qu size should be 1, actual: #{Qu.size}" 34 | 35 | work_and_die 36 | -------------------------------------------------------------------------------- /examples/instrumentation.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require 'socket' 3 | require_relative './example_setup' 4 | require 'qu/instrumentation/statsd' 5 | 6 | Thread.new do 7 | socket = UDPSocket.new 8 | socket.bind(nil, 6868) 9 | loop do 10 | puts socket.recvfrom(50)[0].chomp 11 | end 12 | end 13 | 14 | # Config that matters to a user using qu 15 | # 1. setup instrumenter 16 | # 2. set client for subscriber 17 | Qu.configure do |config| 18 | config.instrumenter = ActiveSupport::Notifications 19 | end 20 | Qu::Instrumentation::StatsdSubscriber.client = Statsd.new('localhost', 6868) 21 | 22 | class SimpleJob < Qu::Job 23 | end 24 | 25 | SimpleJob.create 26 | SimpleJob.create 27 | 28 | work_and_die 29 | -------------------------------------------------------------------------------- /examples/logging.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require_relative './example_setup' 3 | require 'qu/instrumentation/log_subscriber' 4 | 5 | class SimpleJob < Qu::Job 6 | end 7 | 8 | SimpleJob.create 9 | SimpleJob.create 10 | 11 | work_and_die 12 | -------------------------------------------------------------------------------- /examples/queues_custom_names.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require_relative './example_setup' 3 | 4 | class CallThePresidentJob < Qu::Job 5 | queue :low 6 | 7 | def initialize(message) 8 | @message = message 9 | end 10 | 11 | def perform 12 | logger.info "calling the president: #{@message}" 13 | end 14 | end 15 | 16 | class CallTheNunesJob < Qu::Job 17 | queue :high 18 | 19 | def initialize(message) 20 | @message = message 21 | end 22 | 23 | def perform 24 | logger.info "calling the nunes: #{@message}" 25 | end 26 | end 27 | 28 | CallThePresidentJob.create('blah blah blah...') 29 | CallTheNunesJob.create('blah blah blah...') 30 | 31 | work_and_die 1, :high, :low 32 | -------------------------------------------------------------------------------- /examples/queues_default_name.rb: -------------------------------------------------------------------------------- 1 | # Example of how Qu works with graceful shutdown turned on. 2 | require_relative './example_setup' 3 | 4 | class CallTheNunes < Qu::Job 5 | def perform 6 | logger.info 'calling the nunes' 7 | end 8 | end 9 | 10 | payload = CallTheNunes.create 11 | 12 | Qu.logger.info "Queue used: #{payload.queue}" 13 | 14 | work_and_die 15 | -------------------------------------------------------------------------------- /lib/qu-immediate.rb: -------------------------------------------------------------------------------- 1 | require 'qu' 2 | require 'qu/backend/immediate' 3 | 4 | Qu.backend = Qu::Backend::Immediate.new -------------------------------------------------------------------------------- /lib/qu-mongo.rb: -------------------------------------------------------------------------------- 1 | require 'qu' 2 | require 'qu/backend/mongo' 3 | 4 | Qu.backend = Qu::Backend::Mongo.new -------------------------------------------------------------------------------- /lib/qu-rails.rb: -------------------------------------------------------------------------------- 1 | require 'qu' 2 | 3 | module Qu 4 | class Railtie < Rails::Railtie 5 | rake_tasks do 6 | load "qu/tasks.rb" 7 | end 8 | 9 | initializer "qu.logger" do |app| 10 | Qu.logger = Rails.logger 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/qu-redis.rb: -------------------------------------------------------------------------------- 1 | require 'qu' 2 | require 'qu/backend/redis' 3 | 4 | Qu.backend = Qu::Backend::Redis.new -------------------------------------------------------------------------------- /lib/qu-sqs.rb: -------------------------------------------------------------------------------- 1 | require 'qu' 2 | require 'qu/backend/sqs' 3 | 4 | Qu.backend = Qu::Backend::SQS.new 5 | -------------------------------------------------------------------------------- /lib/qu.rb: -------------------------------------------------------------------------------- 1 | require 'qu/version' 2 | require 'qu/logger' 3 | require 'qu/failure' 4 | require 'qu/hooks' 5 | require 'qu/payload' 6 | require 'qu/job' 7 | require 'qu/backend/base' 8 | require 'qu/backend/instrumented' 9 | require 'qu/instrumenters/noop' 10 | require 'qu/runner/direct' 11 | require 'qu/worker' 12 | 13 | require 'forwardable' 14 | require 'logger' 15 | 16 | module Qu 17 | InstrumentationNamespace = :qu 18 | 19 | extend SingleForwardable 20 | extend self 21 | 22 | @interval = 5 23 | 24 | attr_accessor :logger, :graceful_shutdown, :instrumenter, :interval, :runner 25 | 26 | def_delegators :backend, :push, :pop, :complete, :abort, :fail, :size, :clear 27 | 28 | def backend 29 | @backend || raise("Qu backend not configured. Install one of the backend gems like qu-redis.") 30 | end 31 | 32 | def backend=(backend) 33 | @backend = Backend::Instrumented.wrap(backend) 34 | end 35 | 36 | def configure(&block) 37 | block.call(self) 38 | end 39 | 40 | # Internal: Convert an object to json. 41 | def dump_json(object) 42 | JSON.dump(object) if object 43 | end 44 | 45 | # Internal: Convert json to an object. 46 | def load_json(object) 47 | JSON.load(object) if object 48 | end 49 | end 50 | 51 | Qu.configure do |config| 52 | config.logger = Logger.new(STDOUT) 53 | config.logger.level = Logger::INFO 54 | config.instrumenter = Qu::Instrumenters::Noop 55 | config.runner = Qu::Runner::Direct.new 56 | end 57 | -------------------------------------------------------------------------------- /lib/qu/backend/base.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | module Qu 4 | module Backend 5 | class Base 6 | include Logger 7 | attr_accessor :connection 8 | 9 | # Public: Override in subclass. 10 | def push(payload) 11 | payload.id = SecureRandom.uuid 12 | payload.perform 13 | end 14 | 15 | # Public: Override in subclass. 16 | def complete(payload) 17 | end 18 | 19 | # Public: Override in subclass. 20 | def abort(payload) 21 | end 22 | 23 | # Public: Override in subclass. 24 | def fail(payload) 25 | end 26 | 27 | # Public: Override in subclass. 28 | def pop(queue = 'default') 29 | end 30 | 31 | # Public: Override in subclass. 32 | def size(queue = 'default') 33 | 0 34 | end 35 | 36 | # Public: Override in subclass. 37 | def clear(queue = 'default') 38 | end 39 | 40 | # Public: Override in subclass. 41 | def reconnect 42 | end 43 | 44 | private 45 | 46 | def dump(data) 47 | Qu.dump_json(data) 48 | end 49 | 50 | def load(data) 51 | Qu.load_json(data) 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/qu/backend/immediate.rb: -------------------------------------------------------------------------------- 1 | require 'securerandom' 2 | 3 | module Qu 4 | module Backend 5 | class Immediate < Base 6 | def push(payload) 7 | payload.id = SecureRandom.uuid 8 | payload.perform 9 | end 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/qu/backend/instrumented.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | 3 | module Qu 4 | module Backend 5 | # Internal: Backend that wraps all backends with instrumentation. 6 | class Instrumented < Base 7 | extend Forwardable 8 | include Qu::Instrumenter 9 | 10 | def self.wrap(backend) 11 | if backend.nil? 12 | backend 13 | else 14 | new(backend) 15 | end 16 | end 17 | 18 | def_delegators :@backend, :connection, :connection=, :reconnect 19 | 20 | def initialize(backend) 21 | @backend = backend 22 | end 23 | 24 | def push(payload) 25 | instrument("push.#{InstrumentationNamespace}") { |ipayload| 26 | ipayload[:payload] = payload 27 | @backend.push(payload) 28 | } 29 | end 30 | 31 | def complete(payload) 32 | instrument("complete.#{InstrumentationNamespace}") { |ipayload| 33 | ipayload[:payload] = payload 34 | @backend.complete(payload) 35 | } 36 | end 37 | 38 | def abort(payload) 39 | instrument("abort.#{InstrumentationNamespace}") { |ipayload| 40 | ipayload[:payload] = payload 41 | @backend.abort(payload) 42 | } 43 | end 44 | 45 | def fail(payload) 46 | instrument("fail.#{InstrumentationNamespace}") { |ipayload| 47 | ipayload[:payload] = payload 48 | @backend.fail(payload) 49 | } 50 | end 51 | 52 | def pop(queue_name = 'default') 53 | instrument("pop.#{InstrumentationNamespace}") { |ipayload| 54 | result = @backend.pop(queue_name) 55 | ipayload[:payload] = result 56 | ipayload[:queue_name] = queue_name 57 | result 58 | } 59 | end 60 | 61 | def size(queue_name = 'default') 62 | instrument("size.#{InstrumentationNamespace}") { |ipayload| 63 | ipayload[:queue_name] = queue_name 64 | @backend.size(queue_name) 65 | } 66 | end 67 | 68 | def clear(queue_name = 'default') 69 | instrument("clear.#{InstrumentationNamespace}") { |ipayload| 70 | ipayload[:queue_name] = queue_name 71 | @backend.clear(queue_name) 72 | } 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/qu/backend/memory.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | require 'securerandom' 3 | 4 | module Qu 5 | module Backend 6 | class Memory < Base 7 | extend Forwardable 8 | 9 | def_delegator :@monitor, :synchronize 10 | 11 | def initialize 12 | @monitor = Monitor.new 13 | @queues = {} 14 | @messages = {} 15 | @pending = {} 16 | @connection = @messages 17 | end 18 | 19 | def push(payload) 20 | payload.id = SecureRandom.uuid 21 | queue_for(payload.queue) do |queue| 22 | queue << payload.id 23 | @messages[payload.id] = dump(payload.attributes_for_push) 24 | payload 25 | end 26 | end 27 | 28 | def complete(payload) 29 | synchronize { @pending.delete(payload.id) } 30 | end 31 | 32 | def abort(payload) 33 | synchronize do 34 | @pending.delete(payload.id) 35 | push(payload) 36 | end 37 | end 38 | 39 | alias fail abort 40 | 41 | def pop(queue_name = 'default') 42 | queue_for(queue_name) do |queue| 43 | if id = queue.shift 44 | payload = Payload.new(load(@messages[id])) 45 | @pending[id] = payload 46 | payload 47 | end 48 | end 49 | end 50 | 51 | def size(queue = 'default') 52 | queue_for(queue).size 53 | end 54 | 55 | def clear(queue_name = 'default') 56 | queue_for(queue_name) { |queue| queue.clear } 57 | end 58 | 59 | private 60 | 61 | def queue_for(queue) 62 | if block_given? 63 | synchronize { yield(@queues[queue] ||= []) } 64 | else 65 | synchronize { @queues[queue] ||= [] } 66 | end 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/qu/backend/mongo.rb: -------------------------------------------------------------------------------- 1 | require 'mongo' 2 | 3 | module Qu 4 | module Backend 5 | class Mongo < Base 6 | 7 | # Number of times to retry connection on connection failure (default: 5) 8 | attr_accessor :max_retries 9 | 10 | # Seconds to wait before try to reconnect after connection 11 | # failure (default: 1) 12 | attr_accessor :retry_frequency 13 | 14 | def initialize 15 | self.max_retries = 5 16 | self.retry_frequency = 1 17 | end 18 | 19 | def push(payload) 20 | payload.id = BSON::ObjectId.new 21 | with_connection_retries do 22 | jobs(payload.queue).insert(payload_attributes(payload)) 23 | end 24 | payload 25 | end 26 | 27 | def abort(payload) 28 | with_connection_retries do 29 | jobs(payload.queue).insert(payload_attributes(payload)) 30 | end 31 | end 32 | 33 | def pop(queue = 'default') 34 | begin 35 | doc = with_connection_retries do 36 | jobs(queue).find_and_modify(:remove => true) 37 | end 38 | 39 | if doc 40 | doc['id'] = doc.delete('_id') 41 | return Payload.new(doc) 42 | end 43 | rescue ::Mongo::OperationFailure 44 | # No jobs in the queue (MongoDB <2) 45 | end 46 | end 47 | 48 | def size(queue = 'default') 49 | with_connection_retries do 50 | jobs(queue).count 51 | end 52 | end 53 | 54 | def clear(queue = 'default') 55 | with_connection_retries do 56 | jobs(queue).drop 57 | end 58 | end 59 | 60 | def connection 61 | @connection ||= begin 62 | host_uri = (ENV['MONGOHQ_URL'] || ENV['MONGOLAB_URI'] || ENV['BOXEN_MONGODB_URL']).to_s 63 | if host_uri && !host_uri.empty? 64 | uri = URI.parse(host_uri) 65 | 66 | # path can come in as nil, "", "/", or "/something"; 67 | # this normalizes to empty string or "something" 68 | path = uri.path.to_s[1..-1].to_s 69 | database = path.empty? ? 'qu' : path 70 | uri.path = "/#{database}" 71 | ::Mongo::MongoClient.from_uri(host_uri).db(database) 72 | else 73 | ::Mongo::MongoClient.new.db('qu') 74 | end 75 | end 76 | end 77 | 78 | def reconnect 79 | connection.connection.reconnect 80 | end 81 | 82 | private 83 | 84 | def payload_attributes(payload) 85 | attrs = payload.attributes_for_push 86 | attrs[:_id] = attrs.delete(:id) 87 | attrs 88 | end 89 | 90 | def jobs(queue) 91 | connection["qu:queue:#{queue}"] 92 | end 93 | 94 | def with_connection_retries 95 | retries = 0 96 | begin 97 | yield 98 | rescue ::Mongo::ConnectionFailure => ex 99 | retries += 1 100 | raise ex if retries > max_retries 101 | sleep retry_frequency * retries 102 | retry 103 | end 104 | end 105 | end 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /lib/qu/backend/redis.rb: -------------------------------------------------------------------------------- 1 | require 'redis-namespace' 2 | require 'securerandom' 3 | 4 | module Qu 5 | module Backend 6 | class Redis < Base 7 | attr_accessor :namespace 8 | 9 | def initialize 10 | self.namespace = :qu 11 | end 12 | 13 | def push(payload) 14 | payload.id = SecureRandom.uuid 15 | connection.rpush("queue:#{payload.queue}", dump(payload.attributes_for_push)) 16 | payload 17 | end 18 | 19 | def abort(payload) 20 | connection.rpush("queue:#{payload.queue}", dump(payload.attributes_for_push)) 21 | end 22 | 23 | def complete(payload) 24 | end 25 | 26 | def pop(queue = 'default') 27 | if data = connection.lpop("queue:#{queue}") 28 | data = load(data) 29 | return Payload.new({ 30 | id: data['id'], 31 | klass: data['klass'], 32 | args: data['args'], 33 | }) 34 | end 35 | end 36 | 37 | def size(queue = 'default') 38 | connection.llen("queue:#{queue}") 39 | end 40 | 41 | def clear(queue = 'default') 42 | connection.del("queue:#{queue}") 43 | end 44 | 45 | def connection 46 | @connection ||= ::Redis::Namespace.new(namespace, :redis => ::Redis.connect(:url => ENV['REDISTOGO_URL'] || ENV['BOXEN_REDIS_URL'])) 47 | end 48 | 49 | def reconnect 50 | connection.client.reconnect 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/qu/backend/spec.rb: -------------------------------------------------------------------------------- 1 | class SimpleJob < Qu::Job 2 | end 3 | 4 | class CustomQueue < Qu::Job 5 | queue :custom 6 | end 7 | 8 | shared_examples_for 'a backend interface' do 9 | let(:payload) { Qu::Payload.new(:klass => SimpleJob) } 10 | 11 | it "can push a payload" do 12 | subject.push payload 13 | end 14 | 15 | it "can complete a payload" do 16 | subject.complete payload 17 | end 18 | 19 | it "can abort a payload" do 20 | subject.abort payload 21 | end 22 | 23 | it "can fail a payload" do 24 | subject.fail payload 25 | end 26 | 27 | it "can pop" do 28 | subject.pop 29 | end 30 | 31 | it "can pop from specific queue" do 32 | subject.pop('foo') 33 | end 34 | 35 | it "can get size of default queue" do 36 | subject.size 37 | end 38 | 39 | it "can get size of specific queue" do 40 | subject.size('foo') 41 | end 42 | 43 | it "can clear default queue" do 44 | subject.clear 45 | end 46 | 47 | it "can clear specific queue" do 48 | subject.clear('foo') 49 | end 50 | 51 | it 'can reconnect' do 52 | subject.reconnect 53 | end 54 | 55 | end 56 | 57 | shared_examples_for 'a backend' do 58 | let(:payload) { Qu::Payload.new(:klass => SimpleJob) } 59 | 60 | before do 61 | subject.clear(payload.queue) 62 | end 63 | 64 | describe 'push' do 65 | it 'should return a payload' do 66 | subject.push(payload).should be_instance_of(Qu::Payload) 67 | end 68 | 69 | it 'should set the payload id' do 70 | subject.push(payload) 71 | payload.id.should_not be_nil 72 | end 73 | 74 | it 'should add a job to the queue' do 75 | subject.push(payload) 76 | payload.queue.should == 'default' 77 | subject.size(payload.queue).should == 1 78 | end 79 | 80 | it 'should assign a different job id for the same job pushed multiple times' do 81 | first = subject.push(payload).id 82 | second = subject.push(payload).id 83 | first.should_not eq(second) 84 | end 85 | 86 | it 'should enqueue the attributes for push' do 87 | payload.should_receive(:attributes_for_push).and_return({}) 88 | subject.push(payload) 89 | end 90 | end 91 | 92 | describe 'pop' do 93 | it 'should return next job' do 94 | subject.push(payload) 95 | subject.pop(payload.queue).id.should == payload.id 96 | end 97 | 98 | it 'should not return an already popped job' do 99 | subject.push(payload) 100 | subject.push(payload.dup) 101 | subject.pop(payload.queue).id.should_not == subject.pop(payload.queue).id 102 | end 103 | 104 | it 'should not return job from different queue' do 105 | subject.push(payload) 106 | subject.pop('video').should be_nil 107 | end 108 | 109 | it 'should properly persist args' do 110 | payload.args = ['a', 'b'] 111 | subject.push(payload) 112 | subject.pop(payload.queue).args.should == ['a', 'b'] 113 | end 114 | 115 | it 'should properly persist a hash argument' do 116 | payload.args = [{:a => 1, :b => 2}] 117 | subject.push(payload) 118 | subject.pop(payload.queue).args.should == [{'a' => 1, 'b' => 2}] 119 | end 120 | end 121 | 122 | describe 'complete' do 123 | it 'should be defined and accept payload' do 124 | subject.complete(payload) 125 | end 126 | end 127 | 128 | describe 'abort' do 129 | before do 130 | subject.push(payload) 131 | end 132 | 133 | it 'should add the job back on the queue' do 134 | popped_payload = subject.pop(payload.queue) 135 | popped_payload.id.should == payload.id 136 | subject.size(payload.queue).should == 0 137 | subject.abort(popped_payload) 138 | subject.size(payload.queue).should == 1 139 | end 140 | end 141 | 142 | describe 'fail' do 143 | it 'should be defined and accept payload' do 144 | subject.fail(payload) 145 | end 146 | end 147 | 148 | describe 'size' do 149 | it 'should use the default queue by default' do 150 | subject.size.should == 0 151 | subject.push(payload) 152 | subject.size.should == 1 153 | end 154 | end 155 | 156 | describe 'clear' do 157 | it 'should clear jobs for given queue' do 158 | subject.push(payload) 159 | subject.size(payload.queue).should == 1 160 | subject.clear(payload.queue) 161 | subject.size(payload.queue).should == 0 162 | end 163 | 164 | it 'should not clear jobs for a different queue' do 165 | subject.push(payload) 166 | subject.clear('other') 167 | subject.size(payload.queue).should == 1 168 | end 169 | end 170 | 171 | describe 'connection=' do 172 | it 'should allow setting the connection' do 173 | connection = double('a connection') 174 | subject.connection = connection 175 | subject.connection.should == connection 176 | end 177 | 178 | it 'should provide a default connection' do 179 | subject.connection.should_not be_nil 180 | end 181 | end 182 | end 183 | -------------------------------------------------------------------------------- /lib/qu/backend/sqs.rb: -------------------------------------------------------------------------------- 1 | require 'aws/sqs' 2 | require 'securerandom' 3 | 4 | module Qu 5 | module Backend 6 | class SQS < Base 7 | def push(payload) 8 | payload.id = SecureRandom.uuid 9 | 10 | queue = begin 11 | connection.queues.named(payload.queue) 12 | rescue ::AWS::SQS::Errors::NonExistentQueue 13 | connection.queues.create(payload.queue) 14 | end 15 | 16 | queue.send_message(dump(payload.attributes_for_push)) 17 | payload 18 | end 19 | 20 | def complete(payload) 21 | payload.message.delete if payload.message 22 | end 23 | 24 | def abort(payload) 25 | payload.message.visibility_timeout = 0 if payload.message 26 | end 27 | 28 | def fail(payload) 29 | payload.message.visibility_timeout = 0 if payload.message 30 | end 31 | 32 | def pop(queue_name = 'default') 33 | begin 34 | queue = connection.queues.named(queue_name) 35 | 36 | if message = queue.receive_message 37 | doc = load(message.body) 38 | payload = Payload.new(doc) 39 | payload.message = message 40 | return payload 41 | end 42 | rescue ::AWS::SQS::Errors::NonExistentQueue 43 | end 44 | end 45 | 46 | def size(queue_name = 'default') 47 | begin 48 | connection.queues.named(queue_name).visible_messages 49 | rescue ::AWS::SQS::Errors::NonExistentQueue 50 | 0 51 | end 52 | end 53 | 54 | def clear(queue_name = 'default') 55 | begin 56 | queue = connection.queues.named(queue_name) 57 | messages = [] 58 | begin 59 | begin 60 | messages = queue.receive_message(:limit => 10) 61 | queue.batch_delete(messages) 62 | rescue ::AWS::SQS::Errors::BatchDeleteSend 63 | end 64 | end while messages.size > 0 65 | rescue ::AWS::SQS::Errors::NonExistentQueue 66 | # doesn't exist so no need to flush 67 | end 68 | end 69 | 70 | def connection 71 | @connection ||= ::AWS::SQS.new 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/qu/failure.rb: -------------------------------------------------------------------------------- 1 | require "qu/failure/log" 2 | require "qu/instrumenter" 3 | 4 | module Qu 5 | module Failure 6 | extend Qu::Instrumenter 7 | 8 | # Public: Creates a failure for the given payload and exception using the 9 | # current failure backend. 10 | # 11 | # payload - The Qu::Payload that raised an exception when performing. 12 | # exception - The exception raised. 13 | # 14 | # Returns nothing. 15 | def self.create(payload, exception) 16 | instrument("failure_create.#{InstrumentationNamespace}") do |ipayload| 17 | ipayload[:payload] = payload 18 | ipayload[:exception] = exception 19 | 20 | backend.create(payload, exception) 21 | end 22 | end 23 | 24 | # Public: Allows user to change failure backend. 25 | def self.backend=(backend) 26 | @backend = backend 27 | end 28 | 29 | # Private: Returns the current failure backend. 30 | def self.backend 31 | @backend ||= Failure::Log 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/qu/failure/log.rb: -------------------------------------------------------------------------------- 1 | require "qu/logger" 2 | 3 | module Qu 4 | module Failure 5 | class Log 6 | extend ::Qu::Logger 7 | 8 | def self.create(payload, exception) 9 | logger.fatal "Qu failure #{payload.to_s}" 10 | log_exception(exception) 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/qu/hooks.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Hooks 3 | def self.included(base) 4 | base.extend ClassMethods 5 | base.send :include, InstanceMethods 6 | end 7 | 8 | module ClassMethods 9 | def define_hooks(*hooks) 10 | hooks.each do |hook| 11 | define_hook_by_kinds(hook, *%w(before after around)) 12 | end 13 | end 14 | 15 | def define_hook_by_kinds( hook, *kinds ) 16 | kinds.each do |kind| 17 | class_eval <<-end_eval, __FILE__, __LINE__ 18 | def self.#{kind}_#{hook}(*methods) 19 | hooks(:#{hook}).add(:#{kind}, *methods) 20 | end 21 | end_eval 22 | end 23 | end 24 | 25 | def hooks(name) 26 | @hooks ||= {} 27 | @hooks[name] ||= Chain.new 28 | end 29 | end 30 | 31 | module InstanceMethods 32 | 33 | def run_hook(name, *args, &block) 34 | find_hooks_for(name).run(self, args, &block) 35 | end 36 | 37 | def run_before_hook( name, *args ) 38 | run_hook_by_type(name, *args, :before) 39 | end 40 | 41 | def run_after_hook( name, *args ) 42 | run_hook_by_type(name, *args, :after) 43 | end 44 | 45 | def find_hooks_for(name) 46 | if self.class.superclass < Qu::Hooks 47 | self.class.superclass.hooks(name).dup.concat self.class.hooks(name) 48 | else 49 | self.class.hooks(name) 50 | end 51 | end 52 | 53 | def halt 54 | throw :halt 55 | end 56 | 57 | private 58 | 59 | def run_hook_by_type( name, type, *args, &block ) 60 | if hook = find_hooks_for(name).find { |h| h.type == type } 61 | hook.call(self, args, &block) 62 | end 63 | end 64 | 65 | end 66 | 67 | class Chain < Array 68 | def run(object, args, &block) 69 | catch :halt do 70 | reverse.inject(block) do |chain, hook| 71 | lambda { hook.call(object, args, &chain) } 72 | end.call 73 | end 74 | end 75 | 76 | def add(kind, *methods) 77 | methods.each {|method| self << Hook.new(kind, method) } 78 | end 79 | 80 | end 81 | 82 | class Hook 83 | attr_reader :type, :method 84 | 85 | def initialize(type, method) 86 | @type, @method = type, method 87 | end 88 | 89 | def call(obj, args, &chain) 90 | if type == :around 91 | obj.send method, *args, &chain 92 | else 93 | obj.send method, *args if type == :before 94 | chain.call if chain 95 | obj.send method, *args if type == :after 96 | end 97 | end 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/qu/instrumentation/log_subscriber.rb: -------------------------------------------------------------------------------- 1 | require 'securerandom' 2 | require 'active_support/notifications' 3 | require 'active_support/log_subscriber' 4 | 5 | module Qu 6 | module Instrumentation 7 | class LogSubscriber < ::ActiveSupport::LogSubscriber 8 | 9 | def logger 10 | LogSubscriber.logger 11 | end 12 | 13 | def pop(event) 14 | log_event(:pop, event) 15 | end 16 | 17 | def push(event) 18 | log_event(:push, event) 19 | end 20 | 21 | def perform(event) 22 | log_event(:perform, event) 23 | end 24 | 25 | def complete(event) 26 | log_event(:complete, event) 27 | end 28 | 29 | def abort(event) 30 | log_event(:abort, event) 31 | end 32 | 33 | def fail(event) 34 | log_event(:fail, event) 35 | end 36 | 37 | private 38 | 39 | def log_event(type, event) 40 | return unless logger.debug? 41 | 42 | description = "Qu #{type}" 43 | details = [] 44 | 45 | if queue_name = event.payload[:queue_name] 46 | details << "queue_name=#{queue_name}" 47 | end 48 | 49 | if payload = event.payload[:payload] 50 | details << "payload=#{payload}" 51 | end 52 | 53 | name = '%s (%.1fms)' % [description, event.duration] 54 | name_color = odd? ? CYAN : MAGENTA 55 | 56 | debug " #{color(name, name_color, true)} [ #{details.join(' ')} ]" 57 | end 58 | 59 | def odd? 60 | @odd_or_even = !@odd_or_even 61 | end 62 | end 63 | 64 | LogSubscriber.logger = Qu.logger 65 | LogSubscriber.attach_to :qu 66 | end 67 | end 68 | 69 | Qu.configure do |config| 70 | config.instrumenter = ActiveSupport::Notifications 71 | end 72 | -------------------------------------------------------------------------------- /lib/qu/instrumentation/statsd.rb: -------------------------------------------------------------------------------- 1 | require 'securerandom' 2 | require 'active_support/notifications' 3 | require 'qu/instrumentation/statsd_subscriber' 4 | 5 | ActiveSupport::Notifications.subscribe /\.#{Qu::InstrumentationNamespace}$/, 6 | Qu::Instrumentation::StatsdSubscriber 7 | -------------------------------------------------------------------------------- /lib/qu/instrumentation/statsd_subscriber.rb: -------------------------------------------------------------------------------- 1 | require 'qu/instrumentation/subscriber' 2 | require 'statsd' 3 | 4 | module Qu 5 | module Instrumentation 6 | class StatsdSubscriber < Subscriber 7 | class << self 8 | attr_accessor :client 9 | end 10 | 11 | def update_timer(metric) 12 | if self.class.client 13 | self.class.client.timing metric, (@duration * 1_000).round 14 | end 15 | end 16 | 17 | def update_counter(metric) 18 | if self.class.client 19 | self.class.client.increment metric 20 | end 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/qu/instrumentation/subscriber.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Instrumentation 3 | class Subscriber 4 | # Public: Use this as the subscribed block. 5 | def self.call(name, start, ending, transaction_id, payload) 6 | new(name, start, ending, transaction_id, payload).update 7 | end 8 | 9 | # Private: Initializes a new event processing instance. 10 | def initialize(name, start, ending, transaction_id, payload) 11 | @name = name 12 | @start = start 13 | @ending = ending 14 | @payload = payload 15 | @duration = ending - start 16 | @transaction_id = transaction_id 17 | end 18 | 19 | # Internal: Override in subclass. 20 | def update_timer(metric) 21 | raise 'not implemented' 22 | end 23 | 24 | # Internal: Override in subclass. 25 | def update_counter(metric) 26 | raise 'not implemented' 27 | end 28 | 29 | # Private 30 | def update 31 | op = @name.split('.', 2).first 32 | return unless op 33 | 34 | update_timer "qu.op.#{op}" 35 | 36 | case op 37 | when "push" 38 | if payload = @payload[:payload] 39 | update_timer "qu.queue.#{payload.queue}.#{op}" 40 | update_timer "qu.job.#{payload.klass}.#{op}" 41 | end 42 | when "pop" 43 | if queue_name = @payload[:queue_name] 44 | update_timer "qu.queue.#{queue_name}.#{op}" 45 | end 46 | else 47 | if payload = @payload[:payload] 48 | update_timer "qu.job.#{payload.klass}.#{op}" 49 | end 50 | end 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/qu/instrumenter.rb: -------------------------------------------------------------------------------- 1 | require "forwardable" 2 | 3 | module Qu 4 | module Instrumenter 5 | extend Forwardable 6 | 7 | def_delegator :"Qu.instrumenter", :instrument 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/qu/instrumenters/memory.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Instrumenters 3 | # Instrumentor that is useful for tests as it stores each of the events that 4 | # are instrumented. 5 | class Memory 6 | Event = Struct.new(:name, :payload, :result) 7 | 8 | attr_reader :events 9 | 10 | def initialize 11 | @events = [] 12 | end 13 | 14 | def instrument(name, payload = {}) 15 | result = if block_given? 16 | yield payload 17 | else 18 | nil 19 | end 20 | @events << Event.new(name, payload, result) 21 | result 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/qu/instrumenters/noop.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Instrumenters 3 | class Noop 4 | def self.instrument(name, payload = {}) 5 | yield payload if block_given? 6 | end 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/qu/job.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | class Job 3 | include Qu::Hooks 4 | define_hooks :push, :perform, :complete, :abort, :fail 5 | define_hook_by_kinds :fork, :before, :after 6 | 7 | attr_accessor :payload 8 | 9 | def self.queue(name = nil) 10 | @queue = name.to_s if name 11 | @queue ||= 'default' 12 | end 13 | 14 | def self.load(payload) 15 | allocate.tap do |job| 16 | job.payload = payload 17 | job.send :initialize, *payload.args 18 | end 19 | end 20 | 21 | def self.create(*args) 22 | Payload.new(:klass => self, :args => args).tap { |payload| payload.push } 23 | end 24 | 25 | # Public: Feel free to override this in your class with specific arg names 26 | # and all that. 27 | def initialize(*) 28 | end 29 | 30 | # Public: Feel free to override this as well. 31 | def perform 32 | end 33 | 34 | # Public: Gives you access to Qu's logger in your job. 35 | def logger 36 | Qu.logger 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/qu/logger.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Logger 3 | def logger 4 | Qu.logger 5 | end 6 | 7 | def log_exception(exception) 8 | message = "\n#{exception.class} (#{exception.message}):\n " 9 | message << clean_backtrace(exception).join("\n ") << "\n\n" 10 | logger.fatal(message) 11 | end 12 | 13 | def clean_backtrace(exception) 14 | defined?(Rails) && Rails.respond_to?(:backtrace_cleaner) ? 15 | Rails.backtrace_cleaner.clean(exception.backtrace) : 16 | exception.backtrace 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/qu/payload.rb: -------------------------------------------------------------------------------- 1 | require 'ostruct' 2 | 3 | module Qu 4 | class Payload < OpenStruct 5 | include Qu::Instrumenter 6 | include Logger 7 | 8 | undef_method(:id) if method_defined?(:id) 9 | 10 | def initialize(options = {}) 11 | super 12 | self.args ||= [] 13 | end 14 | 15 | def klass 16 | @klass ||= constantize(super) 17 | end 18 | 19 | def job 20 | @job ||= klass.load(self) 21 | end 22 | 23 | def queue 24 | @queue ||= (klass.instance_variable_get(:@queue) || 'default').to_s 25 | end 26 | 27 | def perform 28 | job.run_hook(:perform) do 29 | instrument("perform.#{InstrumentationNamespace}") do |ipayload| 30 | ipayload[:payload] = self 31 | job.perform 32 | end 33 | end 34 | 35 | job.run_hook(:complete) { Qu.complete(self) } 36 | rescue Qu::Worker::Abort 37 | abort 38 | rescue => exception 39 | fail(exception) 40 | end 41 | 42 | def abort 43 | job.run_hook(:abort) { Qu.abort(self) } 44 | raise 45 | end 46 | 47 | def fail(exception) 48 | job.run_hook(:fail, exception) { Qu.fail(self) } 49 | Qu::Failure.create(self, exception) 50 | end 51 | 52 | # Internal: Pushes payload to backend. 53 | def push 54 | self.pushed_at = Time.now.utc 55 | 56 | job.run_hook(:push) { Qu.push(self) } 57 | end 58 | 59 | def attributes 60 | { 61 | :id => id, 62 | :klass => klass.to_s, 63 | :args => args, 64 | } 65 | end 66 | 67 | def attributes_for_push 68 | attributes 69 | end 70 | 71 | def to_s 72 | "#{id}:#{klass}:#{args.inspect}" 73 | end 74 | 75 | private 76 | 77 | def constantize(class_name) 78 | return unless class_name 79 | return class_name if class_name.is_a?(Class) 80 | constant = Object 81 | class_name.split('::').each do |name| 82 | constant = constant.const_get(name) || constant.const_missing(name) 83 | end 84 | constant 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/qu/runner/base.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Runner 3 | RunnerLimitReached = Class.new(StandardError) 4 | 5 | class Base 6 | # Public: Override in subclass. 7 | def run(worker, payload) 8 | payload.perform 9 | end 10 | 11 | # Public: Override in subclass. 12 | def stop 13 | end 14 | 15 | # Public: Override in subclass. 16 | def full? 17 | false 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/qu/runner/direct.rb: -------------------------------------------------------------------------------- 1 | require 'qu/runner/base' 2 | 3 | module Qu 4 | module Runner 5 | class Direct < Base 6 | def run(worker, payload) 7 | @full = true 8 | payload.perform 9 | ensure 10 | @full = false 11 | end 12 | 13 | def full? 14 | @full 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/qu/runner/forking.rb: -------------------------------------------------------------------------------- 1 | require 'qu/runner/base' 2 | require 'qu/util/signal_handler' 3 | require 'qu/util/thread_safe_hash' 4 | require 'qu/util/process_wrapper' 5 | 6 | module Qu 7 | module Runner 8 | class Forking < Base 9 | attr_reader :fork_limit, :forks 10 | 11 | def initialize(fork_limit = 1) 12 | @fork_limit = fork_limit 13 | @forks = Qu::Util::ThreadSafeHash.new 14 | end 15 | 16 | def full? 17 | forks.size == fork_limit 18 | end 19 | 20 | def run(worker, payload) 21 | raise RunnerLimitReached.new("#{self.class.name} is already running #{fork_limit} jobs") if full? 22 | 23 | process = Qu::Util::ProcessWrapper.new( forks, worker, payload ) 24 | process.fork 25 | end 26 | 27 | def stop 28 | forks.values.each do |process| 29 | process.stop 30 | end 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/qu/runner/spec.rb: -------------------------------------------------------------------------------- 1 | require 'qu/backend/redis' 2 | 3 | class RunnerJob < Qu::Job 4 | end 5 | 6 | class RedisPusherJob < Qu::Job 7 | def self.client 8 | @client ||= Qu.backend.connection 9 | end 10 | 11 | def initialize(list, value) 12 | @list = list 13 | @value = value 14 | end 15 | 16 | def perform 17 | self.class.client.lpush(@list, @value) 18 | end 19 | end 20 | 21 | class SleepJob < Qu::Job 22 | def initialize(sleep_time = 5) 23 | @sleep = sleep_time 24 | end 25 | 26 | def perform 27 | sleep(@sleep) 28 | end 29 | end 30 | 31 | shared_examples_for 'a runner interface' do 32 | 33 | let(:payload) { Qu::Payload.new(:klass => RunnerJob) } 34 | 35 | it 'can run a payload' do 36 | subject.run(double("worker"), payload) 37 | end 38 | 39 | it 'can check if if it is full' do 40 | subject.full? 41 | end 42 | 43 | it 'can be stopped' do 44 | subject.stop 45 | end 46 | end 47 | 48 | shared_examples_for 'a single job runner' do 49 | 50 | let(:list) { 'push-test-list' } 51 | let(:payload) { Qu::Payload.new(:klass => RedisPusherJob, :args => [list, '1']) } 52 | let(:timeout) { 5 } 53 | 54 | before do 55 | Qu.backend = Qu::Backend::Redis.new 56 | RedisPusherJob.client.del(list) 57 | end 58 | 59 | def expect_values(*args) 60 | timeout.times do 61 | break unless subject.full? 62 | sleep(1) 63 | end 64 | 65 | result = RedisPusherJob.client.lrange(list, 0, -1) 66 | if result.size == args.size 67 | return expect(result).to eq(args) 68 | end 69 | end 70 | 71 | it 'can execute a payload' do 72 | subject.run(double('worker'), payload) 73 | expect_values('1') 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/qu/tasks.rb: -------------------------------------------------------------------------------- 1 | namespace :qu do 2 | desc "Start a worker" 3 | task :work => :environment do 4 | queues = (ENV['QUEUES'] || ENV['QUEUE'] || 'default').to_s.split(',') 5 | worker = Qu::Worker.new(*queues) 6 | begin 7 | worker.start 8 | rescue Qu::Worker::Stop 9 | Qu.logger.debug "Worker #{worker.id} stopped" 10 | rescue Qu::Worker::Abort 11 | Qu.logger.debug "Worker #{worker.id} aborted" 12 | exit(1) 13 | end 14 | end 15 | end 16 | 17 | # Convenience tasks compatibility 18 | task 'jobs:work' => 'qu:work' 19 | task 'resque:work' => 'qu:work' 20 | 21 | # No-op task in case it doesn't already exist 22 | task :environment -------------------------------------------------------------------------------- /lib/qu/util/process_wrapper.rb: -------------------------------------------------------------------------------- 1 | require 'qu/util/signal_handler' 2 | require 'qu/util/procline' 3 | 4 | module Qu 5 | module Util 6 | FailedToForkError = Class.new(StandardError) 7 | ExitFailureError = Class.new(StandardError) 8 | 9 | class ProcessWrapper 10 | attr_reader :pid, :payload, :worker, :kill_timeout 11 | 12 | def initialize(process_collection, worker, payload, kill_timeout = 5) 13 | @process_collection = process_collection 14 | @worker = worker 15 | @payload = payload 16 | @kill_timeout = kill_timeout 17 | end 18 | 19 | def fork 20 | payload.job.run_before_hook(:fork) 21 | parent_pid = Process.pid 22 | @pid = Kernel.fork do 23 | begin 24 | $stdout.sync = true 25 | $stderr.sync = true 26 | Qu::Util::Procline.set("fork of #{parent_pid} working on #{payload.id} from #{payload.queue}") 27 | Qu.backend.reconnect 28 | SignalHandler.clear(*Qu::Worker::SIGNALS) 29 | payload.job.run_after_hook(:fork) 30 | payload.perform 31 | ensure 32 | exit! 33 | end 34 | end 35 | 36 | if @pid 37 | setup_wait_watcher 38 | else 39 | raise FailedToForkError.new("Could not fork process") 40 | end 41 | 42 | @pid 43 | end 44 | 45 | def setup_wait_watcher 46 | @process_collection[pid] = self 47 | Thread.new do 48 | begin 49 | Process.waitpid(pid) rescue SystemCallError 50 | payload.fail(ExitFailureError.new($?.to_s)) if $?.signaled? 51 | rescue => e 52 | logger.error("Failed waiting for process #{e.message}\n#{e.backtrace.join("\n")}") 53 | ensure 54 | @process_collection.delete(pid) 55 | end 56 | end 57 | end 58 | 59 | def stop 60 | return unless pid 61 | 62 | if Process.waitpid(pid, Process::WNOHANG) 63 | logger.info "Child #{pid} already quit." 64 | return 65 | end 66 | 67 | signal_child("TERM") 68 | signal_child("KILL") unless quit_gracefully? 69 | rescue SystemCallError 70 | logger.info "Child #{pid} already quit and reaped." 71 | ensure 72 | @process_collection.delete(pid) 73 | end 74 | 75 | def signal_child(signal) 76 | logger.info "Sending #{signal} signal to child #{pid}" 77 | Process.kill(signal, pid) 78 | end 79 | 80 | def quit_gracefully? 81 | if Qu.graceful_shutdown 82 | self.kill_timeout.times do 83 | sleep(1) 84 | return true if Process.waitpid(pid, Process::WNOHANG) 85 | end 86 | end 87 | return false 88 | end 89 | 90 | def logger 91 | Qu.logger 92 | end 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /lib/qu/util/procline.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Util 3 | class Procline 4 | def self.set(message) 5 | $0 = "qu-#{Qu::VERSION}: #{message}" 6 | end 7 | end 8 | end 9 | end -------------------------------------------------------------------------------- /lib/qu/util/signal_handler.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | module Util 3 | class SignalHandler 4 | def self.trap( *signals, &block ) 5 | signals.each do |signal| 6 | Signal.trap(signal) do 7 | block.call(signal) 8 | end 9 | end 10 | end 11 | 12 | def self.clear(*signals) 13 | signals.each do |signal| 14 | begin 15 | Signal.trap(signal, 'DEFAULT') 16 | rescue ArgumentError => e 17 | warn "Could not trap signal #{signal} - #{e}" 18 | end 19 | end 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/qu/util/thread_safe_hash.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | require 'monitor' 3 | 4 | module Qu 5 | module Util 6 | class ThreadSafeHash 7 | include Enumerable 8 | extend Forwardable 9 | 10 | def_delegator :@monitor, :synchronize 11 | def_delegators :@items, :size, :[] 12 | 13 | def initialize(original = {}) 14 | @items = original.dup 15 | @monitor = Monitor.new 16 | end 17 | 18 | def each( &block ) 19 | synchronize { @items.each(&block) } 20 | end 21 | 22 | def []=(key,value) 23 | synchronize { @items[key] = value } 24 | end 25 | 26 | def delete(value) 27 | synchronize { @items.delete(value) } 28 | end 29 | 30 | def values 31 | synchronize { @items.values } 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/qu/version.rb: -------------------------------------------------------------------------------- 1 | module Qu 2 | VERSION = "0.2.0" 3 | end 4 | -------------------------------------------------------------------------------- /lib/qu/worker.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | require 'qu/util/signal_handler' 3 | 4 | module Qu 5 | class Worker 6 | include Logger 7 | 8 | SIGNALS = [:INT, :TERM] 9 | 10 | attr_accessor :queues 11 | 12 | # Internal: Raised when signal received, no job is being performed, and 13 | # graceful shutdown is disabled. 14 | class Abort < StandardError 15 | end 16 | 17 | # Internal: Raised when signal received and no job is being performed. 18 | class Stop < StandardError 19 | end 20 | 21 | def initialize(*queues) 22 | @queues = queues.flatten.map { |q| q.to_s.strip } 23 | @queues << 'default' if @queues.empty? 24 | @running = false 25 | @performing = false 26 | end 27 | 28 | def id 29 | @id ||= "#{hostname}:#{pid}:#{queues.join(',')}" 30 | end 31 | 32 | def work 33 | did_work = false 34 | 35 | unless Qu.runner.full? 36 | queues.each do |queue_name| 37 | if payload = Qu.pop(queue_name) 38 | begin 39 | @performing = true 40 | Qu.runner.run(self, payload) 41 | ensure 42 | did_work = true 43 | @performing = false 44 | break 45 | end 46 | end 47 | end 48 | end 49 | 50 | did_work 51 | end 52 | 53 | def start 54 | return if running? 55 | @running = true 56 | 57 | logger.warn "Worker #{id} starting" 58 | register_signal_handlers 59 | 60 | loop do 61 | unless running? 62 | break 63 | end 64 | 65 | unless work 66 | sleep Qu.interval 67 | end 68 | end 69 | rescue => e 70 | logger.error("Failed run loop #{e.message}\n#{e.backtrace.join("\n")}") 71 | raise 72 | ensure 73 | stop 74 | end 75 | 76 | def stop 77 | @running = false 78 | Qu.runner.stop 79 | 80 | if performing? 81 | raise Abort unless Qu.graceful_shutdown 82 | else 83 | raise Stop 84 | end 85 | end 86 | 87 | def performing? 88 | @performing 89 | end 90 | 91 | def running? 92 | @running 93 | end 94 | 95 | private 96 | 97 | def pid 98 | @pid ||= Process.pid 99 | end 100 | 101 | def hostname 102 | @hostname ||= Socket.gethostname 103 | end 104 | 105 | def register_signal_handlers 106 | logger.debug "Worker #{id} registering traps for INT and TERM signals" 107 | Qu::Util::SignalHandler.trap( *SIGNALS ) do |signal| 108 | logger.info("Worker #{id} received #{signal}, stopping") 109 | stop 110 | end 111 | end 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /qu-mongo.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "qu/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "qu-mongo" 7 | s.version = Qu::VERSION 8 | s.authors = ["Brandon Keepers"] 9 | s.email = ["brandon@opensoul.org"] 10 | s.homepage = "http://github.com/bkeepers/qu" 11 | s.summary = "Mongo backend for qu" 12 | s.description = "Mongo backend for qu" 13 | 14 | s.files = `git ls-files -- lib | grep mongo`.split("\n") 15 | s.require_paths = ["lib"] 16 | 17 | s.add_dependency 'mongo', '~> 1.9.0' 18 | s.add_dependency 'qu', Qu::VERSION 19 | 20 | s.add_development_dependency 'bson_ext', '~> 1.9.0' 21 | end 22 | -------------------------------------------------------------------------------- /qu-rails.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "qu/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "qu-rails" 7 | s.version = Qu::VERSION 8 | s.authors = ["Brandon Keepers"] 9 | s.email = ["brandon@opensoul.org"] 10 | s.homepage = "http://github.com/bkeepers/qu" 11 | s.summary = "Rails integration for qu" 12 | s.description = "Rails integration for qu" 13 | 14 | s.files = `git ls-files -- lib | grep rails`.split("\n") 15 | s.require_paths = ["lib"] 16 | 17 | s.add_dependency 'railties', '>= 3.2', '< 5' 18 | s.add_dependency 'qu', Qu::VERSION 19 | end 20 | -------------------------------------------------------------------------------- /qu-redis.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "qu/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "qu-redis" 7 | s.version = Qu::VERSION 8 | s.authors = ["Brandon Keepers"] 9 | s.email = ["brandon@opensoul.org"] 10 | s.homepage = "http://github.com/bkeepers/qu" 11 | s.summary = "Redis backend for qu" 12 | s.description = "Redis backend for qu" 13 | 14 | s.files = `git ls-files -- lib | grep redis`.split("\n") 15 | s.require_paths = ["lib"] 16 | 17 | s.add_dependency 'redis-namespace' 18 | s.add_dependency 'qu', Qu::VERSION 19 | end 20 | -------------------------------------------------------------------------------- /qu-sqs.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "qu/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "qu-sqs" 7 | s.version = Qu::VERSION 8 | s.authors = ["John Nunemaker"] 9 | s.email = ["nunemaker@gmail.com"] 10 | s.homepage = "http://github.com/bkeepers/qu" 11 | s.summary = "SQS backend for qu" 12 | s.description = "SQS backend for qu" 13 | 14 | s.files = `git ls-files -- lib | grep sqs`.split("\n") 15 | s.require_paths = ["lib"] 16 | 17 | s.add_dependency 'aws-sdk', '~> 1.0' 18 | s.add_dependency 'qu', Qu::VERSION 19 | 20 | s.add_development_dependency 'fake_sqs', '~> 0.1.0' 21 | end 22 | -------------------------------------------------------------------------------- /qu.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "qu/version" 4 | 5 | plugin_files = Dir['qu-*.gemspec'].map { |gemspec| 6 | eval(File.read(gemspec)).files 7 | }.flatten.uniq 8 | 9 | Gem::Specification.new do |s| 10 | s.name = "qu" 11 | s.version = Qu::VERSION 12 | s.authors = ["Brandon Keepers"] 13 | s.email = ["brandon@opensoul.org"] 14 | s.homepage = "http://github.com/bkeepers/qu" 15 | s.summary = %q{a Ruby library for queuing and processing background jobs.} 16 | s.description = %q{a Ruby library for queuing and processing background jobs with configurable backends.} 17 | 18 | s.files = `git ls-files`.split("\n") - plugin_files 19 | s.test_files = `git ls-files -- spec`.split("\n") 20 | s.executables = `git ls-files -- bin`.split("\n").map{ |f| File.basename(f) } 21 | s.require_paths = ["lib"] 22 | end 23 | -------------------------------------------------------------------------------- /script/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #/ Usage: bootstrap [bundle options] 3 | #/ 4 | #/ Bundle install the dependencies. 5 | #/ 6 | #/ Examples: 7 | #/ 8 | #/ bootstrap 9 | #/ bootstrap --local 10 | #/ 11 | 12 | set -e 13 | cd $(dirname "$0")/.. 14 | 15 | [ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && { 16 | grep '^#/' <"$0"| cut -c4- 17 | exit 0 18 | } 19 | 20 | rm -rf .bundle/{binstubs,config} 21 | bundle install --binstubs .bundle/binstubs --path .bundle --quiet "$@" 22 | -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #/ Usage: test [individual test file] 3 | #/ 4 | #/ Bootstrap and run all tests or an individual test. 5 | #/ 6 | #/ Examples: 7 | #/ 8 | #/ # run all tests 9 | #/ test 10 | #/ 11 | #/ # run individual test 12 | #/ test spec/qu_spec.rb 13 | #/ 14 | 15 | set -e 16 | cd $(dirname "$0")/.. 17 | 18 | [ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && { 19 | grep '^#/' <"$0"| cut -c4- 20 | exit 0 21 | } 22 | 23 | specs="spec/" 24 | 25 | if [ $# -gt 0 ] 26 | then 27 | specs=$@ 28 | fi 29 | 30 | script/bootstrap && bundle exec rspec $specs 31 | -------------------------------------------------------------------------------- /spec/qu/backend/immediate_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu-immediate' 3 | 4 | describe Qu::Backend::Immediate do 5 | let(:payload) { Qu::Payload.new(:klass => SimpleJob) } 6 | 7 | it 'performs immediately' do 8 | payload.should_receive(:perform) 9 | subject.push(payload) 10 | end 11 | 12 | it_should_behave_like 'a backend interface' 13 | end 14 | -------------------------------------------------------------------------------- /spec/qu/backend/instrumented_spec.rb: -------------------------------------------------------------------------------- 1 | # To run the tests for this you need to have fake_sqs running. 2 | # You can fire it up like this: 3 | # 4 | # bundle exec fake_sqs -p 5111 5 | # 6 | require 'spec_helper' 7 | require 'qu/backend/instrumented' 8 | require 'qu-redis' 9 | 10 | describe Qu::Backend::Instrumented do 11 | subject { 12 | described_class.new(Qu::Backend::Redis.new) 13 | } 14 | 15 | it_should_behave_like 'a backend', :services => :redis 16 | it_should_behave_like 'a backend interface' 17 | end 18 | -------------------------------------------------------------------------------- /spec/qu/backend/memory_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/backend/memory' 3 | 4 | describe Qu::Backend::Memory do 5 | it_should_behave_like 'a backend' 6 | it_should_behave_like 'a backend interface' 7 | end -------------------------------------------------------------------------------- /spec/qu/backend/mongo_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu-mongo' 3 | 4 | describe Qu::Backend::Mongo do 5 | if Qu::Specs.perform?(described_class, :mongo) 6 | it_should_behave_like 'a backend' 7 | it_should_behave_like 'a backend interface' 8 | 9 | describe 'connection' do 10 | it 'should default the qu database' do 11 | subject.connection.should be_instance_of(Mongo::DB) 12 | subject.connection.name.should == 'qu' 13 | end 14 | 15 | it 'should use MONGOHQ_URL from heroku' do 16 | begin 17 | Mongo::MongoClient.any_instance.stub(:connect) 18 | ENV['MONGOHQ_URL'] = 'mongodb://user:pw@host:10060/quspec' 19 | subject.connection.name.should == 'quspec' 20 | # debugger 21 | subject.connection.connection.host_port.should == ['host', 10060] 22 | subject.connection.connection.auths.should satisfy { |v| 23 | # Not happy about this, but the format of the auths attribute differs depending on your installed mongo version 24 | # mongo >= 1.8.4 uses symbols for hash keys 25 | # mongo < 1.8.4 uses strings for hash keys 26 | v == [{:db_name => 'quspec', :username => 'user', :password => 'pw'}] or 27 | v == [{'db_name' => 'quspec', 'username' => 'user', 'password' => 'pw'}] 28 | } 29 | ensure 30 | ENV.delete('MONGOHQ_URL') 31 | end 32 | end 33 | 34 | context "Connection Failure" do 35 | let(:retries_number) { 3 } 36 | let(:retries_frequency) { 5 } 37 | 38 | before do 39 | subject.max_retries = retries_number 40 | subject.retry_frequency = retries_frequency 41 | 42 | Mongo::Collection.any_instance.stub(:count).and_raise(Mongo::ConnectionFailure) 43 | subject.stub(:sleep) 44 | end 45 | 46 | it "raise error" do 47 | expect { subject.size }.to raise_error(Mongo::ConnectionFailure) 48 | end 49 | 50 | it "trying to reconnect" do 51 | subject.connection.should_receive(:[]).exactly(4).times.and_raise(Mongo::ConnectionFailure) 52 | expect { subject.size }.to raise_error 53 | end 54 | 55 | it "sleep between tries" do 56 | subject.should_receive(:sleep).with(5).ordered 57 | subject.should_receive(:sleep).with(10).ordered 58 | subject.should_receive(:sleep).with(15).ordered 59 | 60 | expect { subject.size }.to raise_error 61 | end 62 | 63 | end 64 | end 65 | 66 | describe 'pop' do 67 | let(:worker) { Qu::Worker.new } 68 | 69 | describe "on mongo >=2" do 70 | it 'should return nil when no jobs exist' do 71 | subject.clear 72 | Mongo::Collection.any_instance.should_receive(:find_and_modify).and_return(nil) 73 | lambda { subject.pop(worker).should be_nil }.should_not raise_error 74 | end 75 | end 76 | 77 | describe 'on mongo <2' do 78 | it 'should return nil when no jobs exist' do 79 | subject.clear 80 | Mongo::Collection.any_instance.should_receive(:find_and_modify).and_raise(Mongo::OperationFailure) 81 | lambda { subject.pop(worker).should be_nil }.should_not raise_error 82 | end 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/qu/backend/redis_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu-redis' 3 | 4 | describe Qu::Backend::Redis do 5 | if Qu::Specs.perform?(described_class, :redis) 6 | it_should_behave_like 'a backend' 7 | it_should_behave_like 'a backend interface' 8 | 9 | describe 'connection' do 10 | it 'should create default connection if one not provided' do 11 | subject.connection.should be_instance_of(Redis::Namespace) 12 | subject.connection.namespace.should == :qu 13 | end 14 | 15 | it 'should use REDISTOGO_URL from heroku with namespace' do 16 | begin 17 | ENV['REDISTOGO_URL'] = 'redis://0.0.0.0:9876' 18 | subject.connection.client.host.should == '0.0.0.0' 19 | subject.connection.client.port.should == 9876 20 | subject.connection.namespace.should == :qu 21 | ensure 22 | ENV.delete 'REDISTOGO_URL' 23 | end 24 | end 25 | 26 | it 'should allow customizing the namespace' do 27 | subject.namespace = :foobar 28 | subject.connection.namespace.should == :foobar 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/qu/backend/sqs_spec.rb: -------------------------------------------------------------------------------- 1 | # To run the tests for this you need to have fake_sqs running. 2 | # You can fire it up like this: 3 | # 4 | # bundle exec fake_sqs -p 5111 5 | # 6 | require 'spec_helper' 7 | require 'net/http' 8 | require 'qu-sqs' 9 | 10 | AWS.config( 11 | use_ssl: false, 12 | sqs_endpoint: 'localhost', 13 | sqs_port: 5111, 14 | access_key_id: 'asdf', 15 | secret_access_key: 'asdf', 16 | ) 17 | 18 | describe Qu::Backend::SQS do 19 | def reset_service(service) 20 | host = AWS.config.send("#{service}_endpoint") 21 | port = AWS.config.send("#{service}_port") 22 | Net::HTTP.new(host, port).request(Net::HTTP::Delete.new("/")) 23 | end 24 | 25 | if Qu::Specs.perform?(described_class, :sqs) 26 | before(:each) do 27 | reset_service(:sqs) 28 | end 29 | 30 | it_should_behave_like 'a backend' 31 | it_should_behave_like 'a backend interface' 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /spec/qu/failure_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Qu::Failure do 4 | describe "#create" do 5 | it "calls create on backend" do 6 | payload = Qu::Payload.new(:klass => SimpleJob) 7 | exception = StandardError.new 8 | Qu::Failure.backend.should_receive(:create).with(payload, exception) 9 | Qu::Failure.create(payload, exception) 10 | end 11 | end 12 | 13 | describe "#backend" do 14 | it "defaults to log" do 15 | described_class.backend.should eq(Qu::Failure::Log) 16 | end 17 | end 18 | 19 | describe "#backend=" do 20 | before do 21 | @original_backend = Qu::Failure.backend 22 | end 23 | 24 | after do 25 | Qu::Failure.backend = @original_backend 26 | end 27 | 28 | it "changes the backend" do 29 | new_backend = double("backend") 30 | Qu::Failure.backend = new_backend 31 | Qu::Failure.backend.should eq(new_backend) 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /spec/qu/hooks_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Qu::Hooks do 4 | before do 5 | class Pirate 6 | include Qu::Hooks 7 | define_hooks :pillage, :plunder 8 | 9 | attr_reader :events 10 | 11 | def initialize 12 | @events = [] 13 | end 14 | 15 | def pillage 16 | run_hook(:pillage) do 17 | @events << :pillage 18 | end 19 | end 20 | 21 | private 22 | 23 | def drink 24 | @events << :drink 25 | end 26 | 27 | def be_merry 28 | @events << :be_merry 29 | end 30 | 31 | def rest 32 | @events << :rest_before 33 | yield 34 | @events << :rest_after 35 | end 36 | end 37 | 38 | class Captain < Pirate 39 | def fight_peter_pan 40 | @events << :fight 41 | halt 42 | end 43 | end 44 | end 45 | 46 | after do 47 | Object.send :remove_const, :Captain 48 | Object.send :remove_const, :Pirate 49 | end 50 | 51 | let(:captain) { Captain.new } 52 | 53 | describe 'define_hooks' do 54 | it 'should create an empty chain' do 55 | Captain.hooks(:pillage).should be_instance_of(Qu::Hooks::Chain) 56 | Captain.hooks(:pillage).size.should == 0 57 | end 58 | 59 | it 'should define before, after and around methods' do 60 | Captain.respond_to?(:before_pillage).should be_true 61 | Captain.respond_to?(:after_pillage).should be_true 62 | Captain.respond_to?(:around_pillage).should be_true 63 | end 64 | end 65 | 66 | describe 'before_hook' do 67 | it 'should add hook with given method' do 68 | Captain.before_pillage :drink 69 | captain.pillage 70 | captain.events.should == [:drink, :pillage] 71 | end 72 | 73 | it 'should add hook with multiple methods' do 74 | Captain.before_pillage :drink, :be_merry 75 | captain.pillage 76 | captain.events.should == [:drink, :be_merry, :pillage] 77 | end 78 | 79 | it 'should inherit hooks from parent class' do 80 | Captain.before_pillage :be_merry 81 | Pirate.before_pillage :drink 82 | 83 | captain.pillage 84 | captain.events.should == [:drink, :be_merry, :pillage] 85 | end 86 | end 87 | 88 | describe 'after_hook' do 89 | it 'should add hook with given method' do 90 | Captain.after_pillage :drink 91 | captain.pillage 92 | captain.events.should == [:pillage, :drink] 93 | end 94 | 95 | it 'should add hook with multiple methods' do 96 | Captain.after_pillage :drink, :be_merry 97 | captain.pillage 98 | captain.events.should == [:pillage, :be_merry, :drink] 99 | end 100 | 101 | it 'should run declared hooks in reverse order' do 102 | Captain.after_pillage :drink 103 | Captain.after_pillage :be_merry 104 | captain.pillage 105 | captain.events.should == [:pillage, :be_merry, :drink] 106 | end 107 | end 108 | 109 | describe 'around_hook' do 110 | it 'should add hook with given method' do 111 | Captain.around_pillage :rest 112 | captain.pillage 113 | captain.events.should == [:rest_before, :pillage, :rest_after] 114 | end 115 | 116 | it 'should maintain order with before and after hooks' do 117 | Captain.around_pillage :rest 118 | Captain.before_pillage :drink 119 | Captain.after_pillage :be_merry 120 | captain.pillage 121 | captain.events.should == [:rest_before, :drink, :pillage, :be_merry, :rest_after] 122 | end 123 | 124 | it 'should halt chain if it does not yield' do 125 | Captain.around_pillage :drink 126 | Captain.before_pillage :be_merry 127 | captain.pillage 128 | captain.events.should == [:drink] 129 | end 130 | end 131 | 132 | describe 'run_hook' do 133 | it 'should call block when no hooks are declared' do 134 | captain.pillage 135 | captain.events.should == [:pillage] 136 | end 137 | 138 | it 'should pass args to method' do 139 | Captain.before_pillage :drink 140 | captain.should_receive(:drink).with(:rum) 141 | captain.run_hook(:pillage, :rum) { } 142 | end 143 | 144 | describe 'with a halt before' do 145 | before do 146 | Captain.before_pillage :fight_peter_pan, :drink 147 | end 148 | 149 | it 'should not call other hooks' do 150 | captain.should_not_receive :drink 151 | captain.run_hook(:pillage) {} 152 | end 153 | 154 | it 'should not invoke block' do 155 | called = false 156 | captain.run_hook(:pillage) { called = true } 157 | called.should be_false 158 | end 159 | end 160 | 161 | describe 'with a halt after' do 162 | before do 163 | Captain.after_pillage :drink, :fight_peter_pan 164 | end 165 | 166 | it 'should not call other hooks' do 167 | captain.should_not_receive :drink 168 | captain.run_hook(:pillage) {} 169 | end 170 | 171 | it 'should invoke block' do 172 | called = false 173 | captain.run_hook(:pillage) { called = true } 174 | called.should be_true 175 | end 176 | end 177 | end 178 | 179 | describe 'run_hook_by_type' do 180 | before do 181 | Captain.before_pillage :be_merry 182 | Captain.after_pillage :drink 183 | end 184 | 185 | it 'should not call the before hook' do 186 | expect(captain).not_to receive(:be_merry).with(no_args()) 187 | captain.run_after_hook(:pillage) 188 | expect(captain.events).to eq([:drink]) 189 | end 190 | 191 | it 'should not call the after hook' do 192 | expect(captain).not_to receive(:drink).with(no_args()) 193 | captain.run_before_hook(:pillage) 194 | expect(captain.events).to eq([:be_merry]) 195 | end 196 | 197 | it 'should not run hooks if they are not defined' do 198 | expect { captain.run_before_hook(:some_hook) }.to_not raise_error 199 | end 200 | 201 | end 202 | 203 | end 204 | -------------------------------------------------------------------------------- /spec/qu/instrumentation/log_subscriber_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/backend/redis' 3 | require 'qu/instrumentation/log_subscriber' 4 | 5 | describe Qu::Instrumentation::LogSubscriber do 6 | let(:io) { StringIO.new } 7 | let(:log) { io.string } 8 | 9 | before(:each) do 10 | Qu.backend = Qu::Backend::Redis.new 11 | Qu.clear 12 | @original_instrumenter = Qu.instrumenter 13 | Qu.instrumenter = ActiveSupport::Notifications 14 | described_class.logger = Logger.new(io).tap { |logger| 15 | logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" } 16 | } 17 | end 18 | 19 | after(:each) do 20 | Qu.instrumenter = @original_instrumenter 21 | described_class.logger = nil 22 | end 23 | 24 | it "logs empty pop" do 25 | worker = Qu::Worker.new 26 | worker.work 27 | line = find_line('Qu pop') 28 | line.should include("queue_name=default") 29 | end 30 | 31 | it "logs pop with payload" do 32 | payload = SimpleJob.create 33 | worker = Qu::Worker.new 34 | worker.work 35 | line = find_line('Qu pop') 36 | line.should include("queue_name=default payload=#{payload}") 37 | end 38 | 39 | it "logs push" do 40 | payload = SimpleJob.create 41 | line = find_line('Qu push') 42 | line.should include(payload.to_s) 43 | end 44 | 45 | it "logs perform" do 46 | payload = SimpleJob.create 47 | payload.perform 48 | line = find_line('Qu perform') 49 | line.should include(payload.to_s) 50 | end 51 | 52 | it "logs complete" do 53 | payload = SimpleJob.create 54 | payload.perform 55 | line = find_line('Qu complete') 56 | line.should include(payload.to_s) 57 | 58 | # should not get to abort 59 | expect { find_line('Qu abort') }.to raise_error 60 | end 61 | 62 | it "logs abort" do 63 | payload = SimpleJob.create 64 | payload.job.stub(:perform).and_raise(Qu::Worker::Abort.new) 65 | begin 66 | payload.perform 67 | flunk # should not get here 68 | rescue Qu::Worker::Abort 69 | line = find_line('Qu abort') 70 | line.should include(payload.to_s) 71 | 72 | # should not get to complete 73 | expect { find_line('Qu complete') }.to raise_error 74 | end 75 | end 76 | 77 | it "logs fail" do 78 | payload = SimpleJob.create 79 | payload.job.stub(:perform).and_raise(StandardError.new) 80 | 81 | begin 82 | payload.perform 83 | flunk # should not get here 84 | rescue => exception 85 | line = find_line('Qu fail') 86 | line.should include(payload.to_s) 87 | 88 | # should not get to complete 89 | expect { find_line('Qu complete') }.to raise_error 90 | end 91 | end 92 | 93 | def find_line(str) 94 | regex = /#{Regexp.escape(str)}/ 95 | lines = log.split("\n") 96 | lines.detect { |line| line =~ regex } || 97 | raise("Could not find line matching #{str.inspect} in #{lines.inspect}") 98 | end 99 | 100 | def clear_logs 101 | io.string = '' 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /spec/qu/instrumentation/statsd_subscriber_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/instrumentation/statsd' 3 | 4 | describe Qu::Instrumentation::StatsdSubscriber do 5 | let(:statsd_client) { Statsd.new } 6 | let(:socket) { FakeUDPSocket.new } 7 | 8 | before do 9 | Qu.backend = Qu::Backend::Redis.new 10 | Qu.clear 11 | @original_instrumenter = Qu.instrumenter 12 | Qu.instrumenter = ActiveSupport::Notifications 13 | described_class.client = statsd_client 14 | Thread.current[:statsd_socket] = socket 15 | end 16 | 17 | after do 18 | Qu.instrumenter = @original_instrumenter 19 | described_class.client = nil 20 | Thread.current[:statsd_socket] = nil 21 | end 22 | 23 | def assert_timer(metric) 24 | regex = /#{Regexp.escape metric}\:\d+\|ms/ 25 | socket.buffer.detect { |op| op.first =~ regex }.should_not be_nil 26 | end 27 | 28 | def assert_counter(metric) 29 | socket.buffer.detect { |op| op.first == "#{metric}:1|c" }.should_not be_nil 30 | end 31 | 32 | it "instruments pop" do 33 | worker = Qu::Worker.new 34 | worker.work 35 | assert_timer "qu.op.pop" 36 | assert_timer "qu.queue.default.pop" 37 | end 38 | 39 | it "instruments push" do 40 | payload = SimpleJob.create 41 | assert_timer "qu.op.push" 42 | assert_timer "qu.job.SimpleJob.push" 43 | assert_timer "qu.queue.default.push" 44 | end 45 | 46 | it "instruments perform" do 47 | payload = SimpleJob.create 48 | payload.perform 49 | assert_timer "qu.op.perform" 50 | assert_timer "qu.job.SimpleJob.perform" 51 | end 52 | 53 | it "instruments complete" do 54 | payload = SimpleJob.create 55 | payload.perform 56 | assert_timer "qu.op.complete" 57 | assert_timer "qu.job.SimpleJob.complete" 58 | end 59 | 60 | it "instruments abort" do 61 | payload = SimpleJob.create 62 | payload.job.stub(:perform).and_raise(Qu::Worker::Abort.new) 63 | begin 64 | payload.perform 65 | flunk # should not get here 66 | rescue Qu::Worker::Abort 67 | assert_timer "qu.op.abort" 68 | assert_timer "qu.job.SimpleJob.abort" 69 | end 70 | end 71 | 72 | it "instruments fail" do 73 | payload = SimpleJob.create 74 | payload.job.stub(:perform).and_raise(StandardError.new) 75 | begin 76 | payload.perform 77 | flunk # should not get here 78 | rescue => exception 79 | assert_timer "qu.op.fail" 80 | assert_timer "qu.job.SimpleJob.fail" 81 | end 82 | end 83 | 84 | it "instruments failure create" do 85 | payload = SimpleJob.create 86 | payload.job.stub(:perform).and_raise(StandardError.new) 87 | begin 88 | payload.perform 89 | flunk # should not get here 90 | rescue => exception 91 | assert_timer "qu.op.failure_create" 92 | assert_timer "qu.job.SimpleJob.failure_create" 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /spec/qu/instrumenters/memory_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/instrumenters/memory' 3 | 4 | describe Qu::Instrumenters::Memory do 5 | describe "#initialize" do 6 | it "sets events to empty array" do 7 | instrumenter = described_class.new 8 | instrumenter.events.should eq([]) 9 | end 10 | end 11 | 12 | describe "#instrument" do 13 | it "adds to events" do 14 | instrumenter = described_class.new 15 | name = 'user.signup' 16 | payload = {:email => 'john@doe.com'} 17 | block_result = :yielded 18 | 19 | result = instrumenter.instrument(name, payload) { block_result } 20 | result.should eq(block_result) 21 | 22 | event = described_class::Event.new(name, payload, block_result) 23 | instrumenter.events.should eq([event]) 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/qu/instrumenters/noop_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/instrumenters/noop' 3 | 4 | describe Qu::Instrumenters::Noop do 5 | describe ".instrument" do 6 | context "with name" do 7 | it "yields block" do 8 | yielded = false 9 | described_class.instrument(:foo) { yielded = true } 10 | yielded.should be_true 11 | end 12 | end 13 | 14 | context "with name and payload" do 15 | it "yields block" do 16 | yielded = false 17 | described_class.instrument(:foo, {:pay => :load}) { yielded = true } 18 | yielded.should be_true 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/qu/job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Qu::Job do 4 | %w(push perform complete abort fail).each do |hook| 5 | it "should define hooks for #{hooks}" do 6 | Qu::Job.should respond_to("before_#{hook}") 7 | Qu::Job.should respond_to("around_#{hook}") 8 | Qu::Job.should respond_to("after_#{hook}") 9 | end 10 | end 11 | 12 | describe '.queue' do 13 | it 'should allow setting the queue name' do 14 | CustomQueue.queue.should == 'custom' 15 | end 16 | 17 | it 'should default to default' do 18 | SimpleJob.queue.should == 'default' 19 | end 20 | end 21 | 22 | describe '.load' do 23 | let(:payload) { Qu::Payload.new(:klass => 'SimpleJob')} 24 | 25 | it 'should return an instance' do 26 | SimpleJob.load(payload).should be_instance_of(SimpleJob) 27 | end 28 | 29 | it 'should initialize with args' do 30 | payload.args = [:foo] 31 | 32 | c = Class.new(Qu::Job) do 33 | def initialize(arg) 34 | @arg = arg 35 | end 36 | end 37 | c.load(payload).instance_variable_get(:@arg).should == :foo 38 | end 39 | 40 | it 'should assign payload before initializing' do 41 | c = Class.new(Qu::Job) do 42 | def initialize 43 | payload.foo = :bar 44 | end 45 | end 46 | 47 | job = c.load(payload) 48 | job.payload.should == payload 49 | payload.foo.should == :bar 50 | end 51 | end 52 | 53 | describe 'create' do 54 | it 'should call push with a payload' do 55 | Qu.should_receive(:push) do |payload| 56 | payload.queue.should eq('default') 57 | payload.should be_instance_of(Qu::Payload) 58 | payload.klass.should == SimpleJob 59 | payload.args.should == [9] 60 | end 61 | 62 | SimpleJob.create(9) 63 | end 64 | 65 | it 'should run push hoook' do 66 | SimpleJob.any_instance.should_receive(:run_hook).with(:push).and_yield 67 | SimpleJob.create(9) 68 | end 69 | 70 | it 'should not push job if hook halts' do 71 | SimpleJob.any_instance.stub(:run_hook) 72 | Qu.should_not_receive(:push) 73 | 74 | SimpleJob.create(9) 75 | end 76 | end 77 | 78 | it 'delegates logger to Qu.logger' do 79 | SimpleJob.new.logger.should be(Qu.logger) 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/qu/payload_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Qu::Payload do 4 | it 'should default id to nil' do 5 | Qu::Payload.new.id.should == nil 6 | end 7 | 8 | it 'should allow id to be set' do 9 | Qu::Payload.new(:id => 5).id.should == 5 10 | end 11 | 12 | describe 'queue' do 13 | it 'should default to "default"' do 14 | Qu::Payload.new.queue.should == 'default' 15 | end 16 | 17 | it 'should get queue from klass' do 18 | Qu::Payload.new(:klass => CustomQueue).queue.should == 'custom' 19 | end 20 | end 21 | 22 | describe 'klass' do 23 | it 'should constantize string' do 24 | Qu::Payload.new(:klass => 'CustomQueue').klass.should == CustomQueue 25 | end 26 | 27 | it 'should find namespaced class' do 28 | Qu::Payload.new(:klass => 'Qu::Payload').klass.should == Qu::Payload 29 | end 30 | end 31 | 32 | describe 'attributes' do 33 | subject { Qu::Payload.new(:klass => SimpleJob, :args => ['test'], :id => 1) } 34 | 35 | it 'returns hash of attributes' do 36 | subject.attributes.should eq({ 37 | :klass => 'SimpleJob', 38 | :args => ['test'], 39 | :id => 1, 40 | }) 41 | end 42 | end 43 | 44 | describe 'job' do 45 | subject { Qu::Payload.new(:klass => SimpleJob) } 46 | 47 | it 'should load the job' do 48 | SimpleJob.should_receive(:load).with(subject) 49 | subject.job 50 | end 51 | 52 | it 'should return the job' do 53 | subject.job.should be_instance_of(SimpleJob) 54 | end 55 | end 56 | 57 | describe 'perform' do 58 | subject { Qu::Payload.new(:klass => SimpleJob) } 59 | 60 | it 'should call perform on job' do 61 | subject.job.should_receive(:perform) 62 | subject.perform 63 | end 64 | 65 | it 'should run perform hooks' do 66 | subject.job.stub(:run_hook).and_yield 67 | subject.job.should_receive(:run_hook).with(:perform) 68 | subject.perform 69 | end 70 | 71 | it 'should call complete on backend' do 72 | Qu.should_receive(:complete) 73 | subject.perform 74 | end 75 | 76 | it 'should run complete hooks' do 77 | subject.job.stub(:run_hook).and_yield 78 | subject.job.should_receive(:run_hook).with(:complete) 79 | subject.perform 80 | end 81 | 82 | context 'when being aborted' do 83 | before do 84 | SimpleJob.any_instance.stub(:perform).and_raise(Qu::Worker::Abort) 85 | end 86 | 87 | it 'should abort the job and re-raise the error' do 88 | Qu.should_receive(:abort).with(subject) 89 | lambda { subject.perform }.should raise_error(Qu::Worker::Abort) 90 | end 91 | 92 | it 'should not call complete' do 93 | Qu.should_not_receive(:complete) 94 | lambda { subject.perform }.should raise_error(Qu::Worker::Abort) 95 | end 96 | 97 | it 'should run abort hook' do 98 | subject.job.stub(:run_hook).and_yield 99 | subject.job.should_receive(:run_hook).with(:abort) 100 | lambda { subject.perform }.should raise_error(Qu::Worker::Abort) 101 | end 102 | end 103 | 104 | context 'when the job raises an error' do 105 | let(:error) { StandardError.new("Some kind of error") } 106 | 107 | before do 108 | subject.job.stub(:perform).and_raise(error) 109 | end 110 | 111 | it 'should not call complete' do 112 | Qu.should_not_receive(:complete) 113 | subject.perform 114 | end 115 | 116 | it 'should call fail' do 117 | Qu.should_receive(:fail).with(subject) 118 | subject.perform 119 | end 120 | 121 | it 'should run fail hook' do 122 | subject.job.stub(:run_hook).and_yield 123 | subject.job.should_receive(:run_hook).with(:fail, error) 124 | subject.perform 125 | end 126 | 127 | it 'should call create for failure backend' do 128 | Qu::Failure.should_receive(:create).with(subject, error) 129 | subject.perform 130 | end 131 | end 132 | end 133 | 134 | describe "push" do 135 | subject { Qu::Payload.new(:klass => SimpleJob) } 136 | 137 | it "pushes payload to backend" do 138 | Qu.should_receive(:push).with(subject) 139 | subject.push 140 | end 141 | 142 | it "sets pushed_at" do 143 | subject.pushed_at.should be_nil 144 | subject.push 145 | subject.pushed_at.should be_instance_of(Time) 146 | end 147 | end 148 | end 149 | -------------------------------------------------------------------------------- /spec/qu/runner/direct_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/runner/direct' 3 | 4 | describe Qu::Runner::Direct do 5 | it_should_behave_like 'a runner interface' 6 | it_should_behave_like 'a single job runner' 7 | end 8 | -------------------------------------------------------------------------------- /spec/qu/runner/forking_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/runner/forking' 3 | 4 | describe Qu::Runner::Forking do 5 | it_should_behave_like 'a runner interface' 6 | it_should_behave_like 'a single job runner' 7 | end 8 | -------------------------------------------------------------------------------- /spec/qu/util/thread_safe_hash_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/util/thread_safe_hash' 3 | 4 | describe Qu::Util::ThreadSafeHash do 5 | it 'should wrap a hash and dup it' do 6 | options = {options: 'some options'} 7 | hash = Qu::Util::ThreadSafeHash.new(options) 8 | options[:other] = 'some-value' 9 | expect(hash[:other]).to be_nil 10 | end 11 | 12 | it 'should delete by key' do 13 | hash = Qu::Util::ThreadSafeHash.new(options: 'some options') 14 | hash.delete(:options) 15 | expect(hash[:options]).to be_nil 16 | end 17 | 18 | it 'should a copy of the values' do 19 | hash = Qu::Util::ThreadSafeHash.new(options: 'some options', some: 'value') 20 | values = hash.values 21 | hash.delete(:options) 22 | expect(values).to eq(['some options', 'value']) 23 | end 24 | 25 | it 'should have a size' do 26 | hash = Qu::Util::ThreadSafeHash.new(options: 'some options', some: 'value') 27 | expect(hash.size).to eq(2) 28 | end 29 | 30 | it 'should be navigable' do 31 | hash = Qu::Util::ThreadSafeHash.new(options: 'some options', some: 'value') 32 | pairs = [] 33 | hash.each do |key,value| 34 | pairs << [key,value] 35 | end 36 | 37 | expect(pairs).to eq([[:options, 'some options'], [:some, 'value']]) 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/qu/worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'qu/backend/memory' 3 | 4 | describe Qu::Worker do 5 | let(:job) { Qu::Payload.new(:id => '1', :klass => SimpleJob) } 6 | 7 | describe 'queues' do 8 | it 'should use default if none specified' do 9 | Qu::Worker.new.queues.should == ['default'] 10 | Qu::Worker.new('default').queues.should == ['default'] 11 | Qu::Worker.new(['default']).queues.should == ['default'] 12 | end 13 | 14 | it 'should use specified if any' do 15 | Qu::Worker.new('a', 'b').queues.should == ['a', 'b'] 16 | Qu::Worker.new(['a', 'b']).queues.should == ['a', 'b'] 17 | Qu::Worker.new(:a, :b).queues.should == ['a', 'b'] 18 | Qu::Worker.new([:a, :b]).queues.should == ['a', 'b'] 19 | end 20 | 21 | it 'should drop queue name whitespace' do 22 | Qu::Worker.new(' a ', ' b ').queues.should == ['a', 'b'] 23 | end 24 | end 25 | 26 | describe 'id' do 27 | it "should default hostname and pid" do 28 | Socket.stub(:gethostname).and_return("foo") 29 | Process.stub(:pid).and_return(12345) 30 | worker = Qu::Worker.new('a', 'b') 31 | worker.id.should eq("foo:12345:a,b") 32 | end 33 | 34 | it 'should not expand star in queue names' do 35 | Qu::Worker.new('a', '*').id.should =~ /a,*/ 36 | end 37 | end 38 | 39 | describe 'start' do 40 | it 'sleeps for interval if no work performed' do 41 | begin 42 | original_interval = Qu.interval 43 | Qu.stub(:pop).and_return(nil) 44 | 45 | Timeout.timeout(0.1) do 46 | Qu::Worker.new('a', 'b', 'c').start 47 | end 48 | flunk # should never get here as it should timeout 49 | rescue Timeout::Error, Qu::Worker::Stop 50 | # all good 51 | ensure 52 | Qu.interval = original_interval 53 | end 54 | end 55 | 56 | it 'stops worker if work loop is ever broken' do 57 | worker = Qu::Worker.new('a', 'b', 'c') 58 | 59 | begin 60 | Timeout.timeout(0.1) { worker.start } 61 | flunk # should not get here 62 | rescue => e 63 | worker.should_not be_running 64 | end 65 | end 66 | end 67 | 68 | describe 'stop' do 69 | context "when performing a job" do 70 | before do 71 | subject.stub(:performing?).and_return(true) 72 | end 73 | 74 | it 'raises abort with graceful shutdown disabled' do 75 | Qu.should_receive(:graceful_shutdown).and_return(false) 76 | expect { subject.stop }.to raise_exception(Qu::Worker::Abort) 77 | end 78 | 79 | it 'does not raise if graceful shutdown enabled' do 80 | Qu.should_receive(:graceful_shutdown).and_return(true) 81 | expect { subject.stop }.to_not raise_exception 82 | end 83 | end 84 | 85 | context "when not performing a job" do 86 | it 'raises stop' do 87 | subject.should_receive(:performing?).and_return(false) 88 | expect { subject.stop }.to raise_exception(Qu::Worker::Stop) 89 | end 90 | end 91 | end 92 | 93 | describe 'work' do 94 | context 'with job in first queue' do 95 | 96 | before do 97 | expect(Qu).to receive(:pop).with(subject.queues.first).and_return(job) 98 | end 99 | 100 | it 'should pop a payload and perform it' do 101 | expect(job).to receive(:perform) 102 | subject.work 103 | end 104 | 105 | it 'returns true' do 106 | subject.work.should be(true) 107 | end 108 | end 109 | 110 | context 'with job in a middle of queue' do 111 | before do 112 | Qu.stub(:pop).and_return(nil, job) 113 | end 114 | 115 | it 'should not pop once job is found' do 116 | job.should_receive(:perform) 117 | Qu.should_not_receive(:pop).with('c') 118 | Qu::Worker.new('a', 'b', 'c').work.should be(true) 119 | end 120 | end 121 | 122 | context 'with job in last queue' do 123 | before do 124 | Qu.stub(:pop).and_return(nil, nil, nil, job) 125 | end 126 | 127 | it 'pops until job found and performs it' do 128 | job.should_receive(:perform) 129 | Qu::Worker.new('a', 'b', 'c', 'd').work 130 | end 131 | 132 | it 'returns true' do 133 | Qu::Worker.new('a', 'b', 'c', 'd').work.should be(true) 134 | end 135 | end 136 | 137 | context 'with no job in any queue' do 138 | before do 139 | Qu.stub(:pop).and_return(nil) 140 | end 141 | 142 | it 'not error' do 143 | expect { subject.work }.to_not raise_error 144 | end 145 | 146 | it 'pops once for each queue' do 147 | Qu.should_receive(:pop).with('a').once.ordered.and_return(nil) 148 | Qu.should_receive(:pop).with('b').once.ordered.and_return(nil) 149 | Qu.should_receive(:pop).with('c').once.ordered.and_return(nil) 150 | Qu::Worker.new('a', 'b', 'c').work 151 | end 152 | 153 | it 'returns false' do 154 | Qu::Worker.new('a', 'b', 'c').work.should be(false) 155 | end 156 | end 157 | end 158 | end 159 | -------------------------------------------------------------------------------- /spec/qu_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Qu do 4 | %w(push pop complete abort size clear).each do |method| 5 | it "should delegate #{method} to backend" do 6 | Qu.backend.should_receive(method).with(:arg) 7 | Qu.send(method, :arg) 8 | end 9 | end 10 | 11 | describe 'configure' do 12 | it 'should yield Qu' do 13 | Qu.configure do |c| 14 | c.should == Qu 15 | end 16 | end 17 | end 18 | 19 | describe 'backend' do 20 | it 'should raise error if backend not configured' do 21 | Qu.backend = nil 22 | lambda { Qu.backend }.should raise_error 23 | end 24 | end 25 | 26 | describe 'interval' do 27 | it 'defaults to 5' do 28 | Qu.interval.should be(5) 29 | end 30 | end 31 | 32 | describe 'interval=' do 33 | before do 34 | @original_interval = Qu.interval 35 | end 36 | 37 | after do 38 | Qu.interval = @original_interval 39 | end 40 | 41 | it 'updates interval' do 42 | Qu.interval = 1 43 | Qu.interval.should be(1) 44 | end 45 | end 46 | 47 | [ 48 | :instrumenter, 49 | :instrumenter=, 50 | ].each do |method_name| 51 | it "responds to #{method_name}" do 52 | Qu.should respond_to(method_name) 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler.require :test 3 | require 'qu' 4 | require 'qu/backend/spec' 5 | require 'qu/runner/spec' 6 | 7 | root_path = Pathname(__FILE__).dirname.join('..').expand_path 8 | Dir[root_path.join("spec/support/**/*.rb")].each { |f| require f } 9 | 10 | module Qu 11 | module Specs 12 | def self.perform?(class_under_spec, *service_names) 13 | services = service_names.flatten 14 | return true if services.size == 0 15 | 16 | down_services = services.select { |service| !running?(service) } 17 | if down_services.any? 18 | puts "Skipping #{class_under_spec}. Required services are not running (#{down_services.join(', ')})." 19 | else 20 | true 21 | end 22 | end 23 | 24 | def self.running?(service) 25 | case service.to_s 26 | when "sqs" 27 | host = AWS.config.send("#{service}_endpoint") 28 | port = AWS.config.send("#{service}_port") 29 | Net::HTTP.new(host, port).request(Net::HTTP::Get.new("/")) 30 | true 31 | when "mongo" 32 | uri = ENV['MONGOHQ_URL'] || ENV['MONGOLAB_URI'] || ENV['BOXEN_MONGODB_URL'] 33 | 34 | client = if uri.nil? || uri.empty? 35 | Mongo::MongoClient.new 36 | else 37 | Mongo::MongoClient.from_uri(uri) 38 | end 39 | 40 | client.ping 41 | true 42 | when "redis" 43 | uri = ENV['REDISTOGO_URL'] || ENV['BOXEN_REDIS_URL'] 44 | 45 | client = if uri.nil? || uri.empty? 46 | Redis.new 47 | else 48 | Redis.connect(:url => uri) 49 | end 50 | 51 | client.ping 52 | true 53 | else 54 | false 55 | end 56 | rescue => exception 57 | false 58 | end 59 | end 60 | end 61 | 62 | RSpec.configure do |config| 63 | config.before(:each) do 64 | Qu.backend = double('a backend', { 65 | :push => nil, 66 | :pop => nil, 67 | :complete => nil, 68 | :abort => nil, 69 | :fail => nil, 70 | }) 71 | end 72 | end 73 | 74 | log_path = root_path.join("log") 75 | log_path.mkpath 76 | log_file = log_path.join("qu.log") 77 | log_to = ENV.fetch("QU_LOG_STDOUT", false) ? STDOUT : log_file 78 | 79 | Qu.logger = Logger.new(log_to) 80 | -------------------------------------------------------------------------------- /spec/support/fake_udp_socket.rb: -------------------------------------------------------------------------------- 1 | class FakeUDPSocket 2 | attr_reader :buffer 3 | 4 | def initialize 5 | @buffer = [] 6 | end 7 | 8 | def send(message, *rest) 9 | @buffer.push [message] 10 | end 11 | 12 | def recv 13 | @buffer.shift 14 | end 15 | 16 | def clear 17 | @buffer = [] 18 | end 19 | 20 | def to_s 21 | inspect 22 | end 23 | 24 | def inspect 25 | "" 26 | end 27 | end 28 | --------------------------------------------------------------------------------