├── spec ├── .gitkeep ├── support │ └── dj_sqs_sample_jobs.rb ├── serialization_spec.rb ├── spec_helper.rb └── delayed_job_sqs_spec.rb ├── .rspec ├── .gitignore ├── lib ├── delayed │ ├── backend │ │ ├── sqs_config.rb │ │ ├── version.rb │ │ ├── worker.rb │ │ ├── actions.rb │ │ └── sqs.rb │ └── serialization │ │ └── sqs.rb └── delayed_job_sqs.rb ├── CHANGELOG.md ├── Gemfile ├── Rakefile ├── delayed_job_sqs.gemspec ├── LICENSE └── README.md /spec/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --colour 2 | --format documentation 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.tmproj 3 | .DS_Store 4 | *.esproj/ 5 | .idea/ 6 | /coverage 7 | /rdoc 8 | /pkg 9 | .rvmrc 10 | .bundle/config 11 | Gemfile.lock 12 | -------------------------------------------------------------------------------- /lib/delayed/backend/sqs_config.rb: -------------------------------------------------------------------------------- 1 | class SqsConfig 2 | attr_accessor :project_id, :default_queue_name, :delay_seconds, :visibility_timeout, :message_retention_period, :aws_config 3 | end 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## 0.2.0 4 | 5 | * Adds support for bulk messaging to SQS. 6 | 7 | ## 0.2.1 8 | 9 | * Adds `batch_delay_jobs` method, a common usage pattern for buffer usage. 10 | -------------------------------------------------------------------------------- /lib/delayed/backend/version.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | module Backend 3 | module Sqs 4 | @@version = nil 5 | 6 | def self.version 7 | @@version ||= "0.2.1" 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | platforms :ruby do 4 | gem 'sqlite3' 5 | end 6 | 7 | group :test do 8 | gem 'activerecord', (ENV['RAILS_VERSION'] || ['>= 3.0', '< 4.2']) 9 | 10 | gem 'fake_sqs', :git => 'git@github.com:mdsol/fake_sqs.git' 11 | end 12 | 13 | gemspec 14 | -------------------------------------------------------------------------------- /spec/support/dj_sqs_sample_jobs.rb: -------------------------------------------------------------------------------- 1 | # This module contains our own sample jobs used for testing purposes. These are available in addition to those used in 2 | # the dj sqs backend shared examples. 3 | module DelayedJobSqs 4 | class SimpleJob 5 | cattr_accessor :runs; self.runs = 0 6 | def perform; @@runs += 1; end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'rubygems' 4 | require 'bundler' 5 | 6 | begin 7 | Bundler.setup(:default, :development) 8 | rescue Bundler::BundlerError => e 9 | $stderr.puts e.message 10 | $stderr.puts "Run `bundle install` to install missing gems" 11 | exit e.status_code 12 | end 13 | 14 | require 'rake' 15 | 16 | -------------------------------------------------------------------------------- /lib/delayed_job_sqs.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require 'aws-sdk-v1' 3 | require 'delayed_job' 4 | require 'active_support' 5 | 6 | module ActiveSupport 7 | Inflector.inflections do |inflect| 8 | inflect.irregular('sqs', 'sqs') 9 | end 10 | end 11 | 12 | require_relative 'delayed/serialization/sqs' 13 | require_relative 'delayed/backend/actions' 14 | require_relative 'delayed/backend/sqs_config' 15 | require_relative 'delayed/backend/worker' 16 | require_relative 'delayed/backend/version' 17 | require_relative 'delayed/backend/sqs' 18 | 19 | Delayed::Worker.backend = :sqs 20 | 21 | -------------------------------------------------------------------------------- /lib/delayed/serialization/sqs.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'zlib' 4 | require 'json' unless defined?(JSON) 5 | require 'base64' 6 | 7 | module DelayedJobSqs 8 | module Document 9 | MAX_SQS_MESSAGE_SIZE_IN_BYTES = 2**18 10 | 11 | def self.sqs_safe_json_dump(obj) 12 | json = JSON.dump(obj) 13 | if json.bytesize >= MAX_SQS_MESSAGE_SIZE_IN_BYTES 14 | JSON.dump(dj_compressed_document: compress(json)) 15 | else 16 | json 17 | end 18 | end 19 | 20 | def self.sqs_safe_json_load(json) 21 | obj = JSON.load(json) 22 | if obj.is_a?(Hash) && obj.key?('dj_compressed_document') 23 | JSON.load(decompress(obj['dj_compressed_document'])) 24 | else 25 | obj 26 | end 27 | end 28 | 29 | def self.compress(string) 30 | Base64.encode64(Zlib::Deflate.deflate(string)) 31 | end 32 | 33 | def self.decompress(string) 34 | Zlib::Inflate.inflate(Base64.decode64(string)) 35 | end 36 | 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /delayed_job_sqs.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'delayed/backend/version' 5 | 6 | Gem::Specification.new do |s| 7 | s.require_paths = ["lib"] 8 | s.name = "delayed_job_sqs" 9 | s.version = Delayed::Backend::Sqs.version 10 | s.authors = ["Eric Hankinson", "Matthew Szenher"] 11 | s.email = ["eric.hankinson@gmail.com", "mszenher@mdsol.com"] 12 | s.description = "Amazon SQS backend for delayed_job" 13 | s.summary = "Amazon SQS backend for delayed_job" 14 | s.homepage = "https://github.com/kumichou/delayed_job_sqs" 15 | s.license = "MIT" 16 | 17 | s.files = `git ls-files`.split($/) 18 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 19 | 20 | s.add_dependency('aws-sdk-v1', '>= 1.11.1') 21 | s.add_dependency('delayed_job', '>= 3.0.0') 22 | 23 | s.add_development_dependency('rspec', '>= 3') 24 | s.add_development_dependency('simplecov', '0.7.1') 25 | end 26 | 27 | -------------------------------------------------------------------------------- /spec/serialization_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../lib/delayed/serialization/sqs' 2 | 3 | describe(DelayedJobSqs::Document) do 4 | 5 | it 'serializes and deserializes an array' do 6 | array = ['a', 'b'] 7 | serialized_array = described_class.sqs_safe_json_dump(array) 8 | deserialized_array = described_class.sqs_safe_json_load(serialized_array) 9 | expect(deserialized_array).to eq(array) 10 | end 11 | 12 | it 'serializes and deserializes a hash' do 13 | hash = { 'a' => 'b' } 14 | serialized_hash = described_class.sqs_safe_json_dump(hash) 15 | deserialized_hash = described_class.sqs_safe_json_load(serialized_hash) 16 | expect(deserialized_hash).to eq(hash) 17 | end 18 | 19 | it 'serializes and deserializes a large array using compression' do 20 | array = ['a', 'b'] * 2**18 21 | serialized_array = described_class.sqs_safe_json_dump(array) 22 | expect(serialized_array.bytesize).to be < 2**18 23 | deserialized_array = described_class.sqs_safe_json_load(serialized_array) 24 | expect(deserialized_array).to eq(array) 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 LeanDog, Inc. and other contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /lib/delayed/backend/worker.rb: -------------------------------------------------------------------------------- 1 | require_relative 'sqs_config' 2 | 3 | module Delayed 4 | class Worker 5 | 6 | class << self 7 | attr_accessor :config, :sqs, :delay, :timeout, :expires_in, :aws_config 8 | 9 | def configure 10 | yield(config) 11 | 12 | self.default_queue_name = if !config.default_queue_name.nil? && config.default_queue_name.length != 0 13 | config.default_queue_name 14 | else 15 | 'default' 16 | end 17 | self.delay = config.delay_seconds || 0 18 | self.timeout = config.visibility_timeout || 5.minutes 19 | self.expires_in = config.message_retention_period || 4.days 20 | end 21 | 22 | def config 23 | @config ||= SqsConfig.new 24 | end 25 | end 26 | end 27 | 28 | module Backend 29 | module Sqs 30 | if Object.const_defined?(:Rails) and Rails.const_defined?(:Railtie) 31 | class Railtie < Rails::Railtie 32 | 33 | # configure our gem after Rails completely boots so that we have 34 | # access to any config/initializers that were run 35 | config.after_initialize do 36 | AWS::Rails.setup 37 | 38 | Delayed::Worker.sqs = AWS::SQS.new 39 | Delayed::Worker.configure {} 40 | end 41 | end 42 | elsif defined?(AWS.config) && AWS.config.access_key_id && AWS.config.secret_access_key 43 | # Use config in AWS.config if it is defined well enough for our sqs-y purposes. 44 | Delayed::Worker.sqs = AWS::SQS.new 45 | Delayed::Worker.configure {} 46 | else 47 | path = Pathname.new(Delayed::Worker.config.aws_config) 48 | 49 | if File.exists?(path) 50 | cfg = YAML::load(File.read(path)) 51 | 52 | unless cfg.keys[0] 53 | raise "AWS Yaml configuration file is missing a section" 54 | end 55 | 56 | AWS.config(cfg.keys[0]) 57 | end 58 | 59 | Delayed::Worker.sqs = AWS::SQS.new 60 | Delayed::Worker.configure {} 61 | end 62 | end 63 | end 64 | end 65 | 66 | 67 | -------------------------------------------------------------------------------- /lib/delayed/backend/actions.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | module Backend 3 | module Sqs 4 | module Actions 5 | def field(name, options = {}) 6 | default = options[:default] || nil 7 | define_method name do 8 | @attributes ||= {} 9 | @attributes[name.to_sym] || default 10 | end 11 | 12 | define_method "#{name}=" do |value| 13 | @attributes ||= {} 14 | @attributes[name.to_sym] = value 15 | end 16 | end 17 | 18 | def before_fork 19 | end 20 | 21 | def after_fork 22 | end 23 | 24 | def db_time_now 25 | Time.now.utc 26 | end 27 | 28 | # Find an available job message on the queue for the given worker to start working on. 29 | # Only jobs which are not failed and which should not be run in the future are given to the worker. 30 | def find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time) 31 | Delayed::Worker.queues.each_with_index do |queue, index| 32 | message = sqs.queues.named(queue_name(index)).receive_message 33 | 34 | return [Delayed::Backend::Sqs::Job.new(message)] if message 35 | end 36 | [] 37 | end 38 | 39 | def delete_all 40 | deleted = 0 41 | 42 | Delayed::Worker.queues.each_with_index do |queue, index| 43 | loop do 44 | msgs = sqs.queues.named(queue_name(index)).receive_message({ :limit => 10}) 45 | break if msgs.blank? 46 | msgs.each do |msg| 47 | msg.delete 48 | deleted += 1 49 | end 50 | end 51 | end 52 | 53 | puts "Messages removed: #{deleted}" 54 | end 55 | 56 | # No need to check locks 57 | def clear_locks!(*args) 58 | true 59 | end 60 | 61 | private 62 | 63 | def sqs 64 | ::Delayed::Worker.sqs 65 | end 66 | 67 | def queue_name(index) 68 | Delayed::Worker.queues[index] 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | SPEC_DIR = File.expand_path("..", __FILE__) 2 | LIB_DIR = File.expand_path("../lib", SPEC_DIR) 3 | 4 | $LOAD_PATH.unshift(LIB_DIR) 5 | $LOAD_PATH.uniq! 6 | 7 | require 'bundler' 8 | require 'simplecov' 9 | 10 | SimpleCov.start do 11 | add_group 'lib', 'lib' 12 | add_filter 'spec' 13 | end 14 | 15 | require 'rspec' 16 | require 'delayed_job' 17 | 18 | Bundler.setup 19 | 20 | require 'fake_sqs/test_integration' 21 | require 'aws-sdk-v1' 22 | 23 | Dir["#{SPEC_DIR}/support/*.rb"].each { |f| require f } 24 | 25 | DEFAULT_QUEUE_NAME = 'default' # A queue name to be used by default by both delayed_jobs and delayed_jobs_sqs. 26 | 27 | # Define AWS config. to be used in tests. This is for the benefit of telling delayed_job_sqs where the sqs endpoint is 28 | # and what credentials to use to talk to it. Here, we use localhost b/c we are using fake_sqs as our SQS endpoint. 29 | AWS.config( 30 | use_ssl: false, 31 | sqs_endpoint: 'localhost', 32 | sqs_port: 4568, 33 | access_key_id: 'fake', 34 | secret_access_key: 'fake', 35 | sqs_queue_name: DEFAULT_QUEUE_NAME, 36 | ) 37 | 38 | require File.join(LIB_DIR, 'delayed_job_sqs') 39 | 40 | # Queue names for queues used by dj_sqs specific tests as well as the dj backend shared examples ('a delayed_job backend'). 41 | QUEUES_TO_CREATE = [DEFAULT_QUEUE_NAME, 'tracking', 'small', 'medium', 'large', 'one', 'two'] 42 | 43 | RSpec.configure do |config| 44 | config.mock_with :rspec 45 | config.treat_symbols_as_metadata_keys_with_true_values = true 46 | 47 | # Before running the test suite, initialize and start fake_sqs. 48 | config.before(:suite) do 49 | $fake_sqs = FakeSQS::TestIntegration.new(database: ':memory:') 50 | $fake_sqs.start 51 | $sqs = AWS::SQS.new 52 | end 53 | 54 | # Each example tagged with :sqs, we reset fake_sqs and recreate the SQS queue in which we store jobs. 55 | config.before(:each, :sqs) do 56 | $fake_sqs.reset 57 | QUEUES_TO_CREATE.each{ |q_name| $sqs.queues.create(q_name) } 58 | end 59 | 60 | config.before(:each) do 61 | Delayed::Worker.logger = Logger.new('/tmp/dj.log') 62 | # TODO: Shouldn't DJ_SQS just set the queue name(s)? 63 | Delayed::Worker.queues = QUEUES_TO_CREATE 64 | end 65 | 66 | # After running the test suite, stop fake_sqs. 67 | config.after(:suite) { $fake_sqs.stop } 68 | end 69 | -------------------------------------------------------------------------------- /spec/delayed_job_sqs_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'delayed/backend/shared_spec' 3 | require 'active_record' 4 | require 'fake_sqs' 5 | 6 | # This story class is a simple class required by the shared specs. It is in fact copied from DJ repo. 7 | # The shared specs expect the Story class to have a lot of ActiveRecord-like qualities so I'm just 8 | # making it an ActiveRecord-based class. 9 | # Used to test interactions between DJ and an ORM 10 | ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:' 11 | ActiveRecord::Base.logger = Delayed::Worker.logger 12 | ActiveRecord::Migration.verbose = false 13 | 14 | ActiveRecord::Schema.define do 15 | create_table :stories, :primary_key => :story_id, :force => true do |table| 16 | table.string :text 17 | table.boolean :scoped, :default => true 18 | end 19 | end 20 | 21 | class Story < ActiveRecord::Base 22 | self.primary_key = 'story_id' 23 | def tell; text; end 24 | def whatever(n, _); tell*n; end 25 | default_scope { where(:scoped => true) } 26 | 27 | handle_asynchronously :whatever 28 | end 29 | 30 | describe Delayed::Backend::Sqs::Job, :sqs do 31 | it_behaves_like 'a delayed_job backend' 32 | 33 | let(:simple_job) { DelayedJobSqs::SimpleJob.new } 34 | 35 | let(:sqs_message) { AWS::SQS::ReceivedMessage.new(AWS::SQS::Queue.new("http://0.0.0.0:#{AWS.config.sqs_port}/#{DEFAULT_QUEUE_NAME}"), 36 | 1, "", opts = {body: {job: "New job"}.to_json}) } 37 | 38 | let(:sqs_job) { Delayed::Backend::Sqs::Job.new(sqs_message) } 39 | 40 | [1, 2].each do |num_jobs| 41 | it "delays #{num_jobs} simple job(s) successfully" do 42 | before_runs_count = DelayedJobSqs::SimpleJob.runs 43 | 44 | num_jobs.times{ simple_job.delay.perform } 45 | Delayed::Worker.new.work_off 46 | 47 | DelayedJobSqs::SimpleJob.runs.should == (before_runs_count + num_jobs) 48 | end 49 | end 50 | 51 | after do 52 | $fake_sqs.start 53 | $fake_sqs.clear_failure 54 | end 55 | 56 | describe 'enqueue' do 57 | 58 | it 'raises if AWS SQS returns non ok status' do 59 | $fake_sqs.api_fail('send_message') 60 | expect {described_class.enqueue(payload_object: SimpleJob.new)}.to raise_error(FakeSQS::InvalidAction) 61 | end 62 | 63 | it 'raises if AWS SQS fails to respond' do 64 | $fake_sqs.stop 65 | expect {described_class.enqueue(payload_object: SimpleJob.new)}.to raise_error(Errno::ECONNREFUSED) 66 | end 67 | 68 | end 69 | 70 | describe 'fail' do 71 | 72 | context 'with sqs message' do 73 | 74 | after do 75 | sqs_job.fail! 76 | end 77 | 78 | it 'destroys the job' do 79 | sqs_message.should_receive(:delete) 80 | end 81 | 82 | it 'logs the failure' do 83 | sqs_job.should_receive(:puts).with(/Job with attributes/) 84 | sqs_job.should_receive(:puts).with(/Job destroyed!/) 85 | end 86 | end 87 | 88 | it 'logs the failure to destroy the job without sqs message' do 89 | sqs_job_no_msg = Delayed::Backend::Sqs::Job.new 90 | sqs_job_no_msg.should_receive(:puts).with(/Job with attributes/) 91 | sqs_job_no_msg.should_receive(:puts).with(/Could not destroy job/) 92 | sqs_job_no_msg.fail! 93 | end 94 | 95 | end 96 | 97 | describe 'retry' do 98 | 99 | it 'does not delete the job if failed to resend it' do 100 | $fake_sqs.api_fail('send_message') 101 | sqs_message.should_not_receive(:delete) 102 | expect { sqs_job.save }.to raise_error(AWS::SQS::Errors::InvalidAction) 103 | end 104 | end 105 | 106 | end 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is an [Amazon SQS](http://aws.amazon.com/sqs/) backend for [delayed_job](http://github.com/collectiveidea/delayed_job) 2 | 3 | # Getting Started 4 | 5 | ## Get credentials 6 | 7 | To start using delayed_job_sqs, you need to sign up for an AWS account and setup your credentials. 8 | 9 | 1. Go to https://portal.aws.amazon.com/gp/aws/developer/registration/index.html and sign up. 10 | 2. Get your AWS Access Key Id & Secret Access Key for your account. 11 | 3. Either `require 'aws-sdk'` and call `AWS.config()` prior to `require 'delayed_job_sqs'`, or create an aws.yml that will get loaded with your app. You will need to configure Delayed::Worker below to pass in the YAML file location: 12 | 13 | ```yaml 14 | [aws] 15 | access_key_id: 16 | secret_access_key: 17 | ``` 18 | 19 | Or if using Rails, create config/initializers/aws-sdk.rb and put the following into the file: 20 | 21 | ```ruby 22 | AWS.config({ 23 | access_key_id: '', 24 | secret_access_key: '', 25 | }) 26 | ``` 27 | 28 | ## Installation 29 | 30 | Add the gems to your `Gemfile:` 31 | 32 | ```ruby 33 | gem 'delayed_job' 34 | gem 'delayed_job_sqs' 35 | ``` 36 | 37 | Optionally: Add an initializer (`config/initializers/delayed_job.rb`): 38 | 39 | ```ruby 40 | Delayed::Worker.configure do |config| 41 | # optional params: 42 | config.aws_config = '~/stuff/things/aws.yml' # Specify the file location of the AWS configuration YAML if you're not using Rails and you want to use a YAML file instead of calling AWS.config 43 | config.default_queue_name = 'default' # Specify an alternative default queue name 44 | config.delay_seconds = # Sets the default delay in seconds for messages sent to the queue. 45 | config.message_retention_period = 345600 # The number of seconds Amazon SQS retains a message. Must be an integer from 3600 (1 hour) to 1209600 (14 days). The default for this attribute is 345600 (4 days). 46 | config.visibility_timeout = 30 # The length of time (in seconds) that a message received from a queue will be invisible to other receiving components when they ask to receive messages. Valid values: integers from 0 to 43200 (12 hours). 47 | config.wait_time_seconds = # How many seconds to wait for a response 48 | end 49 | ``` 50 | 51 | ## Usage 52 | 53 | That's it. Use [delayed_job as normal](http://github.com/collectiveidea/delayed_job). 54 | 55 | Example: 56 | 57 | ```ruby 58 | class User 59 | def background_stuff 60 | puts "I run in the background" 61 | end 62 | end 63 | ``` 64 | 65 | Then in one of your controllers: 66 | 67 | ```ruby 68 | user = User.new 69 | user.delay.background_stuff 70 | ``` 71 | 72 | ### Batch SQS 73 | You can send multiple messages to SQS in a single request by using: 74 | ```ruby 75 | Delayed::Job.start_buffering! 76 | ``` 77 | 78 | Once you want to stop buffering, simply use: 79 | ```ruby 80 | Delayed::Job.stop_buffering! 81 | ``` 82 | 83 | Ensure that messages are sent by persisting the messages at the end of your transaction: 84 | ```ruby 85 | Delayed::Job.persist_buffer! 86 | ``` 87 | This clears the buffer once the messages are sent. 88 | 89 | Finally, you may explicitly clear the buffer at any time with: 90 | ```ruby 91 | Delayed::Job.clear_buffer! 92 | ``` 93 | 94 | Or simply wrap your transaction with `batch_delay_jobs` 95 | ```ruby 96 | Delayed::Job.batch_delay_jobs do 97 | . 98 | . 99 | . 100 | end 101 | # => Sends generated SQS messages in batch, then stops buffering. 102 | ``` 103 | 104 | ## Start worker process 105 | 106 | rake jobs:work 107 | 108 | If you want to process a specific queue that's not in your initializer or called `default`, use the `QUEUE` or `QUEUES` environment variable: 109 | 110 | QUEUE=tracking rake jobs:work 111 | 112 | That will start pulling jobs off the default queue and processing them. The gem will also handle multiple named queues if you have configured `rake` or `scripts/delayed_job` accordingly however be sure to name the queue when putting objects on the queue: 113 | 114 | ```ruby 115 | user = User.new 116 | user.delay(queue: 'bestest_queue').background_stuff 117 | ``` 118 | 119 | -------------------------------------------------------------------------------- /lib/delayed/backend/sqs.rb: -------------------------------------------------------------------------------- 1 | module Delayed 2 | module Backend 3 | module Sqs 4 | class Job 5 | include ::DelayedJobSqs::Document 6 | include Delayed::Backend::Base 7 | extend Delayed::Backend::Sqs::Actions 8 | 9 | field :priority, :type => Integer, :default => 0 10 | field :attempts, :type => Integer, :default => 0 11 | field :handler, :type => String 12 | field :run_at, :type => Time # TODO: implement run_at 13 | field :locked_at, :type => Time 14 | field :locked_by, :type => String 15 | field :failed_at, :type => Time 16 | field :last_error, :type => String 17 | field :queue, :type => String 18 | 19 | MAX_MESSAGES_IN_BATCH = 10 20 | 21 | class << self 22 | # Wrap transactions in this method to automatically send SQS messages generated in the transaction in batches. 23 | def batch_delay_jobs 24 | nested_block = true if buffering? 25 | begin 26 | clear_buffer! unless nested_block 27 | start_buffering! 28 | yield 29 | persist_buffer! 30 | ensure 31 | stop_buffering! unless nested_block 32 | clear_buffer! 33 | end 34 | end 35 | 36 | def buffering? 37 | @buffering 38 | end 39 | 40 | def start_buffering! 41 | @buffering = true 42 | end 43 | 44 | def stop_buffering! 45 | @buffering = false 46 | end 47 | 48 | def clear_buffer! 49 | @buffer = nil 50 | end 51 | 52 | def buffer 53 | @buffer ||= {} 54 | end 55 | 56 | def persist_buffer! 57 | buffer.each do |queue_name, message_batches| 58 | message_batches.each do |message_batch| 59 | sqs.queues.named(queue_name).batch_send(message_batch) if message_batch.size > 0 60 | end 61 | end 62 | end 63 | 64 | def create(attrs = {}) 65 | new(attrs).tap do |o| 66 | o.save 67 | end 68 | end 69 | 70 | def create!(attrs = {}) 71 | new(attrs).tap do |o| 72 | o.save! 73 | end 74 | end 75 | 76 | # Count the total number of jobs in all queues. 77 | def count 78 | num_jobs = 0 79 | Delayed::Worker.queues.each_with_index do |queue, index| 80 | queue = sqs.queues.named(queue_name(index)) 81 | num_jobs += queue.approximate_number_of_messages + queue.approximate_number_of_messages_delayed + queue.approximate_number_of_messages_not_visible 82 | end 83 | num_jobs 84 | end 85 | end 86 | 87 | def buffering? 88 | self.class.buffering? 89 | end 90 | 91 | def buffer 92 | self.class.buffer 93 | end 94 | 95 | def initialize(data = {}) 96 | puts "[init] Delayed::Backend::Sqs" 97 | @msg = nil 98 | 99 | if data.is_a?(AWS::SQS::ReceivedMessage) 100 | @msg = data 101 | data = ::DelayedJobSqs::Document.sqs_safe_json_load(data.body) 102 | end 103 | 104 | data.symbolize_keys! 105 | payload_obj = data.delete(:payload_object) || data.delete(:handler) 106 | 107 | # Ensure that run_at is present and is a Time object. 108 | data[:run_at] = if data[:run_at].nil? 109 | Time.now.utc 110 | elsif data[:run_at].is_a?(String) 111 | Time.parse(data[:run_at]) 112 | else 113 | data[:run_at] 114 | end 115 | 116 | @queue_name = data[:queue] || Delayed::Worker.default_queue_name 117 | @delay = data[:delay] || Delayed::Worker.delay 118 | @timeout = data[:timeout] || Delayed::Worker.timeout 119 | @expires_in = data[:expires_in] || Delayed::Worker.expires_in 120 | @attributes = data 121 | self.payload_object = payload_obj 122 | end 123 | 124 | 125 | def payload_object 126 | @payload_object ||= YAML.load(self.handler) 127 | rescue TypeError, LoadError, NameError, ArgumentError => e 128 | raise Delayed::DeserializationError, 129 | "Job failed to load: #{e.message}. Handler: #{handler.inspect}" 130 | end 131 | 132 | def payload_object=(object) 133 | if object.is_a? String 134 | @payload_object = YAML.load(object) 135 | self.handler = object 136 | else 137 | @payload_object = object 138 | self.handler = object.to_yaml 139 | end 140 | rescue TypeError, LoadError, NameError, ArgumentError => e 141 | puts "Failed to serialize #{object} because #{e.message} (#{e.class})." 142 | # If we have trouble serializing the object, simply assume it is already serialized and store it as is 143 | # in hopes that it can be deserialized when the time comes. This is what the dj lint calls for. 144 | self.handler = object 145 | end 146 | 147 | def save 148 | puts "[SAVE] #{@attributes.inspect}" 149 | 150 | if @attributes[:handler].blank? 151 | raise "Handler missing!" 152 | end 153 | payload = ::DelayedJobSqs::Document.sqs_safe_json_dump(@attributes) 154 | 155 | @msg.delete if @msg 156 | 157 | if buffering? 158 | add_to_buffer(message_body: payload, delay_seconds: @delay) 159 | else 160 | sqs.queues.named(queue_name).send_message(payload, delay_seconds: @delay ) 161 | end 162 | true 163 | end 164 | 165 | def save! 166 | save 167 | end 168 | 169 | def add_to_buffer(message) 170 | buffer[@queue_name] = [[]] unless buffer[@queue_name] 171 | current_buffer = buffer[@queue_name] 172 | 173 | if buffer_over_limit?(current_buffer, message[:message_body]) 174 | current_buffer << [message] 175 | else 176 | current_buffer.last << message 177 | end 178 | end 179 | 180 | def destroy 181 | if @msg 182 | message_id = @msg.id 183 | @msg.delete # TODO: need more fault tolerance around this! 184 | puts "Job destroyed! #{message_id} \nWith attributes: #{@attributes.inspect}" 185 | else 186 | puts "Could not destroy job b/c no SQS message provided: #{@attributes.inspect}" 187 | end 188 | end 189 | 190 | # Mark the job as failed (i.e. set failed_at to the current time). 191 | # TODO: Put failed jobs in s3 or onto a failed job queue (if they are set to be retained). 192 | # TODO: Need more fault tolerance in this method. 193 | def fail! 194 | puts "Job with attributes #{@attributes.inspect} failed!" 195 | if Delayed::Worker.destroy_failed_jobs 196 | destroy 197 | else 198 | update_attributes(failed_at: Time.now.utc) 199 | end 200 | end 201 | 202 | def update_attributes(attributes) 203 | attributes.symbolize_keys! 204 | @attributes.merge! attributes 205 | save 206 | end 207 | 208 | # No need to check locks 209 | def lock_exclusively!(*args) 210 | true 211 | end 212 | 213 | # No need to check locks 214 | def unlock(*args) 215 | true 216 | end 217 | 218 | # This method is supposed to reload the payload object. 219 | # NOTE: I can't find evidence that this is actually called in anything but tests by delayed job. 220 | # We are copying the implementation given in delayed_job/spec/delayed/backend/test.rb 221 | def reload(*args) 222 | reset 223 | self 224 | end 225 | 226 | # Must give each job an id. 227 | def id 228 | rand(10e6) 229 | end 230 | 231 | private 232 | 233 | def queue_name 234 | @queue_name 235 | end 236 | 237 | def sqs 238 | ::Delayed::Worker.sqs 239 | end 240 | 241 | def buffer_over_limit?(target_buffer, added_message) 242 | target_buffer_size = target_buffer.last.reduce(0) { |m, msg| m + msg[:message_body].bytesize } 243 | total_buffer_size = target_buffer_size + added_message.bytesize 244 | 245 | total_buffer_size >= ::DelayedJobSqs::Document::MAX_SQS_MESSAGE_SIZE_IN_BYTES || 246 | target_buffer.last.size >= MAX_MESSAGES_IN_BATCH 247 | end 248 | end 249 | end 250 | end 251 | end 252 | --------------------------------------------------------------------------------