├── VERSION ├── .rspec ├── examples ├── amqp │ ├── Gemfile │ ├── Gemfile.lock │ ├── amqp_publisher.rb │ └── amqp_consumer.rb ├── queue_message.rb ├── sandwich_worker.rb ├── sandwich_worker_with_class.rb └── sandwich_client_with_custom_listener.rb ├── .document ├── doc └── cloudist.png ├── lib ├── cloudist │ ├── core_ext │ │ ├── module.rb │ │ ├── string.rb │ │ ├── kernel.rb │ │ ├── object.rb │ │ └── class.rb │ ├── errors.rb │ ├── encoding.rb │ ├── publisher.rb │ ├── queues │ │ ├── reply_queue.rb │ │ ├── job_queue.rb │ │ └── basic_queue.rb │ ├── messaging.rb │ ├── worker.rb │ ├── utils.rb │ ├── application.rb │ ├── request.rb │ ├── job.rb │ ├── payload.rb │ ├── message.rb │ ├── listener.rb │ ├── queue.rb │ └── payload_old.rb ├── em │ ├── iterator.rb │ └── em_timer_utils.rb ├── cloudist_old.rb └── cloudist.rb ├── spec ├── spec_helper.rb ├── cloudist │ ├── queue_spec.rb │ ├── utils_spec.rb │ ├── job_spec.rb │ ├── messaging_spec.rb │ ├── basic_queue_spec.rb │ ├── request_spec.rb │ ├── payload_spec_2_spec.rb │ ├── message_spec.rb │ └── payload_spec.rb ├── core_ext │ └── string_spec.rb ├── support │ └── amqp.rb └── cloudist_spec.rb ├── Gemfile ├── .gitignore ├── LICENSE.txt ├── Gemfile.lock ├── Rakefile ├── README.md └── cloudist.gemspec /VERSION: -------------------------------------------------------------------------------- 1 | 0.5.0 -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | -------------------------------------------------------------------------------- /examples/amqp/Gemfile: -------------------------------------------------------------------------------- 1 | source :rubygems 2 | 3 | gem 'amqp', '~>0.7' -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | bin/* 3 | - 4 | features/**/*.feature 5 | LICENSE.txt 6 | -------------------------------------------------------------------------------- /doc/cloudist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivanvanderbyl/cloudist/HEAD/doc/cloudist.png -------------------------------------------------------------------------------- /examples/amqp/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | amqp (0.7.1) 5 | eventmachine (>= 0.12.4) 6 | eventmachine (0.12.10) 7 | 8 | PLATFORMS 9 | ruby 10 | 11 | DEPENDENCIES 12 | amqp (~> 0.7) 13 | -------------------------------------------------------------------------------- /lib/cloudist/core_ext/module.rb: -------------------------------------------------------------------------------- 1 | class Module 2 | def remove_possible_method(method) 3 | remove_method(method) 4 | rescue NameError 5 | end 6 | 7 | def redefine_method(method, &block) 8 | remove_possible_method(method) 9 | define_method(method, &block) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/cloudist/errors.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Error < RuntimeError; end 3 | class BadPayload < Error; end 4 | class EnqueueError < Error; end 5 | class StaleHeadersError < BadPayload; end 6 | class UnknownReplyTo < RuntimeError; end 7 | class ExpiredMessage < RuntimeError; end 8 | end 9 | -------------------------------------------------------------------------------- /lib/cloudist/core_ext/string.rb: -------------------------------------------------------------------------------- 1 | class String 2 | # Returns true iff +other+ appears exactly at the start of +self+. 3 | def starts_with? other 4 | self[0, other.length] == other 5 | end 6 | 7 | # Returns true iff +other+ appears exactly at the end of +self+. 8 | def ends_with? other 9 | self[-other.length, other.length] == other 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/cloudist/core_ext/kernel.rb: -------------------------------------------------------------------------------- 1 | module Kernel 2 | # Returns the object's singleton class. 3 | def singleton_class 4 | class << self 5 | self 6 | end 7 | end unless respond_to?(:singleton_class) # exists in 1.9.2 8 | 9 | # class_eval on an object acts like singleton_class.class_eval. 10 | def class_eval(*args, &block) 11 | singleton_class.class_eval(*args, &block) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /examples/queue_message.rb: -------------------------------------------------------------------------------- 1 | $:.unshift File.dirname(__FILE__) + '/../lib' 2 | require "rubygems" 3 | require "cloudist" 4 | 5 | Cloudist.signal_trap! 6 | # 7 | # This demonstrates how to send a message to a listener 8 | # 9 | Cloudist.start { 10 | 11 | payload = Cloudist::Payload.new(:event => :started, :message_type => 'event') 12 | 13 | q = Cloudist::ReplyQueue.new('temp.reply.make.sandwich') 14 | q.publish(payload) 15 | 16 | stop 17 | } 18 | -------------------------------------------------------------------------------- /lib/cloudist/encoding.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | module Encoding 3 | def encode(message) 4 | # Marshal.dump(message) 5 | # JSON.dump(message.to_hash) 6 | message.to_json 7 | end 8 | 9 | def decode(message) 10 | raise ArgumentError, "First argument can't be nil" if message.nil? 11 | return message unless message.is_a?(String) 12 | # Marshal.load(message) 13 | JSON.load(message) 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 2 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 3 | require 'rspec' 4 | require "moqueue" 5 | require 'cloudist' 6 | 7 | # Requires supporting files with custom matchers and macros, etc, 8 | # in ./support/ and its subdirectories. 9 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f} 10 | 11 | RSpec.configure do |config| 12 | config.mock_with :rspec 13 | end 14 | -------------------------------------------------------------------------------- /spec/cloudist/queue_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../spec_helper', __FILE__) 2 | 3 | # describe Cloudist::Queue do 4 | # before(:each) do 5 | # stub_amqp! 6 | # end 7 | # 8 | # it "should cache new queues" do 9 | # q1 = Cloudist::Queue.new("test.queue") 10 | # q2 = Cloudist::Queue.new("test.queue") 11 | # 12 | # # q1.cached_queues.should == {} 13 | # q1.q.should == q2.q 14 | # Cloudist::Queue.cached_queues.should == {} 15 | # end 16 | # end 17 | -------------------------------------------------------------------------------- /lib/cloudist/publisher.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Publisher 3 | 4 | class << self 5 | def enqueue(queue_name, data) 6 | payload = Cloudist::Payload.new(data) 7 | 8 | queue = Cloudist::JobQueue.new(queue_name) 9 | 10 | queue.setup 11 | 12 | # send_message = proc { 13 | queue.publish(payload) 14 | # } 15 | # EM.next_tick(&send_message) 16 | 17 | return Job.new(payload) 18 | end 19 | end 20 | 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/core_ext/string_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | require "cloudist/core_ext/string" 4 | 5 | describe "String" do 6 | it "should support ends_with?" do 7 | "started!".ends_with?('!').should be_true 8 | "started!".ends_with?('-').should be_false 9 | end 10 | 11 | it "should support starts_with?" do 12 | "event-started".starts_with?("event").should be_true 13 | "event-started".starts_with?("reply").should be_false 14 | end 15 | 16 | end 17 | -------------------------------------------------------------------------------- /spec/support/amqp.rb: -------------------------------------------------------------------------------- 1 | def stub_amqp! 2 | AMQP.stub(:start) 3 | mock_queue = mock("AMQP::Queue") 4 | mock_queue.stub(:bind) 5 | mock_queue.stub(:name).and_return("test.queue") 6 | 7 | mock_ex = mock("AMQP::Exchange") 8 | mock_ex.stub(:name).and_return("test.queue") 9 | 10 | mock_channel = mock("AMQP::Channel") 11 | mock_channel.stub(:prefetch).with(1) 12 | mock_channel.stub(:queue).and_return(mock_queue) 13 | mock_channel.stub(:direct).and_return(mock_ex) 14 | 15 | AMQP::Channel.stub(:new).and_return(mock_channel) 16 | end 17 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source :rubygems 2 | 3 | gem 'amqp', '~> 0.8.1' 4 | gem "json", "~> 1.4" 5 | gem "i18n" 6 | gem "activesupport", "> 3.0.0" 7 | gem "hashie" 8 | gem "uuid" 9 | 10 | # Add dependencies to develop your gem here. 11 | # Include everything needed to run rake, tests, features, etc. 12 | group :development do 13 | gem "rspec", "~> 2.4.0" 14 | gem "moqueue", :git => "git://github.com/ivanvanderbyl/moqueue.git" 15 | gem "jeweler", "~> 1.6.4" 16 | gem "reek", "~> 1.2.8" 17 | gem "roodi", "~> 2.1.0" 18 | end 19 | -------------------------------------------------------------------------------- /lib/cloudist/queues/reply_queue.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class ReplyQueue < Cloudist::Queues::BasicQueue 3 | def initialize(queue_name, options={}) 4 | options[:auto_delete] = true 5 | options[:nowait] = false 6 | 7 | @prefetch = Cloudist.listener_prefetch 8 | 9 | super(queue_name, options) 10 | end 11 | 12 | # def subscribe(&block) 13 | # super do |request| 14 | # yield request if block_given? 15 | # teardown 16 | # end 17 | # end 18 | 19 | # def teardown 20 | # queue.delete 21 | # super 22 | # end 23 | 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/cloudist/messaging.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | autoload :Singleton, 'singleton' 3 | 4 | class Messaging 5 | include Singleton 6 | 7 | class << self 8 | 9 | def active_queues 10 | instance.active_queues 11 | end 12 | 13 | def add_queue(queue) 14 | (instance.active_queues ||= {}).merge!({queue.name.to_s => queue}) 15 | instance.active_queues 16 | end 17 | 18 | def remove_queue(queue_name) 19 | (instance.active_queues ||= {}).delete(queue_name.to_s) 20 | instance.active_queues 21 | end 22 | end 23 | 24 | attr_accessor :active_queues 25 | 26 | 27 | 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/em/iterator.rb: -------------------------------------------------------------------------------- 1 | module EventMachine 2 | class Iterator 3 | 4 | def initialize(container) 5 | @container = container 6 | end 7 | 8 | def each(work, done=proc{}) 9 | do_work = proc { 10 | if @container && !@container.empty? 11 | work.call(@container.shift) 12 | EM.next_tick(&do_work) 13 | else 14 | done.call 15 | end 16 | } 17 | EM.next_tick(&do_work) 18 | end 19 | 20 | def map(work, done=proc{}) 21 | mapped = [] 22 | map_work = proc { |n| mapped << work.call(n) } 23 | map_done = proc { done.call(mapped) } 24 | each(map_work, map_done) 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /spec/cloudist/utils_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | describe Cloudist::Utils do 4 | it "should return reply queue name" do 5 | Cloudist::Utils.reply_prefix('eat.sandwich').should == 'temp.reply.eat.sandwich' 6 | end 7 | 8 | it "should return log queue name" do 9 | Cloudist::Utils.log_prefix('eat.sandwich').should == 'temp.log.eat.sandwich' 10 | end 11 | 12 | it "should return stats queue name" do 13 | Cloudist::Utils.stats_prefix('eat.sandwich').should == 'temp.stats.eat.sandwich' 14 | end 15 | 16 | # it "should generate queue name" do 17 | # Cloudist::Utils.generate_queue('test').should == '' 18 | # end 19 | end 20 | -------------------------------------------------------------------------------- /lib/cloudist/queues/job_queue.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class JobQueue < Cloudist::Queues::BasicQueue 3 | 4 | def initialize(queue_name, options={}) 5 | options[:auto_delete] = false 6 | options[:nowait] = false 7 | 8 | @prefetch = Cloudist.worker_prefetch 9 | puts "Prefetch: #{@prefetch}" 10 | super(queue_name, options) 11 | end 12 | 13 | # def initialize(queue_name, options={}) 14 | # @prefetch = 1 15 | # # opts[:auto_delete] = false 16 | # 17 | # super(queue_name, options) 18 | # end 19 | 20 | # def setup_exchange 21 | # @exchange = channel.direct(queue_name) 22 | # queue.bind(exchange) 23 | # end 24 | 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/cloudist/job_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | describe Cloudist::Job do 4 | before(:each) do 5 | @payload = Cloudist::Payload.new({:bread => 'white'}) 6 | end 7 | 8 | it "should be constructable with payload" do 9 | job = Cloudist::Job.new(@payload) 10 | job.payload.should == @payload 11 | end 12 | 13 | it "should be constructable with payload and return ID" do 14 | job = Cloudist::Job.new(@payload) 15 | job.id.should == @payload.id 16 | end 17 | 18 | it "should be constructable with payload and return data" do 19 | job = Cloudist::Job.new(@payload) 20 | job.data.should == @payload.body 21 | end 22 | 23 | end 24 | -------------------------------------------------------------------------------- /spec/cloudist/messaging_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../spec_helper', __FILE__) 2 | 3 | # describe Cloudist::Messaging do 4 | # 5 | # it "should add queue to queues list" do 6 | # queue = mock("Cloudist::Queue") 7 | # queue.stubs(:name).returns("test.queue") 8 | # Cloudist::Messaging.add_queue(queue) 9 | # Cloudist::Messaging.active_queues.keys.should include('test.queue') 10 | # end 11 | # 12 | # it "should be able to remove queues from list" do 13 | # queue = mock("Cloudist::Queue") 14 | # queue.stubs(:name).returns("test.queue") 15 | # Cloudist::Messaging.add_queue(queue).keys.should == ['test.queue'] 16 | # Cloudist::Messaging.remove_queue('test.queue').keys.should == [] 17 | # end 18 | # 19 | # end 20 | -------------------------------------------------------------------------------- /lib/cloudist/worker.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Worker 3 | 4 | attr_reader :job, :queue, :payload 5 | 6 | def initialize(job, queue) 7 | @job, @queue, @payload = job, queue, job.payload 8 | 9 | # Do custom initialization 10 | self.setup if self.respond_to?(:setup) 11 | end 12 | 13 | def data 14 | job.data 15 | end 16 | 17 | def headers 18 | job.headers 19 | end 20 | 21 | def id 22 | job.id 23 | end 24 | 25 | def process 26 | raise NotImplementedError, "Your worker class must subclass this method" 27 | end 28 | 29 | def log 30 | Cloudist.log 31 | end 32 | 33 | end 34 | 35 | class GenericWorker < Worker 36 | def process(&block) 37 | instance_eval(&block) 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # rcov generated 2 | coverage 3 | 4 | # rdoc generated 5 | rdoc 6 | 7 | # yard generated 8 | doc 9 | .yardoc 10 | 11 | # bundler 12 | .bundle 13 | 14 | # jeweler generated 15 | pkg 16 | 17 | # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 18 | # 19 | # * Create a file at ~/.gitignore 20 | # * Include files you want ignored 21 | # * Run: git config --global core.excludesfile ~/.gitignore 22 | # 23 | # After doing this, these files will be ignored in all your git projects, 24 | # saving you from having to 'pollute' every project you touch with them 25 | # 26 | # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line) 27 | # 28 | # For MacOS: 29 | # 30 | #.DS_Store 31 | # 32 | # For TextMate 33 | #*.tmproj 34 | #tmtags 35 | # 36 | # For emacs: 37 | #*~ 38 | #\#* 39 | #.\#* 40 | # 41 | # For vim: 42 | #*.swp 43 | -------------------------------------------------------------------------------- /spec/cloudist/basic_queue_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | describe "Cloudist" do 4 | describe "Cloudist::Queues::BasicQueue" do 5 | before(:each) do 6 | overload_amqp 7 | reset_broker 8 | end 9 | 10 | it "should create a queue and exchange" do 11 | # MQ.stubs(:direct).with(:name).returns(true) 12 | @mq = mock("MQ") 13 | @exchange = mock("MQ Exchange") 14 | @queue = mock("MQ Queue") 15 | 16 | @queue.expects(:bind).with(@exchange) 17 | # @mq.expects(:queue).with("make.sandwich") 18 | 19 | bq = Cloudist::Queues::BasicQueue.new("make.sandwich") 20 | bq.stub(:q).and_return(@queue) 21 | bq.stub(:mq).and_return(@mq) 22 | bq.stub(:ex).and_return(@exchange) 23 | 24 | bq.setup 25 | 26 | bq.q.should_not be_nil 27 | bq.ex.should_not be_nil 28 | bq.mq.should_not be_nil 29 | end 30 | 31 | end 32 | 33 | end 34 | -------------------------------------------------------------------------------- /examples/sandwich_worker.rb: -------------------------------------------------------------------------------- 1 | # Cloudst Example: Sandwich Worker 2 | # 3 | # This example demonstrates receiving a job and sending back events to the client to let it know we've started and finsihed 4 | # making a sandwich. From here you could dispatch an eat.sandwich event. 5 | # 6 | # Be sure to update the Cloudist connection settings if they differ from defaults: 7 | # user: guest 8 | # pass: guest 9 | # port: 5672 10 | # host: localhost 11 | # vhost: / 12 | # 13 | $:.unshift File.dirname(__FILE__) + '/../lib' 14 | require "rubygems" 15 | require "cloudist" 16 | 17 | Cloudist.signal_trap! 18 | 19 | Cloudist.start { 20 | log.info("Started Worker") 21 | 22 | job('make.sandwich') { 23 | log.info("JOB (#{id}) Make sandwich with #{data[:bread]} bread") 24 | 25 | job.started! 26 | 27 | (1..20).each do |i| 28 | job.progress(i * 5) 29 | sleep(1) 30 | 31 | raise ArgumentError, "NOT GOOD!" if i == 4 32 | end 33 | job.finished! 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Ivan Vanderbyl 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /spec/cloudist/request_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | describe Cloudist::Request do 4 | before { 5 | @mq_header = mock("MQ::Header") 6 | @mq_header.stub(:headers).and_return({:published_on=>Time.now.to_i - 60, :event_hash=>"foo", :message_id=>"foo", :ttl=>300}) 7 | 8 | q = Cloudist::JobQueue.new('test.queue') 9 | 10 | @request = Cloudist::Request.new(q, Marshal.dump({:bread => 'white'}), @mq_header) 11 | } 12 | 13 | it "should return ttl" do 14 | @request.ttl.should == 300 15 | end 16 | 17 | it "should have a payload" do 18 | @request.payload.should_not be_nil 19 | @request.payload.should be_a(Cloudist::Payload) 20 | end 21 | 22 | it "should be 1 minute old" do 23 | @request.age.should == 60 24 | end 25 | 26 | it "should not be expired" do 27 | @request.expired?.should_not be_true 28 | end 29 | 30 | it "should not be acked yet" do 31 | @request.acked?.should be_false 32 | end 33 | 34 | it "should be ackable" do 35 | @mq_header.stub(:ack).and_return(true) 36 | 37 | @request.ack.should be_true 38 | @request.acked?.should be_true 39 | end 40 | 41 | end 42 | -------------------------------------------------------------------------------- /examples/amqp/amqp_publisher.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # encoding: utf-8 3 | 4 | require "rubygems" 5 | require 'amqp' 6 | 7 | def amqp_settings 8 | uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/') 9 | { 10 | :vhost => uri.path, 11 | :host => uri.host, 12 | :user => uri.user, 13 | :port => uri.port || 5672, 14 | :pass => uri.password, 15 | :heartbeat => 120, 16 | :logging => false 17 | } 18 | rescue Object => e 19 | raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}" 20 | end 21 | 22 | p amqp_settings 23 | 24 | def log(*args) 25 | puts args.inspect 26 | end 27 | 28 | EM.run do 29 | puts "Running..." 30 | AMQP.start(amqp_settings) do |connection| 31 | log "Connected to AMQP broker" 32 | 33 | channel = AMQP::Channel.new(connection) 34 | channel.prefetch(1) 35 | queue = channel.queue("test.hello.world") 36 | exchange = channel.direct 37 | queue.bind(exchange) 38 | 39 | EM.defer do 40 | 10000.times { |i| 41 | log "Publishing message #{i+1}" 42 | if i % 1000 == 0 43 | puts "Sleeping..." 44 | sleep(1) 45 | end 46 | exchange.publish "Hello, world! - #{i+1}"#, :routing_key => queue.name 47 | } 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /examples/amqp/amqp_consumer.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # encoding: utf-8 3 | 4 | require "rubygems" 5 | require 'amqp' 6 | 7 | def amqp_settings 8 | uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/') 9 | { 10 | :vhost => uri.path, 11 | :host => uri.host, 12 | :user => uri.user, 13 | :port => uri.port || 5672, 14 | :pass => uri.password, 15 | :heartbeat => 120, 16 | :logging => false 17 | } 18 | rescue Object => e 19 | raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}" 20 | end 21 | 22 | p amqp_settings 23 | 24 | def log(*args) 25 | puts args.inspect 26 | end 27 | 28 | EM.run do 29 | puts "Running..." 30 | AMQP.start(amqp_settings) do |connection| 31 | log "Connected to AMQP broker" 32 | 33 | channel = AMQP::Channel.new(connection) 34 | channel.prefetch(1) 35 | queue = channel.queue("test.hello.world") 36 | exchange = channel.direct 37 | queue.bind(exchange) 38 | 39 | @count = 0 40 | 41 | queue.subscribe(:ack => true) do |h, payload| 42 | puts "--" 43 | EM.defer { 44 | # sleep(1) 45 | @count += 1 46 | log "Received a message: #{payload} - #{@count}" 47 | h.ack 48 | } 49 | end 50 | 51 | # queue.subscribe(:ack => false) do |h, payload| 52 | # @count += 1 53 | # log "Received a message: #{payload} - #{@count}" 54 | # end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/cloudist/utils.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | module Utils 3 | extend self 4 | 5 | def reply_prefix(name) 6 | "temp.reply.#{name}" 7 | end 8 | 9 | def log_prefix(name) 10 | "temp.log.#{name}" 11 | end 12 | 13 | def stats_prefix(name) 14 | "temp.stats.#{name}" 15 | end 16 | 17 | def generate_queue(exchange_name, second_name=nil) 18 | second_name ||= $$ 19 | "#{generate_name_for_instance(exchange_name)}.#{second_name}" 20 | end 21 | 22 | def generate_name_for_instance(name) 23 | "#{name}.#{Socket.gethostname}" 24 | end 25 | 26 | # DEPRECATED 27 | def generate_reply_to(name) 28 | "#{reply_prefix(name)}.#{generate_sym}" 29 | end 30 | 31 | def generate_sym 32 | values = [ 33 | rand(0x0010000), 34 | rand(0x0010000), 35 | rand(0x0010000), 36 | rand(0x0010000), 37 | rand(0x0010000), 38 | rand(0x1000000), 39 | rand(0x1000000), 40 | ] 41 | "%04x%04x%04x%04x%04x%06x%06x" % values 42 | end 43 | 44 | def encode_message(object) 45 | Marshal.dump(object).to_s 46 | end 47 | 48 | def decode_message(string) 49 | Marshal.load(string) 50 | end 51 | 52 | def decode_json(string) 53 | if defined? ActiveSupport::JSON 54 | ActiveSupport::JSON.decode string 55 | else 56 | JSON.load string 57 | end 58 | end 59 | 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /examples/sandwich_worker_with_class.rb: -------------------------------------------------------------------------------- 1 | # Cloudst Example: Sandwich Worker 2 | # 3 | # This example demonstrates receiving a job and sending back events to the client to let it know we've started and finsihed 4 | # making a sandwich. From here you could dispatch an eat.sandwich event. 5 | # 6 | # Be sure to update the Cloudist connection settings if they differ from defaults: 7 | # user: guest 8 | # pass: guest 9 | # port: 5672 10 | # host: localhost 11 | # vhost: / 12 | # 13 | $:.unshift File.dirname(__FILE__) + '/../lib' 14 | require "rubygems" 15 | require "cloudist" 16 | 17 | class SandwichWorker < Cloudist::Worker 18 | def process 19 | log.info("Processing #{queue.name} job: #{id}") 20 | 21 | # This will trigger the start event 22 | # Appending ! to the end of a method will trigger an 23 | # event reply with its name 24 | # 25 | # e.g. job.working! 26 | # 27 | job.started! 28 | 29 | (1..5).each do |i| 30 | # This sends a progress reply, you could use this to 31 | # update a progress bar in your UI 32 | # 33 | # usage: #progress([INTEGER 0 - 100]) 34 | job.progress(i * 20) 35 | 36 | # Work hard! 37 | sleep(1) 38 | 39 | # Uncomment this to test error handling in Listener 40 | # raise ArgumentError, "NOT GOOD!" if i == 4 41 | end 42 | 43 | # Trigger finished event 44 | job.finished! 45 | end 46 | end 47 | 48 | Cloudist.signal_trap! 49 | 50 | Cloudist.start(:logging => false, :worker_prefetch => 2) { 51 | Cloudist.handle('make.sandwich').with(SandwichWorker) 52 | } 53 | -------------------------------------------------------------------------------- /lib/cloudist/application.rb: -------------------------------------------------------------------------------- 1 | require "singleton" 2 | 3 | module Cloudist 4 | class Application 5 | include Singleton 6 | 7 | class << self 8 | def start(options = {}, &block) 9 | options = instance.settings.update(options) 10 | AMQP.start(options) do 11 | instance.setup_reconnect_hook! 12 | 13 | instance.instance_eval(&block) if block_given? 14 | end 15 | end 16 | 17 | def signal_trap! 18 | ::Signal.trap('INT') { Cloudist.stop } 19 | ::Signal.trap('TERM'){ Cloudist.stop } 20 | end 21 | end 22 | 23 | def settings 24 | @@settings ||= default_settings 25 | end 26 | 27 | def settings=(settings_hash) 28 | @@settings = default_settings.update(settings_hash) 29 | end 30 | 31 | def default_settings 32 | uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/') 33 | { 34 | :vhost => uri.path, 35 | :host => uri.host, 36 | :user => uri.user, 37 | :port => uri.port || 5672, 38 | :pass => uri.password, 39 | :heartbeat => 5, 40 | :logging => false 41 | } 42 | rescue Object => e 43 | raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}" 44 | end 45 | 46 | private 47 | 48 | def setup_reconnect_hook! 49 | AMQP.conn.connection_status do |status| 50 | 51 | log.debug("AMQP connection status changed: #{status}") 52 | 53 | if status == :disconnected 54 | AMQP.conn.reconnect(true) 55 | end 56 | end 57 | end 58 | 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/cloudist/request.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Request 3 | include Cloudist::Encoding 4 | 5 | attr_reader :queue_header, :qobj, :payload, :start, :headers, :body 6 | 7 | def initialize(queue, encoded_body, queue_header) 8 | @qobj, @queue_header = queue, queue_header 9 | 10 | @body = decode(encoded_body) 11 | @headers = parse_custom_headers(queue_header) 12 | 13 | @payload = Cloudist::Payload.new(encoded_body, queue_header.headers.dup) 14 | @headers = @payload.headers 15 | 16 | @start = Time.now.utc.to_f 17 | end 18 | 19 | def parse_custom_headers(amqp_headers) 20 | h = amqp_headers.headers.dup 21 | 22 | h[:published_on] = h[:published_on].to_i 23 | 24 | h[:ttl] = h[:ttl].to_i rescue -1 25 | h[:ttl] = -1 if h[:ttl] == 0 26 | 27 | h 28 | end 29 | 30 | def for_message 31 | [body.dup, queue_header.headers.dup] 32 | end 33 | 34 | def q 35 | qobj.queue 36 | end 37 | 38 | def ex 39 | qobj.exchange 40 | end 41 | 42 | def mq 43 | qobj.channel 44 | end 45 | 46 | def channel 47 | mq 48 | end 49 | 50 | def age 51 | return -1 unless headers[:published_on] 52 | start - headers[:published_on].to_f 53 | end 54 | 55 | def ttl 56 | headers[:ttl] || -1 57 | end 58 | 59 | def expired? 60 | return false if ttl == -1 61 | age > ttl 62 | end 63 | 64 | def acked? 65 | @acked == true 66 | end 67 | 68 | def ack 69 | return if acked? 70 | queue_header.ack 71 | @acked = true 72 | rescue AMQP::ChannelClosedError => e 73 | Cloudist.handle_error(e) 74 | end 75 | 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/em/em_timer_utils.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | module EMTimerUtils 3 | def self.included(base) 4 | base.extend(ClassMethods) 5 | end 6 | 7 | module ClassMethods 8 | # Trap exceptions leaving the block and log them. Do not re-raise 9 | def trap_exceptions 10 | yield 11 | rescue => e 12 | em_exception(e) 13 | end 14 | 15 | def em_exception(e) 16 | msg = format_em_exception(e) 17 | log.error "[EM.timer] #{msg}", :exception => e 18 | end 19 | 20 | def format_em_exception(e) 21 | # avoid backtrace in /usr or vendor if possible 22 | system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ } 23 | reordered_backtrace = app + system 24 | 25 | # avoid "/" as the method name (we want the controller action) 26 | row = 0 27 | row = 1 if reordered_backtrace[row].match(/in `\/'$/) 28 | 29 | # get file and method name 30 | begin 31 | file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2] 32 | file.gsub!(/.*\//, '') 33 | "#{e.class} in #{file} #{method}: #{e.message}" 34 | rescue 35 | "#{e.class} in #{e.backtrace.first}: #{e.message}" 36 | end 37 | end 38 | 39 | # One-shot timer 40 | def timer(duration, &blk) 41 | EM.add_timer(duration) { trap_exceptions(&blk) } 42 | end 43 | 44 | # Add a periodic timer. If the now argument is true, run the block 45 | # immediately in addition to scheduling the periodic timer. 46 | def periodic_timer(duration, now=false, &blk) 47 | timer(1, &blk) if now 48 | EM.add_periodic_timer(duration) { trap_exceptions(&blk) } 49 | end 50 | end 51 | 52 | def timer(*args, &blk); self.class.timer(*args, &blk); end 53 | def periodic_timer(*args, &blk); self.class.periodic_timer(*args, &blk); end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GIT 2 | remote: git://github.com/ivanvanderbyl/moqueue.git 3 | revision: 5d20cc7bceed3cf4f40c02e638f4398b9f0a2e00 4 | specs: 5 | moqueue (0.2.1) 6 | amqp (>= 0.8.0.rc14) 7 | 8 | GEM 9 | remote: http://rubygems.org/ 10 | specs: 11 | activesupport (3.2.6) 12 | i18n (~> 0.6) 13 | multi_json (~> 1.0) 14 | amq-client (0.8.7) 15 | amq-protocol (>= 0.8.4) 16 | eventmachine 17 | amq-protocol (0.8.4) 18 | amqp (0.8.4) 19 | amq-client (~> 0.8.7) 20 | amq-protocol (~> 0.8.4) 21 | eventmachine 22 | diff-lcs (1.1.3) 23 | eventmachine (0.12.10) 24 | git (1.2.5) 25 | hashie (1.2.0) 26 | i18n (0.6.0) 27 | jeweler (1.6.4) 28 | bundler (~> 1.0) 29 | git (>= 1.2.5) 30 | rake 31 | json (1.7.3) 32 | macaddr (1.6.1) 33 | systemu (~> 2.5.0) 34 | multi_json (1.3.6) 35 | rake (0.9.2.2) 36 | reek (1.2.12) 37 | ripper_ruby_parser (~> 0.0.7) 38 | ruby2ruby (~> 1.2.5) 39 | ruby_parser (~> 2.0) 40 | sexp_processor (~> 3.0) 41 | ripper_ruby_parser (0.0.8) 42 | sexp_processor (~> 3.0) 43 | roodi (2.1.0) 44 | ruby_parser 45 | rspec (2.4.0) 46 | rspec-core (~> 2.4.0) 47 | rspec-expectations (~> 2.4.0) 48 | rspec-mocks (~> 2.4.0) 49 | rspec-core (2.4.0) 50 | rspec-expectations (2.4.0) 51 | diff-lcs (~> 1.1.2) 52 | rspec-mocks (2.4.0) 53 | ruby2ruby (1.2.5) 54 | ruby_parser (~> 2.0) 55 | sexp_processor (~> 3.0) 56 | ruby_parser (2.3.1) 57 | sexp_processor (~> 3.0) 58 | sexp_processor (3.2.0) 59 | systemu (2.5.1) 60 | uuid (2.3.5) 61 | macaddr (~> 1.0) 62 | 63 | PLATFORMS 64 | ruby 65 | 66 | DEPENDENCIES 67 | activesupport (> 3.0.0) 68 | amqp (~> 0.8.1) 69 | hashie 70 | i18n 71 | jeweler (~> 1.6.4) 72 | json (~> 1.4) 73 | moqueue! 74 | reek (~> 1.2.8) 75 | roodi (~> 2.1.0) 76 | rspec (~> 2.4.0) 77 | uuid 78 | -------------------------------------------------------------------------------- /examples/sandwich_client_with_custom_listener.rb: -------------------------------------------------------------------------------- 1 | # Cloudst Example: Sandwich Client with custom listener class 2 | # 3 | # This example demonstrates dispatching a job to the worker and receiving event callbacks. 4 | # 5 | # Be sure to update the Cloudist connection settings if they differ from defaults: 6 | # user: guest 7 | # pass: guest 8 | # port: 5672 9 | # host: localhost 10 | # vhost: / 11 | # 12 | $:.unshift File.dirname(__FILE__) + '/../lib' 13 | require "rubygems" 14 | require "cloudist" 15 | 16 | $total_jobs = 0 17 | 18 | class SandwichListener < Cloudist::Listener 19 | listen_to "make.sandwich" 20 | 21 | before :find_job 22 | 23 | def find_job 24 | puts "--- #{payload.id}" 25 | end 26 | 27 | def progress(i) 28 | puts "Progress: %1d%" % i 29 | end 30 | 31 | def runtime(seconds) 32 | puts "#{id} Finished job in #{seconds} seconds" 33 | $total_jobs -= 1 34 | puts "--- #{$total_jobs} jobs remaining" 35 | end 36 | 37 | # def started 38 | # puts "Started" 39 | # end 40 | 41 | def event(type) 42 | puts "Event: #{type}" 43 | end 44 | 45 | def finished 46 | puts "*** Finished ***" 47 | 48 | if $total_jobs == 0 49 | puts "Completed all jobs" 50 | Cloudist.stop 51 | end 52 | end 53 | 54 | def error(e) 55 | puts "#{e.exception}: #{e.message} (#{e.backtrace.first})" 56 | end 57 | end 58 | 59 | 60 | Cloudist.signal_trap! 61 | 62 | Cloudist.start(:logging => true) { 63 | puts AMQP.settings.inspect 64 | 65 | if ARGV.empty? 66 | puts "Please specify a number of workers to start as your first argument" 67 | Cloudist.stop 68 | else 69 | puts "*** Please ensure you have a worker running ***" 70 | 71 | job_count = ARGV.pop.to_i 72 | $total_jobs = job_count 73 | job_count.times { |i| 74 | log.info("Dispatching sandwich making job...") 75 | puts "Queued job: " + enqueue('make.sandwich', {:bread => 'white', :sandwich_number => i}).id 76 | } 77 | end 78 | 79 | add_listener(SandwichListener) 80 | } 81 | -------------------------------------------------------------------------------- /spec/cloudist_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/spec_helper') 2 | require "moqueue" 3 | 4 | describe "Cloudist" do 5 | before(:each) do 6 | stub_amqp! 7 | end 8 | it "should start an AMQP instance" do 9 | AMQP.should_receive(:start).once 10 | 11 | Cloudist.start do 12 | 13 | end 14 | end 15 | 16 | # before(:each) do 17 | # overload_amqp 18 | # reset_broker 19 | # Cloudist.remove_workers 20 | # 21 | # @mq = mock("MQ") 22 | # @queue, @exchange = mock_queue_and_exchange('make.sandwich') 23 | # 24 | # @qobj = Cloudist::JobQueue.any_instance 25 | # @qobj.stubs(:q).returns(@queue) 26 | # @qobj.stubs(:mq).returns(@mq) 27 | # @qobj.stubs(:ex).returns(@exchange) 28 | # @qobj.stubs(:setup) 29 | # end 30 | # 31 | # it "should register a worker" do 32 | # Cloudist.register_worker('make.sandwich', SandwichWorker) 33 | # Cloudist.workers.should have_key("make.sandwich") 34 | # Cloudist.workers["make.sandwich"].size.should == 1 35 | # end 36 | # 37 | # it "should support handle syntax" do 38 | # Cloudist.workers.should == {} 39 | # Cloudist.handle('make.sandwich').with(SandwichWorker) 40 | # Cloudist.workers.should have_key("make.sandwich") 41 | # Cloudist.workers["make.sandwich"].size.should == 1 42 | # end 43 | # 44 | # # it "should support handle syntax with multiple queues" do 45 | # # Cloudist.workers.should == {} 46 | # # Cloudist.handle('make.sandwich', 'eat.sandwich').with(SandwichWorker) 47 | # # # Cloudist.workers.should == {"make.sandwich"=>[SandwichWorker], "eat.sandwich"=>[SandwichWorker]} 48 | # # end 49 | # 50 | # it "should call process on worker when job arrives" do 51 | # job = Cloudist.enqueue('make.sandwich', {:bread => 'white'}) 52 | # job.payload.published?.should be_true 53 | # SandwichWorker.any_instance.expects(:process) 54 | # Cloudist.handle('make.sandwich').with(SandwichWorker) 55 | # Cloudist.workers.should have_key("make.sandwich") 56 | # Cloudist.workers["make.sandwich"].size.should == 1 57 | # end 58 | 59 | end 60 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | begin 4 | Bundler.setup(:default, :development) 5 | rescue Bundler::BundlerError => e 6 | $stderr.puts e.message 7 | $stderr.puts "Run `bundle install` to install missing gems" 8 | exit e.status_code 9 | end 10 | require 'rake' 11 | 12 | require 'jeweler' 13 | Jeweler::Tasks.new do |gem| 14 | # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options 15 | gem.name = "cloudist" 16 | gem.homepage = "http://github.com/ivanvanderbyl/cloudist" 17 | gem.license = "MIT" 18 | gem.summary = %Q{Super fast job queue using AMQP} 19 | gem.description = %Q{Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, DaemonKit or your own custom application. Refer to github page for examples} 20 | gem.email = "ivanvanderbyl@me.com" 21 | gem.authors = ["Ivan Vanderbyl"] 22 | # Include your dependencies below. Runtime dependencies are required when using your gem, 23 | # and development dependencies are only needed for development (ie running rake tasks, tests, etc) 24 | # gem.add_runtime_dependency 'jabber4r', '> 0.1' 25 | # gem.add_development_dependency 'rspec', '> 1.2.3' 26 | end 27 | Jeweler::RubygemsDotOrgTasks.new 28 | 29 | require 'rspec/core' 30 | require 'rspec/core/rake_task' 31 | RSpec::Core::RakeTask.new(:spec) do |spec| 32 | spec.pattern = FileList['spec/**/*_spec.rb'] 33 | end 34 | 35 | RSpec::Core::RakeTask.new(:rcov) do |spec| 36 | spec.pattern = 'spec/**/*_spec.rb' 37 | spec.rcov = true 38 | end 39 | 40 | require 'reek/rake/task' 41 | Reek::Rake::Task.new do |t| 42 | t.fail_on_error = true 43 | t.verbose = false 44 | t.source_files = 'lib/**/*.rb' 45 | end 46 | 47 | require 'roodi' 48 | require 'roodi_task' 49 | RoodiTask.new do |t| 50 | t.verbose = false 51 | end 52 | 53 | task :default => :spec 54 | 55 | require 'rake/rdoctask' 56 | Rake::RDocTask.new do |rdoc| 57 | version = File.exist?('VERSION') ? File.read('VERSION') : "" 58 | 59 | rdoc.rdoc_dir = 'rdoc' 60 | rdoc.title = "cloudist #{version}" 61 | rdoc.rdoc_files.include('README*') 62 | rdoc.rdoc_files.include('lib/**/*.rb') 63 | end 64 | -------------------------------------------------------------------------------- /lib/cloudist/job.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Job 3 | attr_reader :payload, :reply_queue 4 | 5 | def initialize(payload) 6 | @payload = payload 7 | 8 | if payload.reply_to 9 | @reply_queue = ReplyQueue.new(payload.reply_to) 10 | reply_queue.setup 11 | else 12 | @reply_queue = nil 13 | end 14 | end 15 | 16 | def id 17 | payload.id 18 | end 19 | 20 | def data 21 | payload.body 22 | end 23 | 24 | def body 25 | data 26 | end 27 | 28 | def log 29 | Cloudist.log 30 | end 31 | 32 | def cleanup 33 | # :noop 34 | end 35 | 36 | def reply(body, headers = {}, options = {}) 37 | raise ArgumentError, "Reply queue not ready" unless reply_queue 38 | 39 | options = { 40 | :echo => false 41 | }.update(options) 42 | 43 | headers = { 44 | :message_id => payload.id, 45 | :message_type => "reply" 46 | }.update(headers) 47 | 48 | reply_payload = Payload.new(body, headers) 49 | published_headers = reply_queue.publish(reply_payload) 50 | 51 | reply_payload 52 | end 53 | 54 | # Sends a progress update 55 | # Inputs: percentage - Integer 56 | # Optional description, this could be displayed to the user e.g. Resizing image 57 | def progress(percentage, description = nil) 58 | reply({:progress => percentage, :description => description}, {:message_type => 'progress'}) 59 | end 60 | 61 | def event(event_name, event_data = {}, options = {}) 62 | event_data ||= {} 63 | reply(event_data, {:event => event_name, :message_type => 'event'}, options) 64 | end 65 | 66 | def safely(&blk) 67 | yield 68 | rescue Exception => e 69 | handle_error(e) 70 | end 71 | 72 | def handle_error(e) 73 | reply({:exception => e.class.name.to_s, :message => e.message.to_s, :backtrace => e.backtrace}, {:message_type => 'error'}) 74 | end 75 | 76 | def method_missing(meth, *args, &blk) 77 | if meth.to_s.ends_with?("!") 78 | event(meth.to_s.gsub(/(!)$/, ''), args.shift) 79 | else 80 | super 81 | end 82 | end 83 | 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/cloudist/payload.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Payload 3 | include Utils 4 | include Encoding 5 | 6 | attr_reader :body, :headers, :amqp_headers, :timestamp 7 | 8 | def initialize(body, headers = {}) 9 | @published = false 10 | @timestamp = Time.now.to_f 11 | 12 | body = decode(body) if body.is_a?(String) 13 | @body = Hashie::Mash.new(decode(body)) 14 | @headers = Hashie::Mash.new(headers) 15 | @amqp_headers = {} 16 | # puts "Initialised Payload: #{id}" 17 | 18 | parse_headers! 19 | end 20 | 21 | def find_or_create_id 22 | if headers["message_id"] 23 | headers.message_id 24 | else 25 | UUID.generate 26 | end 27 | end 28 | 29 | def id 30 | find_or_create_id 31 | end 32 | 33 | def to_a 34 | [encode(body), {:headers => encoded_headers}] 35 | end 36 | 37 | def parse_headers! 38 | headers[:published_on] ||= body.delete("timestamp") || timestamp 39 | headers[:message_type] ||= body.delete("message_type") || 'reply' 40 | 41 | headers[:ttl] ||= Cloudist::DEFAULT_TTL 42 | headers[:message_id] = id 43 | 44 | headers[:published_on] = headers[:published_on].to_f 45 | 46 | headers[:ttl] = headers[:ttl].to_i rescue -1 47 | headers[:ttl] = -1 if headers[:ttl] == 0 48 | 49 | # If this payload was received with a timestamp, 50 | # we don't want to override it on #timestamp 51 | if timestamp > headers[:published_on] 52 | @timestamp = headers[:published_on] 53 | end 54 | 55 | headers 56 | end 57 | 58 | def encoded_headers 59 | h = headers.dup 60 | h.each { |k,v| h[k] = v.to_s } 61 | return h 62 | end 63 | 64 | def set_reply_to(queue_name) 65 | headers[:reply_to] = reply_prefix(queue_name) 66 | end 67 | 68 | def reply_to 69 | headers.reply_to 70 | end 71 | 72 | def message_type 73 | headers.message_type 74 | end 75 | 76 | def [](key) 77 | self.body[key.to_s] 78 | end 79 | 80 | def method_missing(meth, *args, &blk) 81 | if body.has_key?(meth.to_s) 82 | return body[meth] 83 | elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last 84 | body.has_key?(key.to_s) 85 | else 86 | super 87 | end 88 | end 89 | 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/cloudist/message.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Message 3 | include Cloudist::Encoding 4 | 5 | attr_reader :body, :headers, :id, :timestamp 6 | 7 | # Expects body to be decoded 8 | def initialize(body, headers = {}) 9 | @body = Hashie::Mash.new(body.dup) 10 | 11 | @id ||= headers[:message_id] || headers[:id] && headers.delete(:id) || UUID.generate 12 | @headers = Hashie::Mash.new(headers.dup) 13 | 14 | @timestamp = Time.now.utc.to_f 15 | 16 | update_headers(headers) 17 | end 18 | 19 | alias_method :data, :body 20 | 21 | def update_headers(new_headers = {}) 22 | update_headers! 23 | headers.merge!(new_headers) 24 | end 25 | 26 | def update_headers! 27 | headers[:ttl] ||= Cloudist::DEFAULT_TTL 28 | headers[:timestamp] = timestamp 29 | headers[:message_id] ||= id 30 | headers[:message_type] = 'message' 31 | headers[:queue_name] ||= 'test' 32 | 33 | headers.each { |k,v| headers[k] = v.to_s } 34 | end 35 | 36 | # Convenience method for replying 37 | # Constructs a reply message and publishes it 38 | def reply(body, reply_headers = {}) 39 | raise RuntimeError, "Cannot reply to an unpublished message" unless published? 40 | 41 | msg = Message.new(body, reply_headers) 42 | msg.set_reply_header 43 | reply_q = Cloudist::ReplyQueue.new(headers[:queue_name]) 44 | msg.publish(reply_q) 45 | end 46 | 47 | # Publishes this message to the exchange or queue 48 | # Queue should be a Cloudist::Queue object responding to #publish 49 | def publish(queue) 50 | raise ArgumentError, "Publish expects a Cloudist::Queue instance" unless queue.is_a?(Cloudist::Queue) 51 | set_queue_name_header(queue) 52 | update_published_date! 53 | update_headers! 54 | queue.publish(self) 55 | end 56 | 57 | def update_published_date! 58 | headers[:published_on] = Time.now.utc.to_f 59 | end 60 | 61 | # This is so we can reply back to the sender 62 | def set_queue_name_header(queue) 63 | update_headers(:queue_name => queue.name) 64 | end 65 | 66 | def published? 67 | @published ||= !!@headers.published_on 68 | end 69 | 70 | def created_at 71 | headers.timestamp ? Time.at(headers.timestamp.to_f) : timestamp 72 | end 73 | 74 | def published_at 75 | headers[:published_on] ? Time.at(headers[:published_on].to_f) : timestamp 76 | end 77 | 78 | def latency 79 | (published_at.to_f - created_at.to_f) 80 | end 81 | 82 | def encoded 83 | [encode(body), {:headers => headers}] 84 | end 85 | 86 | def inspect 87 | "<#{self.class.name} id=#{id}>" 88 | end 89 | 90 | private 91 | 92 | def set_reply_header 93 | headers[:message_type] = 'reply' 94 | end 95 | 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /lib/cloudist/core_ext/object.rb: -------------------------------------------------------------------------------- 1 | # Taken from Rails ActiveSupport 2 | class Object 3 | def remove_subclasses_of(*superclasses) #:nodoc: 4 | Class.remove_class(*subclasses_of(*superclasses)) 5 | end 6 | 7 | begin 8 | ObjectSpace.each_object(Class.new) {} 9 | 10 | # Exclude this class unless it's a subclass of our supers and is defined. 11 | # We check defined? in case we find a removed class that has yet to be 12 | # garbage collected. This also fails for anonymous classes -- please 13 | # submit a patch if you have a workaround. 14 | def subclasses_of(*superclasses) #:nodoc: 15 | subclasses = [] 16 | 17 | superclasses.each do |sup| 18 | ObjectSpace.each_object(class << sup; self; end) do |k| 19 | if k != sup && (k.name.blank? || eval("defined?(::#{k}) && ::#{k}.object_id == k.object_id")) 20 | subclasses << k 21 | end 22 | end 23 | end 24 | 25 | subclasses 26 | end 27 | rescue RuntimeError 28 | # JRuby and any implementations which cannot handle the objectspace traversal 29 | # above fall back to this implementation 30 | def subclasses_of(*superclasses) #:nodoc: 31 | subclasses = [] 32 | 33 | superclasses.each do |sup| 34 | ObjectSpace.each_object(Class) do |k| 35 | if superclasses.any? { |superclass| k < superclass } && 36 | (k.name.blank? || eval("defined?(::#{k}) && ::#{k}.object_id == k.object_id")) 37 | subclasses << k 38 | end 39 | end 40 | subclasses.uniq! 41 | end 42 | subclasses 43 | end 44 | end 45 | 46 | def extended_by #:nodoc: 47 | ancestors = class << self; ancestors end 48 | ancestors.select { |mod| mod.class == Module } - [ Object, Kernel ] 49 | end 50 | 51 | def extend_with_included_modules_from(object) #:nodoc: 52 | object.extended_by.each { |mod| extend mod } 53 | end 54 | 55 | unless defined? instance_exec # 1.9 56 | module InstanceExecMethods #:nodoc: 57 | end 58 | include InstanceExecMethods 59 | 60 | # Evaluate the block with the given arguments within the context of 61 | # this object, so self is set to the method receiver. 62 | # 63 | # From Mauricio's http://eigenclass.org/hiki/bounded+space+instance_exec 64 | def instance_exec(*args, &block) 65 | begin 66 | old_critical, Thread.critical = Thread.critical, true 67 | n = 0 68 | n += 1 while respond_to?(method_name = "__instance_exec#{n}") 69 | InstanceExecMethods.module_eval { define_method(method_name, &block) } 70 | ensure 71 | Thread.critical = old_critical 72 | end 73 | 74 | begin 75 | send(method_name, *args) 76 | ensure 77 | InstanceExecMethods.module_eval { remove_method(method_name) } rescue nil 78 | end 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/cloudist/payload_spec_2_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | describe Cloudist::Payload do 4 | include Cloudist::Encoding 5 | 6 | it "should accept a hash for data" do 7 | pl = Cloudist::Payload.new({:bread => 'white'}) 8 | pl.body.bread.should == "white" 9 | end 10 | 11 | it "should accept encoded message" do 12 | pl = Cloudist::Payload.new(encode({:bread => 'white'})) 13 | pl.body.bread.should == "white" 14 | end 15 | 16 | it "should retrieve id from headers" do 17 | pl = Cloudist::Payload.new({:bread => 'white'}, {:message_id => "12345"}) 18 | pl.id.should == "12345" 19 | end 20 | 21 | it "should prepare headers" do 22 | payload = Cloudist::Payload.new({:bread => 'white'}) 23 | payload.body.bread.should == "white" 24 | payload.headers.has_key?("ttl").should be_true 25 | # payload.headers.has_key?(:content_type).should be_true 26 | # payload.headers[:content_type].should == "application/json" 27 | payload.headers.has_key?("published_on").should be_true 28 | payload.headers.has_key?("message_id").should be_true 29 | end 30 | 31 | it "should extract published_on from data" do 32 | time = Time.now.to_f 33 | payload = Cloudist::Payload.new({:bread => 'white', :timestamp => time}) 34 | payload.headers[:published_on].should == time 35 | end 36 | 37 | it "should not override timestamp if already present in headers" do 38 | time = (Time.now.to_f - 10.0) 39 | payload = Cloudist::Payload.new({:bread => 'white'}, {:published_on => time}) 40 | payload.headers[:published_on].should == time 41 | end 42 | 43 | it "should override timestamp if not present" do 44 | payload = Cloudist::Payload.new({:bread => 'white'}) 45 | payload.headers[:published_on].should be_within(0.1).of Time.now.to_f 46 | payload.timestamp.should be_within(0.1).of Time.now.to_f 47 | end 48 | 49 | it "should parse custom headers" do 50 | payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white'}), {:published_on => 12345, :message_id => "foo"}) 51 | payload.headers.to_hash.should == { "published_on"=>12345, "message_id"=>"foo", "ttl"=>300 } 52 | end 53 | 54 | it "should create a unique event hash" do 55 | payload = Cloudist::Payload.new({:bread => 'white'}) 56 | payload.id.size.should == 36 57 | end 58 | 59 | it "should not create a new message_id unless it doesn't have one" do 60 | payload = Cloudist::Payload.new({:bread => 'white'}) 61 | payload.id.size.should == 36 62 | payload = Cloudist::Payload.new({:bread => 'white'}, {:message_id => 'foo'}) 63 | payload.id.should == 'foo' 64 | end 65 | 66 | it "should format payload for sending" do 67 | payload = Cloudist::Payload.new({:bread => 'white'}, {:message_id => 'foo', :message_type => 'reply'}) 68 | body, popts = payload.to_a 69 | headers = popts[:headers] 70 | 71 | body.should == encode(Hashie::Mash.new({:bread => 'white'})) 72 | headers[:ttl].should == "300" 73 | headers[:message_type].should == 'reply' 74 | end 75 | 76 | 77 | 78 | end 79 | -------------------------------------------------------------------------------- /spec/cloudist/message_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../spec_helper', __FILE__) 2 | 3 | # describe Cloudist::Message do 4 | # before(:each) do 5 | # stub_amqp! 6 | # @queue = Cloudist::Queue.new("test.queue") 7 | # @queue.stubs(:publish) 8 | # @headers = {} 9 | # end 10 | # 11 | # it "should have a unique id when new" do 12 | # msg = Cloudist::Message.new({:hello => "world"}, @headers) 13 | # msg.id.size.should == "57b474f0-496c-012e-6f57-34159e11a916".size 14 | # end 15 | # 16 | # it "should not update id when existing message" do 17 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 18 | # msg.update_headers 19 | # msg.id.should == "not-an-id" 20 | # end 21 | # 22 | # it "should remove id from headers and update with message_id" do 23 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 24 | # msg.update_headers 25 | # msg.id.should == "not-an-id" 26 | # msg.headers.id.should == nil 27 | # 28 | # msg = Cloudist::Message.new({:hello => "world"}, {:message_id => "not-an-id"}) 29 | # msg.update_headers 30 | # msg.id.should == "not-an-id" 31 | # msg.headers.id.should == nil 32 | # end 33 | # 34 | # it "should update headers" do 35 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 36 | # msg.update_headers 37 | # 38 | # msg.headers.keys.should include *["ttl", "timestamp", "message_id"] 39 | # end 40 | # 41 | # it "should allow custom header when updating" do 42 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 43 | # msg.update_headers(:message_type => "reply") 44 | # 45 | # msg.headers.keys.should include *["ttl", "timestamp", "message_id", "message_type"] 46 | # end 47 | # 48 | # it "should not be published if timestamp is not in headers" do 49 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 50 | # msg.published?.should be_false 51 | # end 52 | # 53 | # it "should be published if timestamp is in headers" do 54 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 55 | # msg.publish(@queue) 56 | # msg.published?.should be_true 57 | # end 58 | # 59 | # it "should include ttl in headers" do 60 | # msg = Cloudist::Message.new({:hello => "world"}) 61 | # # msg.publish(@queue) 62 | # msg.headers[:ttl].should == "300" 63 | # end 64 | # 65 | # it "should get created_at date from header" do 66 | # time = Time.now.to_f 67 | # msg = Cloudist::Message.new({:hello => "world"}, {:timestamp => time}) 68 | # msg.created_at.to_f.should == time 69 | # end 70 | # 71 | # it "should set published_at when publishing" do 72 | # time = Time.now.to_f 73 | # msg = Cloudist::Message.new({:hello => "world"}, {:timestamp => time}) 74 | # msg.publish(@queue) 75 | # msg.published_at.to_f.should > time 76 | # end 77 | # 78 | # it "should have latency" do 79 | # time = (Time.now).to_f 80 | # msg = Cloudist::Message.new({:hello => "world"}, {:timestamp => time}) 81 | # sleep(0.1) 82 | # msg.publish(@queue) 83 | # msg.latency.should be_within(0.001).of(0.1) 84 | # end 85 | # 86 | # it "should reply to sender" do 87 | # msg = Cloudist::Message.new({:hello => "world"}, {:id => "not-an-id"}) 88 | # msg.reply(:success => true) 89 | # end 90 | # 91 | # end 92 | -------------------------------------------------------------------------------- /lib/cloudist/listener.rb: -------------------------------------------------------------------------------- 1 | require "active_support" 2 | module Cloudist 3 | class Listener 4 | include ActiveSupport::Callbacks 5 | 6 | attr_reader :job_queue_name, :payload 7 | class_attribute :job_queue_names 8 | 9 | class << self 10 | def listen_to(*job_queue_names) 11 | self.job_queue_names = job_queue_names.map { |q| Utils.reply_prefix(q) } 12 | end 13 | 14 | def subscribe(queue_name) 15 | raise RuntimeError, "You can't subscribe until EM is running" unless EM.reactor_running? 16 | 17 | reply_queue = Cloudist::ReplyQueue.new(queue_name) 18 | reply_queue.subscribe do |request| 19 | instance = Cloudist.listener_instances[queue_name] ||= new 20 | instance.handle_request(request) 21 | end 22 | 23 | queue_name 24 | end 25 | 26 | def before(*args, &block) 27 | set_callback(:call, :before, *args, &block) 28 | end 29 | 30 | def after(*args, &block) 31 | set_callback(:call, :after, *args, &block) 32 | end 33 | end 34 | 35 | define_callbacks :call, :rescuable => true 36 | 37 | def handle_request(request) 38 | @payload = request.payload 39 | key = [payload.message_type.to_s, payload.headers[:event]].compact.join(':') 40 | 41 | meth, *args = handle_key(key) 42 | 43 | if meth && self.respond_to?(meth) 44 | if method(meth).arity <= args.size 45 | call(meth, args.first(method(meth).arity)) 46 | else 47 | raise ArgumentError, "Unable to fire callback (#{meth}) because we don't have enough args" 48 | end 49 | end 50 | end 51 | 52 | def id 53 | payload.id 54 | end 55 | 56 | def data 57 | payload.body 58 | end 59 | 60 | def handle_key(key) 61 | key = key.split(':', 2) 62 | return [nil, nil] if key.empty? 63 | 64 | method_and_args = [key.shift.to_sym] 65 | case method_and_args[0] 66 | when :event 67 | if key.size > 0 && self.respond_to?(key.first) 68 | method_and_args = [key.shift] 69 | end 70 | method_and_args << key 71 | 72 | when :progress 73 | method_and_args << payload.progress 74 | method_and_args << payload.description 75 | 76 | when :runtime 77 | method_and_args << payload.runtime 78 | 79 | when :reply 80 | 81 | when :update 82 | 83 | when :error 84 | # method_and_args << Cloudist::SafeError.new(payload) 85 | method_and_args << Hashie::Mash.new(payload.body) 86 | 87 | when :log 88 | method_and_args << payload.message 89 | method_and_args << payload.level 90 | 91 | else 92 | method_and_args << data if method(method_and_args[0]).arity == 1 93 | end 94 | 95 | return method_and_args 96 | end 97 | 98 | def call(meth, args) 99 | run_callbacks :call do 100 | if args.empty? 101 | send(meth) 102 | else 103 | send(meth, *args) 104 | end 105 | end 106 | end 107 | 108 | def progress(pct) 109 | # :noop 110 | end 111 | 112 | def runtime(seconds) 113 | # :noop 114 | end 115 | 116 | def event(type) 117 | # :noop 118 | end 119 | 120 | def log(message, level) 121 | # :noop 122 | end 123 | 124 | def error(e) 125 | # :noop 126 | end 127 | 128 | end 129 | 130 | class GenericListener < Listener 131 | 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /lib/cloudist/queues/basic_queue.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class UnknownReplyTo < RuntimeError; end 3 | class ExpiredMessage < RuntimeError; end 4 | 5 | module Queues 6 | class BasicQueue 7 | attr_reader :queue_name, :options 8 | attr_reader :queue, :exchange, :channel, :prefetch 9 | 10 | alias :q :queue 11 | alias :ex :exchange 12 | alias :mq :channel 13 | 14 | def initialize(queue_name, options = {}) 15 | @prefetch ||= options.delete(:prefetch) || 1 16 | 17 | options = { 18 | :auto_delete => true, 19 | :durable => false, 20 | :nowait => true 21 | }.update(options) 22 | 23 | @queue_name, @options = queue_name, options 24 | 25 | setup 26 | end 27 | 28 | def inspect 29 | "<#{self.class.name} queue_name=#{queue_name}>" 30 | end 31 | 32 | def setup 33 | return if @setup.eql?(true) 34 | 35 | @channel ||= AMQP::Channel.new(Cloudist.connection) do 36 | channel.prefetch(self.prefetch, false) if self.prefetch 37 | end 38 | 39 | @queue = @channel.queue(queue_name, options) 40 | 41 | setup_exchange 42 | 43 | @setup = true 44 | end 45 | 46 | def setup_exchange 47 | @exchange = channel.direct("") 48 | end 49 | 50 | # def setup_exchange 51 | # @exchange = channel.direct(queue_name) 52 | # queue.bind(exchange) 53 | # end 54 | 55 | def log 56 | Cloudist.log 57 | end 58 | 59 | def tag 60 | s = "queue=#{queue.name}" 61 | s += " exchange=#{exchange.name}" if exchange 62 | s 63 | end 64 | 65 | def subscribe(&block) 66 | queue.subscribe(:ack => true) do |queue_header, encoded_message| 67 | # next if Cloudist.closing? 68 | 69 | request = Cloudist::Request.new(self, encoded_message, queue_header) 70 | 71 | handle_request = proc { 72 | begin 73 | raise Cloudist::ExpiredMessage if request.expired? 74 | # yield request if block_given? 75 | block.call(request) 76 | 77 | rescue Cloudist::ExpiredMessage 78 | log.error "AMQP Message Timeout: #{tag} ttl=#{request.ttl} age=#{request.age}" 79 | 80 | rescue => e 81 | Cloudist.handle_error(e) 82 | ensure 83 | request.ack 84 | # unless Cloudist.closing? 85 | # finished = Time.now.utc.to_i 86 | # log.debug("Finished Job in #{finished - request.start} seconds") 87 | end 88 | } 89 | 90 | handle_ack = proc { 91 | request.ack 92 | } 93 | 94 | EM.defer(handle_request, handle_ack) 95 | end 96 | log.info "AMQP Subscribed: #{tag}" 97 | self 98 | end 99 | 100 | def print_status 101 | # queue.status{ |num_messages, num_consumers| 102 | # log.info("STATUS: #{queue.name}: JOBS: #{num_messages} WORKERS: #{num_consumers+1}") 103 | # } 104 | end 105 | 106 | def publish(payload) 107 | payload.set_reply_to(queue_name) 108 | body, headers = payload.to_a 109 | headers.merge!(:routing_key => queue.name) 110 | exchange.publish(body, headers) 111 | end 112 | 113 | def publish_to_q(payload) 114 | body, headers = payload.to_a 115 | # headers.merge!(:routing_key => queue.name) 116 | queue.publish(body, headers) 117 | return headers 118 | end 119 | 120 | def teardown 121 | @queue.unsubscribe 122 | @channel.close 123 | log.debug "AMQP Unsubscribed: #{tag}" 124 | end 125 | 126 | def destroy 127 | teardown 128 | end 129 | end 130 | end 131 | end 132 | -------------------------------------------------------------------------------- /lib/cloudist/queue.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | # 3 | # NOTE: Queue is Deprecated, please use BasicQueue 4 | # 5 | class Queue 6 | 7 | attr_reader :options, :name, :channel, :q, :ex 8 | 9 | class_attribute :cached_queues 10 | 11 | def initialize(name, options = {}) 12 | self.class.cached_queues ||= {} 13 | 14 | options = { 15 | :auto_delete => false, 16 | :durable => true 17 | }.update(options) 18 | 19 | @name, @options = name, options 20 | 21 | setup 22 | p self.cached_queues.keys 23 | 24 | log.debug(tag) 25 | purge 26 | end 27 | 28 | def purge 29 | q.purge 30 | end 31 | 32 | def inspect 33 | "<#{self.class.name} name=#{name} exchange=#{ex ? ex.name : 'nil'}>" 34 | end 35 | 36 | def log 37 | Cloudist.log 38 | end 39 | 40 | def tag 41 | [].tap { |a| 42 | a << "queue=#{q.name}" if q 43 | a << "exchange=#{ex.name}" if ex 44 | }.join(' ') 45 | end 46 | 47 | def publish(msg) 48 | raise ArgumentError, "Publish expects a Cloudist::Message object" unless msg.is_a?(Cloudist::Message) 49 | 50 | body, headers = msg.encoded 51 | # EM.defer { 52 | publish_to_ex(body, headers) 53 | # } 54 | 55 | p msg.body.to_hash 56 | end 57 | 58 | # def channel 59 | # self.class.channel 60 | # end 61 | # 62 | # def q 63 | # self.class.q 64 | # end 65 | # 66 | # def ex 67 | # self.class.ex 68 | # end 69 | 70 | def publish_to_ex(body, headers = {}) 71 | ex.publish(body, headers) 72 | end 73 | 74 | def publish_to_q(body, headers = {}) 75 | q.publish(body, headers) 76 | end 77 | 78 | def teardown 79 | q.unsubscribe 80 | channel.close 81 | log.debug "AMQP Unsubscribed: #{tag}" 82 | end 83 | 84 | def destroy 85 | teardown 86 | end 87 | 88 | def subscribe(options = {}, &block) 89 | options[:ack] = true 90 | q.subscribe(options) do |queue_header, encoded_message| 91 | request = Cloudist::Request.new(self, encoded_message, queue_header) 92 | 93 | msg = Cloudist::Message.new(*request.for_message) 94 | 95 | EM.defer do 96 | begin 97 | raise Cloudist::ExpiredMessage if request.expired? 98 | yield msg 99 | 100 | rescue Cloudist::ExpiredMessage 101 | log.error "AMQP Message Timeout: #{tag} ttl=#{request.ttl} age=#{request.age}" 102 | 103 | rescue Exception => e 104 | Cloudist.handle_error(e) 105 | 106 | ensure 107 | request.ack unless Cloudist.closing? 108 | end 109 | end 110 | end 111 | end 112 | 113 | private 114 | 115 | def setup 116 | if self.class.cached_queues.keys.include?(name.to_sym) 117 | @q = self.class.cached_queues[name.to_sym][:q] 118 | @ex = self.class.cached_queues[name.to_sym][:ex] 119 | @channel = self.class.cached_queues[name.to_sym][:channel] 120 | setup_binding 121 | else 122 | puts "Setup" 123 | 124 | setup_channel 125 | setup_queue 126 | setup_exchange 127 | 128 | self.class.cached_queues[name.to_sym] = {:q => q, :ex => ex, :channel => channel} 129 | end 130 | setup_binding 131 | end 132 | 133 | def setup_channel 134 | @channel = ::AMQP::Channel.new 135 | 136 | # Set up QOS. If you do not do this then the subscribe in receive_message 137 | # will get overwelmd and the whole thing will collapse in on itself. 138 | channel.prefetch(1) 139 | end 140 | 141 | def setup_queue 142 | @q = channel.queue(name, options) 143 | end 144 | 145 | def setup_exchange 146 | @ex = channel.direct(name) 147 | # setup_binding 148 | end 149 | 150 | def setup_binding 151 | q.bind(ex) 152 | end 153 | 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | **This was an experiment, it's no longer maintained. Go learn Elixir/Phoenix :)** 3 | 4 | ___ 5 | 6 | Cloudist 7 | ======== 8 | 9 | Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, DaemonKit or your own custom application. Cloudist uses AMQP (RabbitMQ mainly) for transport and provides a simple DSL for queuing jobs and receiving responses including logs, exceptions and job progress. 10 | 11 | Cloudist can be used to distribute long running tasks such as encoding a video, generating PDFs, scraping site data 12 | or even just sending emails. Unlike other job queues (DelayedJob etc) Cloudist does not load your entire Rails stack into memory for every worker, and it is not designed to, instead it expects all the data your worker requires to complete a job to be sent in the initial job request. This means your workers stay slim and can scale very quickly and even run on EC2 micros outside your applications environment without any further configuration. 13 | 14 | Installation 15 | ------------ 16 | 17 | ```bash 18 | gem install cloudist 19 | ``` 20 | 21 | Or if your app has a Gemfile: 22 | 23 | ```ruby 24 | gem 'cloudist', '~> 0.4.4' 25 | ``` 26 | 27 | Usage 28 | ----- 29 | 30 | **Refer to examples.** 31 | 32 | Configuration 33 | ------------- 34 | 35 | The only configuration required to get going are the AMQP settings, these can be set in two ways: 36 | 37 | 1. Using the `AMQP_URL` environment variable with value of `amqp://username:password@localhost:5672/vhost` 38 | 39 | 2. Updating the settings hash manually: 40 | 41 | ```ruby 42 | Cloudist.settings = {:user => 'guest', :pass => 'password', :vhost => '/', :host => 'localhost', :port => 5672} 43 | ``` 44 | 45 | Now and what's coming 46 | --------------------- 47 | 48 | Cloudist was developed to provide the messaging layer used within TestPilot [Continuous Integration](http://testpilot.me) service. 49 | 50 | TestPilot still uses [Cloudist](http://testpilot.me/ivan/cloudist) heavily and a number of features will be merged in soon. 51 | 52 | Acknowledgements 53 | ---------------- 54 | 55 | Portions of this gem are based on code from the following projects: 56 | 57 | - Heroku's Droid gem 58 | - Lizzy 59 | - Minion 60 | 61 | Contributing to Cloudist 62 | ------------------------ 63 | 64 | * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet 65 | * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it 66 | * Fork the project 67 | * Start a feature/bugfix branch e.g. git checkout -b feature-my-awesome-idea or bugfix-this-does-not-work 68 | * Commit and push until you are happy with your contribution 69 | * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally. 70 | * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it. 71 | 72 | Authors 73 | ------- 74 | 75 | [IvanVanderbyl](http://testpilot.me/ivan) - [Blog](http://ivanvanderbyl.github.com/) 76 | 77 | Copyright 78 | --------- 79 | 80 | Copyright (c) 2011 Ivan Vanderbyl. 81 | 82 | Permission is hereby granted, free of charge, to any person obtaining 83 | a copy of this software and associated documentation files (the 84 | "Software"), to deal in the Software without restriction, including 85 | without limitation the rights to use, copy, modify, merge, publish, 86 | distribute, sublicense, and/or sell copies of the Software, and to 87 | permit persons to whom the Software is furnished to do so, subject to 88 | the following conditions: 89 | 90 | The above copyright notice and this permission notice shall be 91 | included in all copies or substantial portions of the Software. 92 | 93 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 94 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 95 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 96 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 97 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 98 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 99 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 100 | 101 | -------------------------------------------------------------------------------- /lib/cloudist/payload_old.rb: -------------------------------------------------------------------------------- 1 | module Cloudist 2 | class Payload 3 | include Utils 4 | 5 | attr_reader :body, :publish_opts, :headers, :timestamp 6 | 7 | def initialize(body, headers = {}, publish_opts = {}) 8 | @publish_opts, @headers = publish_opts, Hashie::Mash.new(headers) 9 | @published = false 10 | 11 | body = parse_message(body) if body.is_a?(String) 12 | 13 | # raise Cloudist::BadPayload, "Expected Hash for payload" unless body.is_a?(Hash) 14 | 15 | @timestamp = Time.now.to_f 16 | 17 | @body = body 18 | # Hashie::Mash.new(body) 19 | 20 | update_headers 21 | end 22 | 23 | # Return message formatted as JSON and headers ready for transport in array 24 | def formatted 25 | update_headers 26 | 27 | [encode_message(body), publish_opts] 28 | end 29 | 30 | def id 31 | @id ||= event_hash.to_s 32 | end 33 | 34 | def id=(new_id) 35 | @id = new_id.to_s 36 | update_headers 37 | end 38 | 39 | def frozen? 40 | headers.frozen? 41 | end 42 | 43 | def freeze! 44 | headers.freeze 45 | body.freeze 46 | end 47 | 48 | def update_headers 49 | headers = extract_custom_headers 50 | (publish_opts[:headers] ||= {}).merge!(headers) 51 | end 52 | 53 | def extract_custom_headers 54 | raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published? 55 | headers[:published_on] ||= body.is_a?(Hash) && body.delete(:published_on) || Time.now.utc.to_i 56 | headers[:ttl] ||= body.is_a?(Hash) && body.delete('ttl') || Cloudist::DEFAULT_TTL 57 | headers[:timestamp] = timestamp 58 | # this is the event hash that gets transferred through various publish/reply actions 59 | headers[:event_hash] ||= id 60 | 61 | # this value should be unique for each published/received message pair 62 | headers[:message_id] ||= id 63 | 64 | # We use JSON for message transport exclusively 65 | # headers[:content_type] ||= 'application/json' 66 | 67 | # headers[:headers][:message_type] = 'event' 68 | # ||= body.delete('message_type') || 'reply' 69 | 70 | # headers[:headers] = custom_headers 71 | 72 | # some strange behavior with integers makes it better to 73 | # convert all amqp headers to strings to avoid any problems 74 | headers.each { |k,v| headers[k] = v.to_s } 75 | 76 | headers 77 | end 78 | 79 | def parse_custom_headers 80 | return { } unless headers 81 | 82 | h = headers.dup 83 | 84 | h[:published_on] = h[:published_on].to_i 85 | 86 | h[:ttl] = h[:ttl].to_i rescue -1 87 | h[:ttl] = -1 if h[:ttl] == 0 88 | 89 | h 90 | end 91 | 92 | def set_reply_to(queue_name) 93 | headers["reply_to"] = reply_name(queue_name) 94 | set_master_queue_name(queue_name) 95 | end 96 | 97 | def set_master_queue_name(queue_name) 98 | headers[:master_queue] = queue_name 99 | end 100 | 101 | def reply_name(queue_name) 102 | # "#{queue_name}.#{id}" 103 | Utils.reply_prefix(queue_name) 104 | end 105 | 106 | def reply_to 107 | headers["reply_to"] 108 | end 109 | 110 | def message_type 111 | headers["message_type"] 112 | end 113 | 114 | def event_hash 115 | @event_hash ||= headers["event_hash"] || create_event_hash 116 | end 117 | 118 | def create_event_hash 119 | # s = Time.now.to_s + object_id.to_s + rand(100).to_s 120 | # Digest::MD5.hexdigest(s) 121 | UUID.generate 122 | end 123 | 124 | def parse_message(raw) 125 | # return { } unless raw 126 | # decode_json(raw) 127 | decode_message(raw) 128 | end 129 | 130 | def [](key) 131 | body[key] 132 | end 133 | 134 | def published? 135 | @published == true 136 | end 137 | 138 | def publish 139 | return if published? 140 | @published = true 141 | freeze! 142 | end 143 | 144 | def method_missing(meth, *args, &blk) 145 | if body.is_a?(Hash) && body.has_key?(meth) 146 | return body[meth] 147 | elsif key = meth.to_s.match(/(.+)(?:\?$)/).to_a.last 148 | body.has_key?(key.to_sym) 149 | else 150 | super 151 | end 152 | end 153 | 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /lib/cloudist/core_ext/class.rb: -------------------------------------------------------------------------------- 1 | require 'cloudist/core_ext/kernel' 2 | require 'cloudist/core_ext/module' 3 | 4 | # Extracted from ActiveSupport 3.0 5 | class Class 6 | 7 | # Taken from http://coderrr.wordpress.com/2008/04/10/lets-stop-polluting-the-threadcurrent-hash/ 8 | def thread_local_accessor name, options = {} 9 | m = Module.new 10 | m.module_eval do 11 | class_variable_set :"@@#{name}", Hash.new {|h,k| h[k] = options[:default] } 12 | end 13 | m.module_eval %{ 14 | FINALIZER = lambda {|id| @@#{name}.delete id } 15 | 16 | def #{name} 17 | @@#{name}[Thread.current.object_id] 18 | end 19 | 20 | def #{name}=(val) 21 | ObjectSpace.define_finalizer Thread.current, FINALIZER unless @@#{name}.has_key? Thread.current.object_id 22 | @@#{name}[Thread.current.object_id] = val 23 | end 24 | } 25 | 26 | class_eval do 27 | include m 28 | extend m 29 | end 30 | end 31 | 32 | 33 | # Declare a class-level attribute whose value is inheritable by subclasses. 34 | # Subclasses can change their own value and it will not impact parent class. 35 | # 36 | # class Base 37 | # class_attribute :setting 38 | # end 39 | # 40 | # class Subclass < Base 41 | # end 42 | # 43 | # Base.setting = true 44 | # Subclass.setting # => true 45 | # Subclass.setting = false 46 | # Subclass.setting # => false 47 | # Base.setting # => true 48 | # 49 | # In the above case as long as Subclass does not assign a value to setting 50 | # by performing Subclass.setting = _something_ , Subclass.setting 51 | # would read value assigned to parent class. Once Subclass assigns a value then 52 | # the value assigned by Subclass would be returned. 53 | # 54 | # This matches normal Ruby method inheritance: think of writing an attribute 55 | # on a subclass as overriding the reader method. However, you need to be aware 56 | # when using +class_attribute+ with mutable structures as +Array+ or +Hash+. 57 | # In such cases, you don't want to do changes in places but use setters: 58 | # 59 | # Base.setting = [] 60 | # Base.setting # => [] 61 | # Subclass.setting # => [] 62 | # 63 | # # Appending in child changes both parent and child because it is the same object: 64 | # Subclass.setting << :foo 65 | # Base.setting # => [:foo] 66 | # Subclass.setting # => [:foo] 67 | # 68 | # # Use setters to not propagate changes: 69 | # Base.setting = [] 70 | # Subclass.setting += [:foo] 71 | # Base.setting # => [] 72 | # Subclass.setting # => [:foo] 73 | # 74 | # For convenience, a query method is defined as well: 75 | # 76 | # Subclass.setting? # => false 77 | # 78 | # Instances may overwrite the class value in the same way: 79 | # 80 | # Base.setting = true 81 | # object = Base.new 82 | # object.setting # => true 83 | # object.setting = false 84 | # object.setting # => false 85 | # Base.setting # => true 86 | # 87 | # To opt out of the instance writer method, pass :instance_writer => false. 88 | # 89 | # object.setting = false # => NoMethodError 90 | def class_attribute(*attrs) 91 | instance_writer = !attrs.last.is_a?(Hash) || attrs.pop[:instance_writer] 92 | 93 | attrs.each do |name| 94 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 95 | def self.#{name}() nil end 96 | def self.#{name}?() !!#{name} end 97 | 98 | def self.#{name}=(val) 99 | singleton_class.class_eval do 100 | remove_possible_method(:#{name}) 101 | define_method(:#{name}) { val } 102 | end 103 | 104 | if singleton_class? 105 | class_eval do 106 | remove_possible_method(:#{name}) 107 | def #{name} 108 | defined?(@#{name}) ? @#{name} : singleton_class.#{name} 109 | end 110 | end 111 | end 112 | val 113 | end 114 | 115 | remove_method :#{name} if method_defined?(:#{name}) 116 | def #{name} 117 | defined?(@#{name}) ? @#{name} : self.class.#{name} 118 | end 119 | 120 | def #{name}? 121 | !!#{name} 122 | end 123 | RUBY 124 | 125 | attr_writer name if instance_writer 126 | end 127 | end 128 | 129 | private 130 | def singleton_class? 131 | # in case somebody is crazy enough to overwrite allocate 132 | allocate = Class.instance_method(:allocate) 133 | # object.class always points to a real (non-singleton) class 134 | allocate.bind(self).call.class != self 135 | rescue TypeError 136 | # MRI/YARV/JRuby all disallow creating new instances of a singleton class 137 | true 138 | end 139 | end 140 | -------------------------------------------------------------------------------- /cloudist.gemspec: -------------------------------------------------------------------------------- 1 | # Generated by jeweler 2 | # DO NOT EDIT THIS FILE DIRECTLY 3 | # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' 4 | # -*- encoding: utf-8 -*- 5 | 6 | Gem::Specification.new do |s| 7 | s.name = "cloudist" 8 | s.version = "0.5.0" 9 | 10 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= 11 | s.authors = ["Ivan Vanderbyl"] 12 | s.date = "2012-07-12" 13 | s.description = "Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, DaemonKit or your own custom application. Refer to github page for examples" 14 | s.email = "ivanvanderbyl@me.com" 15 | s.extra_rdoc_files = [ 16 | "LICENSE.txt", 17 | "README.md" 18 | ] 19 | s.files = [ 20 | ".document", 21 | ".rspec", 22 | "Gemfile", 23 | "Gemfile.lock", 24 | "LICENSE.txt", 25 | "README.md", 26 | "Rakefile", 27 | "VERSION", 28 | "cloudist.gemspec", 29 | "doc/cloudist.png", 30 | "examples/amqp/Gemfile", 31 | "examples/amqp/Gemfile.lock", 32 | "examples/amqp/amqp_consumer.rb", 33 | "examples/amqp/amqp_publisher.rb", 34 | "examples/queue_message.rb", 35 | "examples/sandwich_client_with_custom_listener.rb", 36 | "examples/sandwich_worker.rb", 37 | "examples/sandwich_worker_with_class.rb", 38 | "lib/cloudist.rb", 39 | "lib/cloudist/application.rb", 40 | "lib/cloudist/core_ext/class.rb", 41 | "lib/cloudist/core_ext/kernel.rb", 42 | "lib/cloudist/core_ext/module.rb", 43 | "lib/cloudist/core_ext/object.rb", 44 | "lib/cloudist/core_ext/string.rb", 45 | "lib/cloudist/encoding.rb", 46 | "lib/cloudist/errors.rb", 47 | "lib/cloudist/job.rb", 48 | "lib/cloudist/listener.rb", 49 | "lib/cloudist/message.rb", 50 | "lib/cloudist/messaging.rb", 51 | "lib/cloudist/payload.rb", 52 | "lib/cloudist/payload_old.rb", 53 | "lib/cloudist/publisher.rb", 54 | "lib/cloudist/queue.rb", 55 | "lib/cloudist/queues/basic_queue.rb", 56 | "lib/cloudist/queues/job_queue.rb", 57 | "lib/cloudist/queues/reply_queue.rb", 58 | "lib/cloudist/request.rb", 59 | "lib/cloudist/utils.rb", 60 | "lib/cloudist/worker.rb", 61 | "lib/cloudist_old.rb", 62 | "lib/em/em_timer_utils.rb", 63 | "lib/em/iterator.rb", 64 | "spec/cloudist/basic_queue_spec.rb", 65 | "spec/cloudist/job_spec.rb", 66 | "spec/cloudist/message_spec.rb", 67 | "spec/cloudist/messaging_spec.rb", 68 | "spec/cloudist/payload_spec.rb", 69 | "spec/cloudist/payload_spec_2_spec.rb", 70 | "spec/cloudist/queue_spec.rb", 71 | "spec/cloudist/request_spec.rb", 72 | "spec/cloudist/utils_spec.rb", 73 | "spec/cloudist_spec.rb", 74 | "spec/core_ext/string_spec.rb", 75 | "spec/spec_helper.rb", 76 | "spec/support/amqp.rb" 77 | ] 78 | s.homepage = "http://github.com/ivanvanderbyl/cloudist" 79 | s.licenses = ["MIT"] 80 | s.require_paths = ["lib"] 81 | s.rubygems_version = "1.8.15" 82 | s.summary = "Super fast job queue using AMQP" 83 | 84 | if s.respond_to? :specification_version then 85 | s.specification_version = 3 86 | 87 | if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then 88 | s.add_runtime_dependency(%q, ["~> 0.8.1"]) 89 | s.add_runtime_dependency(%q, ["~> 1.4"]) 90 | s.add_runtime_dependency(%q, [">= 0"]) 91 | s.add_runtime_dependency(%q, ["> 3.0.0"]) 92 | s.add_runtime_dependency(%q, [">= 0"]) 93 | s.add_runtime_dependency(%q, [">= 0"]) 94 | s.add_development_dependency(%q, ["~> 2.4.0"]) 95 | s.add_development_dependency(%q, [">= 0"]) 96 | s.add_development_dependency(%q, ["~> 1.6.4"]) 97 | s.add_development_dependency(%q, ["~> 1.2.8"]) 98 | s.add_development_dependency(%q, ["~> 2.1.0"]) 99 | else 100 | s.add_dependency(%q, ["~> 0.8.1"]) 101 | s.add_dependency(%q, ["~> 1.4"]) 102 | s.add_dependency(%q, [">= 0"]) 103 | s.add_dependency(%q, ["> 3.0.0"]) 104 | s.add_dependency(%q, [">= 0"]) 105 | s.add_dependency(%q, [">= 0"]) 106 | s.add_dependency(%q, ["~> 2.4.0"]) 107 | s.add_dependency(%q, [">= 0"]) 108 | s.add_dependency(%q, ["~> 1.6.4"]) 109 | s.add_dependency(%q, ["~> 1.2.8"]) 110 | s.add_dependency(%q, ["~> 2.1.0"]) 111 | end 112 | else 113 | s.add_dependency(%q, ["~> 0.8.1"]) 114 | s.add_dependency(%q, ["~> 1.4"]) 115 | s.add_dependency(%q, [">= 0"]) 116 | s.add_dependency(%q, ["> 3.0.0"]) 117 | s.add_dependency(%q, [">= 0"]) 118 | s.add_dependency(%q, [">= 0"]) 119 | s.add_dependency(%q, ["~> 2.4.0"]) 120 | s.add_dependency(%q, [">= 0"]) 121 | s.add_dependency(%q, ["~> 1.6.4"]) 122 | s.add_dependency(%q, ["~> 1.2.8"]) 123 | s.add_dependency(%q, ["~> 2.1.0"]) 124 | end 125 | end 126 | 127 | -------------------------------------------------------------------------------- /spec/cloudist/payload_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '../../spec_helper') 2 | 3 | describe Cloudist::Payload do 4 | # it "should raise bad payload error unless data is a hash" do 5 | # lambda { 6 | # Cloudist::Payload.new([1,2,3]) 7 | # }.should raise_error(Cloudist::BadPayload) 8 | # end 9 | 10 | it "should accept a hash for data" do 11 | lambda { 12 | Cloudist::Payload.new({:bread => 'white'}) 13 | }.should_not raise_error(Cloudist::BadPayload) 14 | end 15 | 16 | it "should prepare headers" do 17 | payload = Cloudist::Payload.new({:bread => 'white'}) 18 | payload.body[:bread].should == "white" 19 | 20 | payload.headers.has_key?('ttl').should be_true 21 | # payload.headers.has_key?(:content_type).should be_true 22 | # payload.headers[:content_type].should == "application/json" 23 | payload.headers.has_key?('published_on').should be_true 24 | payload.headers.has_key?('message_id').should be_true 25 | end 26 | 27 | it "should extract published_on from data" do 28 | payload = Cloudist::Payload.new({:bread => 'white', :published_on => 12345678}) 29 | payload.headers['published_on'].should == "12345678" 30 | end 31 | 32 | it "should extract custom event hash from data" do 33 | payload = Cloudist::Payload.new({:bread => 'white', :event_hash => 'foo'}) 34 | payload.body.should == {:bread=>"white"} 35 | payload.headers[:event_hash].should == "foo" 36 | end 37 | 38 | it "should parse JSON message" do 39 | payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white', :event_hash => 'foo'})) 40 | payload.body.should == {:bread => "white"} 41 | end 42 | 43 | it "should parse custom headers" do 44 | payload = Cloudist::Payload.new(Marshal.dump({:bread => 'white', :event_hash => 'foo'}), {:published_on => 12345}) 45 | payload.parse_custom_headers.should == {:published_on=>12345, :event_hash=>"foo", :message_id=>"foo", :ttl=>300} 46 | end 47 | 48 | it "should create a unique event hash" do 49 | payload = Cloudist::Payload.new({:bread => 'white'}) 50 | payload.create_event_hash.size.should == 32 51 | end 52 | 53 | it "should not create a new event hash unless it doesn't have one" do 54 | payload = Cloudist::Payload.new({:bread => 'white'}) 55 | payload.event_hash.size.should == 32 56 | payload = Cloudist::Payload.new({:bread => 'white'}, {:event_hash => 'foo'}) 57 | payload.event_hash.should == 'foo' 58 | end 59 | 60 | it "should delegate missing methods to header keys" do 61 | payload = Cloudist::Payload.new({:bread => 'white'}, {:event_hash => 'foo', :ttl => 300}) 62 | payload[:bread].should == 'white' 63 | end 64 | 65 | it "should format payload for sending" do 66 | payload = Cloudist::Payload.new({:bread => 'white'}, {:event_hash => 'foo', :message_type => 'reply'}) 67 | body, popts = payload.to_a 68 | headers = popts[:headers] 69 | 70 | body.should == Marshal.dump({:bread => 'white'}) 71 | headers[:ttl].should == "300" 72 | headers[:message_type].should == 'reply' 73 | end 74 | 75 | it "should generate a unique payload ID" do 76 | payload = Cloudist::Payload.new({:bread => 'white'}) 77 | payload.id.size.should == 32 78 | end 79 | 80 | it "should allow setting of payload ID" do 81 | payload = Cloudist::Payload.new({:bread => 'white'}) 82 | payload.id = "2345" 83 | payload.id.should == "2345" 84 | end 85 | 86 | it "should allow changing of payload after being published" do 87 | payload = Cloudist::Payload.new({:bread => 'white'}) 88 | payload.publish 89 | lambda { payload.id = "12334456" }.should raise_error 90 | end 91 | 92 | it "should freeze" do 93 | payload = Cloudist::Payload.new({:bread => 'white'}) 94 | lambda {payload.body[:bread] = "brown"}.should_not raise_error(TypeError) 95 | payload.body[:bread].should == "brown" 96 | payload.publish 97 | lambda {payload.body[:bread] = "rainbow"}.should raise_error(TypeError) 98 | end 99 | 100 | it "should allow setting of reply header" do 101 | payload = Cloudist::Payload.new({:bread => 'white'}) 102 | 103 | payload.headers[:reply_to].should be_nil 104 | payload.set_reply_to("my_custom_queue") 105 | payload.headers[:reply_to].should_not be_nil 106 | payload.headers[:reply_to].should match /^temp\.reply\.my_custom_queue/ 107 | body, popts = payload.to_a 108 | headers = popts[:headers] 109 | headers[:reply_to].should == payload.headers[:reply_to] 110 | end 111 | 112 | it "should not overwrite passed in headers" do 113 | payload = Cloudist::Payload.new({:bread => 'white'}, {:ttl => 25, :event_hash => 'foo', :published_on => 12345, :message_id => 1}) 114 | payload.headers[:ttl].should == "25" 115 | payload.headers[:event_hash].should == "foo" 116 | payload.headers[:published_on].should == "12345" 117 | payload.headers[:message_id].should == "1" 118 | end 119 | 120 | it "should allow custom headers to be set" do 121 | payload = Cloudist::Payload.new({:bread => 'white'}, {:message_type => 'event'}) 122 | payload.headers[:message_type].should == 'event' 123 | end 124 | 125 | it "should be able to transport an error" do 126 | e = ArgumentError.new("FAILED") 127 | payload = Cloudist::Payload.new(e, {:message_type => 'error'}) 128 | end 129 | 130 | it "should be able to query payload keys with key?" do 131 | payload = Cloudist::Payload.new({:bread => 'white'}, {:ttl => 25, :event_hash => 'foo', :published_on => 12345, :message_id => 1}) 132 | payload.bread?.should be_true 133 | payload.cheese?.should be_false 134 | end 135 | 136 | end -------------------------------------------------------------------------------- /lib/cloudist_old.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'json' unless defined? ActiveSupport::JSON 3 | 4 | require "amqp" 5 | 6 | require "logger" 7 | require "digest/md5" 8 | 9 | $:.unshift File.dirname(__FILE__) 10 | # require "em/iterator" 11 | require "cloudist/core_ext/string" 12 | require "cloudist/core_ext/object" 13 | require "cloudist/core_ext/class" 14 | require "cloudist/errors" 15 | require "cloudist/utils" 16 | require "cloudist/queues/basic_queue" 17 | require "cloudist/queues/sync_queue" 18 | require "cloudist/queues/job_queue" 19 | require "cloudist/queues/sync_job_queue" 20 | require "cloudist/queues/reply_queue" 21 | require "cloudist/queues/sync_reply_queue" 22 | require "cloudist/queues/log_queue" 23 | require "cloudist/publisher" 24 | require "cloudist/payload" 25 | require "cloudist/request" 26 | require "cloudist/callback_methods" 27 | require "cloudist/listener" 28 | require "cloudist/callback" 29 | require "cloudist/callbacks/error_callback" 30 | require "cloudist/job" 31 | require "cloudist/worker" 32 | 33 | module Cloudist 34 | class << self 35 | 36 | @@workers = {} 37 | 38 | # Start the Cloudist loop 39 | # 40 | # Cloudist.start { 41 | # # Do stuff in here 42 | # } 43 | # 44 | # == Options 45 | # * :user => 'name' 46 | # * :pass => 'secret' 47 | # * :host => 'localhost' 48 | # * :port => 5672 49 | # * :vhost => / 50 | # * :heartbeat => 5 51 | # * :logging => false 52 | # 53 | # Refer to default config below for how to set these as defaults 54 | # 55 | def start(options = {}, &block) 56 | config = settings.update(options) 57 | AMQP.start(config) do 58 | AMQP.conn.connection_status do |status| 59 | log.debug("AMQP connection status changed: #{status}") 60 | if status == :disconnected 61 | AMQP.conn.reconnect(true) 62 | end 63 | end 64 | 65 | self.instance_eval(&block) if block_given? 66 | end 67 | end 68 | 69 | # Define a worker. Must be called inside start loop 70 | # 71 | # worker { 72 | # job('make.sandwich') {} 73 | # } 74 | # 75 | # REMOVED 76 | # 77 | def worker(&block) 78 | raise NotImplementedError, "This DSL format has been removed. Please use job('make.sandwich') {} instead." 79 | end 80 | 81 | # Defines a job handler (GenericWorker) 82 | # 83 | # job('make.sandwich') { 84 | # job.started! 85 | # # Work hard 86 | # sleep(5) 87 | # job.finished! 88 | # } 89 | # 90 | # Refer to sandwich_worker.rb example 91 | # 92 | def job(queue_name) 93 | if block_given? 94 | block = Proc.new 95 | register_worker(queue_name, &block) 96 | else 97 | raise ArgumentError, "You must supply a block as the last argument" 98 | end 99 | end 100 | 101 | # Registers a worker class to handle a specific queue 102 | # 103 | # Cloudist.handle('make.sandwich', 'eat.sandwich').with(MyWorker) 104 | # 105 | # A standard worker would look like this: 106 | # 107 | # class MyWorker < Cloudist::Worker 108 | # def process 109 | # log.debug(data.inspect) 110 | # end 111 | # end 112 | # 113 | # A new instance of this worker will be created everytime a job arrives 114 | # 115 | # Refer to examples. 116 | def handle(*queue_names) 117 | class << queue_names 118 | def with(handler) 119 | self.each do |queue_name| 120 | Cloudist.register_worker(queue_name.to_s, handler) 121 | end 122 | end 123 | end 124 | queue_names 125 | end 126 | 127 | def register_worker(queue_name, klass = nil, &block) 128 | job_queue = JobQueue.new(queue_name) 129 | job_queue.subscribe do |request| 130 | j = Job.new(request.payload.dup) 131 | # EM.defer do 132 | begin 133 | if block_given? 134 | worker_instance = GenericWorker.new(j, job_queue.q) 135 | worker_instance.process(&block) 136 | elsif klass 137 | worker_instance = klass.new(j, job_queue.q) 138 | worker_instance.process 139 | else 140 | raise RuntimeError, "Failed to register worker, I need either a handler class or block." 141 | end 142 | rescue Exception => e 143 | j.handle_error(e) 144 | ensure 145 | finished = Time.now.utc.to_f 146 | log.debug("Finished Job in #{finished - request.start} seconds") 147 | j.reply({:runtime => (finished - request.start)}, {:message_type => 'runtime'}) 148 | j.cleanup 149 | end 150 | # end 151 | end 152 | 153 | ((@@workers[queue_name.to_s] ||= []) << job_queue).uniq! 154 | end 155 | 156 | # Accepts either a queue name or a job instance returned from enqueue. 157 | # This method operates in two modes, when given a queue name, it 158 | # will return all responses regardless of job id so you can use the job 159 | # id to lookup a database record to update etc. 160 | # When given a job instance it will only return messages from that job. 161 | # 162 | # DEPRECATED 163 | # 164 | def listen(*queue_names, &block) 165 | raise NotImplementedError, "This DSL method has been removed. Please use add_listener" 166 | 167 | # @@listeners ||= [] 168 | # queue_names.each do |job_or_queue_name| 169 | # _listener = Cloudist::Listener.new(job_or_queue_name) 170 | # _listener.subscribe(&block) 171 | # @@listeners << _listener 172 | # end 173 | # return @@listeners 174 | end 175 | 176 | # Adds a listener class 177 | def add_listener(klass) 178 | @@listeners ||= [] 179 | 180 | raise ArgumentError, "Your listener must extend Cloudist::Listener" unless klass.superclass == Cloudist::Listener 181 | raise ArgumentError, "Your listener must declare at least one queue to listen to. Use listen_to 'queue.name'" if klass.job_queue_names.nil? 182 | 183 | klass.job_queue_names.each do |queue_name| 184 | klass.subscribe(queue_name) 185 | end 186 | 187 | @@listeners << klass 188 | 189 | return @@listeners 190 | end 191 | 192 | # Enqueues a job. 193 | # Takes a queue name and data hash to be sent to the worker. 194 | # Returns Job instance 195 | # Use Job#id to reference job later on. 196 | def enqueue(job_queue_name, data = nil) 197 | raise EnqueueError, "Incorrect arguments, you must include data when enqueuing job" if data.nil? 198 | # TODO: Detect if inside loop, if not use bunny sync 199 | Cloudist::Publisher.enqueue(job_queue_name, data) 200 | end 201 | 202 | # Send a reply synchronously 203 | # This uses bunny instead of AMQP and as such can be run outside 204 | # of EventMachine and the Cloudist start loop. 205 | # 206 | # Usage: Cloudist.reply('make.sandwich', {:sandwhich_id => 12345}) 207 | # 208 | def reply(queue_name, job_id, data, options = {}) 209 | headers = { 210 | :message_id => job_id, 211 | :message_type => "reply", 212 | # :event => 'working', 213 | :message_type => 'reply' 214 | }.update(options) 215 | 216 | payload = Cloudist::Payload.new(data, headers) 217 | 218 | queue = Cloudist::SyncReplyQueue.new(queue_name) 219 | 220 | queue.setup 221 | queue.publish_to_q(payload) 222 | end 223 | 224 | # Call this at anytime inside the loop to exit the app. 225 | def stop_safely 226 | if EM.reactor_running? 227 | ::EM.add_timer(0.2) { 228 | ::AMQP.stop { 229 | ::EM.stop 230 | } 231 | } 232 | end 233 | end 234 | 235 | alias :stop :stop_safely 236 | 237 | def closing? 238 | ::AMQP.closing? 239 | end 240 | 241 | def log 242 | @@log ||= Logger.new($stdout) 243 | end 244 | 245 | def log=(log) 246 | @@log = log 247 | end 248 | 249 | def handle_error(e) 250 | log.error "#{e.class}: #{e.message}"#, :exception => e 251 | log.error e.backtrace.join("\n") 252 | end 253 | 254 | def version 255 | @@version ||= File.read(File.dirname(__FILE__) + '/../VERSION').strip 256 | end 257 | 258 | # EM beta 259 | 260 | def default_settings 261 | uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/') 262 | { 263 | :vhost => uri.path, 264 | :host => uri.host, 265 | :user => uri.user, 266 | :port => uri.port || 5672, 267 | :pass => uri.password, 268 | :heartbeat => 5, 269 | :logging => false 270 | } 271 | rescue Object => e 272 | raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}" 273 | end 274 | 275 | def settings 276 | @@settings ||= default_settings 277 | end 278 | 279 | def settings=(settings_hash) 280 | @@settings = default_settings.update(settings_hash) 281 | end 282 | 283 | def signal_trap! 284 | ::Signal.trap('INT') { Cloudist.stop } 285 | ::Signal.trap('TERM'){ Cloudist.stop } 286 | end 287 | 288 | alias :install_signal_trap :signal_trap! 289 | 290 | def workers 291 | @@workers 292 | end 293 | 294 | def remove_workers 295 | @@workers = {} 296 | end 297 | 298 | end 299 | 300 | end -------------------------------------------------------------------------------- /lib/cloudist.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'json' unless defined? ActiveSupport::JSON 3 | require "amqp" 4 | require "hashie" 5 | require "logger" 6 | require "digest/md5" 7 | require "uuid" 8 | 9 | $:.unshift File.dirname(__FILE__) 10 | 11 | require "em/em_timer_utils" 12 | require "cloudist/core_ext/string" 13 | require "cloudist/core_ext/object" 14 | require "cloudist/core_ext/class" 15 | require "cloudist/errors" 16 | require "cloudist/utils" 17 | require "cloudist/encoding" 18 | require "cloudist/queues/basic_queue" 19 | require "cloudist/queues/job_queue" 20 | require "cloudist/queues/reply_queue" 21 | require "cloudist/publisher" 22 | require "cloudist/payload" 23 | require "cloudist/request" 24 | require "cloudist/listener" 25 | require "cloudist/job" 26 | require "cloudist/worker" 27 | 28 | module Cloudist 29 | DEFAULT_TTL = 300 30 | 31 | class << self 32 | thread_local_accessor :channels, :default => {} 33 | thread_local_accessor :workers, :default => {} 34 | thread_local_accessor :listeners, :default => [] 35 | thread_local_accessor :listener_instances, :default => {} 36 | 37 | thread_local_accessor :worker_prefetch, :default => 1 38 | thread_local_accessor :listener_prefetch, :default => 1 39 | 40 | # Start the Cloudist loop 41 | # 42 | # Cloudist.start { 43 | # # Do stuff in here 44 | # } 45 | # 46 | # == Options 47 | # * :user => 'name' 48 | # * :pass => 'secret' 49 | # * :host => 'localhost' 50 | # * :port => 5672 51 | # * :vhost => / 52 | # * :heartbeat => 0 53 | # * :logging => false 54 | # 55 | # Refer to default config below for how to set these as defaults 56 | # 57 | def start(options_or_connection = {}, &block) 58 | if options_or_connection.is_a?(Hash) 59 | extract_cloudist_options!(options_or_connection) 60 | config = settings.update(options_or_connection) 61 | AMQP.start(config) do 62 | self.instance_eval(&block) if block_given? 63 | end 64 | else 65 | # self.connection = options_or_connection 66 | self.instance_eval(&block) if block_given? 67 | end 68 | end 69 | 70 | def extract_cloudist_options!(options) 71 | self.worker_prefetch = options.delete(:worker_prefetch) || 1 72 | self.listener_prefetch = options.delete(:listener_prefetch) || 1 73 | end 74 | 75 | def connection 76 | AMQP.connection 77 | end 78 | 79 | def connection=(conn) 80 | AMQP.connection = conn 81 | end 82 | 83 | # Define a worker. Must be called inside start loop 84 | # 85 | # worker { 86 | # job('make.sandwich') {} 87 | # } 88 | # 89 | # REMOVED 90 | # 91 | def worker(&block) 92 | raise NotImplementedError, "This DSL format has been removed. Please use job('make.sandwich') {} instead." 93 | end 94 | 95 | # Defines a job handler (GenericWorker) 96 | # 97 | # job('make.sandwich') { 98 | # job.started! 99 | # # Work hard 100 | # sleep(5) 101 | # job.finished! 102 | # } 103 | # 104 | # Refer to sandwich_worker.rb example 105 | # 106 | def job(queue_name) 107 | if block_given? 108 | block = Proc.new 109 | register_worker(queue_name, &block) 110 | else 111 | raise ArgumentError, "You must supply a block as the last argument" 112 | end 113 | end 114 | 115 | # Registers a worker class to handle a specific queue 116 | # 117 | # Cloudist.handle('make.sandwich', 'eat.sandwich').with(MyWorker) 118 | # 119 | # A standard worker would look like this: 120 | # 121 | # class MyWorker < Cloudist::Worker 122 | # def process 123 | # log.debug(data.inspect) 124 | # end 125 | # end 126 | # 127 | # A new instance of this worker will be created everytime a job arrives 128 | # 129 | # Refer to examples. 130 | def handle(*queue_names) 131 | class << queue_names 132 | def with(handler) 133 | self.each do |queue_name| 134 | Cloudist.register_worker(queue_name.to_s, handler) 135 | end 136 | end 137 | end 138 | queue_names 139 | end 140 | 141 | def register_worker(queue_name, klass = nil, &block) 142 | job_queue = JobQueue.new(queue_name) 143 | job_queue.subscribe do |request| 144 | j = Job.new(request.payload.dup) 145 | begin 146 | if block_given? 147 | worker_instance = GenericWorker.new(j, job_queue.q) 148 | worker_instance.process(&block) 149 | elsif klass 150 | worker_instance = klass.new(j, job_queue.q) 151 | worker_instance.process 152 | else 153 | raise RuntimeError, "Failed to register worker, I need either a handler class or block." 154 | end 155 | rescue Exception => e 156 | j.handle_error(e) 157 | ensure 158 | finished = Time.now.utc.to_f 159 | log.debug("Finished Job in #{finished - request.start} seconds") 160 | j.reply({:runtime => (finished - request.start)}, {:message_type => 'runtime'}) 161 | j.cleanup 162 | end 163 | end 164 | 165 | ((self.workers[queue_name.to_s] ||= []) << job_queue).uniq! 166 | end 167 | 168 | # Accepts either a queue name or a job instance returned from enqueue. 169 | # This method operates in two modes, when given a queue name, it 170 | # will return all responses regardless of job id so you can use the job 171 | # id to lookup a database record to update etc. 172 | # When given a job instance it will only return messages from that job. 173 | # 174 | # DEPRECATED 175 | # 176 | def listen(*queue_names, &block) 177 | raise NotImplementedError, "This DSL method has been removed. Please use add_listener" 178 | end 179 | 180 | # Adds a listener class 181 | def add_listener(klass) 182 | raise ArgumentError, "Your listener must extend Cloudist::Listener" unless klass.superclass == Cloudist::Listener 183 | raise ArgumentError, "Your listener must declare at least one queue to listen to. Use listen_to 'queue.name'" if klass.job_queue_names.nil? 184 | 185 | klass.job_queue_names.each do |queue_name| 186 | klass.subscribe(queue_name) 187 | end 188 | 189 | self.listeners << klass 190 | 191 | return self.listeners 192 | end 193 | 194 | # Enqueues a job. 195 | # Takes a queue name and data hash to be sent to the worker. 196 | # Returns Job instance 197 | # Use Job#id to reference job later on. 198 | def enqueue(job_queue_name, data = nil) 199 | raise EnqueueError, "Incorrect arguments, you must include data when enqueuing job" if data.nil? 200 | # TODO: Detect if inside loop, if not use bunny sync 201 | Cloudist::Publisher.enqueue(job_queue_name, data) 202 | end 203 | 204 | # Send a reply synchronously 205 | # This uses bunny instead of AMQP and as such can be run outside 206 | # of EventMachine and the Cloudist start loop. 207 | # 208 | # Usage: Cloudist.reply('make.sandwich', {:sandwhich_id => 12345}) 209 | # 210 | # def reply(queue_name, job_id, data, options = {}) 211 | # headers = { 212 | # :message_id => job_id, 213 | # :message_type => "reply", 214 | # # :event => 'working', 215 | # :message_type => 'reply' 216 | # }.update(options) 217 | # 218 | # payload = Cloudist::Payload.new(data, headers) 219 | # 220 | # queue = Cloudist::SyncReplyQueue.new(queue_name) 221 | # 222 | # queue.setup 223 | # queue.publish_to_q(payload) 224 | # end 225 | 226 | # Call this at anytime inside the loop to exit the app. 227 | def stop_safely 228 | if EM.reactor_running? 229 | ::EM.add_timer(0.2) { 230 | ::AMQP.stop { 231 | ::EM.stop 232 | puts "\n" 233 | } 234 | } 235 | end 236 | end 237 | 238 | alias :stop :stop_safely 239 | 240 | def handle_error(e) 241 | log.error "#{e.class}: #{e.message}"#, :exception => e 242 | e.backtrace.each do |line| 243 | log.error line 244 | end 245 | end 246 | 247 | def version 248 | @@version ||= File.read(File.dirname(__FILE__) + '/../VERSION').strip 249 | end 250 | 251 | def default_settings 252 | uri = URI.parse(ENV["AMQP_URL"] || 'amqp://guest:guest@localhost:5672/') 253 | { 254 | :vhost => uri.path, 255 | :host => uri.host, 256 | :user => uri.user, 257 | :port => uri.port || 5672, 258 | :pass => uri.password, 259 | :heartbeat => 0, 260 | :logging => false 261 | } 262 | rescue Object => e 263 | raise "invalid AMQP_URL: (#{uri.inspect}) #{e.class} -> #{e.message}" 264 | end 265 | 266 | def settings 267 | @@settings ||= default_settings 268 | end 269 | 270 | def settings=(settings_hash) 271 | @@settings = default_settings.update(settings_hash) 272 | end 273 | 274 | def signal_trap! 275 | ::Signal.trap('INT') { Cloudist.stop } 276 | ::Signal.trap('TERM'){ Cloudist.stop } 277 | end 278 | 279 | def log 280 | @@log ||= Logger.new($stdout) 281 | end 282 | 283 | def log=(log) 284 | @@log = log 285 | end 286 | 287 | alias :install_signal_trap :signal_trap! 288 | 289 | def remove_workers 290 | self.workers.keys.each do |worker| 291 | self.workers.delete(worker) 292 | end 293 | end 294 | 295 | end 296 | 297 | include Cloudist::EMTimerUtils 298 | end --------------------------------------------------------------------------------