├── README.md ├── evented_magick ├── README.md ├── examples │ └── processor.rb └── lib │ ├── evented_magick.rb │ └── image_temp_file.rb ├── qanat ├── Capfile ├── Gemfile ├── README.md ├── Rakefile ├── bin │ └── qanat ├── config │ ├── arguments.rb │ ├── boot.rb │ ├── deploy.rb │ ├── deploy │ │ ├── production.rb │ │ └── staging.rb │ ├── environment.rb │ ├── environments │ │ ├── development.rb │ │ ├── production.rb │ │ ├── staging.rb │ │ └── test.rb │ ├── initializers │ │ └── qanat.rb │ ├── post-daemonize │ │ └── readme │ ├── pre-daemonize │ │ ├── eager_load.rb │ │ └── readme │ └── sqs.yml ├── lib │ ├── authentication.rb │ ├── dispatch.rb │ ├── qanat.rb │ ├── s3.rb │ ├── sdb.rb │ └── sqs.rb ├── libexec │ └── qanat-daemon.rb ├── script │ ├── console │ ├── destroy │ └── generate ├── spec │ ├── qanat_spec.rb │ ├── spec.opts │ └── spec_helper.rb └── tasks │ ├── rspec.rake │ └── testing.rake └── thin └── thumbnailer.rb /README.md: -------------------------------------------------------------------------------- 1 | Evented 2 | ----------- 3 | 4 | A repository for EventMachine (or anything based on EventMachine) examples 5 | and so much more! 6 | 7 | Because the event-driven programming model is so different to 'normal' programming, 8 | good examples are critical to learning how to solve problems the 'event' way. 9 | 10 | Got a non-trivial example? Please submit it to me for inclusion. It should 11 | integrate more than one event-driven subsystem; no more 10 line echo server 12 | examples! They are junk and don't teach much beyond a 30 second overview. 13 | 14 | I'd like to see examples of calling: 15 | 16 | - 3rd party web services 17 | - database (mysql or postgresql) 18 | - memcached 19 | - message queue processing 20 | 21 | Examples 22 | ========== 23 | 24 | **thin/thumbnailer.rb** 25 | 26 | A Thin-based thumbnail service which transparently pulls original images off S3 and thumbnails them according to URL parameters. 27 | 28 | **qanat** 29 | 30 | A SQS-based message queue processor. Qanat will process up to 10 messages concurrently. 31 | 32 | Sidenote: Qanat is one of the few official Scrabble words which does 33 | not contain a U. It is the Arabic word for an underground irrigation canal. 34 | 35 | **evented_magick** 36 | 37 | An eventmachine-aware version of `MiniMagick` which uses the `EM.system` call to increase performance. 38 | 39 | Your Host 40 | ============= 41 | 42 | Mike Perham, mperham AT gmail.com 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /evented_magick/README.md: -------------------------------------------------------------------------------- 1 | EventedMagick 2 | ---------------- 3 | 4 | A EventMachine-aware wrapper for the ImageMagick command line. Uses EM.system to execute if available. 5 | Requires Ruby 1.9 since it uses Fibers. The internals have also been rewritten to reduce the number 6 | of system() calls. These changes together reduced the time required to run my test from 20sec to 5sec versus the stock MiniMagick library. 7 | 8 | Thanks 9 | ========== 10 | 11 | Based on mini_magick by 12 | 13 | Author 14 | ========== 15 | 16 | Mike Perham, mperham AT gmail.com, 17 | [Github](http://github.com/mperham), 18 | [Twitter](http://twitter.com/mperham), 19 | [Blog](http://mikeperham.com) 20 | 21 | -------------------------------------------------------------------------------- /evented_magick/examples/processor.rb: -------------------------------------------------------------------------------- 1 | # Change this to a directory full of random images 2 | GLOB='/Users/mike/junk/test_images/*.jpg' 3 | 4 | require 'evented_magick' 5 | 6 | # Execute using Ruby's normal system() call. 7 | a = Time.now 8 | files = Dir[GLOB] 9 | files.each do |filename| 10 | image = EventedMagick::Image.new(filename) 11 | image['dimensions'] 12 | end 13 | 14 | puts "Processed #{files.size} in #{Time.now - a} sec" 15 | 16 | # Use Fibers in Ruby 1.9 and EventMachine to run the system 17 | # calls in parallel. We only run up to 5 system() calls in parallel 18 | # to prevent a fork bomb. 19 | if defined? Fiber 20 | require 'fiber' 21 | require 'eventmachine' 22 | 23 | NUM = 5 24 | items = EM::Queue.new 25 | total = 0 26 | process = proc do |filename| 27 | begin 28 | Fiber.new do 29 | image = EventedMagick::Image.new(filename) 30 | image['dimensions'] 31 | total = total + 1 32 | items.pop(process) 33 | end.resume 34 | rescue Exception => ex 35 | puts ex.message 36 | puts ex.backtrace.join("\n") 37 | end 38 | end 39 | 40 | EM.run do 41 | a = Time.now 42 | files = Dir[GLOB] 43 | files.each do |filename| 44 | items.push filename 45 | end 46 | 47 | NUM.times{ items.pop(process) } 48 | EM.add_periodic_timer(1) do 49 | if items.empty? 50 | puts "Processed #{total} in #{Time.now - a} sec" 51 | EM.stop 52 | end 53 | end 54 | end 55 | 56 | end -------------------------------------------------------------------------------- /evented_magick/lib/evented_magick.rb: -------------------------------------------------------------------------------- 1 | require "open-uri" 2 | require "stringio" 3 | require "fileutils" 4 | require "open3" 5 | 6 | require File.join(File.dirname(__FILE__), '/image_temp_file') 7 | 8 | module EventedMagick 9 | class MiniMagickError < RuntimeError; end 10 | 11 | class Image 12 | attr :path 13 | attr :tempfile 14 | attr :output 15 | 16 | # Class Methods 17 | # ------------- 18 | class << self 19 | def from_blob(blob, ext = nil) 20 | begin 21 | tempfile = ImageTempFile.new(ext) 22 | tempfile.binmode 23 | tempfile.write(blob) 24 | ensure 25 | tempfile.close if tempfile 26 | end 27 | 28 | return self.new(tempfile.path, tempfile) 29 | end 30 | 31 | # Use this if you don't want to overwrite the image file 32 | def open(image_path) 33 | File.open(image_path, "rb") do |f| 34 | self.from_blob(f.read, File.extname(image_path)) 35 | end 36 | end 37 | alias_method :from_file, :open 38 | end 39 | 40 | # Instance Methods 41 | # ---------------- 42 | def initialize(input_path, tempfile=nil) 43 | @path = input_path 44 | @tempfile = tempfile # ensures that the tempfile will stick around until this image is garbage collected. 45 | @method = defined?(::EM) && EM.reactor_running? ? :evented_execute : :blocking_execute 46 | 47 | # Ensure that the file is an image 48 | output = run_command("identify", "-format", format_option("%m %w %h"), @path) 49 | (format, width, height) = output.split 50 | @values = { 'format' => format, 'width' => width.to_i, 'height' => height.to_i, 'dimensions' => [width.to_i, height.to_i] } 51 | end 52 | 53 | # For reference see http://www.imagemagick.org/script/command-line-options.php#format 54 | def [](value) 55 | key = value.to_s 56 | return @values[key] if %w(format width height dimensions).include? key 57 | if key == "size" 58 | File.size(@path) 59 | else 60 | run_command('identify', '-format', "\"#{key}\"", @path).split("\n")[0] 61 | end 62 | end 63 | 64 | # Sends raw commands to imagemagick's mogrify command. The image path is automatically appended to the command 65 | def <<(*args) 66 | run_command("mogrify", *args << @path) 67 | end 68 | 69 | # This is a 'special' command because it needs to change @path to reflect the new extension 70 | # Formatting an animation into a non-animated type will result in ImageMagick creating multiple 71 | # pages (starting with 0). You can choose which page you want to manipulate. We default to the 72 | # first page. 73 | def format(format, page=0) 74 | run_command("mogrify", "-format", format, @path) 75 | 76 | old_path = @path.dup 77 | @path.sub!(/(\.\w+)?$/, ".#{format}") 78 | File.delete(old_path) unless old_path == @path 79 | 80 | unless File.exists?(@path) 81 | begin 82 | FileUtils.copy_file(@path.sub(".#{format}", "-#{page}.#{format}"), @path) 83 | rescue e 84 | raise MiniMagickError, "Unable to format to #{format}; #{e}" unless File.exist?(@path) 85 | end 86 | end 87 | ensure 88 | Dir[@path.sub(/(\.\w+)?$/, "-[0-9]*.#{format}")].each do |fname| 89 | File.unlink(fname) 90 | end 91 | end 92 | 93 | # Writes the temporary image that we are using for processing to the output path 94 | def write(output_path) 95 | FileUtils.copy_file @path, output_path 96 | run_command "identify", output_path # Verify that we have a good image 97 | end 98 | 99 | # Give you raw data back 100 | def to_blob 101 | f = File.new @path 102 | f.binmode 103 | f.read 104 | ensure 105 | f.close if f 106 | end 107 | 108 | # If an unknown method is called then it is sent through the morgrify program 109 | # Look here to find all the commands (http://www.imagemagick.org/script/mogrify.php) 110 | def method_missing(symbol, *args) 111 | args.push(@path) # push the path onto the end 112 | run_command("mogrify", "-#{symbol}", *args) 113 | self 114 | end 115 | 116 | # You can use multiple commands together using this method 117 | def combine_options(&block) 118 | c = CommandBuilder.new 119 | block.call c 120 | run_command("mogrify", *c.args << @path) 121 | end 122 | 123 | # Check to see if we are running on win32 -- we need to escape things differently 124 | def windows? 125 | !(RUBY_PLATFORM =~ /win32/).nil? 126 | end 127 | 128 | # Outputs a carriage-return delimited format string for Unix and Windows 129 | def format_option(format) 130 | windows? ? "#{format}\\n" : "#{format}\\\\n" 131 | end 132 | 133 | def run_command(command, *args) 134 | full_args = args.collect do |arg| 135 | # args can contain characters like '>' so we must escape them, but don't quote switches 136 | if arg !~ /^[\+\-]/ 137 | "\"#{arg}\"" 138 | else 139 | arg.to_s 140 | end 141 | end.join(' ') 142 | 143 | full_cmd = "#{command} #{full_args}" 144 | (output, status) = send(@method, full_cmd) 145 | 146 | if status.exitstatus == 0 147 | output 148 | else 149 | raise MiniMagickError, "ImageMagick command (#{full_cmd.inspect}) failed: #{{:status_code => status, :output => output}.inspect}" 150 | end 151 | end 152 | 153 | def evented_execute(cmd) 154 | fiber = Fiber.current 155 | EM::system(cmd) do |output, status| 156 | fiber.resume([output, status]) 157 | end 158 | 159 | Fiber.yield 160 | end 161 | 162 | def blocking_execute(cmd) 163 | output = `#{cmd}` 164 | [output, $?] 165 | end 166 | end 167 | 168 | class CommandBuilder 169 | attr :args 170 | 171 | def initialize 172 | @args = [] 173 | end 174 | 175 | def method_missing(symbol, *args) 176 | @args << "-#{symbol}" 177 | @args += args 178 | end 179 | 180 | def +(value) 181 | @args << "+#{value}" 182 | end 183 | end 184 | end 185 | -------------------------------------------------------------------------------- /evented_magick/lib/image_temp_file.rb: -------------------------------------------------------------------------------- 1 | require "tempfile" 2 | 3 | module EventedMagick 4 | class ImageTempFile < Tempfile 5 | def make_tmpname(ext, n) 6 | 'mini_magick%d-%d%s' % [$$, n, ext ? ".#{ext}" : ''] 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /qanat/Capfile: -------------------------------------------------------------------------------- 1 | unless respond_to?(:namespace) # cap2 differentiator 2 | $stderr.puts "Requires capistrano version 2" 3 | exit 1 4 | end 5 | 6 | require 'config/boot' 7 | load DaemonKit.framework_root + '/lib/daemon_kit/deployment/capistrano.rb' 8 | 9 | Dir['config/deploy/recipes/*.rb'].each { |plugin| load(plugin) } 10 | load 'config/deploy.rb' 11 | -------------------------------------------------------------------------------- /qanat/Gemfile: -------------------------------------------------------------------------------- 1 | #disable_system_gems 2 | 3 | gem 'nokogiri', '>= 1.4.1' 4 | gem 'daemon-kit', '>= 0.1.7.12' 5 | gem 'em-http-request', '>= 0.2.6' 6 | gem 'mime-types', '>= 1.16' 7 | gem 'activerecord', '= 2.3.5' 8 | gem 'postgres-pr', '>= 0.6.0' 9 | gem 'eventmachine', '>= 0.10.12' 10 | gem 'em_postgresql', '>= 0.1.1' 11 | 12 | group :test do 13 | gem "rspec" 14 | gem "mocha" 15 | gem 'right_aws', :git => "git://github.com/mperham/right_aws.git" 16 | end -------------------------------------------------------------------------------- /qanat/README.md: -------------------------------------------------------------------------------- 1 | QANAT 2 | ====== 3 | 4 | "Scalable AWS processing for Ruby" 5 | 6 | Qanat is one of the few words recognized by Scrabble that begin with Q and don't require a U following. 7 | 8 | Context 9 | --------- 10 | 11 | The Ruby 1.8 and 1.9 VM implementations do not scale well. Ruby 1.8 threads can't execute more than one at a time and on more than one core. Many Ruby extensions are not thread-safe and Ruby itself places a GIL around many operations such that multiple Ruby threads can't execute concurrently. JRuby is the only exception to this limitation currently but threaded code itself has issues - thread-safe code is notoriously difficult to write, debug and test. 12 | 13 | At my current employer, we use S3, SQS and SimpleDB to store data. Those services scale very well to huge volumes of data but don't have incredible response times so when you write code which grabs a message from a queue, performs a SimpleDB lookup, makes a change and stores some other data to S3, that entire process might take 2 seconds, where 0.1 sec is actually spent performing calculations with the CPU and the other 1.9 seconds is spent blocked, doing nothing and waiting for the various Amazon web services to respond. 14 | 15 | Qanat is an SQS queue processor which uses an event-driven architecture to work around these issues. It works well for processing messages which spend a lot of time performing I/O, e.g. messages which require calling 3rd party web services, scraping other web sites, making long database queries, etc. 16 | 17 | 18 | Design 19 | ------- 20 | 21 | Qanat will process up to N messages concurrently, using EventMachine to manage the overall processing. Ruby 1.9 is required. 22 | 23 | Qanat provides basic implementations of SQS, SimpleDB and S3 event-based clients. These clients can be used in your own message processing code. 24 | 25 | Install 26 | --------- 27 | 28 | gem install qanat 29 | 30 | You will need to put a file with your Amazon credentials in either `~/.qanat.amzn.yml` or `QANAT_ROOT/config/amzn.yml`. The contents should look like this: 31 | 32 | defaults: &defaults 33 | access_key: 34 | secret_key: 35 | timeout: 5 36 | 37 | development: 38 | <<: *defaults 39 | 40 | test: 41 | <<: *defaults 42 | 43 | production: 44 | <<: *defaults 45 | 46 | 47 | 48 | Author 49 | -------- 50 | 51 | Mike Perham, @mperham, http://mikeperham.com -------------------------------------------------------------------------------- /qanat/Rakefile: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/config/boot' 2 | 3 | require 'rake' 4 | require 'daemon_kit/tasks' 5 | 6 | Dir[File.join(File.dirname(__FILE__), 'tasks/*.rake')].each { |rake| load rake } 7 | -------------------------------------------------------------------------------- /qanat/bin/qanat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Stub executable for qanat 4 | 5 | if RUBY_VERSION > '1.9.0' 6 | 7 | require File.dirname(__FILE__) + '/../config/environment' 8 | 9 | DaemonKit::Application.exec( DAEMON_ROOT + '/libexec/qanat-daemon.rb' ) 10 | 11 | else 12 | puts 'Qanat requires Ruby 1.9.x.' 13 | end -------------------------------------------------------------------------------- /qanat/config/arguments.rb: -------------------------------------------------------------------------------- 1 | # Argument handling for your daemon is configured here. 2 | # 3 | # You have access to two variables when this file is 4 | # parsed. The first is +opts+, which is the object yielded from 5 | # +OptionParser.new+, the second is +@options+ which is a standard 6 | # Ruby hash that is later accessible through 7 | # DaemonKit.arguments.options and can be used in your daemon process. 8 | 9 | opts.on('-q', '--queue NAME', 'Name of SQS to process') do |n| 10 | @options[:queue_name] = n 11 | end 12 | -------------------------------------------------------------------------------- /qanat/config/boot.rb: -------------------------------------------------------------------------------- 1 | # Don't change this file! 2 | # Configure your daemon in config/environment.rb 3 | 4 | DAEMON_ROOT = "#{File.expand_path(File.dirname(__FILE__))}/.." unless defined?( DAEMON_ROOT ) 5 | 6 | begin 7 | require File.expand_path('../../.bundle/environment', __FILE__) 8 | rescue LoadError 9 | puts "Please install bundler:" 10 | puts " gem install bundler -v '~>0.9.3'" 11 | puts "and then initialize Qanat's gem environment:" 12 | puts " bundle install && bundle lock" 13 | exit 14 | end 15 | 16 | module DaemonKit 17 | class << self 18 | def boot! 19 | unless booted? 20 | pick_boot.run 21 | end 22 | end 23 | 24 | def booted? 25 | defined? DaemonKit::Initializer 26 | end 27 | 28 | def pick_boot 29 | (vendor_kit? ? VendorBoot : GemBoot).new 30 | end 31 | 32 | def vendor_kit? 33 | File.exists?( "#{DAEMON_ROOT}/vendor/daemon_kit" ) 34 | end 35 | end 36 | 37 | class Boot 38 | def run 39 | load_initializer 40 | DaemonKit::Initializer.run 41 | end 42 | end 43 | 44 | class VendorBoot < Boot 45 | def load_initializer 46 | require "#{DAEMON_ROOT}/vendor/daemon_kit/lib/daemon_kit/initializer" 47 | end 48 | end 49 | 50 | class GemBoot < Boot 51 | def load_initializer 52 | begin 53 | require 'rubygems' 54 | gem 'kennethkalmer-daemon-kit' 55 | require 'daemon_kit/initializer' 56 | rescue Gem::LoadError 57 | begin 58 | gem 'daemon-kit' 59 | require 'daemon_kit/initializer' 60 | rescue Gem::LoadError => e 61 | msg = <=#{DaemonKit::VERSION}" 39 | end 40 | 41 | # Hook into capistrano's events 42 | before "deploy:update_code", "deploy:check" 43 | 44 | # Create some tasks related to deployment 45 | namespace :deploy do 46 | 47 | desc "Get the current revision of the deployed code" 48 | task :get_current_version do 49 | run "cat #{current_path}/REVISION" do |ch, stream, out| 50 | puts "Current revision: " + out.chomp 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /qanat/config/deploy/production.rb: -------------------------------------------------------------------------------- 1 | #set :deploy_to, "/svc/qanat" # defaults to "/u/apps/#{application}" 2 | #set :user, "" # defaults to the currently logged in user 3 | set :daemon_env, 'production' 4 | 5 | set :domain, 'example.com' 6 | server domain 7 | -------------------------------------------------------------------------------- /qanat/config/deploy/staging.rb: -------------------------------------------------------------------------------- 1 | #set :deploy_to, "/svc/qanat" # defaults to "/u/apps/#{application}" 2 | #set :user, "" # defaults to the currently logged in user 3 | set :daemon_env, 'staging' 4 | 5 | set :domain, 'example.com' 6 | server domain 7 | -------------------------------------------------------------------------------- /qanat/config/environment.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your daemon when you modify this file 2 | 3 | # Uncomment below to force your daemon into production mode 4 | #ENV['DAEMON_ENV'] ||= 'production' 5 | 6 | # Boot up 7 | require File.join(File.dirname(__FILE__), 'boot') 8 | 9 | DaemonKit::Initializer.run do |config| 10 | # The name of the daemon as reported by process monitoring tools 11 | config.daemon_name = 'qanat' 12 | 13 | # Force the daemon to be killed after X seconds from asking it to 14 | config.force_kill_wait = 30 15 | 16 | # Log backraces when a thread/daemon dies (Recommended) 17 | config.backtraces = true 18 | 19 | # Configure the safety net (see DaemonKit::Safety) 20 | # config.safety_net.handler = :mail # (or :hoptoad ) 21 | # config.safety_net.mail.host = 'localhost' 22 | end -------------------------------------------------------------------------------- /qanat/config/environments/development.rb: -------------------------------------------------------------------------------- 1 | # This is the same context as the environment.rb file, it is only 2 | # loaded afterwards and only in the development environment 3 | -------------------------------------------------------------------------------- /qanat/config/environments/production.rb: -------------------------------------------------------------------------------- 1 | # This is the same context as the environment.rb file, it is only 2 | # loaded afterwards and only in the production environment 3 | -------------------------------------------------------------------------------- /qanat/config/environments/staging.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mperham/evented/2940dbcf5aa1c0a43a5febd11e4987e5cd12406b/qanat/config/environments/staging.rb -------------------------------------------------------------------------------- /qanat/config/environments/test.rb: -------------------------------------------------------------------------------- 1 | # This is the same context as the environment.rb file, it is only 2 | # loaded afterwards and only in the test environment 3 | -------------------------------------------------------------------------------- /qanat/config/initializers/qanat.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mperham/evented/2940dbcf5aa1c0a43a5febd11e4987e5cd12406b/qanat/config/initializers/qanat.rb -------------------------------------------------------------------------------- /qanat/config/post-daemonize/readme: -------------------------------------------------------------------------------- 1 | # You can place files in here to be loaded after the code is daemonized. 2 | # 3 | # All the files placed here will just be required into the running 4 | # process. This is the correct place to open any IO objects, establish 5 | # database connections, etc. 6 | -------------------------------------------------------------------------------- /qanat/config/pre-daemonize/eager_load.rb: -------------------------------------------------------------------------------- 1 | require "cgi" 2 | require "base64" 3 | require "openssl" 4 | require "digest/sha1" 5 | require 'digest/md5' 6 | require 'fiber' 7 | require 'yaml' 8 | require 'time' 9 | 10 | require 'nokogiri' 11 | require 'em-http' 12 | require 'authentication' 13 | 14 | require 'sqs' 15 | require 'sdb' 16 | require 's3' 17 | 18 | class Fiber 19 | def self.sleep(sec) 20 | f = Fiber.current 21 | EM.add_timer(sec) do 22 | f.resume 23 | end 24 | Fiber.yield 25 | end 26 | end -------------------------------------------------------------------------------- /qanat/config/pre-daemonize/readme: -------------------------------------------------------------------------------- 1 | # You can place files in here to be loaded before the code is daemonized. 2 | # 3 | # DaemonKit looks for a file named '.rb' and loads 4 | # that file first, and inside a DaemonKit::Initializer block. The 5 | # remaning files then simply required into the running process. 6 | # 7 | # These files are mostly useful for operations that should fail blatantly 8 | # before daemonizing, like loading gems. 9 | # 10 | # Be careful not to open any form of IO in here and expecting it to be 11 | # open inside the running daemon since all IO instances are closed when 12 | # daemonizing (including STDIN, STDOUT & STDERR). 13 | -------------------------------------------------------------------------------- /qanat/config/sqs.yml: -------------------------------------------------------------------------------- 1 | # Copy this file to ~/.qanat.sqs.yml and add your AWS key data. 2 | defaults: &defaults 3 | access_key: invalid_access_key 4 | secret_key: invalid_secret_key 5 | timeout: 5 6 | 7 | development: 8 | <<: *defaults 9 | 10 | test: 11 | <<: *defaults 12 | 13 | production: 14 | <<: *defaults 15 | -------------------------------------------------------------------------------- /qanat/lib/authentication.rb: -------------------------------------------------------------------------------- 1 | module Amazon 2 | module Authentication 3 | SIGNATURE_VERSION = "2" 4 | @@digest = OpenSSL::Digest::Digest.new("sha256") 5 | 6 | def sign(auth_string) 7 | Base64.encode64(OpenSSL::HMAC.digest(digester, aws_secret_access_key, auth_string)).strip 8 | end 9 | 10 | def digester 11 | @@digest 12 | end 13 | 14 | def aws_access_key_id 15 | @config['access_key'] 16 | end 17 | 18 | def aws_secret_access_key 19 | @config['secret_key'] 20 | end 21 | 22 | def amz_escape(param) 23 | param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do 24 | '%' + $1.unpack('H2' * $1.size).join('%').upcase 25 | end 26 | end 27 | 28 | def signed_parameters(hash, verb, host, path) 29 | data = hash.keys.sort.map do |key| 30 | "#{amz_escape(key)}=#{amz_escape(hash[key])}" 31 | end.join('&') 32 | sig = amz_escape(sign("#{verb}\n#{host}\n#{path}\n#{data}")) 33 | "#{data}&Signature=#{sig}" 34 | end 35 | 36 | def generate_request_hash(action, params={}) 37 | request_hash = { 38 | "Action" => action, 39 | "SignatureMethod" => 'HmacSHA256', 40 | "AWSAccessKeyId" => aws_access_key_id, 41 | "SignatureVersion" => SIGNATURE_VERSION, 42 | } 43 | # request_hash["MessageBody"] = message if message 44 | request_hash.merge(default_parameters).merge(params) 45 | end 46 | end 47 | end -------------------------------------------------------------------------------- /qanat/lib/dispatch.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'erb' 3 | require 'active_record' 4 | 5 | RAILS_ENV=DaemonKit.configuration.environment 6 | 7 | ActiveRecord::Base.configurations = YAML::load(ERB.new(File.read(File.join(DAEMON_ROOT, 'config', 'database.yml'))).result) 8 | ActiveRecord::Base.default_timezone = :utc 9 | ActiveRecord::Base.logger = DaemonKit.logger 10 | ActiveRecord::Base.logger.level = Logger::INFO 11 | ActiveRecord::Base.time_zone_aware_attributes = true 12 | Time.zone = 'UTC' 13 | ActiveRecord::Base.establish_connection 14 | 15 | # Your custom message dispatch logic goes below. This is 16 | # a sample of how to do it but you can modify as necessary. 17 | # 18 | # Our message processing classes go in lib/processors. 19 | # 20 | # Example message: 21 | # { :msg_type => 'index_page', :page_id => 15412323 } 22 | # 23 | # Qanat will execute: 24 | # processor = IndexPage.new 25 | # processor.process(hash) 26 | # to handle the message. 27 | 28 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), 'processors'))) 29 | # require 'your_message_processor1' 30 | # require 'your_message_processor2' 31 | 32 | module Qanat 33 | 34 | def self.dispatch(msg) 35 | hash = YAML::load(msg) 36 | name = hash.fetch(:msg_type).to_s.camelize 37 | profile(hash) do 38 | name.constantize.new.process(hash) 39 | end 40 | end 41 | 42 | def self.profile(hash) 43 | a = Time.now 44 | return yield 45 | ensure 46 | DaemonKit.logger.info("Processed message: #{hash.inspect} in #{Time.now - a} sec") 47 | end 48 | 49 | end 50 | 51 | -------------------------------------------------------------------------------- /qanat/lib/qanat.rb: -------------------------------------------------------------------------------- 1 | module Qanat 2 | 3 | def self.run(&block) 4 | # Ensure graceful shutdown of the connection to the broker 5 | DaemonKit.trap('INT') { ::EM.stop } 6 | DaemonKit.trap('TERM') { ::EM.stop } 7 | 8 | # Start our event loop 9 | DaemonKit.logger.debug("EM.run") 10 | EM.run(&block) 11 | end 12 | 13 | def self.load(config) 14 | config = config.to_s 15 | config += '.yml' unless config =~ /\.yml$/ 16 | 17 | hash = {} 18 | path = File.join( DAEMON_ROOT, 'config', config ) 19 | hash.merge!(YAML.load_file( path )) if File.exists?(path) 20 | 21 | path = File.join( ENV['HOME'], ".qanat.#{config}" ) 22 | hash.merge!(YAML.load_file( path )) if File.exists?(path) 23 | 24 | raise ArgumentError, "Can't find #{path}" if hash.size == 0 25 | 26 | hash[DAEMON_ENV] 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /qanat/lib/s3.rb: -------------------------------------------------------------------------------- 1 | require 'mime/types' 2 | 3 | module S3 4 | USE_100_CONTINUE_PUT_SIZE = 1_000_000 5 | DEFAULT_HOST = 's3.amazonaws.com' 6 | AMAZON_HEADER_PREFIX = 'x-amz-' 7 | AMAZON_METADATA_PREFIX = 'x-amz-meta-' 8 | 9 | class Bucket 10 | 11 | include Amazon::Authentication 12 | 13 | def initialize(bucket) 14 | @config = Qanat.load('amzn') 15 | @bucket = bucket 16 | end 17 | 18 | def put(key, data=nil, headers={}) 19 | result = async_operation(:put, 20 | headers.merge(:key => CGI::escape(key), 21 | "content-md5" => Base64.encode64(Digest::MD5.digest(data)).strip), 22 | data) 23 | code = result.response_header.status 24 | if code != 200 25 | raise RuntimeError, "S3 put failed: #{code} #{result.response}" 26 | end 27 | code == 200 28 | end 29 | 30 | def get(key, headers={}, &block) 31 | result = async_operation(:get, headers.merge(:key => CGI::escape(key))) 32 | code = result.response_header.status 33 | if code == 404 34 | return nil 35 | end 36 | if code != 200 37 | raise RuntimeError, "S3 get failed: #{result.response}" 38 | end 39 | result.response 40 | end 41 | 42 | def head(key, headers={}) 43 | result = async_operation(:head, headers.merge(:key => CGI::escape(key))) 44 | code = result.response_header.status 45 | if code == 404 46 | return nil 47 | end 48 | result.response_header 49 | end 50 | 51 | def delete(key, headers={}) 52 | result = async_operation(:delete, headers.merge(:key => CGI::escape(key))) 53 | code = result.response_header.status 54 | code == 200 55 | end 56 | 57 | def put_file(file_path, data) 58 | digest = Digest::MD5.hexdigest(data) 59 | headers = head(file_path) 60 | if headers and headers[MD5SUM] == digest 61 | logger.info "[S3] Skipping upload of #{file_path}, unchanged..." 62 | # skip push to S3 63 | else 64 | logger.info "[S3] Pushing #{file_path}" 65 | options = { MD5SUM => digest } 66 | type = guess_mimetype(file_path) 67 | options["Content-Type"] = type if !type.blank? 68 | options["Content-Encoding"] = 'gzip' if file_path =~ /\.gz$/ 69 | put(file_path, data, { 'x-amz-acl' => 'public-read' }.merge(options)) 70 | end 71 | file_path 72 | end 73 | 74 | private 75 | 76 | def logger 77 | DaemonKit.logger 78 | end 79 | 80 | MD5SUM = 'x-amz-meta-md5sum' 81 | 82 | def guess_mimetype(filepath) 83 | MIME::Types.type_for(filepath)[0].to_s 84 | end 85 | 86 | def async_operation(method, headers={}, body=nil) 87 | f = Fiber.current 88 | path = generate_rest_request(method.to_s.upcase, headers) 89 | args = { :head => headers } 90 | args[:body] = body if body 91 | http = EventMachine::HttpRequest.new("http://#{DEFAULT_HOST}#{path}").send(method, args) 92 | http.callback { f.resume(http) } 93 | http.errback { f.resume(http) } 94 | 95 | return Fiber.yield 96 | end 97 | 98 | def canonical_string(method, path, headers={}, expires=nil) # :nodoc: 99 | s3_headers = {} 100 | headers.each do |key, value| 101 | key = key.downcase 102 | s3_headers[key] = value.to_s.strip if key[/^#{AMAZON_HEADER_PREFIX}|^content-md5$|^content-type$|^date$/o] 103 | end 104 | s3_headers['content-type'] ||= '' 105 | s3_headers['content-md5'] ||= '' 106 | s3_headers['date'] = '' if s3_headers.has_key? 'x-amz-date' 107 | s3_headers['date'] = expires if expires 108 | # prepare output string 109 | out_string = "#{method}\n" 110 | s3_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value| 111 | out_string << (key[/^#{AMAZON_HEADER_PREFIX}/o] ? "#{key}:#{value}\n" : "#{value}\n") 112 | end 113 | # ignore everything after the question mark... 114 | out_string << path.gsub(/\?.*$/, '') 115 | out_string 116 | end 117 | 118 | def generate_rest_request(method, headers) # :nodoc: 119 | # calculate request data 120 | path = "/#{@bucket}/#{headers[:key]}" 121 | headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) } 122 | headers['content-type'] ||= '' 123 | headers['date'] = Time.now.httpdate 124 | auth_string = canonical_string(method, path, headers) 125 | signature = sign(auth_string) 126 | headers['Authorization'] = "AWS #{aws_access_key_id}:#{signature}" 127 | path 128 | end 129 | 130 | def digester 131 | @@digest1 132 | end 133 | 134 | @@digest1 = OpenSSL::Digest::Digest.new("sha1") 135 | end 136 | end 137 | -------------------------------------------------------------------------------- /qanat/lib/sdb.rb: -------------------------------------------------------------------------------- 1 | module SDB 2 | DEFAULT_HOST = 'sdb.amazonaws.com' 3 | API_VERSION = '2009-04-15' 4 | 5 | class Database 6 | include Amazon::Authentication 7 | 8 | def initialize(domain) 9 | @config = Qanat.load('amzn') 10 | @domain = domain 11 | end 12 | 13 | =begin 14 | 15 | 16 | 17 | 18 | alt 19 | alife-40-below-4.jpg 20 | 21 | 22 | fetch_url 23 | http://thekaoseffect.com/blog/wp-content/uploads/2009/11/alife-40-below-4.jpg 24 | 25 | 26 | created_at 27 | 20091105173159 28 | 29 | 30 | 31 | 4842bf3b-cbdd-3f35-406d-1becc842b18c 32 | 0.0000093382 33 | 34 | 35 | =end 36 | def get(id_or_array) 37 | request_hash = generate_request_hash("GetAttributes", 'ItemName' => id_or_array) 38 | http = async_operation(:get, request_hash, :timeout => timeout) 39 | code = http.response_header.status 40 | if code != 200 41 | logger.error "SDB got an error response: #{code} #{http.response}" 42 | return nil 43 | end 44 | to_attributes(http.response) 45 | end 46 | 47 | def put(id, attribs) 48 | hash = { 'ItemName' => id } 49 | idx = 0 50 | attribs.each_pair do |k, v| 51 | hash["Attribute.#{idx}.Name"] = CGI::escape(k.to_s) 52 | hash["Attribute.#{idx}.Value"] = CGI::escape(v.to_s) 53 | idx = idx + 1 54 | end 55 | request_hash = generate_request_hash("PutAttributes", hash) 56 | http = async_operation(:post, request_hash, :timeout => timeout) 57 | end 58 | 59 | private 60 | 61 | def to_attributes(doc) 62 | attributes = {} 63 | xml = Nokogiri::XML(doc) 64 | xml.xpath('//xmlns:Attribute').each do |node| 65 | k = node.at_xpath('.//xmlns:Name').content 66 | v = node.at_xpath('.//xmlns:Value').content 67 | if attributes.has_key?(k) 68 | if !attributes[k].is_a?(Array) 69 | attributes[k] = Array(attributes[k]) 70 | end 71 | attributes[k] << v 72 | else 73 | attributes[k] = v 74 | end 75 | end 76 | attributes 77 | end 78 | 79 | def default_parameters 80 | #http://sdb.amazonaws.com/?AWSAccessKeyId=nosuchkey 81 | # &Action=GetAttributes 82 | # &DomainName=images-test 83 | # &ItemName=0000000000000000000000000000000000000001 84 | # &SignatureMethod=HmacSHA256 85 | # &SignatureVersion=2 86 | # &Timestamp=2009-12-12T20%3A30%3A03.000Z 87 | # &Version=2007-11-07 88 | # &Signature=P0wPnG7pbXjJ%2F0X8Uoclj4ZJXUl32%2Fog2ouegjGtIBU%3D 89 | { 90 | "Timestamp" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"), 91 | "Version" => API_VERSION, 92 | 'DomainName' => @domain, 93 | } 94 | end 95 | 96 | def async_operation(method, parameters, opts) 97 | f = Fiber.current 98 | data = signed_parameters(parameters, method.to_s.upcase, DEFAULT_HOST, '/') 99 | args = if method == :get 100 | { :query => data }.merge(opts) 101 | else 102 | { :body => data }.merge(opts) 103 | end 104 | http = EventMachine::HttpRequest.new("http://#{DEFAULT_HOST}/").send(method, args) 105 | http.callback { f.resume(http) } 106 | http.errback { f.resume(http) } 107 | 108 | return Fiber.yield 109 | end 110 | 111 | def logger 112 | DaemonKit.logger 113 | end 114 | 115 | def timeout 116 | Integer(@config['timeout'] || 10) 117 | end 118 | 119 | def parse_response(string) 120 | parser = XML::Parser.string(string) 121 | doc = parser.parse 122 | # doc.root.namespaces.default_prefix = 'sdb' 123 | return doc 124 | end 125 | end 126 | end -------------------------------------------------------------------------------- /qanat/lib/sqs.rb: -------------------------------------------------------------------------------- 1 | module SQS 2 | DEFAULT_HOST = URI.parse("http://queue.amazonaws.com/") 3 | API_VERSION = "2009-02-01" 4 | 5 | class Queue 6 | REQUEST_TTL = 30 7 | 8 | include Amazon::Authentication 9 | 10 | def initialize(name) 11 | @config = Qanat.load('amzn') 12 | @uri = URI.parse(url_for(name)) 13 | end 14 | 15 | def poll(concurrency, &block) 16 | concurrency.times do 17 | Fiber.new do 18 | while true 19 | receive_msg do |msg| 20 | block.call msg 21 | end 22 | end 23 | end.resume 24 | end 25 | end 26 | 27 | def push(msg) 28 | request_hash = generate_request_hash("SendMessage", 'MessageBody' => msg) 29 | http = async_operation(:get, @uri, request_hash, :timeout => timeout) 30 | code = http.response_header.status 31 | if code != 200 32 | logger.error "SQS send_message returned an error response: #{code} #{http.response}" 33 | end 34 | end 35 | 36 | private 37 | 38 | def create(name) 39 | request_hash = generate_request_hash("CreateQueue", 'QueueName' => name) 40 | http = async_operation(:post, DEFAULT_HOST, request_hash, :timeout => timeout) 41 | code = http.response_header.status 42 | if code != 200 43 | logger.error "SQS send_message returned an error response: #{code} #{http.response}" 44 | end 45 | end 46 | 47 | def url_for(name, recur=false) 48 | raise ArgumentError, "No queue given" if !name || name.strip == '' 49 | request_hash = generate_request_hash("ListQueues", 'QueueNamePrefix' => name) 50 | http = async_operation(:get, DEFAULT_HOST, request_hash, :timeout => timeout) 51 | code = http.response_header.status 52 | if code == 200 53 | doc = Nokogiri::XML(http.response) 54 | tag = doc.xpath('//xmlns:QueueUrl').first 55 | if !tag 56 | if !recur 57 | create(name) 58 | return url_for(name, true) 59 | else 60 | raise ArgumentError, "Unable to create queue '#{name}'" 61 | end 62 | end 63 | url = tag.content 64 | logger.info "Queue #{name} at #{url}" 65 | return url 66 | end 67 | end 68 | 69 | def delete_msg(handle) 70 | logger.info "Deleting #{handle}" 71 | request_hash = generate_request_hash("DeleteMessage", 'ReceiptHandle' => handle) 72 | http = async_operation(:get, @uri, request_hash, :timeout => timeout) 73 | code = http.response_header.status 74 | if code != 200 75 | logger.error "SQS delete returned an error response: #{code} #{http.response}" 76 | end 77 | end 78 | 79 | def receive_msg(count=1, &block) 80 | request_hash = generate_request_hash("ReceiveMessage", 'MaxNumberOfMessages' => count, 81 | 'VisibilityTimeout' => 600) 82 | http = async_operation(:get, @uri, request_hash, :timeout => timeout) 83 | code = http.response_header.status 84 | if code == 200 85 | doc = Nokogiri::XML(http.response) 86 | msgs = doc.xpath('//xmlns:Message') 87 | if msgs.size > 0 88 | msgs.each do |msg| 89 | handle_el = msg.at_xpath('.//xmlns:ReceiptHandle') 90 | (logger.info msg; next) if !handle_el 91 | 92 | handle = msg.at_xpath('.//xmlns:ReceiptHandle').content 93 | message_id = msg.at_xpath('.//xmlns:MessageId').content 94 | checksum = msg.at_xpath('.//xmlns:MD5OfBody').content 95 | body = msg.at_xpath('.//xmlns:Body').content 96 | 97 | if checksum != Digest::MD5.hexdigest(body) 98 | logger.info "SQS message does not match checksum, ignoring..." 99 | else 100 | block.call body 101 | delete_msg(handle) 102 | end 103 | end 104 | else 105 | logger.info "Queue #{@uri} is empty" 106 | Fiber.sleep(5) 107 | end 108 | else 109 | logger.error "SQS returned an error response: #{code} #{http.response}" 110 | Fiber.sleep(5) 111 | # TODO parse the response and print something useful 112 | # TODO retry a few times with exponentially increasing delay 113 | end 114 | end 115 | 116 | def async_operation(method, uri, parameters, opts) 117 | f = Fiber.current 118 | data = signed_parameters(parameters, method.to_s.upcase, uri.host, uri.path) 119 | args = if method == :get 120 | { :query => data }.merge(opts) 121 | else 122 | { :body => data }.merge(opts) 123 | end 124 | http = EventMachine::HttpRequest.new(uri).send(method, args) 125 | http.callback { f.resume(http) } 126 | http.errback { f.resume(http) } 127 | 128 | return Fiber.yield 129 | end 130 | 131 | def default_parameters 132 | request_hash = { "Expires" => (Time.now + REQUEST_TTL).utc.strftime("%Y-%m-%dT%H:%M:%SZ"), 133 | "Version" => API_VERSION } 134 | end 135 | 136 | def logger 137 | DaemonKit.logger 138 | end 139 | 140 | def timeout 141 | Integer(@config['timeout']) 142 | end 143 | end 144 | end -------------------------------------------------------------------------------- /qanat/libexec/qanat-daemon.rb: -------------------------------------------------------------------------------- 1 | # Do your post daemonization configuration here 2 | # At minimum you need just the first line (without the block), or a lot 3 | # of strange things might start happening... 4 | DaemonKit::Application.running! do |config| 5 | # Trap signals with blocks or procs 6 | config.trap( 'HUP' ) do 7 | # Dump the REE stack traces 8 | p caller_for_all_threads if Object.respond_to? :caller_for_all_threads 9 | end 10 | config.trap('TERM') do 11 | # TODO print out the number of messages waiting to be processed 12 | end 13 | end 14 | 15 | require 'dispatch' 16 | 17 | Qanat.run do 18 | DaemonKit.logger.info "start" 19 | 20 | Fiber.new do 21 | sqs = SQS::Queue.new(DaemonKit.arguments.options[:queue_name]) 22 | sqs.poll(3) do |msg| 23 | Fiber.new do 24 | Qanat.dispatch(msg) 25 | end.resume 26 | end 27 | end.resume 28 | end 29 | 30 | 31 | # Images 32 | # IMAGE_SETS = 33 | # [ 34 | # ["85740637ed7ac07ce1444845ec02368fa636d395", "1b4c4a534b318a1c447691c5a13c79b3606350da", "02af3198b237091d8b5753ce3412456d75292384", "e27c68aba619350e53de129b70b5976715765854"], 35 | # ["93cb06618e6c1e3a7fbe68be42e1ee50c995a997", "ca98bfcf1b4b35dcb714701edfacb868984f5761", "a4f11d132853e56aec9817af8d1b0bb733a5e664", "349c788e1dcd16d2f81199b4fbff1dc79ff69eeb"], 36 | # ["fcf55f50662621c980f1e6d492d053f641744b96", "13f799e03d554c0566473e61a26b7abac9e3d9e0", "a55dbd9926ac7f68a5c9faa8077a0711dd3e0669", "b7b7ee7bcd0b644d21baf352378dab024a8ba297"], 37 | # ["f7f65b4bb8b9cb0dd5934e5a58dc0fe3f5aaa615", "1f23391992bf661e4d36391f924f322ac568a69d", "3ae7cf197b5722929fabfa8f5b29e12eefba95d1", "3c5465eb47b0bfd0a3d46fe344f8bbeefe613d9e"], 38 | # ["617fee9c94cc2ddd3f32d519087d08b9a2898cda", "ec1833e79cf9acc66dd0bcc8d1f5e002b5afb128", "7f8ed92c083144d47bc41e762cac8f55efc9dfff", "32b87a0b4e7d99fd30e1652eeb521da6b2e55303"], 39 | # ["140cec0d609c4a00e093f4f37565e2aba2056c15", "7bb7a6d81d2efea86a085cf18054554852c1af1b", "94445b78c5f715104322de5470b7dbd347f0b7ab", "1fb702a09c7164da7a87580a899cdb96825caec6"], 40 | # ["3768ec9758996e74417562ebb10e5d9b4ebfdf17", "dd850940107913fd0ea2e8ab439f9e4b72380c4f", "0e1a429fb4847370a5a99a364827b7becd795a2a", "4992808d11c603017af68ae2bca432641b343b85"], 41 | # ["8748af8ba5e58ae352fecb85e577232ff7f148d6", "dec43a1d45fe6c01ca92d616e46f52415f35b077", "1ec5a121d6a54f04ed47b3972ef7bd75038a2c46", "f2b6de6c8ece6e676a0f92e8fc44e5f3a90a077b"], 42 | # ["23fefcac4410ce26307ae0cc2975bc9bb32e0cf9", "1e3821afd31b84b839528f7819310d04da1d65df", "e89f004792528e2fa9b6bc2e1179cf3da3806be8", "838b2e0674162956ed41e98c0feb68a91bb42838"], 43 | # ["be610c0141988b8c03d0e2bad90d3336d80afe8b", "25f4ce7d8662ec83dbe7175d0ce8270f67186327", "788f2ba821200ac9762e443bb84e3f3f814c1285", "7887750e3429354460d850c9c13a9cf370feec62"], 44 | # ["6e353eebef1a51c9f00854e072c6ce645d0881f1"], 45 | # ] 46 | # 47 | # Qanat.run do 48 | # DaemonKit.logger.info "start" 49 | # 50 | # sdb = SDB::Database.new('images-staging') 51 | # IMAGE_SETS.each_with_index do |images, idx| 52 | # Fiber.new do 53 | # images.each do |iid| 54 | # p [idx, sdb.get(iid)] 55 | # end 56 | # end.resume 57 | # end 58 | # 59 | # Fiber.new do 60 | # sqs = SQS::Queue.new('test') 61 | # 62 | # sqs.poll(5) do |msg| 63 | # DaemonKit.logger.info "Processing #{msg}" 64 | # 65 | # # obj = YAML::load(msg) 66 | # # dispatch(obj, priority) 67 | # end 68 | # end.resume 69 | # 70 | # s3 = S3::Bucket.new('onespot-test') 71 | # Fiber.new do 72 | # s3.put('sqs.rb', File.read(File.dirname(__FILE__) + '/../lib/sqs.rb')) 73 | # puts s3.get('sqs.rb') 74 | # end.resume 75 | # end 76 | # 77 | -------------------------------------------------------------------------------- /qanat/script/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require File.dirname(__FILE__) + '/../config/boot' 3 | require 'daemon_kit/commands/console' 4 | -------------------------------------------------------------------------------- /qanat/script/destroy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) 3 | 4 | begin 5 | require 'rubigen' 6 | rescue LoadError 7 | require 'rubygems' 8 | require 'rubigen' 9 | end 10 | require 'rubigen/scripts/destroy' 11 | 12 | ARGV.shift if ['--help', '-h'].include?(ARGV[0]) 13 | RubiGen::Base.use_component_sources! [:daemon, :test_unit] 14 | RubiGen::Scripts::Destroy.new.run(ARGV) 15 | -------------------------------------------------------------------------------- /qanat/script/generate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) 3 | 4 | begin 5 | require 'rubigen' 6 | rescue LoadError 7 | require 'rubygems' 8 | require 'rubigen' 9 | end 10 | require 'rubigen/scripts/generate' 11 | 12 | ARGV.shift if ['--help', '-h'].include?(ARGV[0]) 13 | RubiGen::Base.use_component_sources! [:daemon] 14 | RubiGen::Scripts::Generate.new.run(ARGV) 15 | -------------------------------------------------------------------------------- /qanat/spec/qanat_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/spec_helper.rb' 2 | 3 | # Time to add your specs! 4 | # http://rspec.info/ 5 | describe "Place your specs here" do 6 | 7 | it "find this spec in spec directory" do 8 | violated "Be sure to write your specs" 9 | end 10 | 11 | end 12 | -------------------------------------------------------------------------------- /qanat/spec/spec.opts: -------------------------------------------------------------------------------- 1 | --colour -------------------------------------------------------------------------------- /qanat/spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | begin 2 | require 'spec' 3 | rescue LoadError 4 | require 'rubygems' 5 | gem 'rspec' 6 | require 'spec' 7 | end 8 | 9 | require File.dirname(__FILE__) + '/../config/environment' 10 | DaemonKit::Application.running! 11 | 12 | Spec::Runner.configure do |config| 13 | # == Mock Framework 14 | # 15 | # RSpec uses it's own mocking framework by default. If you prefer to 16 | # use mocha, flexmock or RR, uncomment the appropriate line: 17 | # 18 | config.mock_with :mocha 19 | # config.mock_with :flexmock 20 | # config.mock_with :rr 21 | end 22 | -------------------------------------------------------------------------------- /qanat/tasks/rspec.rake: -------------------------------------------------------------------------------- 1 | begin 2 | require 'spec' 3 | rescue LoadError 4 | require 'rubygems' 5 | require 'spec' 6 | end 7 | begin 8 | require 'spec/rake/spectask' 9 | rescue LoadError 10 | puts <<-EOS 11 | To use rspec for testing you must install rspec gem: 12 | gem install rspec 13 | EOS 14 | exit(0) 15 | end 16 | 17 | desc "Run the specs under spec/" 18 | Spec::Rake::SpecTask.new do |t| 19 | t.spec_opts = ['--options', "spec/spec.opts"] 20 | t.spec_files = FileList['spec/**/*_spec.rb'] 21 | end 22 | -------------------------------------------------------------------------------- /qanat/tasks/testing.rake: -------------------------------------------------------------------------------- 1 | require 'right_aws' 2 | require 'qanat' 3 | 4 | # Some sample Rake tasks to perform common queue tasks. 5 | namespace :msg do 6 | task :push do 7 | hash = Qanat.load('amzn') 8 | 9 | ACCESS_KEY = hash['access_key'] 10 | SECRET_KEY = hash['secret_key'] 11 | queue = ENV['QUEUE'] || 'test' 12 | count = Integer(ENV['COUNT'] || '5') 13 | sqs = RightAws::SqsGen2.new(ACCESS_KEY, SECRET_KEY, :protocol => 'http', :port => 80) 14 | q = sqs.queue(queue) 15 | count.times do 16 | q.push(Time.now.to_s) 17 | end 18 | end 19 | 20 | task :clone do 21 | hash = Qanat.load('amzn') 22 | 23 | ACCESS_KEY = hash['access_key'] 24 | SECRET_KEY = hash['secret_key'] 25 | to = ENV['TO'] || 'images' 26 | from = ENV['FROM'] || 'tasks_production_lowest' 27 | count = Integer(ENV['COUNT'] || '10') 28 | sqs = RightAws::SqsGen2.new(ACCESS_KEY, SECRET_KEY, :protocol => 'http', :port => 80) 29 | from_q = sqs.queue(from) 30 | to_q = sqs.queue(to) 31 | msgs = from_q.receive_messages(count) 32 | raise RuntimeError, "No messages recv'd from #{from}" if msgs.size == 0 33 | msgs.each do |msg| 34 | p msg.body 35 | next if msg.body !~ /crawl_images/ 36 | to_q.push(msg.body) 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /thin/thumbnailer.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | require 'digest/sha2' 5 | require 'fileutils' 6 | 7 | # port install freeimage 8 | # gem install image_science 9 | require 'image_science' 10 | 11 | # gem install thin 12 | require 'thin' 13 | # gem install em-http-request 14 | require 'em-http' 15 | 16 | class DeferrableBody 17 | include EventMachine::Deferrable 18 | 19 | def call(body) 20 | body.each do |chunk| 21 | @body_callback.call(chunk) 22 | end 23 | end 24 | 25 | def each &blk 26 | @body_callback = blk 27 | end 28 | end 29 | 30 | # Implements a thumbnail service for displaying thumbnails based on original images 31 | # stored in S3. The request comes in like this: 32 | # http://localhost:3000/t/20090801/1234567890123456789012345678901234567890/630x477-5316.jpg 33 | # We: 34 | # 1) do some sanity checking on the URL, break it into parameters and look for a cached version already generated, 35 | # this is all sync and returns immediately if something is wrong. 36 | # 2) async'ly pull the original from S3 using em-http-request. 37 | # 3) if successful, resize the original to the requested size, save it to local disk and return it. 38 | # 39 | # Please note, this example WILL NOT WORK without you setting up an S3 bucket and changing the 40 | # S3HOST constant below. 41 | # 42 | class Thumbnailer 43 | AsyncResponse = [-1, {}, []].freeze 44 | 45 | S3HOST = "your-image-bucket.s3.amazonaws.com" 46 | 47 | def call(env) 48 | url = env["PATH_INFO"] 49 | img = image_request_for(url) 50 | 51 | return invalid! unless img 52 | return invalid! 'Tampered URL' unless img.valid_checksum? or development?(env) 53 | 54 | return [200, 55 | {'Content-Type' => content_type(img.extension), 'X-Accel-Redirect' => img.nginx_location}, 56 | # If development, send the bytes as the response body in addition to the nginx header, so 57 | # we can see the image in our local browser. 58 | development?(env) ? [File.read(img.file_location)] : []] if img.cached? 59 | 60 | # We've done all we can synchronously. Next we need to pull the data from S3 and 61 | # return a response based on the result. 62 | EventMachine.next_tick do 63 | body = DeferrableBody.new 64 | http = EventMachine::HttpRequest.new("http://#{S3HOST}/#{img.s3_location}").get(:timeout => 5) 65 | http.errback do 66 | code = http.response_header.status 67 | log("Error!!! #{code} #{url}") 68 | env['async.callback'].call [code, {'Content-Type' => 'text/plain'}, body] 69 | body.call [http.response] 70 | body.succeed 71 | end 72 | http.callback do 73 | code = http.response_header.status 74 | log("Fetched #{url}: #{code}") 75 | img.to_thumb(http.response) 76 | 77 | # Now that we've recv'd enough data from S3, we can start the async response 78 | # back to the browser by calling the async callback in Thin. 79 | env['async.callback'].call [200, {'Content-Type' => content_type(img.extension), 'X-Accel-Redirect' => img.nginx_location}, body] 80 | 81 | # If development, send the bytes as the response body in addition to the nginx header, so 82 | # we can see the image in our local browser. 83 | if development?(env) 84 | body.call [File.read(img.file_location)] 85 | else 86 | body.call [] 87 | end 88 | body.succeed 89 | end 90 | end 91 | AsyncResponse 92 | end 93 | 94 | def content_type(extension) 95 | case extension 96 | when 'jpg' 97 | return 'image/jpeg' 98 | when 'gif' 99 | return 'image/gif' 100 | when 'png' 101 | return 'image/png' 102 | else 103 | return 'image/jpeg' 104 | end 105 | end 106 | 107 | def invalid!(msg='Invalid URL') 108 | [400, {"Content-Type" => "text/html"}, [msg]] 109 | end 110 | 111 | def not_found! 112 | [404, {"Content-Type" => "text/html"}, ["404 Not Found"]] 113 | end 114 | 115 | def internal_error!(msg) 116 | [500, {"Content-Type" => "text/html"}, ["Internal Error: #{msg}"]] 117 | end 118 | 119 | def image_request_for(url) 120 | if url =~ /^\/[a-z]\/(\d{8})\/(\w{40})\/(\d{2,3})x(\d{2,3})-(\w{4}).(\w{3})(?:\?(.*))?$/ 121 | ImageRequest.new($1, $2, $3, $4, $5, $6, $7) 122 | end 123 | end 124 | 125 | def development?(env) 126 | env['SERVER_NAME'] == 'localhost' 127 | end 128 | 129 | def log(msg) 130 | puts msg 131 | end 132 | end 133 | 134 | class ImageRequest 135 | SECRET_SALT = 'hello world' 136 | THUMB_ROOT = '/tmp/thumbs' 137 | FileUtils.mkdir_p THUMB_ROOT 138 | 139 | attr_accessor :extension 140 | 141 | def initialize(date, image_id, width, height, checksum, extension, params) 142 | @date = date 143 | @image_id = image_id 144 | @width = width 145 | @height = height 146 | @checksum = checksum 147 | @extension = extension 148 | @params = params 149 | end 150 | 151 | def valid_checksum? 152 | data = "#{SECRET_SALT}|#{@date}|#{@image_id}|#{@width}|#{@height}" 153 | code = Digest::SHA2.hexdigest(data)[0..3] 154 | @checksum == code 155 | end 156 | 157 | def to_thumb(data) 158 | ImageScience.with_image_from_memory(data) do |img| 159 | img.resize(Integer(@width), Integer(@height)) do |thumb| 160 | thumb.save(file_location) 161 | end 162 | end 163 | end 164 | 165 | def s3_location 166 | "#{@image_id}.#{@extension}" 167 | end 168 | 169 | def cached? 170 | File.exist? file_location 171 | end 172 | 173 | def file_location 174 | @file_path ||= begin 175 | dir = "#{THUMB_ROOT}/#{@image_id[0..1]}/#{@image_id[2..3]}" 176 | FileUtils.mkdir_p dir unless File.directory? dir 177 | "#{dir}/#{@image_id}-#{@width}x#{@height}.#{@extension}" 178 | end 179 | end 180 | 181 | def nginx_location 182 | file_location.slice(4..-1) 183 | end 184 | 185 | end 186 | 187 | if $0 == __FILE__ 188 | Thin::Server.start('0.0.0.0', 3000) do 189 | use Rack::CommonLogger 190 | run Thumbnailer.new 191 | end 192 | end --------------------------------------------------------------------------------