├── .gitignore ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── asap.gemspec ├── bin ├── asap-em-test-server └── asap-mongrel-test-server ├── examples ├── followers-example-serial.rb ├── followers-example.out └── followers-example.rb ├── java └── netty-3.2.4.Final.jar ├── lib ├── asap.rb └── asap │ ├── fetch_context.rb │ ├── netty.rb │ ├── netty │ ├── http_response_handler.rb │ └── pipeline_factory.rb │ └── version.rb └── spec ├── asap └── netty_spec.rb ├── asap_spec.rb └── spec_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | .rvmrc 3 | Gemfile.lock 4 | *.gem 5 | .bundle 6 | pkg/* 7 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source :rubygems 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Greg Spurrier, Avik Das 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ASAP - a Parallel Fetch Library 2 | =============================== 3 | 4 | by Greg Spurrier, Avik Das 5 | 6 | DESCRIPTION 7 | ----------- 8 | 9 | ASAP is a JRuby library built on top of Netty and Java's NIO classes. It 10 | provides an embedded domain specific language for specifying a list of 11 | resources to fetch, all in parallel, as well specify a dependency tree 12 | in order to use previous results in calculating subsequent ones. The 13 | results of all the requests are then automatically collected into a 14 | simple tree-like data structure that has a one-to-one mapping to the 15 | tree structure specified by the code, despite the fact that results may 16 | arrive in an unspecified order. 17 | 18 | EXAMPLE 19 | ------- 20 | 21 | Assume that a server is running on `http://0.0.0.0:1234`, which maps the 22 | paths `/user/followers` and `/user/followers/N` to a list of five 23 | numbers and data related to the Nth follower respectively. (This is 24 | implemented by `asap-mongrel-test-server`.) 25 | 26 | require 'rubygems' 27 | require 'asap' 28 | 29 | data = Asap do 30 | get 'http://0.0.0.0:1234/user/followers' do |followers| 31 | followers = followers.split("\n").map(&:to_i) 32 | 33 | # get the first 3 followers 34 | get "http://0.0.0.0:1234/user/followers/#{followers[0]}" 35 | get "http://0.0.0.0:1234/user/followers/#{followers[1]}" 36 | get "http://0.0.0.0:1234/user/followers/#{followers[2]}" 37 | 38 | # or you can use a map 39 | followers[3,2].each do |fi| 40 | get "http://0.0.0.0:1234/user/followers/#{fi}" 41 | end 42 | end 43 | end 44 | 45 | p data 46 | 47 | # => [['93\n5\n64\n74\n11', 48 | ['Follower #93', 49 | 'Follower #5' , 50 | 'Follower #64', 51 | 'Follower #74', 52 | 'Follower #11']]] 53 | 54 | QUICK START 55 | ----------- 56 | 57 | gem install asap 58 | 59 | DEVELOPMENT QUICK START 60 | ----------------------- 61 | 62 | # Check out repository 63 | git clone git://github.com/avik-das/asap.git 64 | cd asap 65 | 66 | # Make sure you're using JRuby 67 | 68 | # Install the dependencies 69 | gem install bundler 70 | bundle install 71 | rake install 72 | 73 | # start the server (run in a separate window) 74 | asap-mongrel-test-server 75 | 76 | # run the tests 77 | rake spec 78 | 79 | # compare the times required to fetch the same data if it is retrieved 80 | # serially versus if it is retrieved with ASAP. 81 | time examples/followers-example-serial.rb 82 | time examples/followers-example.rb 83 | 84 | # There is an alternate, EventMachine-based server, but it does not 85 | # implement the /user/followers/ routes. EventMachine does not run 86 | # well with JRuby, while the main library requires JRuby. However, 87 | # the EventMachine-based server runs well on MRI, assuming the 88 | # correct gems are installed (see the gemspec). 89 | script/em_test_server.rb 90 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'bundler/gem_tasks' 3 | 4 | require 'rspec/core/rake_task' 5 | RSpec::Core::RakeTask.new 6 | -------------------------------------------------------------------------------- /asap.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "asap/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "asap" 7 | s.version = Asap::Gem::VERSION 8 | s.platform = Gem::Platform::RUBY 9 | s.authors = ["Greg Spurrier", "Avik Das"] 10 | s.email = ["gspurrier@linkedin.com", "adas@linkedin.com"] 11 | s.homepage = "http://rubygems.org/gems/didactic_clock" 12 | s.summary = %q{A JRuby library for parallel fetches.} 13 | s.description = %q{A JRuby library for parallel fetches. Provides an embedded-domain specific language for declaring dependencies between resources.} 14 | 15 | s.files = `git ls-files`.split("\n") 16 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 17 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 18 | s.require_paths = ["lib"] 19 | 20 | s.add_runtime_dependency("mongrel") 21 | 22 | s.add_development_dependency("rspec") 23 | s.add_development_dependency("rake") 24 | 25 | # Event machine does not run well on JRuby, but the main library requires 26 | # JRuby. The following gems should be installed separately on an MRI 27 | # instance if you wish to run bin/asap-em-test-server 28 | # s.add_runtime_dependency('eventmachine') 29 | # s.add_runtime_dependency('eventmachine_httpserver') 30 | end 31 | -------------------------------------------------------------------------------- /bin/asap-em-test-server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'rubygems' 3 | require 'eventmachine' 4 | require 'evma_httpserver' 5 | require 'cgi' 6 | 7 | class MyHttpServer < EM::Connection 8 | include EM::HttpServer 9 | 10 | def post_init 11 | super 12 | no_environment_strings 13 | end 14 | 15 | def process_http_request 16 | # the http request details are available via the following instance variables: 17 | # @http_protocol 18 | # @http_request_method 19 | # @http_cookie 20 | # @http_if_none_match 21 | # @http_content_type 22 | # @http_path_info 23 | # @http_request_uri 24 | # @http_query_string 25 | # @http_post_content 26 | # @http_headers 27 | 28 | empty, time, message = @http_path_info.split('/') 29 | 30 | response = EM::DelegatedHttpResponse.new(self) 31 | response.status = 200 32 | response.content_type 'text/html' 33 | response.content = CGI.unescape(message) 34 | 35 | EventMachine::Timer.new(time.to_i) do 36 | response.send_response 37 | end 38 | end 39 | end 40 | 41 | EventMachine.run{ 42 | EventMachine.start_server '0.0.0.0', 1234, MyHttpServer 43 | } 44 | -------------------------------------------------------------------------------- /bin/asap-mongrel-test-server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | require 'mongrel' 5 | 6 | class TimeMessageHandler < Mongrel::HttpHandler 7 | def process request, response 8 | response.start 200 do |head,out| 9 | params = get_params request 10 | 11 | wait_time = params[:wait_time].to_i 12 | puts "Sleeping for #{wait_time} seconds" 13 | sleep wait_time 14 | 15 | head["Content-Type"] = "text/plain" 16 | out.write params[:message] # no newline 17 | out.flush 18 | end 19 | end 20 | 21 | private 22 | 23 | def get_params request 24 | uri = request.params['REQUEST_URI'] 25 | _, wait_time, message = *uri.split("/") 26 | message = Mongrel::HttpRequest.unescape message 27 | {:wait_time => wait_time, :message => message} 28 | end 29 | end 30 | 31 | class FollowersHandler < Mongrel::HttpHandler 32 | def process request, response 33 | sleep 5 34 | 35 | id = get_follower_id(request) 36 | message = id ? get_one_follower(id.to_i) : get_all_followers 37 | 38 | response.start 200 do |head,out| 39 | head["Content-Type"] = "text/plain" 40 | out.write message # no newline 41 | out.flush 42 | end 43 | end 44 | 45 | private 46 | 47 | MAX_FOLLOW_ID = 100 48 | 49 | def get_follower_id request 50 | uri = request.params['REQUEST_URI'] 51 | _, _, _, id = *uri.split("/") 52 | id 53 | end 54 | 55 | def get_all_followers 56 | (1..5).map {|i| rand(MAX_FOLLOW_ID)}.join("\n") 57 | end 58 | 59 | def get_one_follower i 60 | "Follower ##{i}" 61 | end 62 | end 63 | 64 | h = Mongrel::HttpServer.new "0.0.0.0", "1234" 65 | h.register "/user/followers", FollowersHandler.new 66 | h.register "/", TimeMessageHandler.new 67 | 68 | puts "Starting server" 69 | h.run.join 70 | puts "Shutting down server" 71 | -------------------------------------------------------------------------------- /examples/followers-example-serial.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'open-uri' 4 | 5 | data = [] 6 | open('http://0.0.0.0:1234/user/followers') do |fresp| 7 | data << fresp.read 8 | followers = data[0].split("\n").map(&:to_i) 9 | 10 | # get the first 3 followers 11 | open("http://0.0.0.0:1234/user/followers/#{followers[0]}") do |resp| 12 | data << resp.read 13 | end 14 | open("http://0.0.0.0:1234/user/followers/#{followers[1]}") do |resp| 15 | data << resp.read 16 | end 17 | open("http://0.0.0.0:1234/user/followers/#{followers[2]}") do |resp| 18 | data << resp.read 19 | end 20 | 21 | # or you can use a map 22 | followers[3,2].each do |fi| 23 | open("http://0.0.0.0:1234/user/followers/#{fi}") do |resp| 24 | data << resp.read 25 | end 26 | end 27 | end 28 | 29 | p data 30 | -------------------------------------------------------------------------------- /examples/followers-example.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | data = 5 | [ 6 | ["44\n64\n6\n21\n87", # /user/followers 7 | 8 | 9 | 10 | ["Follower #44", # /user/followers/44 11 | "Follower #64", # /user/followers/64 12 | "Follower #6" , # /user/followers/6 13 | 14 | 15 | 16 | "Follower #21", "Follower #87"]]] # etc. 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /examples/followers-example.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | require 'asap' 5 | 6 | data = Asap do 7 | get 'http://0.0.0.0:1234/user/followers' do |followers| 8 | followers = followers.split("\n").map(&:to_i) 9 | 10 | # get the first 3 followers 11 | get "http://0.0.0.0:1234/user/followers/#{followers[0]}" 12 | get "http://0.0.0.0:1234/user/followers/#{followers[1]}" 13 | get "http://0.0.0.0:1234/user/followers/#{followers[2]}" 14 | 15 | # or you can use a map 16 | followers[3,2].each do |fi| 17 | get "http://0.0.0.0:1234/user/followers/#{fi}" 18 | end 19 | end 20 | end 21 | 22 | p data 23 | -------------------------------------------------------------------------------- /java/netty-3.2.4.Final.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avik-das/asap/bc376c6628ce3e1d490580a417873db458778f41/java/netty-3.2.4.Final.jar -------------------------------------------------------------------------------- /lib/asap.rb: -------------------------------------------------------------------------------- 1 | require 'asap/fetch_context' 2 | 3 | def Asap *args, &blk 4 | context = Asap::FetchContext.new 5 | context.instance_exec *args, &blk 6 | context.join 7 | context.result 8 | end 9 | -------------------------------------------------------------------------------- /lib/asap/fetch_context.rb: -------------------------------------------------------------------------------- 1 | require 'asap/netty' 2 | 3 | module Asap 4 | class FetchContext 5 | def initialize 6 | @result = [] 7 | @semaphore = java.util.concurrent.Semaphore.new(0) 8 | end 9 | 10 | def get(url, &blk) 11 | target_index = @result.size 12 | @result << nil 13 | Asap::Netty.get(url) do |result| 14 | if blk 15 | Thread.new do 16 | nested = Asap(result, &blk) 17 | @result[target_index] = [result, nested] 18 | @semaphore.release 19 | end 20 | else 21 | @result[target_index] = result 22 | @semaphore.release 23 | end 24 | end 25 | end 26 | 27 | def join 28 | @semaphore.acquire(@result.size) 29 | end 30 | 31 | attr_reader :result 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/asap/netty.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'jruby' 3 | require 'java' 4 | $CLASSPATH << File.expand_path('../../java/netty-3.2.4.Final.jar', File.dirname(__FILE__)) 5 | 6 | require 'asap/netty/http_response_handler' 7 | require 'asap/netty/pipeline_factory' 8 | 9 | module Asap 10 | module Netty 11 | java_import java.net.InetSocketAddress 12 | java_import java.util.concurrent.Executors 13 | java_import org.jboss.netty.bootstrap.ClientBootstrap 14 | java_import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory 15 | java_import org.jboss.netty.handler.codec.http.DefaultHttpRequest 16 | java_import org.jboss.netty.handler.codec.http.HttpVersion 17 | java_import org.jboss.netty.handler.codec.http.HttpMethod 18 | java_import org.jboss.netty.handler.codec.http.HttpHeaders 19 | 20 | 21 | def self.get(url, &callback) 22 | uri = URI.parse(url) 23 | 24 | bootstrap.set_pipeline_factory(PipelineFactory.new(callback)) 25 | 26 | # Open a connection 27 | future = bootstrap.connect(InetSocketAddress.new(uri.host, uri.port)) 28 | channel = future.awaitUninterruptibly.get_channel 29 | raise 'connection failed' unless future.is_success 30 | 31 | # Send the request 32 | request = DefaultHttpRequest.new(HttpVersion::HTTP_1_0, HttpMethod::GET, uri.path) 33 | request.set_header(HttpHeaders::Names::HOST, uri.host) 34 | channel.write(request) 35 | end 36 | 37 | private 38 | 39 | def self.bootstrap 40 | if not @bootstrap 41 | @bootstrap = ClientBootstrap.new( 42 | NioClientSocketChannelFactory.new( 43 | Executors.newCachedThreadPool, 44 | Executors.newCachedThreadPool)) 45 | 46 | at_exit { @bootstrap.release_external_resources } 47 | end 48 | @bootstrap 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/asap/netty/http_response_handler.rb: -------------------------------------------------------------------------------- 1 | module Asap 2 | module Netty 3 | class HttpResponseHandler < org.jboss.netty.channel.SimpleChannelUpstreamHandler 4 | attr_reader :callback 5 | 6 | def initialize(callback) 7 | super() 8 | @callback = callback 9 | end 10 | 11 | def messageReceived(ctxt, e) 12 | response = e.get_message 13 | if response.get_status.get_code == 200 14 | callback.call(response.get_content.to_string(org.jboss.netty.util.CharsetUtil::UTF_8)) 15 | else 16 | raise "Request failed with #{response.get_status.get_code}" 17 | end 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/asap/netty/pipeline_factory.rb: -------------------------------------------------------------------------------- 1 | module Asap 2 | module Netty 3 | class PipelineFactory 4 | include org.jboss.netty.channel.ChannelPipelineFactory 5 | 6 | attr_reader :callback 7 | 8 | def initialize(callback) 9 | @callback = callback 10 | end 11 | 12 | def get_pipeline 13 | org.jboss.netty.channel.Channels.pipeline.tap do |pipeline| 14 | pipeline.add_last("codec", org.jboss.netty.handler.codec.http.HttpClientCodec.new) 15 | pipeline.add_last("handler", Asap::Netty::HttpResponseHandler.new(callback)) 16 | end 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/asap/version.rb: -------------------------------------------------------------------------------- 1 | module Asap 2 | module Gem 3 | VERSION = "0.0.5" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /spec/asap/netty_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Asap::Netty, '.get' do 4 | it 'fetches the specified resource and invokes the callback with it' do 5 | semaphore = java.util.concurrent.Semaphore.new(0) 6 | result = nil 7 | Asap::Netty.get("http://localhost:1234/0/hello") do |data| 8 | result = data 9 | semaphore.release 10 | end 11 | semaphore.acquire 12 | result.should == 'hello' 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /spec/asap_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | def url path 4 | "http://localhost:1234" + path 5 | end 6 | 7 | describe Asap do 8 | it 'should be a module' do 9 | Asap.class.should == Module 10 | end 11 | 12 | it 'should be callable as a function with a block' do 13 | lambda { Asap do; end }.should_not raise_error 14 | end 15 | 16 | it 'should require a block' do 17 | lambda { Asap() }.should raise_error 18 | end 19 | 20 | context 'given no gets' do 21 | it 'should have an empty result' do 22 | Asap do 23 | end.should == [] 24 | end 25 | end 26 | 27 | context 'given one get' do 28 | it 'should return one result' do 29 | Asap do 30 | get url("/0/hello") 31 | end.should == ["hello"] 32 | end 33 | end 34 | 35 | context 'given two gets' do 36 | it 'should return two results' do 37 | Asap do 38 | get url("/0/hello") 39 | get url("/0/world") 40 | end.should == ["hello","world"] 41 | end 42 | end 43 | 44 | context 'given three gets' do 45 | it 'should return three results' do 46 | Asap do 47 | get url("/1/goodbye") 48 | get url("/1/cruel") 49 | get url("/1/world") 50 | end.should == ["goodbye","cruel","world"] 51 | end 52 | 53 | it 'should run in parallel' do 54 | start = Time.now 55 | Asap do 56 | get url("/1/goodbye") 57 | get url("/1/cruel") 58 | get url("/1/world") 59 | end 60 | (Time.now - start).should < 2 61 | end 62 | end 63 | 64 | context 'given one nested get' do 65 | it 'should return one nested result' do 66 | Asap do 67 | get url("/0/%2F0%2Fhello") do |resp| 68 | get url(resp) 69 | end 70 | end.should == [["/0/hello", ["hello"]]] 71 | end 72 | end 73 | 74 | context 'given two single-nested gets' do 75 | it 'should return two single-nested results' do 76 | Asap do 77 | get url("/0/%2F0%2Fhello") do |resp| 78 | get url(resp) 79 | end 80 | get url("/0/%2F0%2Fworld") do |resp| 81 | get url(resp) 82 | end 83 | end.should == [["/0/hello", ["hello"]], ["/0/world", ["world"]]] 84 | end 85 | end 86 | 87 | context 'given a combination of nested and flat gets' do 88 | it 'should return the same combination of nested and flat results' do 89 | Asap do 90 | get url("/0/hello0") 91 | get url("/0/%2F0%2Fhello1") do |resp| 92 | get url(resp) 93 | end 94 | get url("/0/world0") 95 | get url("/0/%2F0%2Fworld1") do |resp| 96 | get url(resp) 97 | end 98 | end.should == ["hello0", 99 | ["/0/hello1", ["hello1"]], 100 | "world0", 101 | ["/0/world1", ["world1"]]] 102 | end 103 | end 104 | 105 | context 'given deep nesting' do 106 | it 'should return a deeply-nested result' do 107 | Asap do 108 | get url("/0/%2F0%2F%252F0%252F%25252F0%25252Fhello") do |r1| 109 | get url(r1) do |r2| 110 | get url(r2) do |r3| 111 | get url(r3) 112 | end 113 | end 114 | end 115 | end.should == 116 | [["/0/%2F0%2F%252F0%252Fhello", [ 117 | ["/0/%2F0%2Fhello", [ 118 | ["/0/hello", ["hello"]]]]]]] 119 | end 120 | end 121 | 122 | context 'given additional arguments' do 123 | it 'should pass the arguments to its block' do 124 | Asap("/0/hello") do |path| 125 | get url(path) 126 | end.should == ["hello"] 127 | end 128 | end 129 | 130 | context 'given a map over the gets' do 131 | it 'should behave like the gets are listed out' do 132 | Asap do 133 | (1..10).each do |i| 134 | get url("/1/#{i}") 135 | end 136 | end.should == (1..10).to_a.map(&:to_s) 137 | end 138 | 139 | it 'should behave like the gets are listed out' do 140 | start = Time.now 141 | Asap do 142 | (1..10).each do |i| 143 | get url("/1/#{i}") 144 | end 145 | end 146 | (Time.now - start).should < 2 147 | end 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler/setup' 3 | 4 | require 'asap' 5 | --------------------------------------------------------------------------------