├── VERSION ├── spec ├── helper.rb └── proxy_spec.rb ├── lib ├── em-proxy.rb └── em-proxy │ ├── proxy.rb │ ├── backend.rb │ └── connection.rb ├── examples ├── appserver.rb ├── line_interceptor.rb ├── port_forward.rb ├── beanstalkd_interceptor.rb ├── relay_port_forward.rb ├── smtp_whitelist.rb ├── duplex.rb ├── selective_forward.rb ├── smtp_spam_filter.rb └── schemaless-mysql │ └── mysql_interceptor.rb ├── Rakefile └── README.rdoc /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.3 2 | -------------------------------------------------------------------------------- /spec/helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'spec' 3 | require 'pp' 4 | require 'em-http' 5 | 6 | require 'lib/em-proxy' -------------------------------------------------------------------------------- /lib/em-proxy.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.dirname(__FILE__) + '/../lib') 2 | 3 | 4 | require "rubygems" 5 | require "eventmachine" 6 | require "socket" 7 | 8 | %w[ backend proxy connection ].each do |file| 9 | require "em-proxy/#{file}" 10 | end 11 | -------------------------------------------------------------------------------- /examples/appserver.rb: -------------------------------------------------------------------------------- 1 | require "rubygems" 2 | require "rack" 3 | 4 | app = lambda { 5 | p [:serving, ARGV[0]] 6 | 7 | sleep(ARGV[1].to_i) 8 | [200, {"Content-Type" => "text/plain"}, ["hello world: #{ARGV[1]}"]] 9 | } 10 | 11 | Rack::Handler::Mongrel.run(app, {:Host => "0.0.0.0", :Port => ARGV[0]}) 12 | -------------------------------------------------------------------------------- /lib/em-proxy/proxy.rb: -------------------------------------------------------------------------------- 1 | class Proxy 2 | 3 | def self.start(options, &blk) 4 | EM.epoll 5 | EM.run do 6 | 7 | trap("TERM") { stop } 8 | trap("INT") { stop } 9 | 10 | EventMachine::start_server(options[:host], options[:port], 11 | EventMachine::ProxyServer::Connection, options) do |c| 12 | blk.call(c) 13 | end 14 | end 15 | end 16 | 17 | def self.stop 18 | puts "Terminating ProxyServer" 19 | EventMachine.stop 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /examples/line_interceptor.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 81 5 | 6 | conn.on_data do |data| 7 | data 8 | end 9 | 10 | conn.on_response do |backend, resp| 11 | # substitute all mentions of hello to 'good bye', aka intercepting proxy 12 | resp.gsub(/hello/, 'good bye') 13 | end 14 | end 15 | 16 | # 17 | # ruby examples/appserver.rb 81 18 | # ruby examples/line_interceptor.rb 19 | # curl localhost 20 | # 21 | # > good bye world: 0 22 | # -------------------------------------------------------------------------------- /examples/port_forward.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 81 5 | 6 | # modify / process request stream 7 | conn.on_data do |data| 8 | p [:on_data, data] 9 | data 10 | end 11 | 12 | # modify / process response stream 13 | conn.on_response do |backend, resp| 14 | p [:on_response, backend, resp] 15 | # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!" 16 | resp 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | 3 | begin 4 | require 'jeweler' 5 | Jeweler::Tasks.new do |gemspec| 6 | gemspec.name = "em-proxy" 7 | gemspec.summary = "EventMachine Proxy DSL" 8 | gemspec.description = gemspec.summary 9 | gemspec.email = "ilya@igvita.com" 10 | gemspec.homepage = "http://github.com/igrigorik/em-proxy" 11 | gemspec.authors = ["Ilya Grigorik"] 12 | gemspec.add_dependency('eventmachine', '>= 0.12.9') 13 | gemspec.rubyforge_project = "em-proxy" 14 | end 15 | 16 | Jeweler::GemcutterTasks.new 17 | rescue LoadError 18 | puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com" 19 | end 20 | -------------------------------------------------------------------------------- /examples/beanstalkd_interceptor.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 11300) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 11301 5 | 6 | # put \r\n 7 | PUT_CMD = /put (\d+) (\d+) (\d+) (\d+)\r\n/ 8 | 9 | conn.on_data do |data| 10 | if put = data.match(PUT_CMD) 11 | 12 | # archive any job > 10 minutes away 13 | if put[2].to_i > 600 14 | p [:put, :archive] 15 | # INSERT INTO .... 16 | 17 | conn.send_data "INSERTED 9999\r\n" 18 | data = nil 19 | end 20 | end 21 | 22 | data 23 | end 24 | 25 | conn.on_response do |backend, resp| 26 | p [:resp, resp] 27 | resp 28 | end 29 | end 30 | 31 | # 32 | # beanstalkd -p 11301 -d 33 | # ruby examples/beanstalkd_interceptor.rb 34 | # 35 | # irb 36 | # >> require 'beanstalk-client' 37 | # >> beanstalk = Beanstalk::Pool.new(['127.0.0.1']) 38 | # >> beanstalk.put("job1") 39 | # => 1 40 | # >> beanstalk.put("job2") 41 | # => 2 42 | # >> beanstalk.put("job3", 0, 1000) 43 | # => 9999 44 | -------------------------------------------------------------------------------- /examples/relay_port_forward.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => false) do |conn| 4 | # Specifying :relay_server or :relay_client is useful if only requests or responses 5 | # need to be processed. The proxy throughput will roughly double. 6 | conn.server :srv, :host => "127.0.0.1", :port => 81, :relay_client => true, :relay_server => true 7 | 8 | conn.on_connect do 9 | p [:on_connect, "#{conn.peer.join(':')} connected"] 10 | end 11 | 12 | # modify / process request stream 13 | # on_data will not be called when :relay_server => true is passed as server option 14 | conn.on_data do |data| 15 | p [:on_data, data] 16 | data 17 | end 18 | 19 | # modify / process response stream 20 | # on_response will not be called when :relay_client => true is passed as server option 21 | conn.on_response do |backend, resp| 22 | p [:on_response, backend, resp] 23 | # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!" 24 | resp 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /examples/smtp_whitelist.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 2525 5 | 6 | # RCPT TO:\r\n 7 | RCPT_CMD = /RCPT TO:<(.*)?>\r\n/ 8 | 9 | conn.on_data do |data| 10 | 11 | if rcpt = data.match(RCPT_CMD) 12 | if rcpt[1] != "ilya@igvita.com" 13 | conn.send_data "550 No such user here\n" 14 | data = nil 15 | end 16 | end 17 | 18 | data 19 | end 20 | 21 | conn.on_response do |backend, resp| 22 | resp 23 | end 24 | end 25 | 26 | 27 | # mailtrap run -p 2525 -f /tmp/mailtrap.log 28 | # ruby examples/smtp_whitelist.rb 29 | # 30 | # >> require 'net/smtp' 31 | # >> smtp = Net::SMTP.start("localhost", 2524) 32 | # >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com" 33 | # => # 34 | # >> smtp.finish 35 | # => # 36 | # 37 | # >> smtp.send_message "Hello World!", "ilya@aiderss.com", "missing_user@igvita.com" 38 | # => Net::SMTPFatalError: 550 No such user here 39 | # 40 | -------------------------------------------------------------------------------- /examples/duplex.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn| 4 | @start = Time.now 5 | @data = Hash.new("") 6 | 7 | conn.server :test, :host => "127.0.0.1", :port => 81 # production, will render resposne 8 | conn.server :prod, :host => "127.0.0.1", :port => 82 # testing, internal only 9 | 10 | conn.on_data do |data| 11 | # rewrite User-Agent 12 | data.gsub(/User-Agent: .*?\r\n/, "User-Agent: em-proxy/0.1\r\n") 13 | end 14 | 15 | conn.on_response do |server, resp| 16 | # only render response from production 17 | @data[server] += resp 18 | resp if server == :prod 19 | end 20 | 21 | conn.on_finish do |name| 22 | p [:on_finish, name, Time.now - @start] 23 | p @data 24 | end 25 | end 26 | 27 | # 28 | # ruby examples/appserver.rb 81 29 | # ruby examples/appserver.rb 82 30 | # ruby examples/line_interceptor.rb 31 | # curl localhost 32 | # 33 | # > [:on_finish, 1.008561] 34 | # > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0", 35 | # :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"} 36 | # -------------------------------------------------------------------------------- /lib/em-proxy/backend.rb: -------------------------------------------------------------------------------- 1 | module EventMachine 2 | module ProxyServer 3 | class Backend < EventMachine::Connection 4 | attr_accessor :plexer, :data, :name, :debug 5 | 6 | def initialize(debug = false) 7 | @debug = debug 8 | @connected = EM::DefaultDeferrable.new 9 | @data = [] 10 | end 11 | 12 | def connection_completed 13 | debug [@name, :conn_complete] 14 | @plexer.connected(@name) 15 | @connected.succeed 16 | end 17 | 18 | def receive_data(data) 19 | debug [@name, data] 20 | @data.push data 21 | @plexer.relay_from_backend(@name, data) 22 | end 23 | 24 | # Buffer data until the connection to the backend server 25 | # is established and is ready for use 26 | def send(data) 27 | @connected.callback { send_data data } 28 | end 29 | 30 | # Notify upstream plexer that the backend server is done 31 | # processing the request 32 | def unbind 33 | debug [@name, :unbind] 34 | @plexer.unbind_backend(@name) 35 | end 36 | 37 | private 38 | 39 | def debug(*data) 40 | return unless @debug 41 | require 'pp' 42 | pp data 43 | puts 44 | end 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /examples/selective_forward.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80) do |conn| 4 | @start = Time.now 5 | @data = Hash.new("") 6 | 7 | conn.server :prod, :host => "127.0.0.1", :port => 81 # production, will render resposne 8 | conn.server :test, :host => "127.0.0.1", :port => 82 # testing, internal only 9 | 10 | conn.on_data do |data| 11 | # rewrite User-Agent 12 | [data.gsub(/User-Agent: .*?\r\n/, 'User-Agent: em-proxy/0.1\r\n'), [:prod]] 13 | end 14 | 15 | conn.on_response do |server, resp| 16 | # only render response from production 17 | @data[server] += resp 18 | resp if server == :prod 19 | end 20 | 21 | conn.on_finish do |name| 22 | p [:on_finish, name, Time.now - @start] 23 | unbind if name == :prod # terminate connection once prod is done 24 | 25 | p @data if name == :done 26 | end 27 | end 28 | 29 | # 30 | # ruby examples/appserver.rb 81 31 | # ruby examples/appserver.rb 82 32 | # ruby examples/line_interceptor.rb 33 | # curl localhost 34 | # 35 | # > [:on_finish, 1.008561] 36 | # > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0", 37 | # :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"} 38 | # -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = EM-Proxy 2 | 3 | EventMachine Proxy DSL for writing high-performance transparent / intercepting proxies in Ruby. 4 | 5 | - Slides from RailsConf 2009: http://bit.ly/D7oWB 6 | - GoGaRuCo notes & Slides: http://www.igvita.com/2009/04/20/ruby-proxies-for-scale-and-monitoring/ 7 | 8 | == Getting started 9 | 10 | # install & configure gemcutter repos 11 | gem update --system 12 | gem install gemcutter 13 | gem tumble 14 | 15 | gem install em-proxy 16 | 17 | irb:0> require 'em-proxy' 18 | 19 | == Simple port forwarding proxy 20 | 21 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn| 22 | conn.server :srv, :host => "127.0.0.1", :port => 81 23 | 24 | # modify / process request stream 25 | conn.on_data do |data| 26 | p [:on_data, data] 27 | data 28 | end 29 | 30 | # modify / process response stream 31 | conn.on_response do |backend, resp| 32 | p [:on_response, backend, resp] 33 | resp 34 | end 35 | 36 | # termination logic 37 | conn.on_finish do |backend, name| 38 | p [:on_finish, name] 39 | 40 | # terminate connection (in duplex mode, you can terminate when prod is done) 41 | unbind if backend == :srv 42 | end 43 | end 44 | 45 | For more examples see the /examples directory. 46 | - SMTP Spam Filtering 47 | - Duplicating traffic 48 | - Selective forwarding 49 | - Beanstalkd interceptor 50 | - etc. 51 | 52 | A schema-free MySQL proof of concept, via an EM-Proxy server: 53 | - http://www.igvita.com/2010/03/01/schema-free-mysql-vs-nosql/ 54 | - Code in examples/schemaless-mysql 55 | 56 | == License 57 | 58 | (The MIT License) 59 | 60 | Copyright (c) 2010 Ilya Grigorik 61 | 62 | Permission is hereby granted, free of charge, to any person obtaining 63 | a copy of this software and associated documentation files (the 64 | 'Software'), to deal in the Software without restriction, including 65 | without limitation the rights to use, copy, modify, merge, publish, 66 | distribute, sublicense, and/or sell copies of the Software, and to 67 | permit persons to whom the Software is furnished to do so, subject to 68 | the following conditions: 69 | 70 | The above copyright notice and this permission notice shall be 71 | included in all copies or substantial portions of the Software. 72 | 73 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 74 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 75 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 76 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 77 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 78 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 79 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 80 | -------------------------------------------------------------------------------- /lib/em-proxy/connection.rb: -------------------------------------------------------------------------------- 1 | module EventMachine 2 | module ProxyServer 3 | class Connection < EventMachine::Connection 4 | attr_accessor :debug 5 | 6 | ##### Proxy Methods 7 | def on_data(&blk); @on_data = blk; end 8 | def on_response(&blk); @on_response = blk; end 9 | def on_finish(&blk); @on_finish = blk; end 10 | def on_connect(&blk); @on_connect = blk; end 11 | 12 | ##### EventMachine 13 | def initialize(options) 14 | @debug = options[:debug] || false 15 | @servers = {} 16 | end 17 | 18 | def receive_data(data) 19 | debug [:connection, data] 20 | processed = @on_data.call(data) if @on_data 21 | 22 | return if processed == :async or processed.nil? 23 | relay_to_servers(processed) 24 | end 25 | 26 | def relay_to_servers(processed) 27 | if processed.is_a? Array 28 | data, servers = *processed 29 | 30 | # guard for "unbound" servers 31 | servers = servers.collect {|s| @servers[s]}.compact 32 | else 33 | data = processed 34 | servers ||= @servers.values.compact 35 | end 36 | 37 | servers.each do |s| 38 | s.send_data data unless data.nil? 39 | end 40 | end 41 | 42 | # 43 | # initialize connections to backend servers 44 | # 45 | def server(name, opts) 46 | srv = EventMachine::connect(opts[:host], opts[:port], EventMachine::ProxyServer::Backend, @debug) do |c| 47 | c.name = name 48 | c.plexer = self 49 | c.proxy_incoming_to(self, 10240) if opts[:relay_server] 50 | end 51 | self.proxy_incoming_to(srv, 10240) if opts[:relay_client] 52 | 53 | @servers[name] = srv 54 | end 55 | 56 | # 57 | # [ip, port] of the connected client 58 | # 59 | def peer 60 | peername = get_peername 61 | @peer ||= peername ? Socket.unpack_sockaddr_in(peername).reverse : nil 62 | end 63 | 64 | # 65 | # relay data from backend server to client 66 | # 67 | def relay_from_backend(name, data) 68 | debug [:relay_from_backend, name, data] 69 | 70 | data = @on_response.call(name, data) if @on_response 71 | send_data data unless data.nil? 72 | end 73 | 74 | def connected(name) 75 | debug [:connected] 76 | @on_connect.call(name) if @on_connect 77 | end 78 | 79 | def unbind 80 | debug [:unbind, :connection] 81 | 82 | # terminate any unfinished connections 83 | @servers.values.compact.each do |s| 84 | s.close_connection 85 | end 86 | end 87 | 88 | def unbind_backend(name) 89 | debug [:unbind_backend, name] 90 | @servers[name] = nil 91 | 92 | # if all connections are terminated downstream, then notify client 93 | close_connection_after_writing if @servers.values.compact.size.zero? 94 | 95 | if @on_finish 96 | @on_finish.call(name) 97 | 98 | # not sure if this is required 99 | # @on_finish.call(:done) if @servers.values.compact.size.zero? 100 | end 101 | end 102 | 103 | private 104 | 105 | def debug(*data) 106 | if @debug 107 | require 'pp' 108 | pp data 109 | puts 110 | end 111 | end 112 | end 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /examples/smtp_spam_filter.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | require 'em-http' 3 | require 'yaml' 4 | require 'net/http' 5 | 6 | Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn| 7 | conn.server :srv, :host => "127.0.0.1", :port => 2525 8 | 9 | RCPT_CMD = /RCPT TO:<(.*)?>\r\n/ # RCPT TO:\r\n 10 | FROM_CMD = /MAIL FROM:<(.*)?>\r\n/ # MAIL FROM:\r\n 11 | MSG_CMD = /354 Start your message/ # 354 Start your message 12 | MSGEND_CMD = /^.\r\n/ 13 | 14 | conn.on_data do |data| 15 | @from = data.match(FROM_CMD)[1] if data.match(FROM_CMD) 16 | @rcpt = data.match(RCPT_CMD)[1] if data.match(RCPT_CMD) 17 | @done = true if data.match(MSGEND_CMD) 18 | 19 | if @buffer 20 | @msg += data 21 | data = nil 22 | end 23 | 24 | if @done 25 | @buffer = false 26 | res = Net::HTTP.post_form(URI.parse('http://api.defensio.com/app/1.2/audit-comment/77ca297d7546705ee2b5136fad0dcaf8.yaml'), { 27 | "owner-url" => "http://www.github.com/igrigorik/em-http-request", 28 | "user-ip" => "216.16.254.254", 29 | "article-date" => "2009/04/01", 30 | "comment-author" => @from, 31 | "comment-type" => "comment", 32 | "comment-content" => @msg}) 33 | 34 | defensio = YAML.load(res.body)['defensio-result'] 35 | p [:defensio, "SPAM: #{defensio['spam']}, Spaminess: #{defensio['spaminess']}"] 36 | 37 | if defensio['spam'] 38 | conn.send_data "550 No such user here\n" 39 | else 40 | data = @msg 41 | end 42 | end 43 | 44 | data 45 | end 46 | 47 | conn.on_response do |server, resp| 48 | p [:resp, resp] 49 | 50 | if resp.match(MSG_CMD) 51 | @buffer = true 52 | @msg = "" 53 | end 54 | 55 | resp 56 | end 57 | end 58 | 59 | # mailtrap run -p 2525 -f /tmp/mailtrap.log 60 | # ruby examples/smtp_spam_filter.rb 61 | # 62 | # >> require 'net/smtp' 63 | # >> smtp = Net::SMTP.start("localhost", 2524) 64 | # >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com" 65 | 66 | 67 | # Protocol trace 68 | # 69 | # [:srv, :conn_complete] 70 | # [:srv, "220 localhost MailTrap ready ESTMP\n"] 71 | # [:relay_from_backend, :srv, "220 localhost MailTrap ready ESTMP\n"] 72 | # [:resp, "220 localhost MailTrap ready ESTMP\n"] 73 | # [:connection, "EHLO localhost.localdomain\r\n"] 74 | # [:srv, "250-localhost offers just ONE extension my pretty"] 75 | # [:relay_from_backend, :srv, "250-localhost offers just ONE extension my pretty"] 76 | # [:resp, "250-localhost offers just ONE extension my pretty"] 77 | # [:srv, "\n250 HELP\n"] 78 | # [:relay_from_backend, :srv, "\n250 HELP\n"] 79 | # [:resp, "\n250 HELP\n"] 80 | # [:connection, "MAIL FROM:\r\n"] 81 | # [:srv, "250 OK\n"] 82 | # [:relay_from_backend, :srv, "250 OK\n"] 83 | # [:resp, "250 OK\n"] 84 | # [:connection, "RCPT TO:\r\n"] 85 | # [:srv, "250 OK"] 86 | # [:relay_from_backend, :srv, "250 OK"] 87 | # [:resp, "250 OK"] 88 | # [:srv, "\n"] 89 | # [:relay_from_backend, :srv, "\n"] 90 | # [:resp, "\n"] 91 | # [:connection, "DATA\r\n"] 92 | # [:srv, "354 Start your message"] 93 | # [:relay_from_backend, :srv, "354 Start your message"] 94 | # [:resp, "354 Start your message"] 95 | # [:srv, "\n"] 96 | # [:relay_from_backend, :srv, "\n"] 97 | # [:resp, "\n"] 98 | # [:connection, "Hello World\r\n"] 99 | # [:connection, ".\r\n"] 100 | # 101 | # [:defensio, "SPAM: false, Spaminess: 0.4"] 102 | # 103 | # [:srv, "250 OK\n"] 104 | # [:relay_from_backend, :srv, "250 OK\n"] 105 | # [:resp, "250 OK\n"] 106 | # 107 | 108 | -------------------------------------------------------------------------------- /spec/proxy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec/helper' 2 | 3 | describe Proxy do 4 | 5 | def failed 6 | EventMachine.stop 7 | fail 8 | end 9 | 10 | it "should recieve data on port 8080" do 11 | EM.run do 12 | EventMachine.add_timer(0.1) do 13 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1}) 14 | end 15 | 16 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 17 | conn.on_data do |data| 18 | data.should =~ /GET \/test/ 19 | EventMachine.stop 20 | end 21 | end 22 | end 23 | end 24 | 25 | it "should call the on_connect callback" do 26 | connected = false 27 | EM.run do 28 | EventMachine.add_timer(0.1) do 29 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 30 | end 31 | 32 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 33 | conn.server :goog, :host => "google.com", :port => 80 34 | 35 | conn.on_connect do |name| 36 | connected = true 37 | EventMachine.stop 38 | end 39 | end 40 | end 41 | connected.should == true 42 | end 43 | 44 | 45 | it "should transparently redirect TCP traffic to google" do 46 | EM.run do 47 | EventMachine.add_timer(0.1) do 48 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 49 | end 50 | 51 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 52 | conn.server :goog, :host => "google.com", :port => 80 53 | conn.on_data { |data| data } 54 | 55 | conn.on_response do |backend, resp| 56 | backend.should == :goog 57 | resp.should =~ /google/ 58 | EventMachine.stop 59 | end 60 | end 61 | end 62 | end 63 | 64 | it "should duplex TCP traffic to two backends google & yahoo" do 65 | EM.run do 66 | EventMachine.add_timer(0.1) do 67 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1}) 68 | end 69 | 70 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 71 | conn.server :goog, :host => "google.com", :port => 80 72 | conn.server :yhoo, :host => "yahoo.com", :port => 80 73 | conn.on_data { |data| data } 74 | 75 | seen = [] 76 | conn.on_response do |backend, resp| 77 | case backend 78 | when :goog then 79 | resp.should =~ /404/ 80 | seen.push backend 81 | when :yhoo 82 | resp.should =~ /404/ 83 | seen.push backend 84 | end 85 | seen.uniq! 86 | 87 | EventMachine.stop if seen.size == 2 88 | end 89 | end 90 | end 91 | end 92 | 93 | it "should intercept & alter response from Google" do 94 | EM.run do 95 | EventMachine.add_timer(0.1) do 96 | http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 97 | http.errback { failed } 98 | http.callback { 99 | http.response_header.status.should == 404 100 | EventMachine.stop 101 | } 102 | end 103 | 104 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 105 | conn.server :goog, :host => "google.com", :port => 80 106 | conn.on_data { |data| data } 107 | conn.on_response do |backend, data| 108 | data.gsub(/^HTTP\/1.1 200/, 'HTTP/1.1 404') 109 | end 110 | end 111 | end 112 | end 113 | 114 | it "should invoke on_finish callback when connection is terminated" do 115 | EM.run do 116 | EventMachine.add_timer(0.1) do 117 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 118 | end 119 | 120 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 121 | conn.server :goog, :host => "google.com", :port => 80 122 | conn.on_data { |data| data } 123 | conn.on_response { |backend, resp| resp } 124 | conn.on_finish do |backend| 125 | backend.should == :goog 126 | EventMachine.stop 127 | end 128 | end 129 | end 130 | end 131 | 132 | it "should not invoke on_data when :relay_client is passed as server option" do 133 | lambda { 134 | EM.run do 135 | EventMachine.add_timer(0.1) do 136 | http =EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 137 | http.callback { EventMachine.stop } 138 | end 139 | 140 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 141 | conn.server :goog, :host => "google.com", :port => 80, :relay_client => true 142 | conn.on_data { |data| raise "Should not be here"; data } 143 | conn.on_response { |backend, resp| resp } 144 | 145 | end 146 | end 147 | }.should_not raise_error 148 | end 149 | 150 | it "should not invoke on_response when :relay_server is passed as server option" do 151 | lambda { 152 | EM.run do 153 | EventMachine.add_timer(0.1) do 154 | http =EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 155 | http.callback { EventMachine.stop } 156 | end 157 | 158 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 159 | conn.server :goog, :host => "google.com", :port => 80, :relay_server => true 160 | conn.on_data { |data| data } 161 | conn.on_response { |backend, resp| raise "Should not be here"; } 162 | 163 | end 164 | end 165 | }.should_not raise_error 166 | end 167 | end -------------------------------------------------------------------------------- /examples/schemaless-mysql/mysql_interceptor.rb: -------------------------------------------------------------------------------- 1 | require "lib/em-proxy" 2 | require "em-mysql" 3 | require "stringio" 4 | require "fiber" 5 | 6 | Proxy.start(:host => "0.0.0.0", :port => 3307) do |conn| 7 | conn.server :mysql, :host => "127.0.0.1", :port => 3306, :relay_server => true 8 | 9 | QUERY_CMD = 3 10 | MAX_PACKET_LENGTH = 2**24-1 11 | 12 | # open a direct connection to MySQL for the schema-free coordination logic 13 | @mysql = EventMachine::MySQL.new(:host => 'localhost', :database => 'noschema') 14 | 15 | conn.on_data do |data| 16 | fiber = Fiber.new { 17 | p [:original_request, data] 18 | 19 | overhead, chunks, seq = data[0,4].unpack("CvC") 20 | type, sql = data[4, data.size].unpack("Ca*") 21 | 22 | p [:request, [overhead, chunks, seq], [type, sql]] 23 | 24 | if type == QUERY_CMD 25 | query = sql.downcase.split 26 | p [:query, query] 27 | 28 | # TODO: can probably switch to http://github.com/omghax/sql 29 | # for AST query parsing & mods. 30 | 31 | case query.first 32 | when "create" then 33 | # Allow schemaless table creation, ex: 'create table posts' 34 | # By creating a table with a single id for key storage, aka 35 | # rewrite to: 'create table posts (id varchar(255))'. All 36 | # future attribute tables will be created on demand at 37 | # insert time of a new record 38 | overload = "(id varchar(255), UNIQUE(id));" 39 | query += [overload] 40 | overhead += overload.size + 1 41 | 42 | p [:create_new_schema_free_table, query, data] 43 | 44 | when "insert" then 45 | # Overload the INSERT syntax to allow for nested parameters 46 | # inside the statement. ex: 47 | # INSERT INTO posts (id, author, nickname, ...) VALUES ( 48 | # 'ilya', 'Ilya Grigorik', 'igrigorik' 49 | # ) 50 | # 51 | # The following query will be mapped into 3 distinct tables: 52 | # => 'posts' table will store the key 53 | # => 'posts_author' will store key, value 54 | # => 'posts_nickname' will store key, value 55 | # 56 | # or, in SQL.. 57 | # 58 | # => insert into posts values("ilya"); 59 | # => create table posts_author (id varchar(40), value varchar(255), UNIQUE(id)); 60 | # => insert into posts_author values("ilya", "Ilya Grigorik"); 61 | # => ... repeat for every attribute 62 | # 63 | # If the table post_value has not been seen before, it will 64 | # be created on the fly. Hence allowing us to add and remove 65 | # keys and values at will. :-) 66 | # 67 | # P.S. There is probably cleaner syntax for this, but hey... 68 | 69 | 70 | if insert = sql.match(/\((.*?)\).*?\((.*?)\)/) 71 | data = {} 72 | table = query[2] 73 | keys = insert[1].split(',').map!{|s| s.strip} 74 | values = insert[2].scan(/([^\'|\"]+)/).flatten.reject {|s| s.strip == ','} 75 | keys.each_with_index {|k,i| data[k] = values[i]} 76 | 77 | data.each do |key, value| 78 | next if key == 'id' 79 | attr_sql = "insert into #{table}_#{key} values('#{data['id']}', '#{value}')" 80 | 81 | q = @mysql.query(attr_sql) 82 | q.errback { |res| 83 | # if the attribute table for this model does not yet exist then create it! 84 | # - yes, there is a race condition here, add fiber logic later 85 | if res.is_a?(Mysql::Error) and res.message =~ /Table.*doesn\'t exist/ 86 | 87 | table_sql = "create table #{table}_#{key} (id varchar(255), value varchar(255), UNIQUE(id))" 88 | tc = @mysql.query(table_sql) 89 | tc.callback { @mysql.query(attr_sql) } 90 | end 91 | } 92 | 93 | p [:inserted_attr, table, key, value] 94 | end 95 | 96 | # override the query to insert the key into posts table 97 | query = query[0,3] + ["VALUES('#{data['id']}')"] 98 | overhead = query.join(" ").size + 1 99 | 100 | p [:insert, query] 101 | end 102 | 103 | when "select" then 104 | # Overload the select call to perform a multi-join in the background 105 | # and rewrite the attribute names to fool the client into thinking it 106 | # all came from the same table. 107 | # 108 | # To figure out which tables we need to join on, do the simple / dumb 109 | # approach and issue a 'show tables like key_%' to do 'runtime 110 | # introspection'. Could easily cache this, but that's for later. 111 | # 112 | # Ex, a 'select * from posts' query with one value (author) would be 113 | # rewritten into the following query: 114 | # 115 | # SELECT posts.id as id, posts_author.value as author FROM posts 116 | # LEFT OUTER JOIN posts_author ON posts_author.id = posts.id 117 | # WHERE posts.id = "ilya"; 118 | 119 | select = sql.match(/select(.*?)from\s([^\s]+)/) 120 | where = sql.match(/where\s([^=]+)\s?=\s?'?"?([^\s'"]+)'?"?/) 121 | attrs, table = select[1].strip.split(','), select[2] if select 122 | key = where[2] if where 123 | 124 | if select 125 | p [:select, select, attrs, where] 126 | 127 | tables = @mysql.query("show tables like '#{table}_%'") 128 | tables.callback { |res| 129 | fiber.resume(res.all_hashes.collect(&:values).flatten.collect{ |c| 130 | c.split('_').last 131 | }) 132 | } 133 | tables = Fiber.yield 134 | 135 | p [:select_tables, tables] 136 | 137 | # build the select statements, hide the tables behind each attribute 138 | join = "select #{table}.id as id " 139 | tables.each do |column| 140 | join += " , #{table}_#{column}.value as #{column} " 141 | end 142 | 143 | # add the joins to stich it all together 144 | join += " FROM #{table} " 145 | tables.each do |column| 146 | join += " LEFT OUTER JOIN #{table}_#{column} ON #{table}_#{column}.id = #{table}.id " 147 | end 148 | 149 | join += " WHERE #{table}.id = '#{key}' " if key 150 | 151 | query = [join] 152 | overhead = join.size + 1 153 | 154 | p [:join_query, join] 155 | end 156 | end 157 | 158 | # repack the query data and forward to server 159 | # - have to split message on packet boundaries 160 | 161 | seq, data = 0, [] 162 | query = StringIO.new([type, query.join(" ")].pack("Ca*")) 163 | while q = query.read(MAX_PACKET_LENGTH) 164 | data.push [q.length % 256, q.length / 256, seq].pack("CvC") + q 165 | seq = (seq + 1) % 256 166 | end 167 | 168 | p [:final_query, data, chunks, overhead] 169 | puts "-" * 100 170 | end 171 | 172 | [data].flatten.each do |chunk| 173 | conn.relay_to_servers(chunk) 174 | end 175 | 176 | :async # we will render results later 177 | } 178 | 179 | fiber.resume 180 | end 181 | end 182 | --------------------------------------------------------------------------------