├── .gitignore ├── .rspec ├── .travis.yml ├── CHANGELOG.md ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── README.md ├── Rakefile ├── bin └── em-proxy ├── em-proxy.gemspec ├── examples ├── appserver.rb ├── balancing-client.rb ├── balancing.rb ├── beanstalkd_interceptor.rb ├── duplex.rb ├── http_proxy.rb ├── line_interceptor.rb ├── port_forward.rb ├── relay_port_forward.rb ├── schemaless-mysql │ └── mysql_interceptor.rb ├── selective_forward.rb ├── simple.rb ├── smtp_spam_filter.rb └── smtp_whitelist.rb ├── lib ├── em-proxy.rb └── em-proxy │ ├── backend.rb │ ├── connection.rb │ └── proxy.rb └── spec ├── balancing_spec.rb ├── helper.rb ├── proxy_spec.rb └── support └── echo_server.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | pkg 3 | 4 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format=documentation 2 | --color 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | rvm: 4 | - 2.3.0 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.1.8 (December 12, 2012) 2 | -------------------- 3 | 4 | * [#28](https://github.com/igrigorik/em-proxy/pull/28) - Fix: bin script - close connections only on relay response - [@bo-chen](https://github.com/bo-chen). 5 | * [#31](https://github.com/igrigorik/em-proxy/pull/31) - Added support for proxying to a unix domain socket - [@dblock](https://github.com/dblock). 6 | * [#34](https://github.com/igrigorik/em-proxy/pull/34) - Fix: duplex TCP traffic to two backends spec race condition - [@dblock](https://github.com/dblock). 7 | 8 | 0.1.7 (October 13, 2012) 9 | ------------------------ 10 | 11 | * Allow force-close on upstream connections - [@igrigoric](https://github.com/igrigorik). 12 | * [#25](https://github.com/igrigorik/em-proxy/pull/25): Added `bind` support - [@kostya](https://github.com/kostya). 13 | * [#27](https://github.com/igrigorik/em-proxy/pull/27): Alias `sock` for `get_sockname` - [@kostya](https://github.com/kostya). 14 | 15 | 0.1.6 (December 27, 2011) 16 | ------------------------- 17 | 18 | * Added HTTP proxy example - [@igrigoric](https://github.com/igrigorik). 19 | * [#11](https://github.com/igrigorik/em-proxy/issues/11) - Fix: closing the client connection immediately after servers connection are closed - [@igrigoric](https://github.com/igrigorik). 20 | * [#13](https://github.com/igrigorik/em-proxy/pull/13): Removed duplicate `unbind_backend` - [@outself](https://github.com/outself). 21 | * [#20](https://github.com/igrigorik/em-proxy/issues/20): Fix: don't buffer data in back-end - [@igrigoric](https://github.com/igrigorik). 22 | 23 | 0.1.5 (January 16, 2011) 24 | ------------------------ 25 | 26 | * Added `em-proxy` bin script for easy proxy debugging & relay use cases - [@igrigoric](https://github.com/igrigorik). 27 | * Replaced Jeweler with Bundler - [@igrigoric](https://github.com/igrigorik). 28 | * Added example of a simple load-balancing proxy - [@karmi](https://github.com/karmi). 29 | 30 | 0.1.4 (October 3, 2010) 31 | ----------------------- 32 | 33 | * Fix: use `instance_eval` to allow unbind - [@igrigoric](https://github.com/igrigorik). 34 | 35 | 0.1.3 (May 29, 2010) 36 | -------------------- 37 | 38 | * Fix: `on_connect` should fire after connection is established to each backend - [@igrigoric](https://github.com/igrigorik). 39 | * Fix: `get_peername` can return nil - [@mdkent](https://github.com/mdkent). 40 | 41 | 0.1.2 (March 26, 2010) 42 | ---------------------- 43 | 44 | * Fix: wait until finishing writing on the frontend - [@eudoxa](https://github.com/eudoxa). 45 | * Removed `:done` callback in `on_finish` - [@igrigoric](https://github.com/igrigorik). 46 | * Ruby 1.9 compatibility - [@dsander](https://github.com/dsander). 47 | * Use EM's `proxy_incomming_to` to do low-level data relaying - [@dsander](https://github.com/dsander). 48 | * Use `Proc#call` instead of `Object#instance_exec` - [@dsander](https://github.com/dsander). 49 | * Added `on_connect` callback, peer helper method - [@dsander](https://github.com/dsander). 50 | * Added schema-free mysql example - [@igrigoric](https://github.com/igrigorik). 51 | * Added support for async processing within the `on_data` callback - [@igrigoric](https://github.com/igrigorik). 52 | 53 | 0.1.1 (October 25, 2009) 54 | ------------------------ 55 | 56 | * Initial public release - [@igrigoric](https://github.com/igrigorik). 57 | * Simple port forwarder - [@igrigoric](https://github.com/igrigorik). 58 | * Duplex, interceptor, smtp whitelist, beanstalkd interceptor, smtp spam filter and selective forward examples - [@igrigoric](https://github.com/igrigorik). 59 | * Control debug output - [@imbriaco](https://github.com/imbriaco). 60 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | em-proxy (0.1.8) 5 | eventmachine 6 | 7 | GEM 8 | remote: http://rubygems.org/ 9 | specs: 10 | addressable (2.4.0) 11 | ansi (1.5.0) 12 | cookiejar (0.3.3) 13 | diff-lcs (1.2.5) 14 | em-http-request (1.1.4) 15 | addressable (>= 2.3.4) 16 | cookiejar (!= 0.3.1) 17 | em-socksify (>= 0.3) 18 | eventmachine (>= 1.0.3) 19 | http_parser.rb (>= 0.6.0) 20 | em-socksify (0.3.1) 21 | eventmachine (>= 1.0.0.beta.4) 22 | eventmachine (1.2.0.1) 23 | http_parser.rb (0.6.0) 24 | posix-spawn (0.3.11) 25 | rake (11.2.2) 26 | rspec (3.4.0) 27 | rspec-core (~> 3.4.0) 28 | rspec-expectations (~> 3.4.0) 29 | rspec-mocks (~> 3.4.0) 30 | rspec-core (3.4.4) 31 | rspec-support (~> 3.4.0) 32 | rspec-expectations (3.4.0) 33 | diff-lcs (>= 1.2.0, < 2.0) 34 | rspec-support (~> 3.4.0) 35 | rspec-mocks (3.4.1) 36 | diff-lcs (>= 1.2.0, < 2.0) 37 | rspec-support (~> 3.4.0) 38 | rspec-support (3.4.1) 39 | 40 | PLATFORMS 41 | ruby 42 | 43 | DEPENDENCIES 44 | ansi 45 | em-http-request 46 | em-proxy! 47 | posix-spawn 48 | rake 49 | rspec 50 | 51 | BUNDLED WITH 52 | 1.12.5 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2009 Ilya Grigorik 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EM-Proxy 2 | 3 | [![Build Status](https://travis-ci.org/igrigorik/em-proxy.png?branch=master)](https://travis-ci.org/igrigorik/em-proxy) 4 | 5 | EventMachine Proxy DSL for writing high-performance transparent / intercepting proxies in Ruby. 6 | 7 | - EngineYard tutorial: [Load testing your environment using em-proxy](http://docs.engineyard.com/em-proxy.html) 8 | - [Slides from RailsConf 2009](http://bit.ly/D7oWB) 9 | - [GoGaRuCo notes & Slides](http://www.igvita.com/2009/04/20/ruby-proxies-for-scale-and-monitoring/) 10 | 11 | ## Getting started 12 | 13 | $> gem install em-proxy 14 | $> em-proxy 15 | Usage: em-proxy [options] 16 | -l, --listen [PORT] Port to listen on 17 | -d, --duplex [host:port, ...] List of backends to duplex data to 18 | -r, --relay [hostname:port] Relay endpoint: hostname:port 19 | -s, --socket [filename] Relay endpoint: unix filename 20 | -v, --verbose Run in debug mode 21 | 22 | $> em-proxy -l 8080 -r localhost:8081 -d localhost:8082,localhost:8083 -v 23 | 24 | The above will start em-proxy on port 8080, relay and respond with data from port 8081, and also (optional) duplicate all traffic to ports 8082 and 8083 (and discard their responses). 25 | 26 | 27 | ## Simple port forwarding proxy 28 | 29 | ```ruby 30 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn| 31 | conn.server :srv, :host => "127.0.0.1", :port => 81 32 | 33 | # modify / process request stream 34 | conn.on_data do |data| 35 | p [:on_data, data] 36 | data 37 | end 38 | 39 | # modify / process response stream 40 | conn.on_response do |backend, resp| 41 | p [:on_response, backend, resp] 42 | resp 43 | end 44 | 45 | # termination logic 46 | conn.on_finish do |backend, name| 47 | p [:on_finish, name] 48 | 49 | # terminate connection (in duplex mode, you can terminate when prod is done) 50 | unbind if backend == :srv 51 | end 52 | end 53 | ``` 54 | 55 | For more examples see the /examples directory. 56 | 57 | - SMTP Spam Filtering 58 | - Duplicating traffic 59 | - Selective forwarding 60 | - Beanstalkd interceptor 61 | - etc. 62 | 63 | A schema-free MySQL proof of concept, via an EM-Proxy server: 64 | 65 | - http://www.igvita.com/2010/03/01/schema-free-mysql-vs-nosql/ 66 | - Code in examples/schemaless-mysql 67 | 68 | ## License 69 | 70 | The MIT License - Copyright (c) 2010 Ilya Grigorik 71 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler::GemHelper.install_tasks 3 | 4 | require 'rspec/core/rake_task' 5 | 6 | desc "Run all RSpec tests" 7 | RSpec::Core::RakeTask.new(:spec) 8 | 9 | task :default => :spec 10 | task :test => [:spec] -------------------------------------------------------------------------------- /bin/em-proxy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | lib = File.expand_path(File.dirname(__FILE__) + '/../lib') 4 | $LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib) 5 | 6 | require 'em-proxy' 7 | require 'optparse' 8 | 9 | ARGV << '--help' if ARGV.empty? 10 | 11 | options = {} 12 | OptionParser.new do |opts| 13 | opts.banner = "Usage: em-proxy [options]" 14 | 15 | opts.on("-l", "--listen [PORT]", Integer, "Port to listen on") do |v| 16 | options[:listen] = v 17 | end 18 | 19 | opts.on("-d", "--duplex [host:port, ...]", Array, "List of backends to duplex data to") do |v| 20 | options[:duplex] = v 21 | end 22 | 23 | opts.on("-r", "--relay [hostname:port]", String, "Relay endpoint: hostname:port") do |v| 24 | options[:relay] = v.split(":") 25 | end 26 | 27 | opts.on("-s", "--socket [filename]", String, "Relay endpoint: unix filename") do |v| 28 | options[:socket] = v 29 | end 30 | 31 | opts.on("-v", "--verbose", "Run in debug mode") do |v| 32 | options[:verbose] = v 33 | end 34 | end.parse! 35 | 36 | 37 | Proxy.start(:host => "0.0.0.0", :port => options[:listen] , :debug => options[:verbose]) do |conn| 38 | if options[:socket] 39 | conn.server :socket, :socket => options[:socket] 40 | else 41 | conn.server :relay, :host => options[:relay].first, :port => options[:relay].last.to_i 42 | end 43 | 44 | options[:duplex].each_with_index do |backend,i| 45 | hostname, port = backend.split(":") 46 | conn.server "backend_#{i}".intern, :host => hostname, :port => port.to_i 47 | end if options[:duplex] 48 | 49 | conn.on_data do |data| 50 | data 51 | end 52 | 53 | conn.on_response do |server, resp| 54 | resp if server == :relay 55 | end 56 | 57 | conn.on_finish do |server| 58 | :close if server == :relay 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /em-proxy.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "em-proxy" 6 | s.version = "0.1.9" 7 | s.platform = Gem::Platform::RUBY 8 | s.authors = ["Ilya Grigorik"] 9 | s.email = ["ilya@igvita.com"] 10 | s.homepage = "http://github.com/igrigorik/em-proxy" 11 | s.summary = %q{EventMachine Proxy DSL} 12 | s.description = s.summary 13 | 14 | s.rubyforge_project = "em-proxy" 15 | 16 | s.add_dependency "eventmachine" 17 | s.add_development_dependency "rspec" 18 | s.add_development_dependency "em-http-request" 19 | s.add_development_dependency "ansi" 20 | s.add_development_dependency "rake" 21 | s.add_development_dependency "posix-spawn" 22 | 23 | s.files = `git ls-files`.split("\n") 24 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 25 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 26 | s.require_paths = ["lib"] 27 | end 28 | -------------------------------------------------------------------------------- /examples/appserver.rb: -------------------------------------------------------------------------------- 1 | require "rubygems" 2 | require "rack" 3 | 4 | app = Proc.new do 5 | p [:serving, ARGV[0]] 6 | 7 | sleep(ARGV[1].to_i) 8 | [200, {"Content-Type" => "text/plain"}, ["hello world: #{ARGV[1]}"]] 9 | end 10 | 11 | Rack::Handler::Mongrel.run(app, {:Host => "0.0.0.0", :Port => ARGV[0]}) 12 | -------------------------------------------------------------------------------- /examples/balancing-client.rb: -------------------------------------------------------------------------------- 1 | # A simple HTTP client, which sends multiple requests to the proxy server 2 | 3 | #!/usr/bin/env ruby 4 | 5 | require 'net/http' 6 | 7 | proxy = Net::HTTP::Proxy('0.0.0.0', '9999') 8 | 9 | count = ENV['COUNT'] || 5 10 | 11 | threads = [] 12 | count.to_i.times do |i| 13 | threads << Thread.new do 14 | proxy.start('www.example.com') do |http| 15 | puts http.get('/').body 16 | puts "^^^ #{i+1} " + '-'*80 + "\n\n" 17 | end 18 | sleep 0.1 19 | end 20 | end 21 | 22 | threads.each { |t| t.join } 23 | -------------------------------------------------------------------------------- /examples/balancing.rb: -------------------------------------------------------------------------------- 1 | $:<< '../lib' << 'lib' 2 | 3 | require 'em-proxy' 4 | require 'ansi/code' 5 | require 'uri' 6 | 7 | # = Balancing Proxy 8 | # 9 | # A simple example of a balancing, reverse/forward proxy such as Nginx or HAProxy. 10 | # 11 | # Given a list of backends, it's able to distribute requests to backends 12 | # via different strategies (_random_, _roundrobin_, _balanced_), see Backend.select. 13 | # 14 | # This example is provided for didactic purposes. Nevertheless, based on some preliminary benchmarks 15 | # and live tests, it performs well in production usage. 16 | # 17 | # You can customize the behaviour of the proxy by changing the BalancingProxy::Callbacks 18 | # callbacks. To give you some ideas: 19 | # 20 | # * Store statistics for the proxy load in Redis (eg.: $redis.incr "proxy>backends>#{backend}>total") 21 | # * Use Redis' _SortedSet_ instead of updating the Backend.list hash to allow for polling from external process 22 | # * Use em-websocket[https://github.com/igrigorik/em-websocket] to build a web frontend for monitoring 23 | # 24 | module BalancingProxy 25 | extend self 26 | 27 | BACKENDS = [ 28 | {:url => "http://0.0.0.0:3000"}, 29 | {:url => "http://0.0.0.0:3001"}, 30 | {:url => "http://0.0.0.0:3002"} 31 | ] 32 | 33 | # Represents a "backend", ie. the endpoint for the proxy. 34 | # 35 | # This could be eg. a WEBrick webserver (see below), so the proxy server works as a _reverse_ proxy. 36 | # But it could also be a proxy server, so the proxy server works as a _forward_ proxy. 37 | # 38 | class Backend 39 | 40 | attr_reader :url, :host, :port 41 | attr_accessor :load 42 | alias :to_s :url 43 | 44 | def initialize(options={}) 45 | raise ArgumentError, "Please provide a :url and :load" unless options[:url] 46 | @url = options[:url] 47 | @load = options[:load] || 0 48 | parsed = URI.parse(@url) 49 | @host, @port = parsed.host, parsed.port 50 | end 51 | 52 | # Select backend: balanced, round-robin or random 53 | # 54 | def self.select(strategy = :balanced) 55 | @strategy = strategy.to_sym 56 | case @strategy 57 | when :balanced 58 | pp [list, list.sort_by { |b| b.load }.first] 59 | backend = list.sort_by { |b| b.load }.first 60 | when :roundrobin 61 | @pool = list.clone if @pool.nil? || @pool.empty? 62 | backend = @pool.shift 63 | when :random 64 | backend = list[ rand(list.size-1) ] 65 | else 66 | raise ArgumentError, "Unknown strategy: #{@strategy}" 67 | end 68 | 69 | Callbacks.on_select.call(backend) 70 | yield backend if block_given? 71 | backend 72 | end 73 | 74 | # List of backends 75 | # 76 | def self.list 77 | @list ||= BACKENDS.map { |backend| new backend } 78 | end 79 | 80 | # Return balancing strategy 81 | # 82 | def self.strategy 83 | @strategy 84 | end 85 | 86 | # Increment "currently serving requests" counter 87 | # 88 | def increment_counter 89 | self.load += 1 90 | end 91 | 92 | # Decrement "currently serving requests" counter 93 | # 94 | def decrement_counter 95 | self.load -= 1 96 | end 97 | 98 | end 99 | 100 | # Callbacks for em-proxy events 101 | # 102 | module Callbacks 103 | include ANSI::Code 104 | extend self 105 | 106 | def on_select 107 | lambda do |backend| 108 | puts black_on_white { 'on_select'.ljust(12) } + " #{backend.inspect}" 109 | backend.increment_counter if Backend.strategy == :balanced 110 | end 111 | end 112 | 113 | def on_connect 114 | lambda do |backend| 115 | puts black_on_magenta { 'on_connect'.ljust(12) } + ' ' + bold { backend } 116 | end 117 | end 118 | 119 | def on_data 120 | lambda do |data| 121 | puts black_on_yellow { 'on_data'.ljust(12) }, data 122 | data 123 | end 124 | end 125 | 126 | def on_response 127 | lambda do |backend, resp| 128 | puts black_on_green { 'on_response'.ljust(12) } + " from #{backend}", resp 129 | resp 130 | end 131 | end 132 | 133 | def on_finish 134 | lambda do |backend| 135 | puts black_on_cyan { 'on_finish'.ljust(12) } + " for #{backend}", '' 136 | backend.decrement_counter if Backend.strategy == :balanced 137 | end 138 | end 139 | 140 | end 141 | 142 | # Wrapping the proxy server 143 | # 144 | module Server 145 | def run(host='0.0.0.0', port=9999) 146 | 147 | puts ANSI::Code.bold { "Launching proxy at #{host}:#{port}...\n" } 148 | 149 | Proxy.start(:host => host, :port => port, :debug => false) do |conn| 150 | 151 | Backend.select do |backend| 152 | 153 | conn.server backend, :host => backend.host, :port => backend.port 154 | 155 | conn.on_connect &Callbacks.on_connect 156 | conn.on_data &Callbacks.on_data 157 | conn.on_response &Callbacks.on_response 158 | conn.on_finish &Callbacks.on_finish 159 | end 160 | 161 | end 162 | end 163 | 164 | module_function :run 165 | end 166 | 167 | end 168 | 169 | if __FILE__ == $0 170 | 171 | require 'rack' 172 | 173 | class Proxy 174 | def self.stop 175 | puts "Terminating ProxyServer" 176 | EventMachine.stop 177 | $servers.each do |pid| 178 | puts "Terminating webserver #{pid}" 179 | Process.kill('KILL', pid) 180 | end 181 | end 182 | end 183 | 184 | # Simple Rack app to run 185 | app = proc { |env| [ 200, {'Content-Type' => 'text/plain'}, ["Hello World!"] ] } 186 | 187 | # Run app on ports 3000-3002 188 | $servers = [] 189 | 3.times do |i| 190 | $servers << Process.fork { Rack::Handler::WEBrick.run(app, {:Host => "0.0.0.0", :Port => "300#{i}"}) } 191 | end 192 | 193 | puts ANSI::Code::green_on_black { "\n=> Send multiple requests to the proxy by running `ruby balancing-client.rb`\n" } 194 | 195 | # Start proxy 196 | BalancingProxy::Server.run 197 | 198 | end 199 | -------------------------------------------------------------------------------- /examples/beanstalkd_interceptor.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 11300) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 11301 5 | 6 | # put \r\n 7 | PUT_CMD = /put (\d+) (\d+) (\d+) (\d+)\r\n/ 8 | 9 | conn.on_data do |data| 10 | if put = data.match(PUT_CMD) 11 | 12 | # archive any job > 10 minutes away 13 | if put[2].to_i > 600 14 | p [:put, :archive] 15 | # INSERT INTO .... 16 | 17 | conn.send_data "INSERTED 9999\r\n" 18 | data = nil 19 | end 20 | end 21 | 22 | data 23 | end 24 | 25 | conn.on_response do |backend, resp| 26 | p [:resp, resp] 27 | resp 28 | end 29 | end 30 | 31 | # 32 | # beanstalkd -p 11301 -d 33 | # ruby examples/beanstalkd_interceptor.rb 34 | # 35 | # irb 36 | # >> require 'beanstalk-client' 37 | # >> beanstalk = Beanstalk::Pool.new(['127.0.0.1']) 38 | # >> beanstalk.put("job1") 39 | # => 1 40 | # >> beanstalk.put("job2") 41 | # => 2 42 | # >> beanstalk.put("job3", 0, 1000) 43 | # => 9999 44 | -------------------------------------------------------------------------------- /examples/duplex.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 8000, :debug => true) do |conn| 4 | @start = Time.now 5 | @data = Hash.new("") 6 | 7 | conn.server :prod, :host => "127.0.0.1", :port => 8100 # production, will render resposne 8 | conn.server :test, :host => "127.0.0.1", :port => 8200 # testing, internal only 9 | 10 | conn.on_data do |data| 11 | # rewrite User-Agent 12 | data.gsub(/User-Agent: .*?\r\n/, "User-Agent: em-proxy/0.1\r\n") 13 | end 14 | 15 | conn.on_response do |server, resp| 16 | # only render response from production 17 | @data[server] += resp 18 | resp if server == :prod 19 | end 20 | 21 | conn.on_finish do |name| 22 | p [:on_finish, name, Time.now - @start] 23 | p @data 24 | :close if name == :prod 25 | end 26 | end 27 | 28 | # 29 | # ruby examples/appserver.rb 8100 1 30 | # ruby examples/appserver.rb 8200 3 31 | # ruby examples/line_interceptor.rb 32 | # curl localhost:8000 33 | # 34 | # > [:on_finish, 1.008561] 35 | # > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0", 36 | # :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"} 37 | # 38 | -------------------------------------------------------------------------------- /examples/http_proxy.rb: -------------------------------------------------------------------------------- 1 | require 'em-proxy' 2 | require 'http/parser' # gem install http_parser.rb 3 | require 'uuid' # gem install uuid 4 | 5 | # > ruby http_proxy.rb 6 | # > curl --proxy localhost:9889 www.google.com 7 | # > curl --proxy x.x.x.x:9889 www.google.com - bind ip example 8 | 9 | host = "0.0.0.0" 10 | port = 9889 11 | puts "listening on #{host}:#{port}..." 12 | 13 | Proxy.start(:host => host, :port => port) do |conn| 14 | @buffer = '' 15 | 16 | @p = Http::Parser.new 17 | @p.on_headers_complete = proc do |h| 18 | session = UUID.generate 19 | puts "New session: #{session} (#{h.inspect})" 20 | 21 | host, port = h['Host'].split(':') 22 | conn.server session, :host => host, :port => (port || 80) #, :bind_host => conn.sock[0] - # for bind ip 23 | conn.relay_to_servers @buffer 24 | 25 | @buffer.clear 26 | end 27 | 28 | conn.on_connect do |data,b| 29 | puts [:on_connect, data, b].inspect 30 | end 31 | 32 | conn.on_data do |data| 33 | @buffer << data 34 | @p << data 35 | 36 | data 37 | end 38 | 39 | conn.on_response do |backend, resp| 40 | puts [:on_response, backend, resp].inspect 41 | resp 42 | end 43 | 44 | conn.on_finish do |backend, name| 45 | puts [:on_finish, name].inspect 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /examples/line_interceptor.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 81 5 | 6 | conn.on_data do |data| 7 | data 8 | end 9 | 10 | conn.on_response do |backend, resp| 11 | # substitute all mentions of hello to 'good bye', aka intercepting proxy 12 | resp.gsub(/hello/, 'good bye') 13 | end 14 | end 15 | 16 | # 17 | # ruby examples/appserver.rb 81 18 | # ruby examples/line_interceptor.rb 19 | # curl localhost 20 | # 21 | # > good bye world: 0 22 | # -------------------------------------------------------------------------------- /examples/port_forward.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 81 5 | 6 | # modify / process request stream 7 | conn.on_data do |data| 8 | p [:on_data, data] 9 | data 10 | end 11 | 12 | # modify / process response stream 13 | conn.on_response do |backend, resp| 14 | p [:on_response, backend, resp] 15 | # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!" 16 | resp 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /examples/relay_port_forward.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80, :debug => false) do |conn| 4 | # Specifying :relay_server or :relay_client is useful if only requests or responses 5 | # need to be processed. The proxy throughput will roughly double. 6 | conn.server :srv, :host => "127.0.0.1", :port => 81, :relay_client => true, :relay_server => true 7 | 8 | conn.on_connect do 9 | p [:on_connect, "#{conn.peer.join(':')} connected"] 10 | end 11 | 12 | # modify / process request stream 13 | # on_data will not be called when :relay_server => true is passed as server option 14 | conn.on_data do |data| 15 | p [:on_data, data] 16 | data 17 | end 18 | 19 | # modify / process response stream 20 | # on_response will not be called when :relay_client => true is passed as server option 21 | conn.on_response do |backend, resp| 22 | p [:on_response, backend, resp] 23 | # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!" 24 | resp 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /examples/schemaless-mysql/mysql_interceptor.rb: -------------------------------------------------------------------------------- 1 | require "lib/em-proxy" 2 | require "em-mysql" 3 | require "stringio" 4 | require "fiber" 5 | 6 | Proxy.start(:host => "0.0.0.0", :port => 3307) do |conn| 7 | conn.server :mysql, :host => "127.0.0.1", :port => 3306, :relay_server => true 8 | 9 | QUERY_CMD = 3 10 | MAX_PACKET_LENGTH = 2**24-1 11 | 12 | # open a direct connection to MySQL for the schema-free coordination logic 13 | @mysql = EventMachine::MySQL.new(:host => 'localhost', :database => 'noschema') 14 | 15 | conn.on_data do |data| 16 | fiber = Fiber.new { 17 | p [:original_request, data] 18 | 19 | overhead, chunks, seq = data[0,4].unpack("CvC") 20 | type, sql = data[4, data.size].unpack("Ca*") 21 | 22 | p [:request, [overhead, chunks, seq], [type, sql]] 23 | 24 | if type == QUERY_CMD 25 | query = sql.downcase.split 26 | p [:query, query] 27 | 28 | # TODO: can probably switch to http://github.com/omghax/sql 29 | # for AST query parsing & mods. 30 | 31 | case query.first 32 | when "create" then 33 | # Allow schemaless table creation, ex: 'create table posts' 34 | # By creating a table with a single id for key storage, aka 35 | # rewrite to: 'create table posts (id varchar(255))'. All 36 | # future attribute tables will be created on demand at 37 | # insert time of a new record 38 | overload = "(id varchar(255), UNIQUE(id));" 39 | query += [overload] 40 | overhead += overload.size + 1 41 | 42 | p [:create_new_schema_free_table, query, data] 43 | 44 | when "insert" then 45 | # Overload the INSERT syntax to allow for nested parameters 46 | # inside the statement. ex: 47 | # INSERT INTO posts (id, author, nickname, ...) VALUES ( 48 | # 'ilya', 'Ilya Grigorik', 'igrigorik' 49 | # ) 50 | # 51 | # The following query will be mapped into 3 distinct tables: 52 | # => 'posts' table will store the key 53 | # => 'posts_author' will store key, value 54 | # => 'posts_nickname' will store key, value 55 | # 56 | # or, in SQL.. 57 | # 58 | # => insert into posts values("ilya"); 59 | # => create table posts_author (id varchar(40), value varchar(255), UNIQUE(id)); 60 | # => insert into posts_author values("ilya", "Ilya Grigorik"); 61 | # => ... repeat for every attribute 62 | # 63 | # If the table post_value has not been seen before, it will 64 | # be created on the fly. Hence allowing us to add and remove 65 | # keys and values at will. :-) 66 | # 67 | # P.S. There is probably cleaner syntax for this, but hey... 68 | 69 | 70 | if insert = sql.match(/\((.*?)\).*?\((.*?)\)/) 71 | data = {} 72 | table = query[2] 73 | keys = insert[1].split(',').map!{|s| s.strip} 74 | values = insert[2].scan(/([^\'|\"]+)/).flatten.reject {|s| s.strip == ','} 75 | keys.each_with_index {|k,i| data[k] = values[i]} 76 | 77 | data.each do |key, value| 78 | next if key == 'id' 79 | attr_sql = "insert into #{table}_#{key} values('#{data['id']}', '#{value}')" 80 | 81 | q = @mysql.query(attr_sql) 82 | q.errback { |res| 83 | # if the attribute table for this model does not yet exist then create it! 84 | # - yes, there is a race condition here, add fiber logic later 85 | if res.is_a?(Mysql::Error) and res.message =~ /Table.*doesn\'t exist/ 86 | 87 | table_sql = "create table #{table}_#{key} (id varchar(255), value varchar(255), UNIQUE(id))" 88 | tc = @mysql.query(table_sql) 89 | tc.callback { @mysql.query(attr_sql) } 90 | end 91 | } 92 | 93 | p [:inserted_attr, table, key, value] 94 | end 95 | 96 | # override the query to insert the key into posts table 97 | query = query[0,3] + ["VALUES('#{data['id']}')"] 98 | overhead = query.join(" ").size + 1 99 | 100 | p [:insert, query] 101 | end 102 | 103 | when "select" then 104 | # Overload the select call to perform a multi-join in the background 105 | # and rewrite the attribute names to fool the client into thinking it 106 | # all came from the same table. 107 | # 108 | # To figure out which tables we need to join on, do the simple / dumb 109 | # approach and issue a 'show tables like key_%' to do 'runtime 110 | # introspection'. Could easily cache this, but that's for later. 111 | # 112 | # Ex, a 'select * from posts' query with one value (author) would be 113 | # rewritten into the following query: 114 | # 115 | # SELECT posts.id as id, posts_author.value as author FROM posts 116 | # LEFT OUTER JOIN posts_author ON posts_author.id = posts.id 117 | # WHERE posts.id = "ilya"; 118 | 119 | select = sql.match(/select(.*?)from\s([^\s]+)/) 120 | where = sql.match(/where\s([^=]+)\s?=\s?'?"?([^\s'"]+)'?"?/) 121 | attrs, table = select[1].strip.split(','), select[2] if select 122 | key = where[2] if where 123 | 124 | if select 125 | p [:select, select, attrs, where] 126 | 127 | tables = @mysql.query("show tables like '#{table}_%'") 128 | tables.callback { |res| 129 | fiber.resume(res.all_hashes.collect(&:values).flatten.collect{ |c| 130 | c.split('_').last 131 | }) 132 | } 133 | tables = Fiber.yield 134 | 135 | p [:select_tables, tables] 136 | 137 | # build the select statements, hide the tables behind each attribute 138 | join = "select #{table}.id as id " 139 | tables.each do |column| 140 | join += " , #{table}_#{column}.value as #{column} " 141 | end 142 | 143 | # add the joins to stich it all together 144 | join += " FROM #{table} " 145 | tables.each do |column| 146 | join += " LEFT OUTER JOIN #{table}_#{column} ON #{table}_#{column}.id = #{table}.id " 147 | end 148 | 149 | join += " WHERE #{table}.id = '#{key}' " if key 150 | 151 | query = [join] 152 | overhead = join.size + 1 153 | 154 | p [:join_query, join] 155 | end 156 | end 157 | 158 | # repack the query data and forward to server 159 | # - have to split message on packet boundaries 160 | 161 | seq, data = 0, [] 162 | query = StringIO.new([type, query.join(" ")].pack("Ca*")) 163 | while q = query.read(MAX_PACKET_LENGTH) 164 | data.push [q.length % 256, q.length / 256, seq].pack("CvC") + q 165 | seq = (seq + 1) % 256 166 | end 167 | 168 | p [:final_query, data, chunks, overhead] 169 | puts "-" * 100 170 | end 171 | 172 | [data].flatten.each do |chunk| 173 | conn.relay_to_servers(chunk) 174 | end 175 | 176 | :async # we will render results later 177 | } 178 | 179 | fiber.resume 180 | end 181 | end 182 | -------------------------------------------------------------------------------- /examples/selective_forward.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 80) do |conn| 4 | @start = Time.now 5 | @data = Hash.new("") 6 | 7 | conn.server :prod, :host => "127.0.0.1", :port => 81 # production, will render resposne 8 | conn.server :test, :host => "127.0.0.1", :port => 82 # testing, internal only 9 | 10 | conn.on_data do |data| 11 | # rewrite User-Agent 12 | [data.gsub(/User-Agent: .*?\r\n/, 'User-Agent: em-proxy/0.1\r\n'), [:prod]] 13 | end 14 | 15 | conn.on_response do |server, resp| 16 | # only render response from production 17 | @data[server] += resp 18 | resp if server == :prod 19 | end 20 | 21 | conn.on_finish do |name| 22 | p [:on_finish, name, Time.now - @start] 23 | unbind if name == :prod # terminate connection once prod is done 24 | 25 | p @data if name == :done 26 | end 27 | end 28 | 29 | # 30 | # ruby examples/appserver.rb 81 31 | # ruby examples/appserver.rb 82 32 | # ruby examples/line_interceptor.rb 33 | # curl localhost 34 | # 35 | # > [:on_finish, 1.008561] 36 | # > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0", 37 | # :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"} 38 | # -------------------------------------------------------------------------------- /examples/simple.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 8080, :debug => true) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 8081 5 | 6 | # modify / process request stream 7 | conn.on_data do |data| 8 | p [:on_data, data] 9 | data 10 | end 11 | 12 | # modify / process response stream 13 | conn.on_response do |backend, resp| 14 | p [:on_response, backend, resp] 15 | resp 16 | end 17 | 18 | # termination logic 19 | conn.on_finish do |backend, name| 20 | p [:on_finish, name] 21 | 22 | # terminate connection (in duplex mode, you can terminate when prod is done) 23 | unbind if backend == :srv 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /examples/smtp_spam_filter.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | require 'em-http' 3 | require 'yaml' 4 | require 'net/http' 5 | 6 | Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn| 7 | conn.server :srv, :host => "127.0.0.1", :port => 2525 8 | 9 | RCPT_CMD = /RCPT TO:<(.*)?>\r\n/ # RCPT TO:\r\n 10 | FROM_CMD = /MAIL FROM:<(.*)?>\r\n/ # MAIL FROM:\r\n 11 | MSG_CMD = /354 Start your message/ # 354 Start your message 12 | MSGEND_CMD = /^.\r\n/ 13 | 14 | conn.on_data do |data| 15 | @from = data.match(FROM_CMD)[1] if data.match(FROM_CMD) 16 | @rcpt = data.match(RCPT_CMD)[1] if data.match(RCPT_CMD) 17 | @done = true if data.match(MSGEND_CMD) 18 | 19 | if @buffer 20 | @msg += data 21 | data = nil 22 | end 23 | 24 | if @done 25 | @buffer = false 26 | res = Net::HTTP.post_form(URI.parse('http://api.defensio.com/app/1.2/audit-comment/77ca297d7546705ee2b5136fad0dcaf8.yaml'), { 27 | "owner-url" => "http://www.github.com/igrigorik/em-http-request", 28 | "user-ip" => "216.16.254.254", 29 | "article-date" => "2009/04/01", 30 | "comment-author" => @from, 31 | "comment-type" => "comment", 32 | "comment-content" => @msg}) 33 | 34 | defensio = YAML.load(res.body)['defensio-result'] 35 | p [:defensio, "SPAM: #{defensio['spam']}, Spaminess: #{defensio['spaminess']}"] 36 | 37 | if defensio['spam'] 38 | conn.send_data "550 No such user here\n" 39 | else 40 | data = @msg 41 | end 42 | end 43 | 44 | data 45 | end 46 | 47 | conn.on_response do |server, resp| 48 | p [:resp, resp] 49 | 50 | if resp.match(MSG_CMD) 51 | @buffer = true 52 | @msg = "" 53 | end 54 | 55 | resp 56 | end 57 | end 58 | 59 | # mailtrap run -p 2525 -f /tmp/mailtrap.log 60 | # ruby examples/smtp_spam_filter.rb 61 | # 62 | # >> require 'net/smtp' 63 | # >> smtp = Net::SMTP.start("localhost", 2524) 64 | # >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com" 65 | 66 | 67 | # Protocol trace 68 | # 69 | # [:srv, :conn_complete] 70 | # [:srv, "220 localhost MailTrap ready ESTMP\n"] 71 | # [:relay_from_backend, :srv, "220 localhost MailTrap ready ESTMP\n"] 72 | # [:resp, "220 localhost MailTrap ready ESTMP\n"] 73 | # [:connection, "EHLO localhost.localdomain\r\n"] 74 | # [:srv, "250-localhost offers just ONE extension my pretty"] 75 | # [:relay_from_backend, :srv, "250-localhost offers just ONE extension my pretty"] 76 | # [:resp, "250-localhost offers just ONE extension my pretty"] 77 | # [:srv, "\n250 HELP\n"] 78 | # [:relay_from_backend, :srv, "\n250 HELP\n"] 79 | # [:resp, "\n250 HELP\n"] 80 | # [:connection, "MAIL FROM:\r\n"] 81 | # [:srv, "250 OK\n"] 82 | # [:relay_from_backend, :srv, "250 OK\n"] 83 | # [:resp, "250 OK\n"] 84 | # [:connection, "RCPT TO:\r\n"] 85 | # [:srv, "250 OK"] 86 | # [:relay_from_backend, :srv, "250 OK"] 87 | # [:resp, "250 OK"] 88 | # [:srv, "\n"] 89 | # [:relay_from_backend, :srv, "\n"] 90 | # [:resp, "\n"] 91 | # [:connection, "DATA\r\n"] 92 | # [:srv, "354 Start your message"] 93 | # [:relay_from_backend, :srv, "354 Start your message"] 94 | # [:resp, "354 Start your message"] 95 | # [:srv, "\n"] 96 | # [:relay_from_backend, :srv, "\n"] 97 | # [:resp, "\n"] 98 | # [:connection, "Hello World\r\n"] 99 | # [:connection, ".\r\n"] 100 | # 101 | # [:defensio, "SPAM: false, Spaminess: 0.4"] 102 | # 103 | # [:srv, "250 OK\n"] 104 | # [:relay_from_backend, :srv, "250 OK\n"] 105 | # [:resp, "250 OK\n"] 106 | # 107 | 108 | -------------------------------------------------------------------------------- /examples/smtp_whitelist.rb: -------------------------------------------------------------------------------- 1 | require 'lib/em-proxy' 2 | 3 | Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn| 4 | conn.server :srv, :host => "127.0.0.1", :port => 2525 5 | 6 | # RCPT TO:\r\n 7 | RCPT_CMD = /RCPT TO:<(.*)?>\r\n/ 8 | 9 | conn.on_data do |data| 10 | 11 | if rcpt = data.match(RCPT_CMD) 12 | if rcpt[1] != "ilya@igvita.com" 13 | conn.send_data "550 No such user here\n" 14 | data = nil 15 | end 16 | end 17 | 18 | data 19 | end 20 | 21 | conn.on_response do |backend, resp| 22 | resp 23 | end 24 | end 25 | 26 | 27 | # mailtrap run -p 2525 -f /tmp/mailtrap.log 28 | # ruby examples/smtp_whitelist.rb 29 | # 30 | # >> require 'net/smtp' 31 | # >> smtp = Net::SMTP.start("localhost", 2524) 32 | # >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com" 33 | # => # 34 | # >> smtp.finish 35 | # => # 36 | # 37 | # >> smtp.send_message "Hello World!", "ilya@aiderss.com", "missing_user@igvita.com" 38 | # => Net::SMTPFatalError: 550 No such user here 39 | # 40 | -------------------------------------------------------------------------------- /lib/em-proxy.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.dirname(__FILE__) + '/../lib') 2 | 3 | 4 | require "rubygems" 5 | require "eventmachine" 6 | require "socket" 7 | 8 | %w[ backend proxy connection ].each do |file| 9 | require "em-proxy/#{file}" 10 | end 11 | -------------------------------------------------------------------------------- /lib/em-proxy/backend.rb: -------------------------------------------------------------------------------- 1 | module EventMachine 2 | module ProxyServer 3 | class Backend < EventMachine::Connection 4 | attr_accessor :plexer, :name, :debug 5 | 6 | def initialize(debug = false) 7 | @debug = debug 8 | @connected = EM::DefaultDeferrable.new 9 | end 10 | 11 | def connection_completed 12 | debug [@name, :conn_complete] 13 | @plexer.connected(@name) 14 | @connected.succeed 15 | end 16 | 17 | def receive_data(data) 18 | debug [@name, data] 19 | @plexer.relay_from_backend(@name, data) 20 | end 21 | 22 | # Buffer data until the connection to the backend server 23 | # is established and is ready for use 24 | def send(data) 25 | @connected.callback { send_data data } 26 | end 27 | 28 | # Notify upstream plexer that the backend server is done 29 | # processing the request 30 | def unbind(reason = nil) 31 | debug [@name, :unbind, reason] 32 | @plexer.unbind_backend(@name) 33 | end 34 | 35 | private 36 | 37 | def debug(*data) 38 | return unless @debug 39 | require 'pp' 40 | pp data 41 | puts 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/em-proxy/connection.rb: -------------------------------------------------------------------------------- 1 | module EventMachine 2 | module ProxyServer 3 | class Connection < EventMachine::Connection 4 | attr_accessor :debug 5 | 6 | ##### Proxy Methods 7 | def on_data(&blk); @on_data = blk; end 8 | def on_response(&blk); @on_response = blk; end 9 | def on_finish(&blk); @on_finish = blk; end 10 | def on_connect(&blk); @on_connect = blk; end 11 | 12 | ##### EventMachine 13 | def initialize(options) 14 | @debug = options[:debug] || false 15 | @tls_key = options[:tls_key] || false 16 | @tls_cert = options[:tls_cert] || false 17 | @servers = {} 18 | end 19 | 20 | def post_init 21 | if @tls_key and @tls_cert 22 | start_tls :private_key_file => @tls_key, :cert_chain_file => @tls_cert, :verify_peer => false 23 | end 24 | end 25 | 26 | def receive_data(data) 27 | debug [:connection, data] 28 | processed = @on_data.call(data) if @on_data 29 | 30 | return if processed == :async or processed.nil? 31 | relay_to_servers(processed) 32 | end 33 | 34 | def relay_to_servers(processed) 35 | if processed.is_a? Array 36 | data, servers = *processed 37 | 38 | # guard for "unbound" servers 39 | servers = servers.collect {|s| @servers[s]}.compact 40 | else 41 | data = processed 42 | servers ||= @servers.values.compact 43 | end 44 | 45 | servers.each do |s| 46 | s.send_data data unless data.nil? 47 | end 48 | end 49 | 50 | # 51 | # initialize connections to backend servers 52 | # 53 | def server(name, opts) 54 | if opts[:socket] 55 | srv = EventMachine::connect_unix_domain(opts[:socket], EventMachine::ProxyServer::Backend, @debug) do |c| 56 | c.name = name 57 | c.plexer = self 58 | c.proxy_incoming_to(self, 10240) if opts[:relay_server] 59 | end 60 | else 61 | srv = EventMachine::bind_connect(opts[:bind_host], opts[:bind_port], opts[:host], opts[:port], EventMachine::ProxyServer::Backend, @debug) do |c| 62 | c.name = name 63 | c.plexer = self 64 | c.proxy_incoming_to(self, 10240) if opts[:relay_server] 65 | end 66 | end 67 | 68 | self.proxy_incoming_to(srv, 10240) if opts[:relay_client] 69 | 70 | @servers[name] = srv 71 | end 72 | 73 | # 74 | # [ip, port] of the connected client 75 | # 76 | def peer 77 | @peer ||= begin 78 | peername = get_peername 79 | peername ? Socket.unpack_sockaddr_in(peername).reverse : nil 80 | end 81 | end 82 | 83 | # 84 | # [ip, port] of the local server connect 85 | # 86 | def sock 87 | @sock ||= begin 88 | sockname = get_sockname 89 | sockname ? Socket.unpack_sockaddr_in(sockname).reverse : nil 90 | end 91 | end 92 | 93 | # 94 | # relay data from backend server to client 95 | # 96 | def relay_from_backend(name, data) 97 | debug [:relay_from_backend, name, data] 98 | 99 | data = @on_response.call(name, data) if @on_response 100 | send_data data unless data.nil? 101 | end 102 | 103 | def connected(name) 104 | debug [:connected] 105 | @on_connect.call(name) if @on_connect 106 | end 107 | 108 | def unbind(reason = nil) 109 | debug [:unbind, :connection, reason] 110 | 111 | # terminate any unfinished connections 112 | @servers.values.compact.each do |s| 113 | s.close_connection_after_writing 114 | end 115 | end 116 | 117 | def unbind_backend(name) 118 | debug [:unbind_backend, name] 119 | @servers[name] = nil 120 | close = :close 121 | 122 | if @on_finish 123 | close = @on_finish.call(name) 124 | end 125 | 126 | # if all connections are terminated downstream, then notify client 127 | if (@servers.values.compact.size.zero? && close != :keep) || (close == :close) 128 | close_connection_after_writing 129 | end 130 | end 131 | 132 | private 133 | 134 | def debug(*data) 135 | if @debug 136 | require 'pp' 137 | pp data 138 | puts 139 | end 140 | end 141 | end 142 | end 143 | end 144 | -------------------------------------------------------------------------------- /lib/em-proxy/proxy.rb: -------------------------------------------------------------------------------- 1 | class Proxy 2 | 3 | def self.start(options, &blk) 4 | EM.epoll 5 | EM.run do 6 | 7 | trap("TERM") { stop } 8 | trap("INT") { stop } 9 | 10 | EventMachine::start_server(options[:host], options[:port], 11 | EventMachine::ProxyServer::Connection, options) do |c| 12 | c.instance_eval(&blk) 13 | end 14 | end 15 | end 16 | 17 | def self.stop 18 | puts "Terminating ProxyServer" 19 | EventMachine.stop 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/balancing_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require File.join(File.dirname(__FILE__), '../', 'examples/balancing') 3 | 4 | describe BalancingProxy do 5 | 6 | before(:each) do 7 | class BalancingProxy::Backend 8 | @list = nil; @pool = nil 9 | end 10 | end 11 | 12 | before(:all) do 13 | @original_stdout = $stdout 14 | # Silence the noisy STDOUT output 15 | $stdout = File.new('/dev/null', 'w') 16 | end 17 | 18 | after(:all) do 19 | $stdout = @original_stdout 20 | end 21 | 22 | context "generally" do 23 | 24 | it "should raise error for unknown strategy" do 25 | lambda { BalancingProxy::Backend.select(:asdf)}.should raise_error(ArgumentError) 26 | end 27 | 28 | end 29 | 30 | context "when using the 'random' strategy" do 31 | 32 | it "should select random backend" do 33 | class BalancingProxy::Backend 34 | def self.list 35 | @list ||= [ 36 | {:url => "http://127.0.0.1:3000"}, 37 | {:url => "http://127.0.0.2:3000"}, 38 | {:url => "http://127.0.0.3:3000"} 39 | ].map { |backend| new backend } 40 | end 41 | end 42 | 43 | srand(0) 44 | BalancingProxy::Backend.select(:random).host.should == '127.0.0.1' 45 | end 46 | 47 | end 48 | 49 | context "when using the 'roundrobin' strategy" do 50 | it "should select backends in rotating order" do 51 | class BalancingProxy::Backend 52 | def self.list 53 | @list ||= [ 54 | {:url => "http://127.0.0.1:3000"}, 55 | {:url => "http://127.0.0.2:3000"}, 56 | {:url => "http://127.0.0.3:3000"} 57 | ].map { |backend| new backend } 58 | end 59 | end 60 | 61 | BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.1' 62 | BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.2' 63 | BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.3' 64 | BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.1' 65 | end 66 | end 67 | 68 | context "when using the 'balanced' strategy" do 69 | 70 | it "should select the first backend when all backends have the same load" do 71 | class BalancingProxy::Backend 72 | def self.list 73 | @list ||= [ 74 | {:url => "http://127.0.0.3:3000", :load => 0}, 75 | {:url => "http://127.0.0.2:3000", :load => 0}, 76 | {:url => "http://127.0.0.1:3000", :load => 0} 77 | ].map { |backend| new backend } 78 | end 79 | end 80 | 81 | BalancingProxy::Backend.select.host.should == '127.0.0.3' 82 | end 83 | 84 | it "should select the least loaded backend" do 85 | class BalancingProxy::Backend 86 | def self.list 87 | @list ||= [ 88 | {:url => "http://127.0.0.3:3000", :load => 4}, 89 | {:url => "http://127.0.0.2:3000", :load => 2}, 90 | {:url => "http://127.0.0.1:3000", :load => 0} 91 | ].map { |backend| new backend } 92 | end 93 | end 94 | 95 | BalancingProxy::Backend.select.host.should == '127.0.0.1' 96 | BalancingProxy::Backend.select.host.should == '127.0.0.1' 97 | BalancingProxy::Backend.select.host.should_not == '127.0.0.3' 98 | BalancingProxy::Backend.select.host.should_not == '127.0.0.3' 99 | end 100 | 101 | end 102 | 103 | end 104 | -------------------------------------------------------------------------------- /spec/helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'em-http' 3 | require 'pp' 4 | require 'tmpdir' 5 | require 'posix/spawn' 6 | 7 | require 'em-proxy' 8 | -------------------------------------------------------------------------------- /spec/proxy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | describe Proxy do 4 | include POSIX::Spawn 5 | 6 | def failed 7 | EventMachine.stop 8 | fail 9 | end 10 | 11 | it "should recieve data on port 8080" do 12 | EM.run do 13 | EventMachine.add_timer(0.1) do 14 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1}) 15 | end 16 | 17 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 18 | conn.on_data do |data| 19 | data.should =~ /GET \/test/ 20 | EventMachine.stop 21 | end 22 | end 23 | end 24 | end 25 | 26 | it "should call the on_connect callback" do 27 | connected = false 28 | EM.run do 29 | EventMachine.add_timer(0.1) do 30 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 31 | end 32 | 33 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 34 | conn.server :goog, :host => "google.com", :port => 80 35 | 36 | conn.on_connect do |name| 37 | connected = true 38 | EventMachine.stop 39 | end 40 | end 41 | end 42 | connected.should == true 43 | end 44 | 45 | 46 | it "should transparently redirect TCP traffic to google" do 47 | EM.run do 48 | EventMachine.add_timer(0.1) do 49 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 50 | end 51 | 52 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 53 | conn.server :goog, :host => "google.com", :port => 80 54 | conn.on_data { |data| data } 55 | 56 | conn.on_response do |backend, resp| 57 | backend.should == :goog 58 | resp.size.should >= 0 59 | EventMachine.stop 60 | end 61 | end 62 | end 63 | end 64 | 65 | it "should duplex TCP traffic to two backends" do 66 | EM.run do 67 | EventMachine.add_timer(0.1) do 68 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1}) 69 | end 70 | 71 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 72 | conn.server :goog1, :host => "google.com", :port => 80 73 | conn.server :goog2, :host => "google.com", :port => 80 74 | conn.on_data { |data| data } 75 | 76 | seen = [] 77 | conn.on_response do |backend, resp| 78 | case backend 79 | when :goog1 then 80 | resp.should =~ /404/ 81 | seen.push backend 82 | when :goog2 83 | resp.should =~ /404/ 84 | seen.push backend 85 | end 86 | seen.uniq! 87 | 88 | EventMachine.stop if seen.size == 2 89 | end 90 | 91 | conn.on_finish do |name| 92 | # keep the connection open if we're still expecting a response 93 | seen.count == 2 ? :close : :keep 94 | end 95 | 96 | end 97 | end 98 | end 99 | 100 | it "should intercept & alter response from Google" do 101 | EM.run do 102 | EventMachine.add_timer(0.1) do 103 | http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 104 | http.errback { failed } 105 | http.callback { 106 | http.response_header.status.should == 404 107 | EventMachine.stop 108 | } 109 | end 110 | 111 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 112 | conn.server :goog, :host => "google.com", :port => 80 113 | conn.on_data { |data| data } 114 | conn.on_response do |backend, data| 115 | data.gsub(/^HTTP\/1.1 301/, 'HTTP/1.1 404') 116 | end 117 | end 118 | end 119 | end 120 | 121 | it "should invoke on_finish callback when connection is terminated" do 122 | EM.run do 123 | EventMachine.add_timer(0.1) do 124 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 125 | end 126 | 127 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 128 | conn.server :goog, :host => "google.com", :port => 80 129 | conn.on_data { |data| data } 130 | conn.on_response { |backend, resp| resp } 131 | conn.on_finish do |backend| 132 | backend.should == :goog 133 | EventMachine.stop 134 | end 135 | end 136 | end 137 | end 138 | 139 | it "should not invoke on_data when :relay_client is passed as server option" do 140 | lambda { 141 | EM.run do 142 | EventMachine.add_timer(0.1) do 143 | http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 144 | http.callback { EventMachine.stop } 145 | end 146 | 147 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 148 | conn.server :goog, :host => "google.com", :port => 80, :relay_client => true 149 | conn.on_data { |data| raise "Should not be here"; data } 150 | conn.on_response { |backend, resp| resp } 151 | 152 | end 153 | end 154 | }.should_not raise_error 155 | end 156 | 157 | it "should not invoke on_response when :relay_server is passed as server option" do 158 | lambda { 159 | EM.run do 160 | EventMachine.add_timer(0.1) do 161 | http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 162 | http.callback { EventMachine.stop } 163 | end 164 | 165 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 166 | conn.server :goog, :host => "google.com", :port => 80, :relay_server => true 167 | conn.on_data { |data| data } 168 | conn.on_response { |backend, resp| raise "Should not be here"; } 169 | 170 | end 171 | end 172 | }.should_not raise_error 173 | end 174 | 175 | context "echo server" do 176 | before :each do 177 | @echo_server = File.expand_path("../../spec/support/echo_server.rb", __FILE__) 178 | end 179 | 180 | context "with a server listening on a TCP port" do 181 | before :each do 182 | @host = '127.0.0.1' 183 | @port = 4242 184 | @pid = spawn("ruby #{@echo_server} #{@host} #{@port}") 185 | sleep 1 # let the server come up 186 | end 187 | after :each do 188 | Process.kill('QUIT', @pid) 189 | end 190 | it "should connect to a unix socket" do 191 | connected = false 192 | EM.run do 193 | EventMachine.add_timer(0.1) do 194 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 195 | end 196 | host = @host 197 | port = @port 198 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 199 | conn.server :local, :host => host, :port => port 200 | conn.on_connect do |name| 201 | connected = true 202 | EventMachine.stop 203 | end 204 | end 205 | end 206 | connected.should == true 207 | end 208 | end 209 | 210 | context "with a server listening on a unix socket" do 211 | before :each do 212 | @socket = File.join(Dir.tmpdir, 'em-proxy.sock') 213 | @pid = spawn("ruby #{@echo_server} #{@socket}") 214 | sleep 1 # let the server come up 215 | end 216 | after :each do 217 | Process.kill('QUIT', @pid) 218 | end 219 | it "should connect to a unix socket" do 220 | connected = false 221 | EM.run do 222 | EventMachine.add_timer(0.1) do 223 | EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1}) 224 | end 225 | socket = @socket 226 | Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn| 227 | conn.server :local, :socket => socket 228 | conn.on_connect do |name| 229 | connected = true 230 | EventMachine.stop 231 | end 232 | end 233 | end 234 | connected.should == true 235 | end 236 | end 237 | end 238 | 239 | end 240 | -------------------------------------------------------------------------------- /spec/support/echo_server.rb: -------------------------------------------------------------------------------- 1 | require 'eventmachine' 2 | 3 | module EchoServer 4 | def receive_data data 5 | send_data ">>>you sent: #{data}" 6 | close_connection if data =~ /quit/i 7 | end 8 | end 9 | 10 | EventMachine.run do 11 | if ARGV.count == 2 12 | EventMachine.start_server ARGV.first, ARGV.last.to_i, EchoServer 13 | elsif ARGV.count == 1 14 | EventMachine.start_server ARGV.first, EchoServer 15 | else 16 | raise "invalid number of params, expected [server] ([port])" 17 | end 18 | end 19 | --------------------------------------------------------------------------------