├── Rakefile ├── lib ├── multibinder │ └── version.rb └── multibinder.rb ├── Gemfile ├── .gitignore ├── haproxy ├── multibinder.service ├── haproxy-multi@.service └── README.md ├── script ├── test ├── cibuild └── deploy ├── test ├── lib-project.sh ├── httpbinder.rb ├── haproxy_shim.rb ├── test-haproxy.sh ├── test-simple.sh └── lib.sh ├── multibinder.gemspec ├── LICENSE.txt ├── README.md └── bin ├── multibinder-haproxy-erb ├── multibinder-haproxy-wrapper └── multibinder /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | -------------------------------------------------------------------------------- /lib/multibinder/version.rb: -------------------------------------------------------------------------------- 1 | module MultiBinder 2 | VERSION = "0.0.5" 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in binder.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.bundle 11 | *.so 12 | *.o 13 | *.a 14 | mkmf.log 15 | *.gem 16 | vendor/gems 17 | -------------------------------------------------------------------------------- /haproxy/multibinder.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Multibinder 3 | After=network.target 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/multibinder /run/multibinder.sock 7 | Restart=always 8 | 9 | [Install] 10 | WantedBy=multi-user.target 11 | -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | SHELL=/bin/sh 6 | 7 | date "+%Y-%m-%dT%H:%M:%SZ" 8 | echo "Beginning tests for multibinder" 9 | echo "===" 10 | 11 | ls -1 test/test-*.sh | xargs -I % -n 1 ${SHELL} % --batch 12 | 13 | echo "===" 14 | date "+%Y-%m-%dT%H:%M:%SZ" 15 | echo "Done." 16 | -------------------------------------------------------------------------------- /script/cibuild: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export RUBY_VERSION=2.1.6-github 4 | export RBENV_VERSION=2.1.6-github 5 | export PATH=/usr/share/rbenv/shims:/usr/sbin:$PATH 6 | export BASE_DIR=/data/multibinder 7 | 8 | bundle install --local --quiet --path vendor/gems 9 | 10 | REALPATH=$(cd $(dirname "$0") && pwd) 11 | 12 | cd $REALPATH/../ 13 | script/test 14 | -------------------------------------------------------------------------------- /script/deploy: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export RUBY_VERSION=2.1.6-github 4 | export RBENV_VERSION=2.1.6-github 5 | export PATH=/usr/share/rbenv/shims:/usr/sbin:$PATH 6 | export BASE_DIR=/data/multibinder 7 | 8 | REALPATH=$(cd $(dirname "$0") && pwd) 9 | cd $REALPATH/../ 10 | 11 | bundle install --local --quiet --path vendor/gems 12 | 13 | sudo sv 1 multibinder 14 | -------------------------------------------------------------------------------- /test/lib-project.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Extends lib.sh for this project 3 | 4 | project_setup () { 5 | local control_sock=${TEMPDIR}/multibinder.sock 6 | launch_service "multibinder" bundle exec multibinder ${control_sock} 7 | 8 | tries=0 9 | while [ ! -S $control_sock ]; do 10 | sleep .1 11 | echo 'Waiting for control socket...' 12 | tries=$((tries + 1)) 13 | if [ $tries -gt 10 ]; then 14 | echo 'Giving up.' 15 | exit 1 16 | fi 17 | done 18 | } 19 | 20 | project_cleanup () { 21 | kill_service "multibinder" 22 | } 23 | -------------------------------------------------------------------------------- /test/httpbinder.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | require 'json' 3 | require 'multibinder' 4 | 5 | server = MultiBinder.bind '127.0.0.1', ARGV[0].to_i 6 | 7 | loop do 8 | socket, _ = server.accept 9 | request = socket.gets 10 | puts request 11 | 12 | begin 13 | socket.print "HTTP/1.0 200 OK\r\n" 14 | socket.print "Content-Type: text/plain\r\n" 15 | socket.print "Connection: close\r\n" 16 | 17 | socket.print "\r\n" 18 | 19 | socket.print "Hello World #{ARGV[1] || ''}!\n" 20 | 21 | socket.close 22 | rescue Errno::EPIPE 23 | puts 'Client unexpectedly closed connection' 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /test/haproxy_shim.rb: -------------------------------------------------------------------------------- 1 | require 'multibinder' 2 | 3 | server = MultiBinder.bind '127.0.0.1', ARGV[0].to_i 4 | 5 | cfg_fn = "#{ENV['TEMPDIR']}/haproxy.cfg" 6 | 7 | File.write(cfg_fn, < false 23 | 24 | Signal.trap("INT") { Process.kill "USR1", pid } 25 | Signal.trap("TERM") { Process.kill "USR1", pid } 26 | 27 | Process.waitpid 28 | -------------------------------------------------------------------------------- /haproxy/haproxy-multi@.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HAProxy Load Balancer 3 | Documentation=man:haproxy(1) 4 | Documentation=file:/usr/share/doc/haproxy/configuration.txt.gz 5 | After=network.target syslog.service multibinder.service 6 | Wants=syslog.service multibinder.service 7 | 8 | [Service] 9 | Environment=CONFIG=/etc/haproxy/%i.cfg.erb 10 | Environment=MULTIBINDER_SOCK=/run/multibinder.sock 11 | EnvironmentFile=-/etc/default/haproxy 12 | ExecStartPre=/usr/local/bin/multibinder-haproxy-erb /usr/sbin/haproxy -f ${CONFIG} -c -q 13 | ExecStart=/usr/local/bin/multibinder-haproxy-wrapper /usr/sbin/haproxy -Ds -f ${CONFIG} -p /run/haproxy-%i.pid $EXTRAOPTS 14 | ExecReload=/bin/sh -c "/usr/local/bin/multibinder-haproxy-erb /usr/sbin/haproxy -c -f ${CONFIG}; /bin/kill -USR2 $MAINPID" 15 | ExecStop=/bin/kill -TERM $MAINPID 16 | KillMode=none 17 | Restart=always 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /multibinder.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'multibinder/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "multibinder" 8 | spec.version = MultiBinder::VERSION 9 | spec.authors = ["Theo Julienne"] 10 | spec.email = ["theojulienne@github.com"] 11 | spec.summary = %q{multibinder is a tiny ruby server that makes writing zero-downtime-reload services simpler.} 12 | spec.homepage = "https://github.com/theojulienne/multibinder" 13 | spec.license = "MIT" 14 | 15 | spec.files = `git ls-files -z`.split("\x0") 16 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 17 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 18 | spec.require_paths = ["lib"] 19 | 20 | spec.add_development_dependency "bundler", "~> 1.5" 21 | spec.add_development_dependency "rake", "~> 10.0" 22 | end 23 | -------------------------------------------------------------------------------- /lib/multibinder.rb: -------------------------------------------------------------------------------- 1 | require 'multibinder/version' 2 | require 'json' 3 | require 'securerandom' 4 | require 'socket' 5 | require 'fcntl' 6 | 7 | module MultiBinder 8 | def self.bind(address, port, options={}) 9 | abort 'MULTIBINDER_SOCK environment variable must be set' if !ENV['MULTIBINDER_SOCK'] 10 | 11 | binder = UNIXSocket.open(ENV['MULTIBINDER_SOCK']) 12 | 13 | # make the request 14 | binder.sendmsg JSON.dump({ 15 | :jsonrpc => '2.0', 16 | :method => 'bind', 17 | :id => SecureRandom.uuid, 18 | :params => [{ 19 | :address => address, 20 | :port => port, 21 | }.merge(options)] 22 | }, 0, nil) 23 | 24 | # get the response 25 | msg, _, _, ctl = binder.recvmsg(:scm_rights=>true) 26 | response = JSON.parse(msg) 27 | if response['error'] 28 | raise response['error']['message'] 29 | end 30 | 31 | binder.close 32 | 33 | socket = ctl.unix_rights[0] 34 | socket.fcntl(Fcntl::F_SETFD, socket.fcntl(Fcntl::F_GETFD) & (-Fcntl::FD_CLOEXEC-1)) 35 | socket 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Theo Julienne 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /test/test-haproxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # test-haproxy.sh: check that we can work with haproxy (if it's installed) 4 | 5 | REALPATH=$(cd $(dirname "$0") && pwd) 6 | . "${REALPATH}/lib.sh" 7 | 8 | TEST_PORT=8000 9 | 10 | tests_use_port $TEST_PORT 11 | 12 | if ! which haproxy >/dev/null 2>&1; then 13 | echo "haproxy not available, skipping tests." 14 | exit 0 15 | fi 16 | 17 | begin_test "haproxy runs with multibinder" 18 | ( 19 | setup 20 | 21 | export TEMPDIR 22 | 23 | launch_service "haproxy" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/haproxy_shim.rb $(offset_port $TEST_PORT) 24 | 25 | wait_for_port "haproxy" $(offset_port $TEST_PORT) 26 | 27 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/ | grep -q 'Request forbidden' 28 | ) 29 | end_test 30 | 31 | begin_test "haproxy can bind to large numbers of file descriptors" 32 | ( 33 | setup 34 | 35 | export TEMPDIR 36 | 37 | echo >$TEMPDIR/haproxy-many-fds.cfg.erb 38 | 39 | for i in $(seq 1000); do 40 | echo " bind <%= bind_tcp('127.0.0.1', $((TEST_PORT + $i))) %>" >>$TEMPDIR/haproxy-many-fds.cfg.erb 41 | done 42 | 43 | bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby bin/multibinder-haproxy-erb -f $TEMPDIR/haproxy-many-fds.cfg.erb --erb-write-only 44 | 45 | if grep 'bind fd@' $TEMPDIR/haproxy-many-fds.cfg | sort | uniq -d | grep -q 'bind fd@'; then 46 | echo 'Expected unique FDs, but found duplicates:' 47 | cat $TEMPDIR/haproxy-many-fds.cfg 48 | exit 1 49 | fi 50 | ) 51 | end_test 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### multibinder 2 | 3 | multibinder is a tiny ruby server that makes writing zero-downtime-reload services simpler. It accepts connections on a UNIX domain socket and binds an arbitrary number of LISTEN sockets given their ip+port combinations. When a bind is requested, the LISTEN socket is sent over the UNIX domain socket using ancillary data. Subsequent identical binds receive the same LISTEN socket. 4 | 5 | multibinder runs on its own, separate from the daemons that use the sockets. multibinder can be re-exec itself to take upgrades by sending it a `SIGUSR1` - existing binds will be retained across re-execs. 6 | 7 | #### Server usage 8 | 9 | After installing multibinder, you can run the multibinder daemon: 10 | 11 | ``` 12 | bundle exec multibinder /path/to/control.sock 13 | ``` 14 | 15 | #### Client usage 16 | 17 | The multibinder library retrieves a socket from a local multibinder server, communicating over the socket you specify in the `MULTIBINDER_SOCK` environment variable (which has to be the same as specified when running multibinder, and the user must have permission to access the file). 18 | 19 | ```ruby 20 | require 'multibinder' 21 | 22 | server = MultiBinder.bind '127.0.0.1', 8000 23 | 24 | # use the server socket 25 | # ... server.accept ... 26 | ``` 27 | 28 | The socket has close-on-exec disabled and is ready to be used in Ruby or passed on to a real service like haproxy via spawn/exec. For an example of using multibinder with haproxy (there are a couple of tricks), see [the haproxy test shim](https://github.com/theojulienne/multibinder/blob/master/test/haproxy_shim.rb). 29 | -------------------------------------------------------------------------------- /bin/multibinder-haproxy-erb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Takes arguments that will be fed to haproxy, but sneakily tweaks the argument 3 | # to '-f' by ERB-parsing it and then providing the result. 4 | 5 | require 'multibinder' 6 | require 'socket' 7 | require 'fcntl' 8 | require 'erb' 9 | 10 | haproxy_call = ARGV 11 | 12 | # find and update '-f' argument to the output of ERB processing 13 | abort 'multibinder-haproxy expects a configuration file to be passed to haproxy' if haproxy_call.index('-f').nil? 14 | config_file_index = haproxy_call.index('-f') + 1 15 | haproxy_erb = haproxy_call[config_file_index] 16 | abort 'Config file must end with .erb or .mb' unless haproxy_erb.end_with? '.erb' or haproxy_erb.end_with? '.mb' 17 | haproxy_cfg = haproxy_erb.sub(/\.(erb|mb)$/, '') 18 | haproxy_call[config_file_index] = haproxy_cfg 19 | 20 | $all_sockets = [] 21 | 22 | def bind_tcp(ip, port) 23 | if ENV['MULTIBINDER_SOCK'] && !ENV['MULTIBINDER_SOCK'].empty? 24 | # if we have multibinder, use it to reuse our binds safely 25 | server = MultiBinder.bind ip, port, { :backlog => 10240 } 26 | $all_sockets.push server # keep a copy while we're running so we don't GC the file descriptors by accident 27 | "fd@#{server.fileno}" 28 | else 29 | # otherwise resort to the old fashioned binds with downtime/packet loss 30 | puts "WARNING: Using haproxy direct binds, restarts will cause error!" 31 | "#{ip}:#{port}" 32 | end 33 | end 34 | 35 | new_data = ERB.new(File.read(haproxy_erb)).result 36 | File.write(haproxy_cfg, new_data) 37 | 38 | if haproxy_call.index('--erb-write-only') 39 | puts 'ERB config write requested without launching command, exiting.' 40 | exit 0 41 | end 42 | 43 | Process.exec *haproxy_call, :close_others => false 44 | -------------------------------------------------------------------------------- /bin/multibinder-haproxy-wrapper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Runs multibinder-haproxy-erb with the same arguments, supporting a USR2 for reload. 3 | 4 | dir = File.expand_path(File.dirname(__FILE__)) 5 | $launch_haproxy = File.join(dir, "multibinder-haproxy-erb") 6 | 7 | SERVICE_DIR = File.basename(dir) 8 | SERVICE_NAME = SERVICE_DIR.split('-').drop(1).join('-') 9 | 10 | abort 'multibinder-haproxy-wrapper expects a pid file to be passed to haproxy' if ARGV.index('-p').nil? 11 | pid_file_index = ARGV.index('-p') + 1 12 | $PID_FILE = ARGV[pid_file_index] 13 | 14 | # launches a new instance. the haproxy-instance script automatically handles 15 | # everything: when no existing pid exists, it starts haproxy normally. when 16 | # an existing haproxy is running, it calls a new copy with `-sf` so that 17 | # haproxy safely hands over execution to the new process. 18 | def launch_instance 19 | args = [$launch_haproxy] + ARGV 20 | if File.exist? $PID_FILE 21 | args << "-sf" 22 | args.concat File.read($PID_FILE).split() 23 | end 24 | 25 | Process.spawn *args 26 | end 27 | 28 | def cleanup_existing 29 | if File.exist? $PID_FILE 30 | `kill -USR1 $(cat #{$PID_FILE}); rm #{$PID_FILE}` 31 | end 32 | end 33 | 34 | # A SIGUSR2 tells us to safely relaunch 35 | Signal.trap("USR2") do 36 | old_pids = File.read($PID_FILE) 37 | 38 | launch_instance 39 | 40 | # wait a while for the pid file to change. after a while, give up and unblock reloads 41 | for i in 0..20 42 | begin 43 | break if File.read($PID_FILE) != old_pids 44 | rescue Errno::ENOENT 45 | end 46 | sleep 1 47 | end 48 | end 49 | 50 | # If we try to kill haproxy, have them gracefully quit rather than terminate immediately 51 | Signal.trap("TERM") do 52 | cleanup_existing 53 | exit 54 | end 55 | 56 | # Start the first process itself 57 | launch_instance 58 | 59 | # Keep waitpid()ing forever. 60 | begin 61 | loop do 62 | Process.waitpid 63 | sleep 10 64 | end 65 | ensure 66 | cleanup_existing 67 | end -------------------------------------------------------------------------------- /test/test-simple.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # test-simple.sh: simple sanity checks 4 | 5 | REALPATH=$(cd $(dirname "$0") && pwd) 6 | . "${REALPATH}/lib.sh" 7 | 8 | TEST_PORT=8000 9 | 10 | tests_use_port $TEST_PORT 11 | 12 | begin_test "server binds and accepts through multibinder" 13 | ( 14 | setup 15 | 16 | launch_service "http" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) 17 | 18 | wait_for_port "binder" $(offset_port $TEST_PORT) 19 | 20 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/ | grep -q 'Hello World' 21 | ) 22 | end_test 23 | 24 | begin_test "server can restart without requests failing while down" 25 | ( 26 | setup 27 | 28 | launch_service "http" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) 29 | 30 | wait_for_port "binder" $(offset_port $TEST_PORT) 31 | 32 | kill_service "http" 33 | 34 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/ | grep -q 'Hello World' & 35 | curl_pid=$! 36 | 37 | sleep 0.5 38 | 39 | # now restart the service 40 | launch_service "http" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) 41 | 42 | # curl should finish, and succeed 43 | wait $curl_pid 44 | ) 45 | end_test 46 | 47 | 48 | begin_test "server can load a second copy then terminate the first" 49 | ( 50 | setup 51 | 52 | launch_service "http" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) "first" 53 | wait_for_port "binder" $(offset_port $TEST_PORT) 54 | 55 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/r1 | grep -q 'Hello World first' 56 | 57 | launch_service "http2" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) "second" 58 | 59 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/r2 | egrep -q 'Hello World (first|second)' 60 | 61 | kill_service "http" 62 | 63 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/r3 | grep -q 'Hello World second' 64 | 65 | kill_service "http2" 66 | ) 67 | end_test 68 | 69 | begin_test "multibinder restarts safely on sigusr1" 70 | ( 71 | setup 72 | 73 | launch_service "http" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) 74 | wait_for_port "binder" $(offset_port $TEST_PORT) 75 | 76 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/r1 | grep -q 'Hello World' 77 | 78 | kill_service "http" 79 | 80 | lsof -p $(service_pid "multibinder") 81 | 82 | kill -USR1 $(service_pid "multibinder") 83 | 84 | # should still be running, should still be listening 85 | lsof -p $(service_pid "multibinder") 86 | lsof -i :$(offset_port $TEST_PORT) -a -p $(service_pid "multibinder") 87 | 88 | # should be able to request the bind again 89 | launch_service "http" bundle exec env MULTIBINDER_SOCK=${TEMPDIR}/multibinder.sock ruby test/httpbinder.rb $(offset_port $TEST_PORT) 90 | wait_for_port "multibinder" $(offset_port $TEST_PORT) 91 | 92 | # requests should work 93 | curl --max-time 5 http://localhost:$(offset_port $TEST_PORT)/r2 | grep -q 'Hello World' 94 | 95 | # and multibinder should have started listening on the control socket twice 96 | grep 'Respawning' $(service_log "multibinder") 97 | grep 'Listening for binds' $(service_log "multibinder") | grep -n 'Listen' | grep "2:" 98 | ) 99 | end_test 100 | -------------------------------------------------------------------------------- /bin/multibinder: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'socket' 4 | require 'json' 5 | require 'fcntl' 6 | 7 | class MultiBinderServer 8 | def initialize(control_file) 9 | @control_file = control_file 10 | @sockets = {} 11 | end 12 | 13 | def handle_client(s) 14 | loop do 15 | msg, _, _, _ = s.recvmsg 16 | break if msg.empty? 17 | request = JSON.parse(msg) 18 | puts "Request: #{request}" 19 | 20 | case request['method'] 21 | when 'bind' 22 | do_bind s, request 23 | else 24 | response = { :error => { :code => -32601, :message => 'Method not found' } } 25 | s.sendmsg JSON.dump(response), 0, nil 26 | end 27 | end 28 | end 29 | 30 | def bind_to_env(bind) 31 | "MULTIBINDER_BIND__tcp__#{bind['address'].sub('.','_')}__#{bind['port']}" 32 | end 33 | 34 | def do_bind(s, request) 35 | bind = request['params'][0] 36 | 37 | begin 38 | name = bind_to_env(bind) 39 | if @sockets[name] 40 | socket = @sockets[name] 41 | elsif ENV[name] 42 | socket = IO.for_fd ENV[name].to_i 43 | else 44 | socket = Socket.new(:INET, :STREAM, 0) 45 | socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true) 46 | socket.fcntl(Fcntl::F_SETFD, socket.fcntl(Fcntl::F_GETFD) & (-Fcntl::FD_CLOEXEC-1)) 47 | socket.bind(Addrinfo.tcp(bind['address'], bind['port'])) 48 | socket.listen(bind['backlog'] || 1000) 49 | ENV[name] = socket.fileno.to_s 50 | end 51 | rescue Exception => e 52 | response = { 53 | :jsonrpc => '2.0', 54 | :id => request['id'], 55 | :error => { 56 | :code => 10000, 57 | :message => "Could not bind: #{e.message}", 58 | :backtrace => e.backtrace, 59 | }, 60 | } 61 | s.sendmsg JSON.dump(response), 0, nil 62 | return 63 | else 64 | @sockets[name] = socket 65 | 66 | response = { 67 | :jsonrpc => '2.0', 68 | :id => request['id'], 69 | :result => true, 70 | } 71 | s.sendmsg JSON.dump(response), 0, nil, Socket::AncillaryData.unix_rights(socket) 72 | end 73 | end 74 | 75 | def bind_accept_loop 76 | UNIXServer.open(@control_file) do |serv| 77 | puts "Listening for binds on control socket: #{@control_file}" 78 | STDOUT.flush 79 | 80 | Signal.trap("USR1") do 81 | serv.close 82 | begin 83 | File.unlink @control_file 84 | rescue Errno::ENOENT 85 | # cool! 86 | end 87 | 88 | # respawn ourselved in an identical way, keeping state through environment. 89 | puts 'Respawning...' 90 | STDOUT.flush 91 | args = [RbConfig.ruby, $0, *ARGV] 92 | args << { :close_others => false } if RUBY_VERSION > '1.9' # RUBBY. 93 | Kernel.exec *args 94 | end 95 | 96 | loop do 97 | s = serv.accept 98 | begin 99 | handle_client s 100 | rescue Exception => e 101 | puts e 102 | puts e.backtrace 103 | ensure 104 | s.close 105 | end 106 | end 107 | end 108 | end 109 | 110 | def serve 111 | begin 112 | File.unlink @control_file 113 | puts "Removed existing control socket: #{@control_file}" 114 | rescue Errno::ENOENT 115 | # :+1: 116 | end 117 | 118 | begin 119 | bind_accept_loop 120 | ensure 121 | begin 122 | File.unlink @control_file 123 | rescue Errno::ENOENT 124 | # cool! 125 | end 126 | end 127 | end 128 | end 129 | 130 | if ARGV.length == 0 131 | abort "Usage: #{$0} " 132 | else 133 | server = MultiBinderServer.new ARGV[0] 134 | server.serve 135 | end 136 | -------------------------------------------------------------------------------- /haproxy/README.md: -------------------------------------------------------------------------------- 1 | ## multibinder + HAProxy 2 | 3 | HAProxy by default doesn't include support for zero-downtime reloads. multibinder was developed to work with HAProxy, and has some useful wrappers included that allow running multiple HAProxy instances with multibinder on the same machine, while enabling zero-downtime reloads. 4 | 5 | ### Installation on Ubuntu 16.04 6 | 7 | Install multibinder and dependencies: 8 | ``` 9 | sudo apt-get install -y haproxy ruby 10 | sudo gem install multibinder 11 | ``` 12 | 13 | The multibinder systemd scripts automatically support multiple haproxy instances, so stop and disable the default haproxy service: 14 | ``` 15 | sudo systemctl stop haproxy 16 | sudo systemctl disable haproxy 17 | ``` 18 | 19 | Install the `multibinder` and `haproxy-multi@` systemd service files, and start multibinder itself: 20 | ``` 21 | sudo cp $(gem environment gemdir)/gems/multibinder-0.0.4/haproxy/*.service /etc/systemd/system/ 22 | sudo systemctl daemon-reload 23 | 24 | sudo systemctl enable multibinder 25 | sudo systemctl start multibinder 26 | ``` 27 | 28 | Create your first haproxy service configuration file, replacing all bind IP/ports with ERB code like the following: 29 | ``` 30 | cat >/etc/haproxy/foo.cfg.erb < 45 | EOF 46 | ``` 47 | 48 | Now start your multibinder-enabled haproxy `foo` service! 49 | ``` 50 | sudo systemctl enable haproxy-multi@foo 51 | sudo systemctl start haproxy-multi@foo 52 | ``` 53 | 54 | You'll have a process tree like the following: 55 | ``` 56 | $ sudo systemctl status haproxy-multi@foo 57 | ● haproxy-multi@foo.service - HAProxy Load Balancer 58 | Loaded: loaded (/etc/systemd/system/haproxy-multi@.service; enabled; vendor preset: enabled) 59 | Active: active (running) since Mon 2016-10-31 20:56:28 UTC; 1s ago 60 | Docs: man:haproxy(1) 61 | file:/usr/share/doc/haproxy/configuration.txt.gz 62 | Process: 3092 ExecStop=/bin/kill -TERM $MAINPID (code=exited, status=0/SUCCESS) 63 | Process: 3076 ExecReload=/bin/sh -c /usr/local/bin/multibinder-haproxy-erb /usr/sbin/haproxy -c -f ${CONFIG}; /bin/kill -USR2 $MAINPID (code=exited, status=0/SUCCESS) 64 | Process: 3105 ExecStartPre=/usr/local/bin/multibinder-haproxy-erb /usr/sbin/haproxy -f ${CONFIG} -c -q (code=exited, status=0/SUCCESS) 65 | Main PID: 3109 (multibinder-hap) 66 | CGroup: /system.slice/system-haproxy\x2dmulti.slice/haproxy-multi@foo.service 67 | ├─3109 /usr/bin/ruby2.3 /usr/local/bin/multibinder-haproxy-wrapper /usr/sbin/haproxy -Ds -f /etc/haproxy/foo.cfg.erb -p /run/haproxy-foo.pid 68 | ├─3113 /usr/sbin/haproxy -Ds -f /etc/haproxy/foo.cfg -p /run/haproxy-foo.pid 69 | └─3115 /usr/sbin/haproxy -Ds -f /etc/haproxy/foo.cfg -p /run/haproxy-foo.pid 70 | 71 | Oct 31 20:56:28 ip-172-31-8-204 systemd[1]: Starting HAProxy Load Balancer... 72 | Oct 31 20:56:28 ip-172-31-8-204 systemd[1]: Started HAProxy Load Balancer. 73 | ``` 74 | 75 | With multibinder running separately from the haproxy process(es): 76 | ``` 77 | $ sudo systemctl status multibinder 78 | ● multibinder.service - Multibinder 79 | Loaded: loaded (/etc/systemd/system/multibinder.service; enabled; vendor preset: enabled) 80 | Active: active (running) since Mon 2016-10-31 20:50:05 UTC; 7min ago 81 | Main PID: 2751 (multibinder) 82 | Tasks: 2 83 | Memory: 4.9M 84 | CPU: 44ms 85 | CGroup: /system.slice/multibinder.service 86 | └─2751 /usr/bin/ruby2.3 /usr/local/bin/multibinder /run/multibinder.sock 87 | 88 | Oct 31 20:50:05 ip-172-31-8-204 systemd[1]: Started Multibinder. 89 | Oct 31 20:50:05 ip-172-31-8-204 multibinder[2751]: Listening for binds on control socket: /run/multibinder.sock 90 | ``` 91 | 92 | Reloading an haproxy instance safely can then be requested through systemctl: 93 | ``` 94 | $ sudo systemctl reload haproxy-multi@foo 95 | ``` 96 | -------------------------------------------------------------------------------- /test/lib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Usage: . lib.sh 3 | # Simple shell command language test library. 4 | # 5 | # Tests must follow the basic form: 6 | # 7 | # begin_test "the thing" 8 | # ( 9 | # set -e 10 | # echo "hello" 11 | # false 12 | # ) 13 | # end_test 14 | # 15 | # When a test fails its stdout and stderr are shown. 16 | # 17 | # Note that tests must `set -e' within the subshell block or failed assertions 18 | # will not cause the test to fail and the result may be misreported. 19 | # 20 | # Copyright (c) 2011-13 by Ryan Tomayko 21 | # License: MIT 22 | 23 | set -e 24 | 25 | TEST_DIR=$(dirname "$0") 26 | ROLE=$(basename $(dirname "$0")) 27 | BASE_DIR=$(cd $(dirname "$0")/../ && pwd) 28 | 29 | TEMPDIR=$(mktemp -d /tmp/test-XXXXXX) 30 | HOME=$TEMPDIR; export HOME 31 | TRASHDIR="${TEMPDIR}" 32 | LOGDIR="$TEMPDIR/log" 33 | 34 | BUILD_DIR=${TEMPDIR}/build 35 | ROLE_DIR=$BUILD_DIR 36 | 37 | # keep track of num tests and failures 38 | tests=0 39 | failures=0 40 | 41 | #mkdir -p $TRASHDIR 42 | mkdir -p $LOGDIR 43 | 44 | # offset port numbers if running in '--batch' mode 45 | TEST_PORT_OFFSET=0 46 | if [ "$1" = "--batch" ]; then 47 | TEST_PORT_OFFSET=$(ls -1 $(dirname "$0")/test-*.sh | grep -n $(basename "$0") | grep -o "^[0-9]*") 48 | TEST_PORT_OFFSET=$(( $TEST_PORT_OFFSET - 1 )) 49 | fi 50 | 51 | # Sanity check up front that nothing is currently using the ports we're 52 | # trying use; port collisions will cause non-obvious test failures. 53 | tests_use_port () { 54 | local p=$(offset_port $1) 55 | 56 | set +e 57 | lsof -n -iTCP:$p | grep -q LISTEN 58 | if [ $? -eq 0 ]; then 59 | echo "**** $(basename "$0") FAIL: Found something using port $p, bailing." 60 | lsof -n -iTCP:$p | grep -e ":$p" | sed -e "s/^/lsof failure $(basename "$0"): /" 61 | exit 1 62 | fi 63 | set -e 64 | } 65 | 66 | # Given a port, increment it by our test number so multiple tests can run 67 | # in parallel without conflicting. 68 | offset_port () { 69 | local base_port="$1" 70 | 71 | echo $(( $base_port + $TEST_PORT_OFFSET )) 72 | } 73 | 74 | # Mark the beginning of a test. A subshell should immediately follow this 75 | # statement. 76 | begin_test () { 77 | test_status=$? 78 | [ -n "$test_description" ] && end_test $test_status 79 | unset test_status 80 | 81 | tests=$(( tests + 1 )) 82 | test_description="$1" 83 | 84 | exec 3>&1 4>&2 85 | out="$TRASHDIR/out" 86 | err="$TRASHDIR/err" 87 | exec 1>"$out" 2>"$err" 88 | 89 | echo "begin_test: $test_description" 90 | 91 | # allow the subshell to exit non-zero without exiting this process 92 | set -x +e 93 | before_time=$(date '+%s') 94 | } 95 | 96 | report_failure () { 97 | msg=$1 98 | desc=$2 99 | failures=$(( failures + 1 )) 100 | printf "test: %-60s $msg\n" "$desc ..." 101 | ( 102 | echo "-- stdout --" 103 | sed 's/^/ /' <"$TRASHDIR/out" 104 | echo "-- stderr --" 105 | grep -a -v -e '^\+ end_test' -e '^+ set +x' <"$TRASHDIR/err" | 106 | sed 's/^/ /' 107 | 108 | for service_log in $(ls $LOGDIR/*.log); do 109 | echo "-- $(basename "$service_log") --" 110 | sed 's/^/ /' <"$service_log" 111 | done 112 | 113 | echo "-- end --" 114 | ) 1>&2 115 | } 116 | 117 | # Mark the end of a test. 118 | end_test () { 119 | test_status="${1:-$?}" 120 | ex_fail="${2:-0}" 121 | after_time=$(date '+%s') 122 | set +x -e 123 | exec 1>&3 2>&4 124 | elapsed_time=$((after_time - before_time)) 125 | 126 | if [ "$test_status" -eq 0 ]; then 127 | if [ "$ex_fail" -eq 0 ]; then 128 | printf "test: %-60s OK (${elapsed_time}s)\n" "$test_description ..." 129 | else 130 | report_failure "OK (unexpected)" "$test_description ..." 131 | fi 132 | else 133 | if [ "$ex_fail" -eq 0 ]; then 134 | report_failure "FAILED (${elapsed_time}s)" "$test_description ..." 135 | else 136 | printf "test: %-60s FAILED (expected)\n" "$test_description ..." 137 | fi 138 | fi 139 | unset test_description 140 | } 141 | 142 | # Mark the end of a test that is expected to fail. 143 | end_test_exfail () { 144 | end_test $? 1 145 | } 146 | 147 | atexit () { 148 | [ -z "$KEEPTRASH" ] && rm -rf "$TEMPDIR" 149 | if [ $failures -gt 0 ]; then 150 | exit 1 151 | else 152 | exit 0 153 | fi 154 | } 155 | trap "atexit" EXIT 156 | 157 | cleanup() { 158 | set +e 159 | 160 | project_cleanup "$@" 161 | 162 | for pid_file in $(ls ${TEMPDIR}/*.pid); do 163 | echo "Cleaning up process in $pid_file ..." 164 | kill $(cat ${pid_file}) || true 165 | done 166 | 167 | echo "Cleaning up any remaining pid files." 168 | rm -rf ${TEMPDIR}/*.pid 169 | 170 | if [ -f "$TEMPDIR/core" ]; then 171 | echo "found a coredump, failing" 172 | exit 1 173 | fi 174 | } 175 | 176 | setup() { 177 | trap cleanup EXIT 178 | trap cleanup INT 179 | trap cleanup TERM 180 | 181 | project_setup "$@" 182 | 183 | set -e 184 | } 185 | 186 | wait_for_file () { 187 | ( 188 | SERVICE="$1" 189 | PID_FILE="$2" 190 | 191 | set +e 192 | 193 | tries=0 194 | 195 | echo "Waiting for $SERVICE to drop $PID_FILE" 196 | while [ ! -e "$PID_FILE" ]; do 197 | tries=$(( $tries + 1 )) 198 | if [ $tries -gt 50 ]; then 199 | echo "FAILED: $SERVICE did not drop $PID_FILE after $tries attempts" 200 | exit 1 201 | fi 202 | echo "Waiting for $SERVICE to drop $PID_FILE" 203 | sleep 0.1 204 | done 205 | echo "OK -- $SERVICE dropped $PID_FILE" 206 | exit 0 207 | ) 208 | } 209 | 210 | # wait for a process to start accepting connections 211 | wait_for_port () { 212 | ( 213 | SERVICE="$1" 214 | SERVICE_PORT="$2" 215 | 216 | set +e 217 | 218 | tries=0 219 | 220 | echo "Waiting for $SERVICE to start accepting connections" 221 | if [ $(uname) = "Linux" ]; then 222 | echo "PROXY TCP4 127.0.0.1 127.0.0.1 123 123\r" | nc -q 0 localhost $SERVICE_PORT 2>&1 >/dev/null 223 | else 224 | echo "PROXY TCP4 127.0.0.1 127.0.0.1 123 123\r" | nc localhost $SERVICE_PORT 2>&1 >/dev/null 225 | fi 226 | while [ $? -ne 0 ]; do 227 | tries=$(( $tries + 1 )) 228 | if [ $tries -gt 50 ]; then 229 | echo "FAILED: $SERVICE not accepting connections after $tries attempts" 230 | exit 1 231 | fi 232 | echo "Waiting for $SERVICE to start accepting connections" 233 | sleep 0.1 234 | if [ $(uname) = "Linux" ]; then 235 | echo "PROXY TCP4 127.0.0.1 127.0.0.1 123 123\r" | nc -q 0 localhost $SERVICE_PORT 2>&1 >/dev/null 236 | else 237 | echo "PROXY TCP4 127.0.0.1 127.0.0.1 123 123\r" | nc localhost $SERVICE_PORT 2>&1 >/dev/null 238 | fi 239 | done 240 | echo "OK -- $SERVICE seems to be accepting connections" 241 | exit 0 242 | ) 243 | } 244 | 245 | # Allow simple launching of a background service, keeping track of the pid 246 | launch_service () { 247 | local service_name=$1 248 | shift 249 | 250 | "$@" >${LOGDIR}/${service_name}.log 2>&1 & 251 | echo "$!" > ${TEMPDIR}/${service_name}.pid 252 | } 253 | 254 | # Clean up after a service launched with launch_service 255 | kill_service () { 256 | local service_name=$1 257 | kill $(cat ${TEMPDIR}/${service_name}.pid) || true 258 | rm -rf ${TEMPDIR}/${service_name}.pid 259 | } 260 | 261 | service_pid () { 262 | local service_name=$1 263 | cat ${TEMPDIR}/${service_name}.pid 264 | } 265 | 266 | service_log () { 267 | local service_name=$1 268 | echo ${LOGDIR}/${service_name}.log 269 | } 270 | 271 | # Stub out functions and let the project extend them 272 | project_setup () { 273 | true 274 | } 275 | 276 | project_cleanup () { 277 | true 278 | } 279 | 280 | if [ -e "$BASE_DIR/test/lib-project.sh" ]; then 281 | . $BASE_DIR/test/lib-project.sh 282 | fi 283 | --------------------------------------------------------------------------------