├── .gitignore ├── bin ├── starling └── starling_top ├── etc ├── sample-config.yml ├── starling.redhat └── starling.ubuntu ├── Rakefile ├── LICENSE ├── starling.gemspec ├── CHANGELOG ├── README.rdoc ├── lib ├── starling │ ├── server.rb │ ├── queue_collection.rb │ ├── persistent_queue.rb │ ├── handler.rb │ └── server_runner.rb └── starling.rb └── spec └── starling_server_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | pkg/* 2 | doc/* 3 | tmp/* 4 | 5 | -------------------------------------------------------------------------------- /bin/starling: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'starling/server_runner' 4 | 5 | StarlingServer::Runner.run 6 | 7 | -------------------------------------------------------------------------------- /etc/sample-config.yml: -------------------------------------------------------------------------------- 1 | starling: 2 | port: 22122 3 | pid_file: /tmp/starling/starling.pid 4 | queue_path: /tmp/starling/spool 5 | timeout: 0 6 | syslog_channel: starling-tampopo 7 | log_level: DEBUG 8 | daemonize: true 9 | 10 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rake/rdoctask' 3 | require 'spec/rake/spectask' 4 | 5 | task :default => :spec 6 | 7 | task :install do 8 | sh %{gem build starling.gemspec} 9 | sh %{sudo gem install starling-*.gem} 10 | end 11 | 12 | Spec::Rake::SpecTask.new do |t| 13 | t.ruby_opts = ['-rtest/unit'] 14 | t.spec_files = FileList['spec/*_spec.rb'] 15 | t.fail_on_error = true 16 | end 17 | 18 | Rake::RDocTask.new do |rd| 19 | rd.main = "README.rdoc" 20 | rd.rdoc_files.include("README.rdoc", "lib/**/*.rb") 21 | rd.rdoc_dir = 'doc' 22 | end 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2007 FIXME full name 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /starling.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = "starling" 3 | s.version = "0.9.8" 4 | s.authors = ["Blaine Cook", "Chris Wanstrath", "Britt Selvitelle", "Glenn Rempe", "Abdul-Rahman Advany"] 5 | s.email = ["blaine@twitter.com", "chris@ozmm.org", "abdulrahman@advany.com"] 6 | s.homepage = "http://github.com/starling/starling/" 7 | s.summary = "Starling is a lightweight, transactional, distributed queue server" 8 | s.description = s.summary 9 | 10 | s.require_path = 'lib' 11 | s.executables = ["starling", "starling_top"] 12 | 13 | # get this easily and accurately by running 'Dir.glob("{lib,test}/**/*")' 14 | # in an IRB session. However, GitHub won't allow that command hence 15 | # we spell it out. 16 | s.files = ["README.rdoc", "LICENSE", "CHANGELOG", "Rakefile", "lib/starling/handler.rb", "lib/starling/persistent_queue.rb", "lib/starling/queue_collection.rb", "lib/starling/server_runner.rb", "lib/starling/server.rb", "lib/starling.rb", "etc/starling.redhat", "etc/starling.ubuntu", "etc/sample-config.yml"] 17 | s.test_files = ["spec/starling_server_spec.rb"] 18 | 19 | s.has_rdoc = true 20 | s.rdoc_options = ["--quiet", "--title", "starling documentation", "--opname", "index.html", "--line-numbers", "--main", "README.rdoc", "--inline-source"] 21 | s.extra_rdoc_files = ["README.rdoc", "CHANGELOG", "LICENSE"] 22 | 23 | s.add_dependency 'memcache-client' 24 | s.add_dependency 'eventmachine', [">= 0.12.0"] 25 | end 26 | -------------------------------------------------------------------------------- /bin/starling_top: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'optparse' 4 | require 'ostruct' 5 | require 'socket' 6 | 7 | @options = OpenStruct.new 8 | @options.hostname = 'localhost' 9 | @options.port = 22122 10 | 11 | op = OptionParser.new do |opts| 12 | opts.banner = "STARLING TOP\nUsage: startop [options]" 13 | opts.separator "General Options:" 14 | opts.on("-h HOSTNAME", "--hostname=HOSTNAME", "Hostname [default: localhost]") do |h| 15 | @options.hostname = h 16 | end 17 | opts.on("-p PORT", "--port=PORT", Integer, "Port [default: 22122]") do |p| 18 | @options.port = p 19 | end 20 | opts.on_tail("--help", "Show this message") do 21 | puts opts 22 | exit 23 | end 24 | end 25 | op.parse! 26 | 27 | 28 | def stats_data 29 | data = '' 30 | sock = TCPSocket.new(@options.hostname, @options.port) 31 | sock.print("stats\r\n") 32 | sock.flush 33 | # memcached does not close the socket once it is done writing 34 | # the stats data. We need to read line by line until we detect 35 | # the END line and then stop/close on our side. 36 | stats = sock.gets 37 | while true 38 | data += stats 39 | break if stats.strip == 'END' 40 | stats = sock.gets 41 | end 42 | sock.close 43 | data 44 | end 45 | 46 | def parse(stats_data) 47 | stats = [] 48 | stats_data.each_line do |line| 49 | stats << "#{$1}: #{$2}" if line =~ /STAT (\w+) (\S+)/ 50 | end 51 | stats.sort 52 | end 53 | 54 | stats = parse(stats_data) 55 | stats.each do |stat| 56 | puts stat 57 | end 58 | -------------------------------------------------------------------------------- /etc/starling.redhat: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Make sure the /var/run/starling, /var/log/starling and /var/spool/starling 4 | # all exist and are owned by starling:starling 5 | # 6 | # starling This shell script takes care of starting and stopping 7 | # the starling server 8 | # chkconfig: 345 98 98 9 | # description: The starling queue server 10 | # processname: starling 11 | # pidfile: /var/run/starling/starling.pid 12 | # logfile: /var/log/starling/starling.log 13 | 14 | # Source function library. 15 | . /etc/rc.d/init.d/functions 16 | 17 | # Source networking configuration. 18 | [ -f /etc/sysconfig/network ] && . /etc/sysconfig/network 19 | 20 | DUSER=starling 21 | DGROUP=starling 22 | LOGFILE=/var/log/starling/starling.log 23 | SPOOLDIR=/var/spool/starling 24 | PORT=22122 25 | LISTEN=0.0.0.0 26 | PIDFILE=/var/run/starling/starling.pid 27 | 28 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 29 | NAME=starling 30 | INSTALL_DIR=/usr/local/bin 31 | DAEMON=$INSTALL_DIR/$NAME 32 | OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE -u $DUSER -g $DGROUP" 33 | 34 | start() { 35 | echo -n $"Starting starling: " 36 | 37 | daemon --pidfile=$PIDFILE $DAEMON $OPTS 38 | echo 39 | } 40 | 41 | stop() { 42 | echo -n $"Stopping starling: " 43 | 44 | killproc -p $PIDFILE starling 45 | echo 46 | } 47 | 48 | case "$1" in 49 | start) 50 | start 51 | ;; 52 | stop) 53 | stop 54 | ;; 55 | restart) 56 | stop 57 | sleep 3 58 | start 59 | ;; 60 | status) 61 | status -p $PIDFILE starling 62 | ;; 63 | *) 64 | echo $"Usage: $0 {start|stop|restart|status}" 65 | exit 1 66 | esac 67 | -------------------------------------------------------------------------------- /etc/starling.ubuntu: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: starling 4 | # Required-Start: $local_fs $remote_fs 5 | # Required-Stop: $local_fs $remote_fs 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: S 0 1 6 8 | # Short-Description: Starling queue server 9 | # Description: The Starling distributed, transactional queue server 10 | ### END INIT INFO 11 | # Author: Twitter 12 | # Version: 0.9.7.7 13 | 14 | set -e 15 | 16 | DUSER=starling 17 | DGROUP=starling 18 | LOGFILE=/var/log/starling/starling.log 19 | SPOOLDIR=/var/spool/starling 20 | PORT=22122 21 | LISTEN=0.0.0.0 22 | PIDFILE=/var/run/starling/starling.pid 23 | 24 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 25 | NAME=starling 26 | DESC="Starling" 27 | INSTALL_DIR=/usr/local/bin 28 | DAEMON=$INSTALL_DIR/$NAME 29 | SCRIPTNAME=/etc/init.d/$NAME 30 | OPTS="-h $LISTEN -p $PORT -d -q $SPOOLDIR -P $PIDFILE -L $LOGFILE" 31 | 32 | . /lib/lsb/init-functions 33 | 34 | 35 | # Gracefully exit if the package has been removed. 36 | test -x $DAEMON || exit 0 37 | 38 | d_start() { 39 | log_begin_msg "Starting Starling Server..." 40 | start-stop-daemon -c $DUSER:$DGROUP --start --quiet --pidfile $PIDFILE --exec $DAEMON \ 41 | -- $OPTS || log_end_msg 1 42 | log_end_msg 0 43 | } 44 | 45 | d_stop() { 46 | log_begin_msg "Stopping Starling Server..." 47 | start-stop-daemon -c $DUSER:$DGROUP --stop --quiet --pidfile $PIDFILE \ 48 | || log_end_msg 1 49 | log_end_msg 0 50 | } 51 | 52 | case "$1" in 53 | start) 54 | d_start 55 | ;; 56 | stop) 57 | d_stop 58 | ;; 59 | restart|force-reload|reload) 60 | d_stop 61 | sleep 2 62 | d_start 63 | ;; 64 | *) 65 | echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 66 | exit 3 67 | ;; 68 | esac 69 | 70 | exit 0 71 | 72 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | == 0.9.9 2 | * remove dependency on SyslogLogger so that starling works on windows. 3 | * fix config file loading so relative paths are expanded properly 4 | * clean up redhat init.d script 5 | * add a 'fetch' command that is non-blocking equivalent to 'get' 6 | * implement the 'delete' method to allow queues to be deleted 7 | 8 | == 0.9.8 9 | * add fix to enable relative paths 10 | * fix tests so they don't run 10 times due to a stupid bug with how the server is forked 11 | * fix some other tests 12 | * fix some error messages 13 | * probably some other things 14 | 15 | == 0.9.7.9 16 | * properly complain if the spool directory isn't writable 17 | * assume group and user privileges in a working order 18 | * support string user / group names in addition to uid/gids 19 | 20 | == 0.9.7.7 21 | * added init.d scripts for redhat and ubuntu by Mike Perham 22 | * fixed dependencies for SyslogLogger, eventmachine and memcache-client by Mike Perham 23 | * added starling_top script to monitor starling server by Mike Perham 24 | * fixed starling_top to use 22122 as port by Abdul-Rahman Advany 25 | 26 | == 0.9.7.6 2008-06-24 27 | * removed client code (will be added to different project) by Abdul-Rahman Advany 28 | 29 | == 0.9.7.5 2008-05-04 30 | * added worker class, using starling client you can now run them in background by Abdul-Rahman Advany 31 | - handles creation of threadpool 32 | - handles fetching of messages and passing these to the threads 33 | - handles pushing of processed messages to starling again if needed 34 | 35 | == 0.9.7 2008-05-03 36 | * merged branch of AnotherBritt and Glenn Rempe by Abdul-Rahman Advany 37 | * rspeced tests by Abdul-Rahman Advany 38 | 39 | == 0.9.6 2008-04-30 40 | * logging of message lifecycle as :age by AnotherBritt 41 | * added some extra logging options by AnotherBritt 42 | * added some test for epoll by AnotherBritt 43 | 44 | == 0.9.5.4 2008-04-28 45 | 46 | * Bumped version number by Glenn Rempe 47 | * Purged all old RubyForge config. Now GitHub friendly by Glenn Rempe 48 | * New gemspec for GitHub gem auto-build-serve by Glenn Rempe 49 | 50 | == 0.9.4 2008-01-31 == 51 | * Evented code added using EventMachine by Chris Wanstrath 52 | 53 | == 2007-11-02 54 | 55 | * Initial release 56 | 57 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = Name 2 | 3 | Starling - a light weight server for reliable distributed message passing. 4 | 5 | = Description 6 | 7 | Starling is a powerful but simple messaging server that enables reliable 8 | distributed queuing with an absolutely minimal overhead. It speaks the 9 | MemCache protocol for maximum cross-platform compatibility. Any language 10 | that speaks MemCache can take advantage of Starling's queue facilities. 11 | 12 | = Installation 13 | 14 | This fork of the Starling source is hosted at GitHub and can be found at: 15 | 16 | http://github.com/starling/starling/tree/master 17 | 18 | The original source was to be found at RubyForge but no longer exists there. 19 | 20 | GitHub serves gems prefixed by a username to differentiate different forks. 21 | This project can be installed with: 22 | 23 | # THIS COMMAND ONE TIME ONLY 24 | gem sources -a http://gems.github.com/ 25 | 26 | # As often as you like 27 | sudo gem install starling-starling 28 | 29 | See http://gems.github.com/ if you want more info about GitHub and gems. 30 | 31 | = Quick Start Usage 32 | 33 | # View the Starling help and usage message 34 | starling --help 35 | 36 | # In a console window start the Starling server. By default 37 | # it runs verbosely in the foreground, listening on 127.0.0.1:22122 38 | # and stores its files under /tmp/starling: 39 | starling 40 | 41 | # In a new console test the put and get of messages on a queue: 42 | 43 | irb 44 | >> require 'starling' 45 | => true 46 | >> starling = Starling.new('127.0.0.1:22122') 47 | => MemCache: 1 servers, 1 buckets, ns: nil, ro: false 48 | >> starling.set('my_queue', 12345) 49 | => nil 50 | >> starling.get('my_queue') 51 | => 12345 52 | 53 | # You can do a simple loop over a queue with something like: 54 | >> loop { puts starling.get('my_queue'); sleep 1 } 55 | 12345 56 | nil 57 | nil 58 | ... 59 | 60 | For more information run the following in a new console: 61 | 62 | 'gem server' 63 | 64 | This will start a gem server on http://localhost:8808/ which you can view in your 65 | browser to see the RDocs for the gem. Or generate rdocs by running the following 66 | in a new console: 67 | 68 | 'rdoc' 69 | 70 | = Using fiveruns memcache-client 71 | 72 | memcache-client from fiveruns has a couple of fixed added like supporting failover and retry on failure. 73 | 74 | This fork of the memcache-client source is hosted at GitHub and can be found at: 75 | 76 | http://github.com/fiveruns/memcache-client/tree/master 77 | 78 | It can be installed using: 79 | 80 | # THIS COMMAND ONE TIME ONLY 81 | gem sources -a http://gems.github.com/ 82 | 83 | # As often as you like 84 | sudo gem install fiveruns-memcache-client 85 | 86 | = Known Issues 87 | 88 | * Starling is "slow" as far as messaging systems are concerned. In practice, 89 | it's fast enough. 90 | 91 | == TODO 92 | 93 | * Implement memcached instead of memcache as a client interface (to make it faster) 94 | 95 | = Authors 96 | 97 | * Blaine Cook 98 | * Chris Wanstrath 99 | * AnotherBritt 100 | * Glenn Rempe 101 | * Abdul-Rahman Advany 102 | 103 | = Copyright 104 | 105 | Starling - a light-weight server for reliable distributed message passing. 106 | Copyright 2007-2008 Blaine Cook , Twitter Inc. 107 | -------------------------------------------------------------------------------- /lib/starling/server.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | require 'logger' 3 | require 'rubygems' 4 | require 'eventmachine' 5 | 6 | here = File.dirname(__FILE__) 7 | 8 | require File.join(here, 'queue_collection') 9 | require File.join(here, 'handler') 10 | 11 | module StarlingServer 12 | 13 | VERSION = "0.9.8" 14 | 15 | class Base 16 | attr_reader :logger 17 | 18 | DEFAULT_HOST = '127.0.0.1' 19 | DEFAULT_PORT = 22122 20 | DEFAULT_PATH = "/tmp/starling/" 21 | DEFAULT_TIMEOUT = 60 22 | 23 | ## 24 | # Initialize a new Starling server and immediately start processing 25 | # requests. 26 | # 27 | # +opts+ is an optional hash, whose valid options are: 28 | # 29 | # [:host] Host on which to listen (default is 127.0.0.1). 30 | # [:port] Port on which to listen (default is 22122). 31 | # [:path] Path to Starling queue logs. Default is /tmp/starling/ 32 | # [:timeout] Time in seconds to wait before closing connections. 33 | # [:logger] A Logger object, an IO handle, or a path to the log. 34 | # [:loglevel] Logger verbosity. Default is Logger::ERROR. 35 | # 36 | # Other options are ignored. 37 | 38 | def self.start(opts = {}) 39 | server = self.new(opts) 40 | server.run 41 | end 42 | 43 | ## 44 | # Initialize a new Starling server, but do not accept connections or 45 | # process requests. 46 | # 47 | # +opts+ is as for +start+ 48 | 49 | def initialize(opts = {}) 50 | @opts = { 51 | :host => DEFAULT_HOST, 52 | :port => DEFAULT_PORT, 53 | :path => DEFAULT_PATH, 54 | :timeout => DEFAULT_TIMEOUT, 55 | :server => self 56 | }.merge(opts) 57 | 58 | @stats = Hash.new(0) 59 | 60 | FileUtils.mkdir_p(@opts[:path]) 61 | 62 | end 63 | 64 | ## 65 | # Start listening and processing requests. 66 | 67 | def run 68 | @stats[:start_time] = Time.now 69 | 70 | if @opts[:syslog_channel] 71 | begin 72 | require 'syslog_logger' 73 | @@logger = SyslogLogger.new(@opts[:syslog_channel]) 74 | rescue LoadError 75 | # SyslogLogger isn't available, so we're just going to use Logger 76 | end 77 | end 78 | 79 | @@logger ||= case @opts[:logger] 80 | when IO, String; Logger.new(@opts[:logger]) 81 | when Logger; @opts[:logger] 82 | else; Logger.new(STDERR) 83 | end 84 | 85 | begin 86 | @opts[:queue] = QueueCollection.new(@opts[:path]) 87 | rescue InaccessibleQueuePath => e 88 | puts "Error: #{e.message}" 89 | exit 1 90 | end 91 | @@logger.level = @opts[:log_level] || Logger::ERROR 92 | 93 | @@logger.info "Starling STARTUP on #{@opts[:host]}:#{@opts[:port]}" 94 | 95 | EventMachine.epoll 96 | EventMachine.set_descriptor_table_size(4096) 97 | EventMachine.run do 98 | EventMachine.start_server(@opts[:host], @opts[:port], Handler, @opts) 99 | end 100 | 101 | # code here will get executed on shutdown: 102 | @opts[:queue].close 103 | end 104 | 105 | def self.logger 106 | @@logger 107 | end 108 | 109 | 110 | ## 111 | # Stop accepting new connections and shutdown gracefully. 112 | 113 | def stop 114 | EventMachine.stop_event_loop 115 | end 116 | 117 | def stats(stat = nil) #:nodoc: 118 | case stat 119 | when nil; @stats 120 | when :connections; 1 121 | else; @stats[stat] 122 | end 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/starling/queue_collection.rb: -------------------------------------------------------------------------------- 1 | require 'thread' 2 | require 'starling/persistent_queue' 3 | 4 | module StarlingServer 5 | class InaccessibleQueuePath < Exception #:nodoc: 6 | end 7 | 8 | ## 9 | # QueueCollection is a proxy to a collection of PersistentQueue instances. 10 | 11 | class QueueCollection 12 | 13 | ## 14 | # Create a new QueueCollection at +path+ 15 | 16 | def initialize(path) 17 | unless File.directory?(path) && File.writable?(path) 18 | raise InaccessibleQueuePath.new("'#{path}' must exist and be read-writable by #{Etc.getpwuid(Process.uid).name}.") 19 | end 20 | 21 | @shutdown_mutex = Mutex.new 22 | 23 | @path = path 24 | @logger = StarlingServer::Base.logger 25 | 26 | @queues = {} 27 | @queue_init_mutexes = {} 28 | 29 | @stats = Hash.new(0) 30 | end 31 | 32 | ## 33 | # Puts +data+ onto the queue named +key+ 34 | 35 | def put(key, data) 36 | queue = queues(key) 37 | return nil unless queue 38 | 39 | @stats[:current_bytes] += data.size 40 | @stats[:total_items] += 1 41 | 42 | queue.push(data) 43 | 44 | return true 45 | end 46 | 47 | ## 48 | # Retrieves data from the queue named +key+ 49 | 50 | def take(key) 51 | queue = queues(key) 52 | if queue.nil? || queue.length == 0 53 | @stats[:get_misses] += 1 54 | return nil 55 | else 56 | @stats[:get_hits] += 1 57 | end 58 | result = queue.pop 59 | @stats[:current_bytes] -= result.size 60 | result 61 | end 62 | 63 | def delete(key) 64 | queue = @queues.delete(key) 65 | return if queue.nil? 66 | queue.purge 67 | end 68 | 69 | ## 70 | # Returns all active queues. 71 | 72 | def queues(key=nil) 73 | return nil if @shutdown_mutex.locked? 74 | 75 | return @queues if key.nil? 76 | 77 | # First try to return the queue named 'key' if it's available. 78 | return @queues[key] if @queues[key] 79 | 80 | # If the queue wasn't available, create or get the mutex that will 81 | # wrap creation of the Queue. 82 | @queue_init_mutexes[key] ||= Mutex.new 83 | 84 | # Otherwise, check to see if another process is already loading 85 | # the queue named 'key'. 86 | if @queue_init_mutexes[key].locked? 87 | # return an empty/false result if we're waiting for the queue 88 | # to be loaded and we're not the first process to request the queue 89 | return nil 90 | else 91 | begin 92 | @queue_init_mutexes[key].lock 93 | # we've locked the mutex, but only go ahead if the queue hasn't 94 | # been loaded. There's a race condition otherwise, and we could 95 | # end up loading the queue multiple times. 96 | if @queues[key].nil? 97 | @queues[key] = PersistentQueue.new(@path, key) 98 | @stats[:current_bytes] += @queues[key].initial_bytes 99 | end 100 | rescue Object => exc 101 | puts "ZOMG There was an exception reading back the queue. That totally sucks." 102 | puts "The exception was: #{exc}. Backtrace: #{exc.backtrace.join("\n")}" 103 | ensure 104 | @queue_init_mutexes[key].unlock 105 | end 106 | end 107 | 108 | return @queues[key] 109 | end 110 | 111 | ## 112 | # Returns statistic +stat_name+ for the QueueCollection. 113 | # 114 | # Valid statistics are: 115 | # 116 | # [:get_misses] Total number of get requests with empty responses 117 | # [:get_hits] Total number of get requests that returned data 118 | # [:current_bytes] Current size in bytes of items in the queues 119 | # [:current_size] Current number of items across all queues 120 | # [:total_items] Total number of items stored in queues. 121 | 122 | def stats(stat_name) 123 | case stat_name 124 | when nil; @stats 125 | when :current_size; current_size 126 | else; @stats[stat_name] 127 | end 128 | end 129 | 130 | ## 131 | # Safely close all queues. 132 | 133 | def close 134 | @shutdown_mutex.lock 135 | @queues.each_pair do |name,queue| 136 | queue.close 137 | @queues.delete(name) 138 | end 139 | end 140 | 141 | private 142 | 143 | def current_size #:nodoc: 144 | @queues.inject(0) { |m, (k,v)| m + v.length } 145 | end 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /lib/starling.rb: -------------------------------------------------------------------------------- 1 | require 'memcache' 2 | 3 | class Starling < MemCache 4 | 5 | WAIT_TIME = 0.25 6 | alias_method :_original_get, :get 7 | alias_method :_original_delete, :delete 8 | 9 | ## 10 | # fetch an item from a queue. 11 | 12 | def get(*args) 13 | loop do 14 | response = _original_get(*args) 15 | return response unless response.nil? 16 | sleep WAIT_TIME 17 | end 18 | end 19 | 20 | ## 21 | # will return the next item or nil 22 | 23 | def fetch(*args) 24 | _original_get(*args) 25 | end 26 | 27 | ## 28 | # Delete the key (queue) from all Starling servers. This is necessary 29 | # because the random way a server is chosen in #get_server_for_key 30 | # implies that the queue could easily be spread across the entire 31 | # Starling cluster. 32 | 33 | def delete(key, expiry = 0) 34 | with_servers do 35 | _original_delete(key, expiry) 36 | end 37 | end 38 | 39 | ## 40 | # Provides a way to work with a specific list of servers by 41 | # forcing all calls to #get_server_for_key to use a specific 42 | # server, and changing that server each time that the call 43 | # yields to the block provided. This helps work around the 44 | # normally random nature of the #get_server_for_key method. 45 | # 46 | # Acquires the mutex for the entire duration of the call 47 | # since unrelated calls to #get_server_for_key might be 48 | # adversely affected by the non_random result. 49 | def with_servers(my_servers = @servers.dup) 50 | return unless block_given? 51 | with_lock do 52 | my_servers.each do |server| 53 | @force_server = server 54 | yield 55 | end 56 | @force_server = nil 57 | end 58 | end 59 | 60 | ## 61 | # insert +value+ into +queue+. 62 | # 63 | # +expiry+ is expressed as a UNIX timestamp 64 | # 65 | # If +raw+ is true, +value+ will not be Marshalled. If +raw+ = :yaml, +value+ 66 | # will be serialized with YAML, instead. 67 | 68 | def set(queue, value, expiry = 0, raw = false) 69 | retries = 0 70 | begin 71 | if raw == :yaml 72 | value = YAML.dump(value) 73 | raw = true 74 | end 75 | 76 | super(queue, value, expiry, raw) 77 | rescue MemCache::MemCacheError => e 78 | retries += 1 79 | sleep WAIT_TIME 80 | retry unless retries > 3 81 | raise e 82 | end 83 | end 84 | 85 | ## 86 | # returns the number of items in +queue+. If +queue+ is +:all+, a hash of all 87 | # queue sizes will be returned. 88 | 89 | def sizeof(queue, statistics = nil) 90 | statistics ||= stats 91 | 92 | if queue == :all 93 | queue_sizes = {} 94 | available_queues(statistics).each do |queue| 95 | queue_sizes[queue] = sizeof(queue, statistics) 96 | end 97 | return queue_sizes 98 | end 99 | 100 | statistics.inject(0) { |m,(k,v)| m + v["queue_#{queue}_items"].to_i } 101 | end 102 | 103 | ## 104 | # returns a list of available (currently allocated) queues. 105 | 106 | def available_queues(statistics = nil) 107 | statistics ||= stats 108 | 109 | statistics.map { |k,v| 110 | v.keys 111 | }.flatten.uniq.grep(/^queue_(.*)_items/).map { |v| 112 | v.gsub(/^queue_/, '').gsub(/_items$/, '') 113 | }.reject { |v| 114 | v =~ /_total$/ || v =~ /_expired$/ 115 | } 116 | end 117 | 118 | ## 119 | # iterator to flush +queue+. Each element will be passed to the provided 120 | # +block+ 121 | 122 | def flush(queue) 123 | sizeof(queue).times do 124 | v = get(queue) 125 | yield v if block_given? 126 | end 127 | end 128 | 129 | private 130 | 131 | def get_server_for_key(key) 132 | raise ArgumentError, "illegal character in key #{key.inspect}" if key =~ /\s/ 133 | raise ArgumentError, "key too long #{key.inspect}" if key.length > 250 134 | raise MemCacheError, "No servers available" if @servers.empty? 135 | return @force_server if @force_server 136 | 137 | bukkits = @buckets.dup 138 | bukkits.nitems.times do |try| 139 | n = rand(bukkits.nitems) 140 | server = bukkits[n] 141 | return server if server.alive? 142 | bukkits.delete_at(n) 143 | end 144 | 145 | raise MemCacheError, "No servers available (all dead)" 146 | end 147 | end 148 | 149 | 150 | class MemCache 151 | 152 | protected 153 | 154 | ## 155 | # Ensure that everything within the given block is executed 156 | # within the locked mutex if this client is multithreaded. 157 | # If the client isn't multithreaded, the block is simply executed. 158 | def with_lock 159 | return unless block_given? 160 | begin 161 | @mutex.lock if @multithread 162 | yield 163 | ensure 164 | @mutex.unlock if @multithread 165 | end 166 | end 167 | 168 | end -------------------------------------------------------------------------------- /lib/starling/persistent_queue.rb: -------------------------------------------------------------------------------- 1 | module StarlingServer 2 | 3 | ## 4 | # PersistentQueue is a subclass of Ruby's thread-safe Queue class. It adds a 5 | # transactional log to the in-memory Queue, which enables quickly rebuilding 6 | # the Queue in the event of a sever outage. 7 | 8 | class PersistentQueue < Queue 9 | 10 | ## 11 | # When a log reaches the SOFT_LOG_MAX_SIZE, the Queue will wait until 12 | # it is empty, and will then rotate the log file. 13 | 14 | SOFT_LOG_MAX_SIZE = 16 * (1024**2) # 16 MB 15 | 16 | TRX_CMD_PUSH = "\000".freeze 17 | TRX_CMD_POP = "\001".freeze 18 | 19 | TRX_PUSH = "\000%s%s".freeze 20 | TRX_POP = "\001".freeze 21 | 22 | attr_reader :initial_bytes 23 | attr_reader :total_items 24 | attr_reader :logsize 25 | attr_reader :current_age 26 | 27 | ## 28 | # Create a new PersistentQueue at +persistence_path+/+queue_name+. 29 | # If a queue log exists at that path, the Queue will be loaded from 30 | # disk before being available for use. 31 | 32 | def initialize(persistence_path, queue_name, debug = false) 33 | @persistence_path = persistence_path 34 | @queue_name = queue_name 35 | @total_items = 0 36 | super() 37 | @initial_bytes = replay_transaction_log(debug) 38 | @current_age = 0 39 | end 40 | 41 | ## 42 | # Pushes +value+ to the queue. By default, +push+ will write to the 43 | # transactional log. Set +log_trx=false+ to override this behaviour. 44 | 45 | def push(value, log_trx = true) 46 | if log_trx 47 | raise NoTransactionLog unless @trx 48 | size = [value.size].pack("I") 49 | transaction sprintf(TRX_PUSH, size, value) 50 | end 51 | 52 | @total_items += 1 53 | super([now_usec, value]) 54 | end 55 | 56 | ## 57 | # Retrieves data from the queue. 58 | 59 | def pop(log_trx = true) 60 | raise NoTransactionLog if log_trx && !@trx 61 | 62 | begin 63 | rv = super(!log_trx) 64 | rescue ThreadError 65 | puts "WARNING: The queue was empty when trying to pop(). Technically this shouldn't ever happen. Probably a bug in the transactional underpinnings. Or maybe shutdown didn't happen cleanly at some point. Ignoring." 66 | rv = [now_usec, ''] 67 | end 68 | transaction "\001" if log_trx 69 | @current_age = (now_usec - rv[0]) / 1000 70 | rv[1] 71 | end 72 | 73 | ## 74 | # Safely closes the transactional queue. 75 | 76 | def close 77 | # Ok, yeah, this is lame, and is *technically* a race condition. HOWEVER, 78 | # the QueueCollection *should* have stopped processing requests, and I don't 79 | # want to add yet another Mutex around all the push and pop methods. So we 80 | # do the next simplest thing, and minimize the time we'll stick around before 81 | # @trx is nil. 82 | @not_trx = @trx 83 | @trx = nil 84 | @not_trx.close 85 | end 86 | 87 | def purge 88 | close 89 | File.delete(log_path) 90 | end 91 | 92 | private 93 | 94 | def log_path #:nodoc: 95 | File.join(@persistence_path, @queue_name) 96 | end 97 | 98 | def reopen_log #:nodoc: 99 | @trx = File.new(log_path, File::CREAT|File::RDWR) 100 | @logsize = File.size(log_path) 101 | end 102 | 103 | def rotate_log #:nodoc: 104 | @trx.close 105 | backup_logfile = "#{log_path}.#{Time.now.to_i}" 106 | File.rename(log_path, backup_logfile) 107 | reopen_log 108 | File.unlink(backup_logfile) 109 | end 110 | 111 | def replay_transaction_log(debug) #:nodoc: 112 | reopen_log 113 | bytes_read = 0 114 | 115 | print "Reading back transaction log for #{@queue_name} " if debug 116 | 117 | while !@trx.eof? 118 | cmd = @trx.read(1) 119 | case cmd 120 | when TRX_CMD_PUSH 121 | print ">" if debug 122 | raw_size = @trx.read(4) 123 | next unless raw_size 124 | size = raw_size.unpack("I").first 125 | data = @trx.read(size) 126 | next unless data 127 | push(data, false) 128 | bytes_read += data.size 129 | when TRX_CMD_POP 130 | print "<" if debug 131 | bytes_read -= pop(false).size 132 | else 133 | puts "Error reading transaction log: " + 134 | "I don't understand '#{cmd}' (skipping)." if debug 135 | end 136 | end 137 | 138 | print " done.\n" if debug 139 | 140 | return bytes_read 141 | end 142 | 143 | def transaction(data) #:nodoc: 144 | raise "no transaction log handle. that totally sucks." unless @trx 145 | 146 | @trx.write_nonblock data 147 | @logsize += data.size 148 | rotate_log if @logsize > SOFT_LOG_MAX_SIZE && self.length == 0 149 | end 150 | 151 | def now_usec 152 | now = Time.now 153 | now.to_i * 1000000 + now.usec 154 | end 155 | end 156 | end 157 | -------------------------------------------------------------------------------- /spec/starling_server_spec.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.join(File.dirname(__FILE__), "..", "lib")) 2 | 3 | require 'rubygems' 4 | require 'fileutils' 5 | require 'memcache' 6 | require 'digest/md5' 7 | require 'starling' 8 | 9 | require 'starling/server' 10 | 11 | class StarlingServer::PersistentQueue 12 | remove_const :SOFT_LOG_MAX_SIZE 13 | SOFT_LOG_MAX_SIZE = 16 * 1024 # 16 KB 14 | end 15 | 16 | def safely_fork(&block) 17 | # anti-race juice: 18 | blocking = true 19 | Signal.trap("USR1") { blocking = false } 20 | 21 | pid = Process.fork(&block) 22 | 23 | while blocking 24 | sleep 0.1 25 | end 26 | 27 | pid 28 | end 29 | 30 | describe "StarlingServer" do 31 | before do 32 | @tmp_path = File.join(File.dirname(__FILE__), "tmp") 33 | 34 | begin 35 | Dir::mkdir(@tmp_path) 36 | rescue Errno::EEXIST 37 | end 38 | 39 | @server_pid = safely_fork do 40 | server = StarlingServer::Base.new(:host => '127.0.0.1', 41 | :port => 22133, 42 | :path => @tmp_path, 43 | :logger => Logger.new(STDERR), 44 | :log_level => Logger::FATAL) 45 | Signal.trap("INT") { 46 | server.stop 47 | exit 48 | } 49 | 50 | Process.kill("USR1", Process.ppid) 51 | server.run 52 | end 53 | 54 | @client = MemCache.new('127.0.0.1:22133') 55 | 56 | end 57 | 58 | it "should test if temp_path exists and is writeable" do 59 | File.exist?(@tmp_path).should be_true 60 | File.directory?(@tmp_path).should be_true 61 | File.writable?(@tmp_path).should be_true 62 | end 63 | 64 | it "should set and get" do 65 | v = rand((2**32)-1) 66 | @client.get('test_set_and_get_one_entry').should be_nil 67 | @client.set('test_set_and_get_one_entry', v) 68 | @client.get('test_set_and_get_one_entry').should eql(v) 69 | end 70 | 71 | it "should respond to delete" do 72 | @client.delete("my_queue").should eql("END\r\n") 73 | starling_client = Starling.new('127.0.0.1:22133') 74 | starling_client.set('my_queue', 50) 75 | starling_client.available_queues.size.should eql(1) 76 | starling_client.delete("my_queue") 77 | starling_client.available_queues.size.should eql(0) 78 | end 79 | 80 | it "should expire entries" do 81 | v = rand((2**32)-1) 82 | @client.get('test_set_with_expiry').should be_nil 83 | now = Time.now.to_i 84 | @client.set('test_set_with_expiry', v + 2, now) 85 | @client.set('test_set_with_expiry', v) 86 | sleep(1.0) 87 | @client.get('test_set_with_expiry').should eql(v) 88 | end 89 | 90 | it "should have age stat" do 91 | now = Time.now.to_i 92 | @client.set('test_age', 'nibbler') 93 | sleep(1.0) 94 | @client.get('test_age').should eql('nibbler') 95 | 96 | stats = @client.stats['127.0.0.1:22133'] 97 | stats.has_key?('queue_test_age_age').should be_true 98 | (stats['queue_test_age_age'] >= 1000).should be_true 99 | end 100 | 101 | it "should rotate log" do 102 | log_rotation_path = File.join(@tmp_path, 'test_log_rotation') 103 | 104 | Dir.glob("#{log_rotation_path}*").each do |file| 105 | File.unlink(file) rescue nil 106 | end 107 | @client.get('test_log_rotation').should be_nil 108 | 109 | v = 'x' * 8192 110 | 111 | @client.set('test_log_rotation', v) 112 | File.size(log_rotation_path).should eql(8207) 113 | @client.get('test_log_rotation') 114 | 115 | @client.get('test_log_rotation').should be_nil 116 | 117 | @client.set('test_log_rotation', v) 118 | @client.get('test_log_rotation').should eql(v) 119 | 120 | File.size(log_rotation_path).should eql(1) 121 | # rotated log should be erased after a successful roll. 122 | Dir.glob("#{log_rotation_path}*").size.should eql(1) 123 | end 124 | 125 | it "should output statistics per server" do 126 | stats = @client.stats 127 | assert_kind_of Hash, stats 128 | assert stats.has_key?('127.0.0.1:22133') 129 | 130 | server_stats = stats['127.0.0.1:22133'] 131 | 132 | basic_stats = %w( bytes pid time limit_maxbytes cmd_get version 133 | bytes_written cmd_set get_misses total_connections 134 | curr_connections curr_items uptime get_hits total_items 135 | rusage_system rusage_user bytes_read ) 136 | 137 | basic_stats.each do |stat| 138 | server_stats.has_key?(stat).should be_true 139 | end 140 | end 141 | 142 | it "should return valid response with unkown command" do 143 | response = @client.add('blah', 1) 144 | response.should eql("CLIENT_ERROR bad command line format\r\n") 145 | end 146 | 147 | it "should disconnect and reconnect again" do 148 | v = rand(2**32-1) 149 | @client.set('test_that_disconnecting_and_reconnecting_works', v) 150 | @client.reset 151 | @client.get('test_that_disconnecting_and_reconnecting_works').should eql(v) 152 | end 153 | 154 | it "should use epoll on linux" do 155 | # this may take a few seconds. 156 | # the point is to make sure that we're using epoll on Linux, so we can 157 | # handle more than 1024 connections. 158 | 159 | unless IO::popen("uname").read.chomp == "Linux" 160 | pending "skipping epoll test: not on Linux" 161 | end 162 | 163 | fd_limit = IO::popen("bash -c 'ulimit -n'").read.chomp.to_i 164 | unless fd_limit > 1024 165 | pending "skipping epoll test: 'ulimit -n' = #{fd_limit}, need > 1024" 166 | end 167 | 168 | v = rand(2**32 - 1) 169 | @client.set('test_epoll', v) 170 | 171 | # we can't open 1024 connections to memcache from within this process, 172 | # because we will hit ruby's 1024 fd limit ourselves! 173 | pid1 = safely_fork do 174 | unused_sockets = [] 175 | 600.times do 176 | unused_sockets << TCPSocket.new("127.0.0.1", 22133) 177 | end 178 | Process.kill("USR1", Process.ppid) 179 | sleep 90 180 | end 181 | pid2 = safely_fork do 182 | unused_sockets = [] 183 | 600.times do 184 | unused_sockets << TCPSocket.new("127.0.0.1", 22133) 185 | end 186 | Process.kill("USR1", Process.ppid) 187 | sleep 90 188 | end 189 | 190 | begin 191 | client = MemCache.new('127.0.0.1:22133') 192 | client.get('test_epoll').should eql(v) 193 | ensure 194 | Process.kill("TERM", pid1) 195 | Process.kill("TERM", pid2) 196 | end 197 | end 198 | 199 | it "should raise error if queue collection is an invalid path" do 200 | invalid_path = nil 201 | while invalid_path.nil? || File.exist?(invalid_path) 202 | invalid_path = File.join('/', Digest::MD5.hexdigest(rand(2**32-1).to_s)[0,8]) 203 | end 204 | 205 | lambda { 206 | StarlingServer::QueueCollection.new(invalid_path) 207 | }.should raise_error(StarlingServer::InaccessibleQueuePath) 208 | end 209 | 210 | after do 211 | Process.kill("INT", @server_pid) 212 | Process.wait(@server_pid) 213 | @client.reset 214 | FileUtils.rm(Dir.glob(File.join(@tmp_path, '*'))) 215 | end 216 | end 217 | -------------------------------------------------------------------------------- /lib/starling/handler.rb: -------------------------------------------------------------------------------- 1 | module StarlingServer 2 | 3 | ## 4 | # This is an internal class that's used by Starling::Server to handle the 5 | # MemCache protocol and act as an interface between the Server and the 6 | # QueueCollection. 7 | 8 | class Handler < EventMachine::Connection 9 | 10 | DATA_PACK_FMT = "Ia*".freeze 11 | 12 | # ERROR responses 13 | ERR_UNKNOWN_COMMAND = "CLIENT_ERROR bad command line format\r\n".freeze 14 | 15 | # GET Responses 16 | GET_COMMAND = /\Aget (.{1,250})\s*\r\n/m 17 | GET_RESPONSE = "VALUE %s %s %s\r\n%s\r\nEND\r\n".freeze 18 | GET_RESPONSE_EMPTY = "END\r\n".freeze 19 | 20 | # SET Responses 21 | SET_COMMAND = /\Aset (.{1,250}) ([0-9]+) ([0-9]+) ([0-9]+)\r\n/m 22 | SET_RESPONSE_SUCCESS = "STORED\r\n".freeze 23 | SET_RESPONSE_FAILURE = "NOT STORED\r\n".freeze 24 | SET_CLIENT_DATA_ERROR = "CLIENT_ERROR bad data chunk\r\nERROR\r\n".freeze 25 | 26 | # DELETE Responses 27 | DELETE_COMMAND = /\Adelete (.{1,250}) ([0-9]+)\r\n/m 28 | DELETE_RESPONSE = "END\r\n".freeze 29 | 30 | # STAT Response 31 | STATS_COMMAND = /\Astats\r\n/m 32 | STATS_RESPONSE = "STAT pid %d\r 33 | STAT uptime %d\r 34 | STAT time %d\r 35 | STAT version %s\r 36 | STAT rusage_user %0.6f\r 37 | STAT rusage_system %0.6f\r 38 | STAT curr_items %d\r 39 | STAT total_items %d\r 40 | STAT bytes %d\r 41 | STAT curr_connections %d\r 42 | STAT total_connections %d\r 43 | STAT cmd_get %d\r 44 | STAT cmd_set %d\r 45 | STAT get_hits %d\r 46 | STAT get_misses %d\r 47 | STAT bytes_read %d\r 48 | STAT bytes_written %d\r 49 | STAT limit_maxbytes %d\r 50 | %sEND\r\n".freeze 51 | QUEUE_STATS_RESPONSE = "STAT queue_%s_items %d\r 52 | STAT queue_%s_total_items %d\r 53 | STAT queue_%s_logsize %d\r 54 | STAT queue_%s_expired_items %d\r 55 | STAT queue_%s_age %d\r\n".freeze 56 | 57 | SHUTDOWN_COMMAND = /\Ashutdown\r\n/m 58 | 59 | 60 | @@next_session_id = 1 61 | 62 | ## 63 | # Creates a new handler for the MemCache protocol that communicates with a 64 | # given client. 65 | 66 | def initialize(options = {}) 67 | @opts = options 68 | end 69 | 70 | ## 71 | # Process incoming commands from the attached client. 72 | 73 | def post_init 74 | @stash = [] 75 | @data = "" 76 | @data_buf = "" 77 | @server = @opts[:server] 78 | @logger = StarlingServer::Base.logger 79 | @expiry_stats = Hash.new(0) 80 | @expected_length = nil 81 | @server.stats[:total_connections] += 1 82 | set_comm_inactivity_timeout @opts[:timeout] 83 | @queue_collection = @opts[:queue] 84 | 85 | @session_id = @@next_session_id 86 | @@next_session_id += 1 87 | 88 | peer = Socket.unpack_sockaddr_in(get_peername) 89 | #@logger.debug "(#{@session_id}) New session from #{peer[1]}:#{peer[0]}" 90 | end 91 | 92 | def receive_data(incoming) 93 | @server.stats[:bytes_read] += incoming.size 94 | @data << incoming 95 | 96 | while data = @data.slice!(/.*?\r\n/m) 97 | response = process(data) 98 | end 99 | 100 | send_data response if response 101 | end 102 | 103 | def process(data) 104 | data = @data_buf + data if @data_buf.size > 0 105 | # our only non-normal state is consuming an object's data 106 | # when @expected_length is present 107 | if @expected_length && data.size == @expected_length 108 | response = set_data(data) 109 | @data_buf = "" 110 | return response 111 | elsif @expected_length 112 | @data_buf = data 113 | return 114 | end 115 | case data 116 | when SET_COMMAND 117 | @server.stats[:set_requests] += 1 118 | set($1, $2, $3, $4.to_i) 119 | when GET_COMMAND 120 | @server.stats[:get_requests] += 1 121 | get($1) 122 | when STATS_COMMAND 123 | stats 124 | when SHUTDOWN_COMMAND 125 | # no point in responding, they'll never get it. 126 | Runner::shutdown 127 | when DELETE_COMMAND 128 | delete $1 129 | else 130 | logger.warn "Unknown command: #{data}." 131 | respond ERR_UNKNOWN_COMMAND 132 | end 133 | rescue => e 134 | logger.error "Error handling request: #{e}." 135 | logger.debug e.backtrace.join("\n") 136 | respond GET_RESPONSE_EMPTY 137 | end 138 | 139 | def unbind 140 | #@logger.debug "(#{@session_id}) connection ends" 141 | end 142 | 143 | private 144 | 145 | def delete(queue) 146 | @queue_collection.delete(queue) 147 | respond DELETE_RESPONSE 148 | end 149 | 150 | def respond(str, *args) 151 | response = sprintf(str, *args) 152 | @server.stats[:bytes_written] += response.length 153 | response 154 | end 155 | 156 | def set(key, flags, expiry, len) 157 | @expected_length = len + 2 158 | @stash = [ key, flags, expiry ] 159 | nil 160 | end 161 | 162 | def set_data(incoming) 163 | key, flags, expiry = @stash 164 | data = incoming.slice(0...@expected_length-2) 165 | @stash = [] 166 | @expected_length = nil 167 | 168 | internal_data = [expiry.to_i, data].pack(DATA_PACK_FMT) 169 | if @queue_collection.put(key, internal_data) 170 | respond SET_RESPONSE_SUCCESS 171 | else 172 | respond SET_RESPONSE_FAILURE 173 | end 174 | end 175 | 176 | def get(key) 177 | now = Time.now.to_i 178 | 179 | while response = @queue_collection.take(key) 180 | expiry, data = response.unpack(DATA_PACK_FMT) 181 | 182 | break if expiry == 0 || expiry >= now 183 | 184 | @expiry_stats[key] += 1 185 | expiry, data = nil 186 | end 187 | 188 | if data 189 | respond GET_RESPONSE, key, 0, data.size, data 190 | else 191 | respond GET_RESPONSE_EMPTY 192 | end 193 | end 194 | 195 | def stats 196 | respond STATS_RESPONSE, 197 | Process.pid, # pid 198 | Time.now - @server.stats(:start_time), # uptime 199 | Time.now.to_i, # time 200 | StarlingServer::VERSION, # version 201 | Process.times.utime, # rusage_user 202 | Process.times.stime, # rusage_system 203 | @queue_collection.stats(:current_size), # curr_items 204 | @queue_collection.stats(:total_items), # total_items 205 | @queue_collection.stats(:current_bytes), # bytes 206 | @server.stats(:connections), # curr_connections 207 | @server.stats(:total_connections), # total_connections 208 | @server.stats(:get_requests), # get count 209 | @server.stats(:set_requests), # set count 210 | @queue_collection.stats(:get_hits), 211 | @queue_collection.stats(:get_misses), 212 | @server.stats(:bytes_read), # total bytes read 213 | @server.stats(:bytes_written), # total bytes written 214 | 0, # limit_maxbytes 215 | queue_stats 216 | end 217 | 218 | def queue_stats 219 | @queue_collection.queues.inject("") do |m,(k,v)| 220 | m + sprintf(QUEUE_STATS_RESPONSE, 221 | k, v.length, 222 | k, v.total_items, 223 | k, v.logsize, 224 | k, @expiry_stats[k], 225 | k, v.current_age) 226 | end 227 | end 228 | 229 | def logger 230 | @logger 231 | end 232 | end 233 | end 234 | -------------------------------------------------------------------------------- /lib/starling/server_runner.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'server') 2 | require 'optparse' 3 | require 'yaml' 4 | 5 | module StarlingServer 6 | class Runner 7 | 8 | attr_accessor :options 9 | private :options, :options= 10 | 11 | def self.run 12 | new 13 | end 14 | 15 | def self.shutdown 16 | @@instance.shutdown 17 | end 18 | 19 | def initialize 20 | @@instance = self 21 | parse_options 22 | 23 | @process = ProcessHelper.new(options[:logger], options[:pid_file], options[:user], options[:group]) 24 | 25 | pid = @process.running? 26 | if pid 27 | STDERR.puts "There is already a Starling process running (pid #{pid}), exiting." 28 | exit(1) 29 | elsif pid.nil? 30 | STDERR.puts "Cleaning up stale pidfile at #{options[:pid_file]}." 31 | end 32 | 33 | start 34 | end 35 | 36 | def load_config_file(filename) 37 | config = YAML.load(File.open(filename)) 38 | 39 | unless config.is_a?(Hash) 40 | STDERR.puts "Config file does not contain a hash: #{filename}, exiting." 41 | exit(1) 42 | end 43 | 44 | if config['starling'].nil? 45 | STDERR.puts "Missing starling section in config file: #{filename}, exiting." 46 | exit(1) 47 | end 48 | 49 | config['starling'].each do |key, value| 50 | # alias some keys 51 | case key 52 | when "queue_path" then key = "path" 53 | when "log_file" then key = "logger" 54 | end 55 | 56 | if %w(logger path pid_file).include?(key) 57 | value = File.expand_path(value) 58 | end 59 | 60 | options[key.to_sym] = value 61 | 62 | if options[:log_level].instance_of?(String) 63 | options[:log_level] = Logger.const_get(options[:log_level]) 64 | end 65 | end 66 | end 67 | 68 | def parse_options 69 | self.options = { :host => '127.0.0.1', 70 | :port => 22122, 71 | :path => File.join('', 'var', 'spool', 'starling'), 72 | :log_level => Logger::INFO, 73 | :daemonize => false, 74 | :timeout => 0, 75 | :pid_file => File.join('', 'var', 'run', 'starling.pid') } 76 | 77 | OptionParser.new do |opts| 78 | opts.summary_width = 25 79 | 80 | opts.banner = "Starling (#{StarlingServer::VERSION})\n\n", 81 | "usage: starling [options...]\n", 82 | " starling --help\n", 83 | " starling --version\n" 84 | 85 | opts.separator "" 86 | opts.separator "Configuration:" 87 | 88 | opts.on("-f", "--config FILENAME", 89 | "Config file (yaml) to load") do |filename| 90 | load_config_file(filename) 91 | end 92 | 93 | opts.on("-q", "--queue_path PATH", 94 | :REQUIRED, 95 | "Path to store Starling queue logs", "(default: #{options[:path]})") do |queue_path| 96 | options[:path] = File.expand_path(queue_path) 97 | end 98 | 99 | opts.separator ""; opts.separator "Network:" 100 | 101 | opts.on("-hHOST", "--host HOST", "Interface on which to listen (default: #{options[:host]})") do |host| 102 | options[:host] = host 103 | end 104 | 105 | opts.on("-pHOST", "--port PORT", Integer, "TCP port on which to listen (default: #{options[:port]})") do |port| 106 | options[:port] = port 107 | end 108 | 109 | opts.separator ""; opts.separator "Process:" 110 | 111 | opts.on("-d", "Run as a daemon.") do 112 | options[:daemonize] = true 113 | end 114 | 115 | opts.on("-PFILE", "--pid FILENAME", "save PID in FILENAME when using -d option.", "(default: #{options[:pid_file]})") do |pid_file| 116 | options[:pid_file] = File.expand_path(pid_file) 117 | end 118 | 119 | opts.on("-u", "--user USER", "User to run as") do |user| 120 | options[:user] = user.to_i == 0 ? Etc.getpwnam(user).uid : user.to_i 121 | end 122 | 123 | opts.on("-gGROUP", "--group GROUP", "Group to run as") do |group| 124 | options[:group] = group.to_i == 0 ? Etc.getgrnam(group).gid : group.to_i 125 | end 126 | 127 | opts.separator ""; opts.separator "Logging:" 128 | 129 | opts.on("-L", "--log [FILE]", "Path to print debugging information.") do |log_path| 130 | options[:logger] = File.expand_path(log_path) 131 | end 132 | 133 | begin 134 | require 'syslog_logger' 135 | 136 | opts.on("-l", "--syslog CHANNEL", "Write logs to the syslog instead of a log file.") do |channel| 137 | options[:syslog_channel] = channel 138 | end 139 | rescue LoadError 140 | end 141 | 142 | opts.on("-v", "Increase logging verbosity (may be used multiple times).") do 143 | options[:log_level] -= 1 144 | end 145 | 146 | opts.on("-t", "--timeout [SECONDS]", Integer, 147 | "Time in seconds before disconnecting inactive clients (0 to disable).", 148 | "(default: #{options[:timeout]})") do |timeout| 149 | options[:timeout] = timeout 150 | end 151 | 152 | opts.separator ""; opts.separator "Miscellaneous:" 153 | 154 | opts.on_tail("-?", "--help", "Display this usage information.") do 155 | puts "#{opts}\n" 156 | exit 157 | end 158 | 159 | opts.on_tail("-V", "--version", "Print version number and exit.") do 160 | puts "Starling #{StarlingServer::VERSION}\n\n" 161 | exit 162 | end 163 | end.parse! 164 | end 165 | 166 | def start 167 | drop_privileges 168 | 169 | @process.daemonize if options[:daemonize] 170 | 171 | setup_signal_traps 172 | @process.write_pid_file 173 | 174 | STDOUT.puts "Starting at #{options[:host]}:#{options[:port]}." 175 | @server = StarlingServer::Base.new(options) 176 | @server.run 177 | 178 | @process.remove_pid_file 179 | end 180 | 181 | def drop_privileges 182 | Process.egid = options[:group] if options[:group] 183 | Process.euid = options[:user] if options[:user] 184 | end 185 | 186 | def shutdown 187 | begin 188 | STDOUT.puts "Shutting down." 189 | StarlingServer::Base.logger.info "Shutting down." 190 | @server.stop 191 | rescue Object => e 192 | STDERR.puts "There was an error shutting down: #{e}" 193 | exit(70) 194 | end 195 | end 196 | 197 | def setup_signal_traps 198 | Signal.trap("INT") { shutdown } 199 | Signal.trap("TERM") { shutdown } 200 | end 201 | end 202 | 203 | class ProcessHelper 204 | 205 | def initialize(log_file = nil, pid_file = nil, user = nil, group = nil) 206 | @log_file = log_file 207 | @pid_file = pid_file 208 | @user = user 209 | @group = group 210 | end 211 | 212 | def safefork 213 | begin 214 | if pid = fork 215 | return pid 216 | end 217 | rescue Errno::EWOULDBLOCK 218 | sleep 5 219 | retry 220 | end 221 | end 222 | 223 | def daemonize 224 | sess_id = detach_from_terminal 225 | exit if pid = safefork 226 | 227 | Dir.chdir("/") 228 | File.umask 0000 229 | 230 | close_io_handles 231 | redirect_io 232 | 233 | return sess_id 234 | end 235 | 236 | def detach_from_terminal 237 | srand 238 | safefork and exit 239 | 240 | unless sess_id = Process.setsid 241 | raise "Couldn't detach from controlling terminal." 242 | end 243 | 244 | trap 'SIGHUP', 'IGNORE' 245 | 246 | sess_id 247 | end 248 | 249 | def close_io_handles 250 | ObjectSpace.each_object(IO) do |io| 251 | unless [STDIN, STDOUT, STDERR].include?(io) 252 | begin 253 | io.close unless io.closed? 254 | rescue Exception 255 | end 256 | end 257 | end 258 | end 259 | 260 | def redirect_io 261 | begin; STDIN.reopen('/dev/null'); rescue Exception; end 262 | 263 | if @log_file 264 | begin 265 | STDOUT.reopen(@log_file, "a") 266 | STDOUT.sync = true 267 | rescue Exception 268 | begin; STDOUT.reopen('/dev/null'); rescue Exception; end 269 | end 270 | else 271 | begin; STDOUT.reopen('/dev/null'); rescue Exception; end 272 | end 273 | 274 | begin; STDERR.reopen(STDOUT); rescue Exception; end 275 | STDERR.sync = true 276 | end 277 | 278 | def rescue_exception 279 | begin 280 | yield 281 | rescue Exception 282 | end 283 | end 284 | 285 | def write_pid_file 286 | return unless @pid_file 287 | FileUtils.mkdir_p(File.dirname(@pid_file)) 288 | File.open(@pid_file, "w") { |f| f.write(Process.pid) } 289 | File.chmod(0644, @pid_file) 290 | end 291 | 292 | def remove_pid_file 293 | return unless @pid_file 294 | File.unlink(@pid_file) if File.exists?(@pid_file) 295 | end 296 | 297 | def running? 298 | return false unless @pid_file 299 | 300 | pid = File.read(@pid_file).chomp.to_i rescue nil 301 | pid = nil if pid == 0 302 | return false unless pid 303 | 304 | begin 305 | Process.kill(0, pid) 306 | return pid 307 | rescue Errno::ESRCH 308 | return nil 309 | rescue Errno::EPERM 310 | return pid 311 | end 312 | end 313 | end 314 | end 315 | --------------------------------------------------------------------------------