├── .gitignore ├── .rspec ├── .travis.yml ├── CHANGELOG ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── gemfiles ├── rails2.gemfile └── rails4.gemfile ├── init.rb ├── lib ├── patches.rb ├── spawnling.rb └── spawnling │ ├── cucumber.rb │ └── version.rb ├── spawnling.gemspec └── spec ├── spawn ├── spawn_spec.rb └── store_spec.rb ├── spec_helper.rb ├── store.rb └── support └── coverage_loader.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.gem 3 | .project 4 | foo 5 | coverage 6 | Gemfile.lock 7 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | --backtrace -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.1 4 | - 2.3.0 5 | gemfile: 6 | - gemfiles/rails4.gemfile 7 | env: 8 | - NONE=true 9 | - RAILS=true 10 | - RAILS=true MEMCACHE=true 11 | script: "bundle exec rake spec" 12 | sudo: false 13 | cache: bundler 14 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | v0.1 - 2007/09/13 2 | 3 | initial version 4 | 5 | -------------------------------------------------- 6 | v0.2 - 2007/09/28 7 | 8 | * return PID of the child process 9 | * added ":detach => false" option 10 | 11 | -------------------------------------------------- 12 | v0.3 - 2007/10/15 13 | 14 | * added ':method => :thread' for threaded spawns 15 | * removed ':detach => false' option in favor of more generic implementation 16 | * added ability to set configuration of the form 'Spawn::method :thread' 17 | * added patch to ActiveRecord::Base to allow for more efficient reconnect in child processes 18 | * added monkey patch for http://dev.rubyonrails.org/ticket/7579 19 | * added wait() method to wait for spawned code blocks 20 | * don't allow threading if allow_concurrency=false 21 | 22 | -------------------------------------------------- 23 | v0.4 - 2008/1/26 24 | 25 | * default to :thread on windows, still :fork on all other platforms 26 | * raise exception when used with :method=>:true and allow_concurrency != true 27 | 28 | -------------------------------------------------- 29 | v0.5 - 2008/3/1 30 | * also default to :thread on JRuby (java) 31 | * added new :method => :yield which doesn't fork or thread, this is useful for testing 32 | * fixed problem with connections piling up on PostgreSQL 33 | 34 | -------------------------------------------------- 35 | v0.6 - 2008/04/21 36 | * only apply clear_reloadable_connections patch on Rails 1.x (7579 fixed in Rails 2.x) 37 | * made it more responsive in more environments by disconnecting from the listener socket in the forked process 38 | 39 | -------------------------------------------------- 40 | v0.7 - 2008/04/24 41 | * more generic mechanism for closing resources after fork 42 | * check for existence of Mongrel before patching it 43 | 44 | -------------------------------------------------- 45 | v0.8 - 2008/05/02 46 | * call exit! within the ensure block so that at_exit handlers aren't called on exceptions 47 | * set logger from RAILS_DEFAULT_LOGGER if available, else STDERR 48 | 49 | -------------------------------------------------- 50 | v0.9 - 2008/05/11 51 | * added ability to set nice level for child process 52 | 53 | -------------------------------------------------- 54 | v1.0 - 2010/10/09 55 | * merged edged to master, let's call this version 1.0 56 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2007 Tom Anderson (tom@squeat.com) 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Spawnling 2 | ========= 3 | 4 | [![Gem Version](https://badge.fury.io/rb/spawnling.png)](https://badge.fury.io/rb/spawnling) 5 | [![Build Status](https://travis-ci.org/tra/spawnling.png?branch=master)](https://travis-ci.org/tra/spawnling) 6 | [![Coverage Status](https://coveralls.io/repos/tra/spawnling/badge.png)](https://coveralls.io/r/tra/spawnling) 7 | [![Dependency Status](https://gemnasium.com/tra/spawnling.png)](https://gemnasium.com/tra/spawnling) 8 | [![Code Climate](https://codeclimate.com/github/tra/spawnling.png)](https://codeclimate.com/github/tra/spawnling) 9 | 10 | # News 11 | 12 | 2013-4-15 gem renamed from "spawn-block" (lame) to "spawnling" (awesome). Sadly the 13 | name "spawn" was taken before I got around to making this into a gem so I decided to 14 | give it a new name and a new home. 15 | 16 | Also, now runs with ruby 1.9 (and later). Because ruby "stole" the name "spawn", this gem 17 | now has been redefined to use "Spawnling.new(&block)" instead of "spawn(&block)". Other 18 | than that nothing has changed in the basic usage. Read below for detailed usage. 19 | 20 | # Spawnling 21 | 22 | This gem provides a 'Spawnling' class to easily fork OR thread long-running sections of 23 | code so that your application can return results to your users more quickly. 24 | It works by creating new database connections in ActiveRecord::Base for the 25 | spawned block so that you don't have to worry about database connections working, 26 | they just do. 27 | 28 | The gem also patches ActiveRecord::Base to handle some known bugs when using 29 | threads if you prefer using the threaded model over forking. 30 | 31 | ## Installation 32 | 33 | The name of the gem is "spawnling" (unfortunately somebody took the name "spawn" before 34 | I was able to convert this to a gem). 35 | 36 | ### git 37 | 38 | If you want to live on the latest master branch, add this to your Gemfile, 39 | 40 | gem 'spawnling', :git => 'https://github.com/tra/spawnling' 41 | 42 | and use bundler to manage it (bundle install, bundle update). 43 | 44 | ### rubygem 45 | 46 | If you'd rather install from the latest gem, 47 | 48 | gem 'spawnling', '~>2.1' 49 | 50 | ### configure 51 | 52 | Make sure that ActiveRecord reconnects to your database automatically when needed, 53 | for instance put 54 | 55 | production/development: 56 | ... 57 | reconnect: true 58 | 59 | into your config/database.yml. 60 | 61 | ## Usage 62 | 63 | Here's a simple example of how to demonstrate the spawn plugin. 64 | In one of your controllers, insert this code (after installing the plugin of course): 65 | ```ruby 66 | Spawnling.new do 67 | logger.info("I feel sleepy...") 68 | sleep 11 69 | logger.info("Time to wake up!") 70 | end 71 | ``` 72 | If everything is working correctly, your controller should finish quickly then you'll see 73 | the last log message several seconds later. 74 | 75 | If you need to wait for the spawned processes/threads, then pass the objects returned by 76 | spawn to Spawnling.wait(), like this: 77 | ```ruby 78 | spawns = [] 79 | N.times do |i| 80 | # spawn N blocks of code 81 | spawns << Spawnling.new do 82 | something(i) 83 | end 84 | end 85 | # wait for all N blocks of code to finish running 86 | Spawnling.wait(spawns) 87 | ``` 88 | ## Options 89 | 90 | The options you can pass to spawn are: 91 | 92 | 93 | 94 | 95 | 96 | 98 | 99 | 102 |
OptionValues
:method:fork, :thread, :yield
:niceinteger value 0-19, 19 = really nice
:killboolean value indicating whether the parent should kill the spawned process 97 | when it exits (only valid when :method => :fork)
:argvstring to override the process name
:detachboolean value indicating whether the parent should Process.detach the 100 | spawned processes. (defaults to true). You *must* Spawnling.wait or Process.wait if you use this. 101 | Changing this allows you to wait for the first child to exit instead of waiting for all of them.
103 | 104 | Any option to spawn can be set as a default so that you don't have to pass them in 105 | to every call of spawn. To configure the spawn default options, add a line to 106 | your configuration file(s) like this: 107 | ```ruby 108 | Spawnling::default_options :method => :thread 109 | ``` 110 | If you don't set any default options, the :method will default to :fork. To 111 | specify different values for different environments, add the default_options call to 112 | he appropriate environment file (development.rb, test.rb). For testing you can set 113 | the default :method to :yield so that the code is run inline. 114 | ```ruby 115 | # in environment.rb 116 | Spawnling::default_options :method => :fork, :nice => 7 117 | # in test.rb, will override the environment.rb setting 118 | Spawnling::default_options :method => :yield 119 | ``` 120 | This allows you to set your production and development environments to use different 121 | methods according to your needs. 122 | 123 | ### be nice 124 | 125 | If you want your forked child to run at a lower priority than the parent process, pass in 126 | the :nice option like this: 127 | ```ruby 128 | Spawnling.new(:nice => 7) do 129 | do_something_nicely 130 | end 131 | ``` 132 | ### fork me 133 | 134 | By default, spawn will use the fork to spawn child processes. You can configure it to 135 | do threading either by telling the spawn method when you call it or by configuring your 136 | environment. 137 | For example, this is how you can tell spawn to use threading on the call, 138 | ```ruby 139 | Spawnling.new(:method => :thread) do 140 | something 141 | end 142 | ``` 143 | When you use threaded spawning, make sure that your application is thread-safe. Rails 144 | can be switched to thread-safe mode with (not sure if this is needed anymore) 145 | ```ruby 146 | # Enable threaded mode 147 | config.threadsafe! 148 | ``` 149 | in environments/your_environment.rb 150 | 151 | ### kill or be killed 152 | 153 | Depending on your application, you may want the children processes to go away when 154 | the parent process exits. By default spawn lets the children live after the 155 | parent dies. But you can tell it to kill the children by setting the :kill option 156 | to true. 157 | 158 | ### a process by any other name 159 | 160 | If you'd like to be able to identify which processes are spawned by looking at the 161 | output of ps then set the :argv option with a string of your choice. 162 | You should then be able to see this string as the process name when 163 | listing the running processes (ps). 164 | 165 | For example, if you do something like this, 166 | ```ruby 167 | 3.times do |i| 168 | Spawnling.new(:argv => "spawn -#{i}-") do 169 | something(i) 170 | end 171 | end 172 | ``` 173 | then in the shell, 174 | ```shell 175 | $ ps -ef | grep spawn 176 | 502 2645 2642 0 0:00.01 ttys002 0:00.02 spawn -0- 177 | 502 2646 2642 0 0:00.02 ttys002 0:00.02 spawn -1- 178 | 502 2647 2642 0 0:00.02 ttys002 0:00.03 spawn -2- 179 | ``` 180 | The length of the process name may be limited by your OS so you might want to experiment 181 | to see how long it can be (it may be limited by the length of the original process name). 182 | 183 | ## Forking vs. Threading 184 | 185 | There are several tradeoffs for using threading vs. forking. Forking was chosen as the 186 | default primarily because it requires no configuration to get it working out of the box. 187 | 188 | Forking advantages: 189 | 190 | - more reliable? - the ActiveRecord code is generally not deemed to be thread-safe. 191 | Even though spawn attempts to patch known problems with the threaded implementation, 192 | there are no guarantees. Forking is heavier but should be fairly reliable. 193 | - keep running - this could also be a disadvantage, but you may find you want to fork 194 | off a process that could have a life longer than its parent. For example, maybe you 195 | want to restart your server without killing the spawned processes. 196 | We don't necessarily condone this (i.e. haven't tried it) but it's technically possible. 197 | - easier - forking works out of the box with spawn, threading requires you set 198 | allow_concurrency=true (for older versions of Rails). 199 | Also, beware of automatic reloading of classes in development 200 | mode (config.cache_classes = false). 201 | 202 | Threading advantages: 203 | - less filling - threads take less resources... how much less? it depends. Some 204 | flavors of Unix are pretty efficient at forking so the threading advantage may not 205 | be as big as you think... but then again, maybe it's more than you think. :wink: 206 | - debugging - you can set breakpoints in your threads 207 | 208 | ## Acknowledgements 209 | 210 | This plugin was initially inspired by Scott Persinger's blog post on how to use fork 211 | in rails for background processing (link no longer available). 212 | 213 | Further inspiration for the threading implementation came from [Jonathon Rochkind's 214 | blog post](http://bibwild.wordpress.com/2007/08/28/threading-in-rails/) on threading in rails. 215 | 216 | Also thanks to all who have helped debug problems and suggest improvements 217 | including: 218 | 219 | - Ahmed Adam, Tristan Schneiter, Scott Haug, Andrew Garfield, Eugene Otto, Dan Sharp, 220 | Olivier Ruffin, Adrian Duyzer, Cyrille Labesse 221 | 222 | - Garry Tan, Matt Jankowski (Rails 2.2.x fixes), Mina Naguib (Rails 2.3.6 fix) 223 | 224 | - Tim Kadom, Mauricio Marcon Zaffari, Danial Pearce, Hongli Lai, Scott Wadden 225 | (passenger fixes) 226 | 227 | - Will Bryant, James Sanders (memcache fix) 228 | 229 | - David Kelso, Richard Hirner, Luke van der Hoeven (gemification and Rails 3 support) 230 | 231 | - Jay Caines-Gooby, Eric Stewart (Unicorn fix) 232 | 233 | - Dan Sharp for the changes that allow it to work with Ruby 1.9 and later 234 | 235 | - <your name here> 236 | 237 | Copyright (c) 2007-present Tom Anderson (tom@squeat.com), see LICENSE 238 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | desc 'Default: run coverage.' 4 | task :default => :spec 5 | 6 | require 'rspec/core/rake_task' 7 | 8 | desc "Run specs" 9 | RSpec::Core::RakeTask.new do |t| 10 | t.pattern = "./spec/**/*_spec.rb" # don't need this, it's default. 11 | # Put spec opts in a file named .rspec in root 12 | end 13 | -------------------------------------------------------------------------------- /gemfiles/rails2.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec :path => "../" 3 | 4 | group :development, :test do 5 | gem 'rails', '~> 2.0' 6 | gem 'mime-types', '< 2.0', :platforms => :ruby_18 7 | end 8 | -------------------------------------------------------------------------------- /gemfiles/rails4.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec :path => "../" 3 | 4 | group :development, :test do 5 | gem 'rails', '~> 4.0' 6 | end 7 | -------------------------------------------------------------------------------- /init.rb: -------------------------------------------------------------------------------- 1 | require 'spawnling' 2 | -------------------------------------------------------------------------------- /lib/patches.rb: -------------------------------------------------------------------------------- 1 | if defined?(ActiveRecord) 2 | # see activerecord/lib/active_record/connection_adaptors/abstract/connection_specification.rb 3 | class ActiveRecord::Base 4 | # reconnect without disconnecting 5 | if ::Spawnling::RAILS_3_x 6 | def self.spawn_reconnect(klass=self) 7 | ActiveRecord::Base.connection.reconnect! 8 | end 9 | elsif ::Spawnling::RAILS_2_2 10 | def self.spawn_reconnect(klass=self) 11 | # keep ancestors' connection_handlers around to avoid them being garbage collected in the forked child 12 | @@ancestor_connection_handlers ||= [] 13 | @@ancestor_connection_handlers << self.connection_handler 14 | self.connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new 15 | 16 | establish_connection 17 | end 18 | else 19 | def self.spawn_reconnect(klass=self) 20 | spec = @@defined_connections[klass.name] 21 | konn = active_connections[klass.name] 22 | # remove from internal arrays before calling establish_connection so that 23 | # the connection isn't disconnected when it calls AR::Base.remove_connection 24 | @@defined_connections.delete_if { |key, value| value == spec } 25 | active_connections.delete_if { |key, value| value == konn } 26 | establish_connection(spec ? spec.config : nil) 27 | end 28 | end 29 | 30 | # this patch not needed on Rails 2.x and later 31 | if ::Spawnling::RAILS_1_x 32 | # monkey patch to fix threading problems, 33 | # see: http://dev.rubyonrails.org/ticket/7579 34 | def self.clear_reloadable_connections! 35 | if @@allow_concurrency 36 | # Hash keyed by thread_id in @@active_connections. Hash of hashes. 37 | @@active_connections.each do |thread_id, conns| 38 | conns.each do |name, conn| 39 | if conn.requires_reloading? 40 | conn.disconnect! 41 | @@active_connections[thread_id].delete(name) 42 | end 43 | end 44 | end 45 | else 46 | # Just one level hash, no concurrency. 47 | @@active_connections.each do |name, conn| 48 | if conn.requires_reloading? 49 | conn.disconnect! 50 | @@active_connections.delete(name) 51 | end 52 | end 53 | end 54 | end 55 | end 56 | end 57 | end 58 | 59 | # just borrowing from the mongrel & passenger patches: 60 | if defined?(Unicorn::HttpServer) && defined?(Unicorn::HttpRequest::REQ) 61 | class Unicorn::HttpServer 62 | REQ = Unicorn::HttpRequest::REQ 63 | alias_method :orig_process_client, :process_client 64 | def process_client(client) 65 | ::Spawnling.resources_to_close(client, REQ) 66 | orig_process_client(client) 67 | end 68 | end 69 | elsif defined? Unicorn::HttpServer 70 | class Unicorn::HttpServer 71 | alias_method :orig_process_client, :process_client 72 | def process_client(client) 73 | ::Spawnling.resources_to_close(client, @request) 74 | orig_process_client(client) 75 | end 76 | end 77 | end 78 | 79 | # see mongrel/lib/mongrel.rb 80 | # it's possible that this is not defined if you're running outside of mongrel 81 | # examples: ./script/runner or ./script/console 82 | if defined? Mongrel::HttpServer 83 | class Mongrel::HttpServer 84 | # redefine Montrel::HttpServer::process_client so that we can intercept 85 | # the socket that is being used so Spawnling can close it upon forking 86 | alias_method :orig_process_client, :process_client 87 | def process_client(client) 88 | ::Spawnling.resources_to_close(client, @socket) 89 | orig_process_client(client) 90 | end 91 | end 92 | end 93 | 94 | need_passenger_patch = true 95 | if defined? PhusionPassenger::VERSION_STRING 96 | # The VERSION_STRING variable was defined sometime after 2.1.0. 97 | # We don't need passenger patch for 2.2.2 or later. 98 | pv = PhusionPassenger::VERSION_STRING.split('.').collect{|s| s.to_i} 99 | need_passenger_patch = pv[0] < 2 || (pv[0] == 2 && (pv[1] < 2 || (pv[1] == 2 && pv[2] < 2))) 100 | end 101 | 102 | if need_passenger_patch 103 | # Patch for work with passenger < 2.1.0 104 | if defined? Passenger::Railz::RequestHandler 105 | class Passenger::Railz::RequestHandler 106 | alias_method :orig_process_request, :process_request 107 | def process_request(headers, input, output) 108 | ::Spawnling.resources_to_close(input, output) 109 | orig_process_request(headers, input, output) 110 | end 111 | end 112 | end 113 | 114 | # Patch for work with passenger >= 2.1.0 115 | if defined? PhusionPassenger::Railz::RequestHandler 116 | class PhusionPassenger::Railz::RequestHandler 117 | alias_method :orig_process_request, :process_request 118 | def process_request(headers, input, output) 119 | ::Spawnling.resources_to_close(input, output) 120 | orig_process_request(headers, input, output) 121 | end 122 | end 123 | end 124 | 125 | # Patch for passenger with Rails >= 2.3.0 (uses rack) 126 | if defined? PhusionPassenger::Rack::RequestHandler 127 | class PhusionPassenger::Rack::RequestHandler 128 | alias_method :orig_process_request, :process_request 129 | def process_request(headers, input, output) 130 | ::Spawnling.resources_to_close(input, output) 131 | orig_process_request(headers, input, output) 132 | end 133 | end 134 | end 135 | end 136 | 137 | if defined?(Rails) 138 | class SpawnlingCache < Rails::Railtie 139 | initializer "cache" do 140 | if defined?(::ActiveSupport::Cache::MemCacheStore) && Rails.cache.class.name == 'ActiveSupport::Cache::MemCacheStore' 141 | ::ActiveSupport::Cache::MemCacheStore.delegate :reset, :to => :@data 142 | end 143 | end 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /lib/spawnling.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | 3 | class Spawnling 4 | if defined? ::Rails 5 | RAILS_1_x = (::Rails::VERSION::MAJOR == 1) unless defined?(RAILS_1_x) 6 | RAILS_2_2 = ((::Rails::VERSION::MAJOR == 2 && ::Rails::VERSION::MINOR >= 2)) unless defined?(RAILS_2_2) 7 | RAILS_3_x = (::Rails::VERSION::MAJOR > 2) unless defined?(RAILS_3_x) 8 | else 9 | RAILS_1_x = nil 10 | RAILS_2_2 = nil 11 | RAILS_3_x = nil 12 | end 13 | 14 | @@default_options = { 15 | # default to forking (unless windows or jruby) 16 | :method => ((RUBY_PLATFORM =~ /(win32|mingw32|java)/) ? :thread : :fork), 17 | :nice => nil, 18 | :kill => false, 19 | :argv => nil, 20 | :detach => true 21 | } 22 | 23 | # things to close in child process 24 | @@resources = [] 25 | 26 | # forked children to kill on exit 27 | @@punks = [] 28 | 29 | # in some environments, logger isn't defined 30 | @@logger = defined?(::Rails) ? ::Rails.logger : ::Logger.new(STDERR) 31 | 32 | def self.logger=(logger) 33 | @@logger = logger 34 | end 35 | 36 | attr_accessor :type 37 | attr_accessor :handle 38 | 39 | # Set the options to use every time spawn is called unless specified 40 | # otherwise. For example, in your environment, do something like 41 | # this: 42 | # Spawnling::default_options = {:nice => 5} 43 | # to default to using the :nice option with a value of 5 on every call. 44 | # Valid options are: 45 | # :method => (:thread | :fork | :yield) 46 | # :nice => nice value of the forked process 47 | # :kill => whether or not the parent process will kill the 48 | # spawned child process when the parent exits 49 | # :argv => changes name of the spawned process as seen in ps 50 | # :detach => whether or not Process.detach is called for spawned child 51 | # processes. You must wait for children on your own if you 52 | # set this to false 53 | def self.default_options(options = {}) 54 | @@default_options.merge!(options) 55 | @@logger.info "spawn> default options = #{options.inspect}" if @@logger 56 | end 57 | 58 | # set the resources to disconnect from in the child process (when forking) 59 | def self.resources_to_close(*resources) 60 | @@resources = resources 61 | end 62 | 63 | # close all the resources added by calls to resource_to_close 64 | def self.close_resources 65 | @@resources.each do |resource| 66 | resource.close if resource && resource.respond_to?(:close) && !resource.closed? 67 | end 68 | # in case somebody spawns recursively 69 | @@resources.clear 70 | end 71 | 72 | def self.alive?(pid) 73 | begin 74 | Process::kill 0, pid 75 | # if the process is alive then kill won't throw an exception 76 | true 77 | rescue Errno::ESRCH 78 | false 79 | end 80 | end 81 | 82 | def self.kill_punks 83 | @@punks.each do |punk| 84 | if alive?(punk) 85 | @@logger.info "spawn> parent(#{Process.pid}) killing child(#{punk})" if @@logger 86 | begin 87 | Process.kill("TERM", punk) 88 | rescue 89 | end 90 | end 91 | end 92 | @@punks = [] 93 | end 94 | # register to kill marked children when parent exits 95 | at_exit { Spawnling.kill_punks } 96 | 97 | # Spawns a long-running section of code and returns the ID of the spawned process. 98 | # By default the process will be a forked process. To use threading, pass 99 | # :method => :thread or override the default behavior in the environment by setting 100 | # 'Spawnling::method :thread'. 101 | def initialize(opts = {}, &block) 102 | @type, @handle = self.class.run(opts, &block) 103 | end 104 | 105 | def self.run(opts = {}, &block) 106 | raise "Must give block of code to be spawned" unless block_given? 107 | options = @@default_options.merge(symbolize_options(opts)) 108 | # setting options[:method] will override configured value in default_options[:method] 109 | case options.fetch(:method) 110 | when :yield 111 | yield 112 | when :thread 113 | # for versions before 2.2, check for allow_concurrency 114 | if allow_concurrency? 115 | return :thread, thread_it(options) { yield } 116 | else 117 | @@logger.error("spawn(:method=>:thread) only allowed when allow_concurrency=true") 118 | raise "spawn requires config.active_record.allow_concurrency=true when used with :method=>:thread" 119 | end 120 | when :fork 121 | return :fork, fork_it(options) { yield } 122 | else 123 | if options[:method].respond_to?(:call) 124 | options[:method].call(proc { yield }) 125 | else 126 | raise ArgumentError, 'method must be :yield, :thread, :fork or respond to method call' 127 | end 128 | end 129 | end 130 | 131 | def self.allow_concurrency? 132 | return true if RAILS_2_2 133 | if defined?(ActiveRecord) && ActiveRecord::Base.respond_to?(:allow_concurrency) 134 | ActiveRecord::Base.allow_concurrency 135 | elsif defined?(Rails) && Rails.application 136 | Rails.application.config.allow_concurrency 137 | else 138 | true # assume user knows what they are doing 139 | end 140 | end 141 | 142 | def self.wait(sids = []) 143 | # wait for all threads and/or forks (if a single sid passed in, convert to array first) 144 | Array(sids).each do |sid| 145 | if sid.type == :thread 146 | sid.handle.join() 147 | else 148 | begin 149 | Process.wait(sid.handle) 150 | rescue 151 | # if the process is already done, ignore the error 152 | end 153 | end 154 | end 155 | # clean up connections from expired threads 156 | clean_connections 157 | end 158 | 159 | protected 160 | 161 | def self.fork_it(options) 162 | # The problem with rails is that it only has one connection (per class), 163 | # so when we fork a new process, we need to reconnect. 164 | @@logger.debug "spawn> parent PID = #{Process.pid}" if @@logger 165 | child = fork do 166 | begin 167 | start = Time.now 168 | @@logger.debug "spawn> child PID = #{Process.pid}" if @@logger 169 | 170 | # this child has no children of it's own to kill (yet) 171 | @@punks = [] 172 | 173 | # set the nice priority if needed 174 | Process.setpriority(Process::PRIO_PROCESS, 0, options[:nice]) if options[:nice] 175 | 176 | # disconnect from the listening socket, et al 177 | Spawnling.close_resources 178 | if defined?(Rails) 179 | # get a new database connection so the parent can keep the original one 180 | ActiveRecord::Base.spawn_reconnect if defined?(ActiveRecord) 181 | # close the memcache connection so the parent can keep the original one 182 | Rails.cache.reset if Rails.cache.respond_to?(:reset) 183 | end 184 | 185 | # set the process name 186 | $0 = options[:argv] if options[:argv] 187 | 188 | # run the block of code that takes so long 189 | yield 190 | 191 | rescue => ex 192 | @@logger.error "spawn> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message}" if @@logger 193 | @@logger.error "spawn> " + ex.backtrace.join("\n") if @@logger 194 | ensure 195 | begin 196 | # to be safe, catch errors on closing the connnections too 197 | ActiveRecord::Base.connection_handler.clear_all_connections! if defined?(ActiveRecord) 198 | ensure 199 | @@logger.info "spawn> child[#{Process.pid}] took #{Time.now - start} sec" if @@logger 200 | # ensure log is flushed since we are using exit! 201 | @@logger.flush if @@logger && @@logger.respond_to?(:flush) 202 | # this child might also have children to kill if it called spawn 203 | Spawnling.kill_punks 204 | # this form of exit doesn't call at_exit handlers 205 | exit!(0) 206 | end 207 | end 208 | end 209 | 210 | # detach from child process (parent may still wait for detached process if they wish) 211 | Process.detach(child) if options[:detach] 212 | 213 | # remove dead children from the target list to avoid memory leaks 214 | @@punks.delete_if {|punk| !Spawn.alive?(punk)} 215 | 216 | # mark this child for death when this process dies 217 | if options[:kill] 218 | @@punks << child 219 | @@logger.debug "spawn> death row = #{@@punks.inspect}" if @@logger 220 | end 221 | 222 | # return Spawnling::Id.new(:fork, child) 223 | return child 224 | end 225 | 226 | def self.thread_it(options) 227 | # clean up stale connections from previous threads 228 | clean_connections 229 | thr = Thread.new do 230 | # run the long-running code block 231 | if defined?(ActiveRecord) 232 | ActiveRecord::Base.connection_pool.with_connection { yield } 233 | else 234 | yield 235 | end 236 | end 237 | thr.priority = -options[:nice] if options[:nice] 238 | return thr 239 | end 240 | 241 | def self.clean_connections 242 | return unless defined? ActiveRecord 243 | ActiveRecord::Base.verify_active_connections! if ActiveRecord::Base.respond_to?(:verify_active_connections!) 244 | ActiveRecord::Base.clear_active_connections! if ActiveRecord::Base.respond_to?(:clear_active_connections!) 245 | end 246 | 247 | # In case we don't have rails, can't call opts.symbolize_keys 248 | def self.symbolize_options(hash) 249 | hash.inject({}) do |new_hash, (key, value)| 250 | new_hash[key.to_sym] = value 251 | new_hash 252 | end 253 | end 254 | end 255 | # backwards compatibility unless someone is using the "other" spawn gem 256 | Spawn = Spawnling unless defined? Spawn 257 | 258 | # patches depends on Spawn so require it after the class 259 | require 'patches' 260 | -------------------------------------------------------------------------------- /lib/spawnling/cucumber.rb: -------------------------------------------------------------------------------- 1 | module SpawnExtensions 2 | # FIXME don't know how to tell Spawn to use #add_spawn_proc without extended 3 | # using extended forces to make methods class methods while this is not very clean 4 | def self.extended(base) 5 | Spawn::method proc{ |block| add_spawn_proc(block) } 6 | end 7 | 8 | # Calls the spawn that was created 9 | # 10 | # Can be used to keep control over forked processes in your tests 11 | def call_last_spawn_proc 12 | spawns = SpawnExtensions.spawn_procs 13 | 14 | raise "No spawn procs left" if spawns.empty? 15 | 16 | spawns.pop.call 17 | end 18 | 19 | private 20 | 21 | def self.spawn_procs 22 | @@spawn_procs ||= [] 23 | end 24 | 25 | def self.add_spawn_proc(block) 26 | spawn_procs << block 27 | end 28 | 29 | end 30 | 31 | # Extend cucumber to take control over spawns 32 | World(SpawnExtensions) -------------------------------------------------------------------------------- /lib/spawnling/version.rb: -------------------------------------------------------------------------------- 1 | class Spawnling 2 | VERSION = '2.1.6' 3 | end 4 | -------------------------------------------------------------------------------- /spawnling.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'spawnling/version' 5 | 6 | Gem::Specification.new do |s| 7 | s.name = "spawnling" 8 | s.version = Spawnling::VERSION 9 | 10 | s.authors = ['Tom Anderson', 'Michael Noack'] 11 | s.email = ['tom@squeat.com', 'michael+spawnling@noack.com.au'] 12 | 13 | s.homepage = %q{https://github.com/tra/spawnling} 14 | s.license = "MIT" 15 | s.summary = %q{Easily fork OR thread long-running sections of code in Ruby} 16 | s.description = %q{This plugin provides a 'Spawnling' class to easily fork OR 17 | thread long-running sections of code so that your application can return 18 | results to your users more quickly. This plugin works by creating new database 19 | connections in ActiveRecord::Base for the spawned block. 20 | 21 | The plugin also patches ActiveRecord::Base to handle some known bugs when using 22 | threads (see lib/patches.rb).} 23 | 24 | s.files = `git ls-files`.split($/) 25 | s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) } 26 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 27 | s.require_paths = ["lib"] 28 | 29 | s.add_development_dependency 'bundler' 30 | s.add_development_dependency 'rake' 31 | s.add_development_dependency 'rspec', '~> 3.0' 32 | s.add_development_dependency 'simplecov' 33 | s.add_development_dependency 'simplecov-rcov' 34 | s.add_development_dependency 'coveralls' 35 | s.add_development_dependency 'rails' 36 | s.add_development_dependency 'activerecord-nulldb-adapter' 37 | s.add_development_dependency 'dalli' 38 | s.add_development_dependency 'travis' 39 | end 40 | -------------------------------------------------------------------------------- /spec/spawn/spawn_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Spawnling do 4 | describe 'defaults' do 5 | context 'when invalid method' do 6 | specify { 7 | expect { 8 | Spawnling.new(method: :threads) { puts "never" } 9 | }.to raise_error( 10 | ArgumentError, 11 | 'method must be :yield, :thread, :fork or respond to method call' 12 | ) 13 | } 14 | end 15 | end 16 | 17 | describe "yields" do 18 | before(:each) do 19 | Spawnling::default_options :method => :yield 20 | end 21 | 22 | it "should work in new block" do 23 | object = double('object') 24 | expect(object).to receive(:do_something) 25 | Spawnling.new do 26 | object.do_something 27 | end 28 | end 29 | 30 | it "should be able to yield directly" do 31 | expect(spawn!).to eq("hello") 32 | end 33 | end 34 | 35 | describe "override" do 36 | before(:each) do 37 | Spawnling::default_options :method => proc{ "foo" } 38 | end 39 | 40 | it "should be able to return a proc" do 41 | expect(spawn!).to eq("foo") 42 | end 43 | 44 | end 45 | 46 | describe "delegate to a proc" do 47 | before(:each) do 48 | Spawnling::default_options :method => proc{ |block| block } 49 | end 50 | 51 | it "should be able to return a proc" do 52 | expect(spawn!).to be_kind_of(Proc) 53 | end 54 | 55 | it "should be able to return a proc" do 56 | expect(spawn!.call).to eq("hello") 57 | end 58 | end 59 | 60 | describe "thread it" do 61 | before(:each) do 62 | Store.reset! 63 | Spawnling::default_options :method => :thread 64 | end 65 | 66 | it "should be able to return a proc" do 67 | expect(Store.flag).to be_falsey 68 | spawn_flag! 69 | sleep(0.1) # wait for file to finish writing 70 | expect(Store.flag).to be_truthy 71 | end 72 | 73 | it "instance should have a type" do 74 | instance = Spawnling.new{} 75 | expect(instance.type).to be(:thread) 76 | end 77 | 78 | it "instance should have a handle" do 79 | instance = Spawnling.new{} 80 | expect(instance.handle).not_to be_nil 81 | end 82 | end 83 | 84 | describe "fork it" do 85 | before(:each) do 86 | Store.reset! 87 | Spawnling::default_options :method => :fork 88 | end 89 | 90 | it "should be able to return a proc" do 91 | expect(Store.flag).to be_falsey 92 | spawn_flag! 93 | sleep(0.1) # wait for file to finish writing 94 | expect(Store.flag).to be_truthy 95 | end 96 | 97 | it "instance should have a type" do 98 | instance = Spawnling.new{} 99 | expect(instance.type).to be(:fork) 100 | end 101 | 102 | it "instance should have a handle" do 103 | instance = Spawnling.new{} 104 | expect(instance.handle).not_to be_nil 105 | end 106 | end 107 | 108 | def spawn! 109 | Spawnling.run do 110 | "hello" 111 | end 112 | end 113 | 114 | def spawn_flag! 115 | Spawnling.new do 116 | Store.flag! 117 | end 118 | end 119 | 120 | end 121 | -------------------------------------------------------------------------------- /spec/spawn/store_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Store do 4 | it 'should flag/unflag' do 5 | Store.reset! 6 | expect(Store.flag).to be_falsey 7 | Store.flag! 8 | expect(Store.flag).to be_truthy 9 | Store.reset! 10 | expect(Store.flag).to be_falsey 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | gem 'rspec' 3 | require 'rspec' 4 | 5 | $:.unshift(File.join(File.dirname(__FILE__), %w[.. lib])) 6 | 7 | require 'store' 8 | if ENV['RAILS'] 9 | require 'rails' 10 | require 'active_record' 11 | end 12 | ActiveRecord::Base.establish_connection :adapter => :nulldb if defined?(ActiveRecord) 13 | 14 | if ENV['RAILS'] 15 | class Application < Rails::Application 16 | config.log_level = :warn 17 | config.logger = Logger.new(STDOUT) 18 | end 19 | Application.initialize! 20 | Application.config.allow_concurrency = true 21 | end 22 | 23 | if ENV['MEMCACHE'] 24 | Application.config.cache_store = :mem_cache_store 25 | end 26 | 27 | require 'support/coverage_loader' 28 | 29 | require 'spawnling' 30 | -------------------------------------------------------------------------------- /spec/store.rb: -------------------------------------------------------------------------------- 1 | class Store 2 | def self.flag! 3 | write('true') 4 | end 5 | 6 | def self.flag 7 | File.read('foo') == 'true' 8 | end 9 | 10 | def self.reset! 11 | write('') 12 | end 13 | 14 | def self.write(message) 15 | f = File.new('foo', "w+") 16 | f.write(message) 17 | f.close 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /spec/support/coverage_loader.rb: -------------------------------------------------------------------------------- 1 | MINIMUM_COVERAGE = 53 2 | 3 | unless ENV['COVERAGE'] == 'off' 4 | require 'simplecov' 5 | require 'simplecov-rcov' 6 | require 'coveralls' 7 | Coveralls.wear! 8 | 9 | SimpleCov.formatters = [ 10 | SimpleCov::Formatter::RcovFormatter, 11 | Coveralls::SimpleCov::Formatter 12 | ] 13 | SimpleCov.start do 14 | add_filter '/vendor/' 15 | add_filter '/spec/' 16 | add_filter '/lib/patches.rb' 17 | add_group 'lib', 'lib' 18 | end 19 | SimpleCov.at_exit do 20 | SimpleCov.result.format! 21 | percent = SimpleCov.result.covered_percent 22 | puts "Coverage is #{"%.2f" % percent}%" 23 | unless percent >= MINIMUM_COVERAGE 24 | puts "Coverage must be above #{MINIMUM_COVERAGE}%" 25 | Kernel.exit(1) 26 | end 27 | end 28 | end 29 | --------------------------------------------------------------------------------