├── VERSION ├── .gitignore ├── .document ├── History.txt ├── spec ├── spec_helper.rb └── lib │ └── mega_mutex_spec.rb ├── LICENSE ├── README.markdown ├── Rakefile ├── mega_mutex.gemspec └── lib ├── mega_mutex └── distributed_mutex.rb └── mega_mutex.rb /VERSION: -------------------------------------------------------------------------------- 1 | 0.4.1 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sw? 2 | .DS_Store 3 | coverage 4 | rdoc 5 | pkg 6 | -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | README.rdoc 2 | lib/**/*.rb 3 | bin/* 4 | features/**/*.feature 5 | LICENSE 6 | -------------------------------------------------------------------------------- /History.txt: -------------------------------------------------------------------------------- 1 | == 0.3.0 2 | 3 | Bugfixes: 4 | * #with_distributed_mutex was not returning the result of the block -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../lib/mega_mutex') 2 | require 'test/unit/assertions' 3 | 4 | # Logging::Logger[:root].add_appenders(Logging::Appenders.stdout) 5 | 6 | module ThreadHelper 7 | def abort_on_thread_exceptions 8 | before(:all) do 9 | @old_abort_on_exception_value = Thread.abort_on_exception 10 | Thread.abort_on_exception = true 11 | end 12 | after(:all) do 13 | Thread.abort_on_exception = @old_abort_on_exception_value 14 | end 15 | end 16 | end 17 | 18 | module ThreadExampleHelper 19 | def threads 20 | @threads ||= [] 21 | end 22 | 23 | def wait_for_threads_to_finish 24 | threads.each{ |t| t.join } 25 | end 26 | end 27 | 28 | Spec::Runner.configure do |config| 29 | config.extend ThreadHelper 30 | config.include ThreadExampleHelper 31 | config.include Test::Unit::Assertions 32 | end 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 Songkick.com 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | 2 | 3 | # mega_mutex 4 | 5 | A distributed mutex for Ruby. 6 | 7 | ## Why 8 | 9 | Sometimes I need to do this: 10 | 11 | unless enough_things? 12 | make_more_things 13 | end 14 | 15 | If I'm running several processes in parallel, I can get a race condition that means two of the processes both think there are not enough things. So we go and make some more, even though we don't need to. 16 | 17 | ## How 18 | 19 | Suppose you have a ThingMaker: 20 | 21 | class ThingMaker 22 | include MegaMutex 23 | 24 | def ensure_just_enough_things 25 | with_distributed_mutex("ThingMaker Mutex ID") do 26 | unless enough_things? 27 | make_more_things 28 | end 29 | end 30 | end 31 | end 32 | 33 | Now, thanks to the magic of MegaMutex, you can be sure that all processes trying to run this code will wait their turn, so each one will have the chance to make exactly the right number of things, without anyone else poking their nose in. 34 | 35 | ## Install 36 | 37 | sudo gem install mega_mutex 38 | 39 | 40 | ## Configure 41 | 42 | MegaMutex uses [memcache-client](http://seattlerb.rubyforge.org/memcache-client/) to store the mutex, so your infrastructure must be set up to use memcache servers. 43 | 44 | By default, MegaMutex will attempt to connect to a memcache on the local machine, but you can configure any number of servers like so: 45 | 46 | MegaMutex.configure do |config| 47 | config.memcache_servers = ['mc1', 'mc2'] 48 | end 49 | 50 | ## Help 51 | 52 | MegaMutex was built by the [Songkick.com](http://www.songkick.com) development team. Come chat to us on [#songkick](irc://chat.freenode.net/#songkick) on freenode.net. 53 | 54 | ## Copyright 55 | 56 | Copyright (c) 2009 Songkick.com. See LICENSE for details. 57 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rake' 3 | 4 | begin 5 | require 'jeweler' 6 | Jeweler::Tasks.new do |gem| 7 | gem.name = "mega_mutex" 8 | gem.summary = %Q{Distributed mutex for Ruby} 9 | gem.description = %Q{Distributed mutex for Ruby} 10 | gem.email = "developers@songkick.com" 11 | gem.homepage = "http://github.com/songkick/mega_mutex" 12 | gem.authors = ["Matt Johnson", "Matt Wynne"] 13 | gem.add_dependency 'memcache-client', '>= 1.7.4' 14 | gem.add_dependency 'logging', '>= 1.1.4' 15 | gem.add_dependency 'reretryable', '>= 0.1.0' 16 | # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings 17 | end 18 | Jeweler::GemcutterTasks.new 19 | rescue LoadError 20 | puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler" 21 | end 22 | 23 | namespace :github do 24 | task :push do 25 | remotes = `git remote`.split("\n") 26 | unless remotes.include?('github') 27 | sh('git remote add github git@github.com:songkick/mega_mutex.git') 28 | end 29 | sh('git push github master') 30 | end 31 | end 32 | 33 | require 'spec/rake/spectask' 34 | Spec::Rake::SpecTask.new(:spec) do |spec| 35 | spec.libs << 'lib' << 'spec' 36 | spec.spec_files = FileList['spec/**/*_spec.rb'] 37 | end 38 | 39 | Spec::Rake::SpecTask.new(:rcov) do |spec| 40 | spec.libs << 'lib' << 'spec' 41 | spec.pattern = 'spec/**/*_spec.rb' 42 | spec.rcov = true 43 | end 44 | 45 | task :spec => :check_dependencies 46 | 47 | task :default => [:spec, 'github:push', 'gemcutter:release'] 48 | 49 | require 'rake/rdoctask' 50 | Rake::RDocTask.new do |rdoc| 51 | if File.exist?('VERSION') 52 | version = File.read('VERSION') 53 | else 54 | version = "" 55 | end 56 | 57 | rdoc.rdoc_dir = 'rdoc' 58 | rdoc.title = "mega_mutex #{version}" 59 | rdoc.rdoc_files.include('README*') 60 | rdoc.rdoc_files.include('lib/**/*.rb') 61 | end 62 | -------------------------------------------------------------------------------- /mega_mutex.gemspec: -------------------------------------------------------------------------------- 1 | # Generated by jeweler 2 | # DO NOT EDIT THIS FILE DIRECTLY 3 | # Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command 4 | # -*- encoding: utf-8 -*- 5 | 6 | Gem::Specification.new do |s| 7 | s.name = %q{mega_mutex} 8 | s.version = "0.4.1" 9 | 10 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= 11 | s.authors = ["Matt Johnson", "Matt Wynne"] 12 | s.date = %q{2010-07-16} 13 | s.description = %q{Distributed mutex for Ruby} 14 | s.email = %q{developers@songkick.com} 15 | s.extra_rdoc_files = [ 16 | "LICENSE", 17 | "README.markdown" 18 | ] 19 | s.files = [ 20 | ".document", 21 | ".gitignore", 22 | "History.txt", 23 | "LICENSE", 24 | "README.markdown", 25 | "Rakefile", 26 | "VERSION", 27 | "lib/mega_mutex.rb", 28 | "lib/mega_mutex/distributed_mutex.rb", 29 | "mega_mutex.gemspec", 30 | "spec/lib/mega_mutex_spec.rb", 31 | "spec/spec_helper.rb" 32 | ] 33 | s.homepage = %q{http://github.com/songkick/mega_mutex} 34 | s.rdoc_options = ["--charset=UTF-8"] 35 | s.require_paths = ["lib"] 36 | s.rubygems_version = %q{1.3.6} 37 | s.summary = %q{Distributed mutex for Ruby} 38 | s.test_files = [ 39 | "spec/lib/mega_mutex_spec.rb", 40 | "spec/spec_helper.rb" 41 | ] 42 | 43 | if s.respond_to? :specification_version then 44 | current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION 45 | s.specification_version = 3 46 | 47 | if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then 48 | s.add_runtime_dependency(%q, [">= 1.7.4"]) 49 | s.add_runtime_dependency(%q, [">= 1.1.4"]) 50 | s.add_runtime_dependency(%q, [">= 0.1.0"]) 51 | else 52 | s.add_dependency(%q, [">= 1.7.4"]) 53 | s.add_dependency(%q, [">= 1.1.4"]) 54 | s.add_dependency(%q, [">= 0.1.0"]) 55 | end 56 | else 57 | s.add_dependency(%q, [">= 1.7.4"]) 58 | s.add_dependency(%q, [">= 1.1.4"]) 59 | s.add_dependency(%q, [">= 0.1.0"]) 60 | end 61 | end 62 | 63 | -------------------------------------------------------------------------------- /lib/mega_mutex/distributed_mutex.rb: -------------------------------------------------------------------------------- 1 | require 'logging' 2 | require 'memcache' 3 | require 'reretryable' 4 | 5 | module MegaMutex 6 | class TimeoutError < Exception; end 7 | 8 | class DistributedMutex 9 | include Retryable 10 | 11 | class << self 12 | def cache 13 | @cache ||= MemCache.new MegaMutex.configuration.memcache_servers, :namespace => MegaMutex.configuration.namespace 14 | end 15 | end 16 | 17 | def initialize(key, options) 18 | @key = key 19 | @lazy = options[:lazy] 20 | @timeout = options[:timeout] || 30 21 | end 22 | 23 | def logger 24 | Logging::Logger[self] 25 | end 26 | 27 | def run(&block) 28 | @start_time = Time.now 29 | log "Attempting to lock mutex..." 30 | lock! 31 | log "Locked. Running critical section..." 32 | result = yield 33 | log "Critical section complete. Unlocking..." 34 | result 35 | ensure 36 | unlock! 37 | log "Unlocking Mutex." 38 | end 39 | 40 | def current_lock 41 | cache.get(@key) 42 | end 43 | 44 | private 45 | def with_retry(&block) 46 | if @lazy 47 | return yield block 48 | else 49 | retryable(:tries => 5, :sleep => 30, :on => MemCache::MemCacheError, :matching => /IO timeout/) do 50 | return yield block 51 | end 52 | end 53 | end 54 | 55 | def with_lazy(&block) 56 | if @lazy 57 | begin 58 | return yield block 59 | rescue MemCache::MemCacheError => exception 60 | log("There was a memcache error that was ignored:\n#{exception.class} (#{exception.message}):\n #{exception.backtrace.join("\n ")}\n\n") 61 | end 62 | else 63 | return yield block 64 | end 65 | end 66 | 67 | def timeout? 68 | return false unless @timeout 69 | Time.now > @start_time + @timeout 70 | end 71 | 72 | def log(message) 73 | logger.debug { "(key:#{@key}) (lock_id:#{my_lock_id}) #{message}" } 74 | end 75 | 76 | def lock! 77 | with_lazy do 78 | until timeout? 79 | with_retry do 80 | return if attempt_to_lock == my_lock_id 81 | sleep 0.1 82 | end 83 | end 84 | raise TimeoutError.new("Failed to obtain a lock within #{@timeout} seconds.") unless @lazy 85 | end 86 | end 87 | 88 | def attempt_to_lock 89 | set_current_lock(my_lock_id) if current_lock.nil? 90 | current_lock 91 | end 92 | 93 | def unlock! 94 | with_lazy do 95 | with_retry do 96 | cache.delete(@key) if locked_by_me? 97 | return #explicit return so we quit out of the block 98 | end 99 | end 100 | end 101 | 102 | def locked_by_me? 103 | current_lock == my_lock_id 104 | end 105 | 106 | def set_current_lock(new_lock) 107 | cache.add(@key, my_lock_id) 108 | end 109 | 110 | def my_lock_id 111 | @my_lock_id ||= "#{Process.pid.to_s}.#{self.object_id.to_s}.#{Time.now.to_i.to_s}" 112 | end 113 | 114 | def cache 115 | self.class.cache 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/mega_mutex.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.push File.expand_path(File.dirname(__FILE__)) unless $:.include?(File.expand_path(File.dirname(__FILE__))) 3 | require 'mega_mutex/distributed_mutex' 4 | 5 | # == Why 6 | # 7 | # Sometimes I need to do this: 8 | # 9 | # unless enough_things? 10 | # make_more_things 11 | # end 12 | # 13 | # If I'm running several processes in parallel, I can get a race condition that means two of the processes both think there are not enough things. So we go and make some more, even though we don't need to. 14 | # 15 | # == How 16 | # 17 | # Suppose you have a ThingMaker: 18 | # 19 | # class ThingMaker 20 | # include MegaMutex 21 | # 22 | # def ensure_just_enough_things 23 | # with_cross_process_mutex("ThingMaker Mutex ID") do 24 | # unless enough_things? 25 | # make_more_things 26 | # end 27 | # end 28 | # end 29 | # end 30 | # 31 | # Now, thanks to the magic of MegaMutex, you can be sure that all processes trying to run this code will wait their turn, so each one will have the chance to make exactly the right number of things, without anyone else poking their nose in. 32 | # 33 | # == Configuration 34 | # 35 | # MegaMutex uses http://seattlerb.rubyforge.org/memcache-client/ to store the mutex, so your infrastructure must be set up to use memcache servers. 36 | # 37 | # By default, MegaMutex will attempt to connect to a memcache on the local machine, but you can configure any number of servers like so: 38 | # 39 | # MegaMutex.configure do |config| 40 | # config.memcache_servers = ['mc1', 'mc2'] 41 | # end 42 | module MegaMutex 43 | 44 | def self.get_current_lock(mutex_id) 45 | DistributedMutex.new(mutex_id).current_lock 46 | end 47 | 48 | ## 49 | # Wraps code that should only be run when the mutex has been obtained. 50 | # 51 | # The mutex_id uniquely identifies the section of code being run. 52 | # 53 | # You can optionally specify a :timeout to control how long to wait for the lock to be released 54 | # before raising a MegaMutex::TimeoutError 55 | # 56 | # with_distributed_mutex('my_mutex_id_1234', :timeout => 20) do 57 | # do_something! 58 | # end 59 | def with_distributed_mutex(mutex_id, options = {}, &block) 60 | mutex = DistributedMutex.new(mutex_id, options) 61 | begin 62 | mutex.run(&block) 63 | rescue Object => e 64 | mega_mutex_insert_into_backtrace( 65 | e, 66 | /mega_mutex\.rb.*with_(distributed|cross_process)_mutex/, 67 | "MegaMutex lock #{mutex_id}" 68 | ) 69 | raise e 70 | end 71 | end 72 | alias :with_cross_process_mutex :with_distributed_mutex 73 | 74 | def with_lazy_distributed_mutex(mutex_id, options = {}, &block) 75 | with_distributed_mutex(mutex_id, options.merge(:lazy => true), &block) 76 | end 77 | 78 | # inserts a line into a backtrace at the correct location 79 | def mega_mutex_insert_into_backtrace(exception, re, newline) 80 | loc = nil 81 | exception.backtrace.each_with_index do |line, index| 82 | if line =~ re 83 | loc = index 84 | break 85 | end 86 | end 87 | if loc 88 | exception.backtrace.insert(loc, newline) 89 | end 90 | end 91 | 92 | class Configuration 93 | attr_accessor :memcache_servers, :namespace 94 | 95 | def initialize 96 | @memcache_servers = 'localhost' 97 | @namespace = 'mega_mutex' 98 | end 99 | end 100 | 101 | class << self 102 | def configure 103 | yield configuration 104 | end 105 | 106 | def configuration 107 | @configuration ||= Configuration.new 108 | end 109 | end 110 | end 111 | 112 | -------------------------------------------------------------------------------- /spec/lib/mega_mutex_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | module MegaMutex 4 | describe MegaMutex do 5 | include MegaMutex 6 | 7 | def logger 8 | Logging::Logger['Specs'] 9 | end 10 | 11 | abort_on_thread_exceptions 12 | 13 | describe "#with_distributed_mutex" do 14 | it "returns the value returned by the block" do 15 | result = with_distributed_mutex(nil) { 12345 } 16 | result.should == 12345 17 | end 18 | end 19 | 20 | describe "two blocks, one fast, one slow" do 21 | before(:each) do 22 | @errors = [] 23 | @mutually_exclusive_block = lambda do 24 | @errors << "Someone else is running this code!" if @running 25 | @running = true 26 | sleep 0.5 27 | @running = nil 28 | end 29 | end 30 | 31 | describe "with no lock" do 32 | it "trying to run the block twice should raise an error" do 33 | threads << Thread.new(&@mutually_exclusive_block) 34 | threads << Thread.new(&@mutually_exclusive_block) 35 | wait_for_threads_to_finish 36 | @errors.should_not be_empty 37 | end 38 | end 39 | 40 | describe "with the same lock key" do 41 | before(:each) do 42 | MemCache.new('localhost').delete(mutex_id) 43 | end 44 | 45 | def mutex_id 46 | 'tests-mutex-key' 47 | end 48 | 49 | [2, 20].each do |n| 50 | describe "when #{n} blocks try to run at the same instant in the same process" do 51 | it "should run each in turn" do 52 | n.times do 53 | threads << Thread.new{ with_distributed_mutex(mutex_id, &@mutually_exclusive_block) } 54 | end 55 | wait_for_threads_to_finish 56 | @errors.should be_empty 57 | end 58 | end 59 | end 60 | 61 | describe "when the first block raises an exception" do 62 | before(:each) do 63 | with_distributed_mutex(mutex_id) do 64 | raise "Something went wrong in my code" 65 | end rescue nil 66 | end 67 | 68 | it "the second block should find that the lock is clear and it can run" do 69 | @success = nil 70 | with_distributed_mutex(mutex_id) do 71 | @success = true 72 | end 73 | @success.should be_true 74 | end 75 | end 76 | 77 | describe "when two blocks try to run at the same instant in different processes" do 78 | before(:each) do 79 | @lock_file = File.expand_path(File.dirname(__FILE__) + '/tmp_lock') 80 | @errors_file = File.expand_path(File.dirname(__FILE__) + '/tmp_errors') 81 | @mutually_exclusive_block = lambda { 82 | File.open(@errors_file, 'w').puts "Someone else is running this code!" if File.exists?(@lock_file) 83 | FileUtils.touch @lock_file 84 | sleep 1 85 | File.delete @lock_file 86 | } 87 | end 88 | 89 | after(:each) do 90 | File.delete @lock_file if File.exists?(@lock_file) 91 | File.delete @errors_file if File.exists?(@errors_file) 92 | end 93 | 94 | it "should run each in turn" do 95 | pids = [] 96 | pids << fork { with_distributed_mutex(mutex_id, &@mutually_exclusive_block); Kernel.exit! } 97 | pids << fork { with_distributed_mutex(mutex_id, &@mutually_exclusive_block); Kernel.exit! } 98 | pids.each{ |p| Process.wait(p) } 99 | if File.exists?(@errors_file) 100 | raise "Expected no errors but found #{File.read(@errors_file)}" 101 | end 102 | end 103 | end 104 | 105 | end 106 | end 107 | 108 | describe "with a timeout" do 109 | 110 | it "should raise an error if the code blocks for longer than the timeout" do 111 | @exception = nil 112 | @first_thread_has_started = false 113 | threads << Thread.new do 114 | with_distributed_mutex('foo') do 115 | @first_thread_has_started = true 116 | sleep 0.2 117 | end 118 | end 119 | threads << Thread.new do 120 | sleep 0.1 until @first_thread_has_started 121 | begin 122 | with_distributed_mutex('foo', :timeout => 0.1 ) do 123 | raise 'this code should never run' 124 | end 125 | rescue Exception => @exception 126 | end 127 | end 128 | wait_for_threads_to_finish 129 | assert @exception.is_a?(MegaMutex::TimeoutError), "Expected TimeoutError to be raised, but wasn't" 130 | end 131 | end 132 | 133 | describe "lazy mutex" do 134 | it "should ignore memcache errors and yield to the block in all cases if the lazy option was supplied" do 135 | `killall memcached` #HAX 136 | 137 | i = 0 138 | with_distributed_mutex('foo', :lazy => true) { i = 1 } 139 | i.should == 1 140 | 141 | `memcached -d` 142 | end 143 | 144 | it "should not ignore memcache errors and raise an exception if the lazy option is not supplied" do 145 | `killall memcached` #HAX 146 | 147 | i = 0 148 | lambda { with_distributed_mutex('foo') { i = 1 } }.should raise_error 149 | i.should == 0 150 | 151 | `memcached -d` 152 | end 153 | end 154 | end 155 | end --------------------------------------------------------------------------------