├── examples ├── rack-worker │ ├── files │ │ └── test.txt │ ├── config.yml │ ├── init.rb │ └── actors │ │ └── rack_worker.rb ├── rabbitmqctl │ ├── init.rb │ └── actors │ │ └── rabbit.rb ├── client-agent │ ├── init.rb │ ├── README │ ├── actors │ │ └── client.rb │ └── cli.rb ├── simpleagent │ ├── init.rb │ ├── config.yml │ ├── actors │ │ └── simple.rb │ └── cli.rb ├── secure │ ├── actors │ │ └── secure.rb │ ├── README │ ├── init.rb │ ├── certs │ │ ├── agent_cert.pem │ │ ├── mapper_cert.pem │ │ ├── agent_key.pem │ │ └── mapper_key.pem │ └── cli.rb ├── crew.rb ├── rabbitconf.rb ├── admin_with_handlers.diff └── async_rack_front │ └── async_rack_front.ru ├── docs ├── JamesRH_Lisa.pdf └── links.txt ├── .gitignore ├── lib ├── nanite │ ├── daemonize.rb │ ├── identity.rb │ ├── security │ │ ├── cached_certificate_store_proxy.rb │ │ ├── distinguished_name.rb │ │ ├── signature.rb │ │ ├── static_certificate_store.rb │ │ ├── rsa_key_pair.rb │ │ ├── certificate_cache.rb │ │ ├── certificate.rb │ │ ├── encrypted_document.rb │ │ └── secure_serializer.rb │ ├── actor_registry.rb │ ├── local_state.rb │ ├── reaper.rb │ ├── console.rb │ ├── log │ │ └── formatter.rb │ ├── pid_file.rb │ ├── util.rb │ ├── security_provider.rb │ ├── actor.rb │ ├── serializer.rb │ ├── amqp.rb │ ├── mapper_proxy.rb │ ├── log.rb │ ├── dispatcher.rb │ ├── job.rb │ ├── config.rb │ ├── streaming.rb │ ├── state.rb │ ├── admin.rb │ ├── cluster.rb │ ├── packets.rb │ ├── agent.rb │ └── mapper.rb └── nanite.rb ├── spec ├── encrypted_document_spec.rb ├── spec_helper.rb ├── certificate_spec.rb ├── distinguished_name_spec.rb ├── signature_spec.rb ├── rsa_key_pair_spec.rb ├── static_certificate_store_spec.rb ├── cached_certificate_store_proxy_spec.rb ├── secure_serializer_spec.rb ├── certificate_cache_spec.rb ├── actor_spec.rb ├── util_spec.rb ├── actor_registry_spec.rb ├── serializer_spec.rb ├── dispatcher_spec.rb ├── local_state_spec.rb ├── packet_spec.rb ├── job_spec.rb ├── agent_spec.rb └── cluster_spec.rb ├── nanite.gemspec ├── TODO ├── bin ├── nanite-mapper ├── nanite-admin └── nanite-agent ├── Rakefile ├── tasks └── rabbitmq.rake └── LICENSE /examples/rack-worker/files/test.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/rabbitmqctl/init.rb: -------------------------------------------------------------------------------- 1 | register(Rabbit.new) -------------------------------------------------------------------------------- /examples/client-agent/init.rb: -------------------------------------------------------------------------------- 1 | register Client.new 2 | -------------------------------------------------------------------------------- /examples/simpleagent/init.rb: -------------------------------------------------------------------------------- 1 | register Simple.new 2 | 3 | #tag "foo" -------------------------------------------------------------------------------- /docs/JamesRH_Lisa.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raphael/nanite/master/docs/JamesRH_Lisa.pdf -------------------------------------------------------------------------------- /examples/simpleagent/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | :user: nanite 3 | :pass: testing 4 | :vhost: /nanite 5 | :log_level: debug -------------------------------------------------------------------------------- /examples/rack-worker/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | :pass: testing 3 | :vhost: /nanite 4 | :user: nanite 5 | :identity: fred 6 | -------------------------------------------------------------------------------- /examples/rack-worker/init.rb: -------------------------------------------------------------------------------- 1 | app = Proc.new do |env| 2 | [200, {'Content-Type'=>'text/html'}, "hello world!"] 3 | end 4 | 5 | register(RackWorker.new(app), 'rack') 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.DS_Store 2 | \#* 3 | .#* 4 | .emacs* 5 | pkg 6 | db 7 | vendor 8 | *.log 9 | rdoc 10 | config.yml 11 | spec/config.yml 12 | *.tmproj 13 | nbproject 14 | .idea 15 | -------------------------------------------------------------------------------- /examples/rack-worker/actors/rack_worker.rb: -------------------------------------------------------------------------------- 1 | class RackWorker 2 | include Nanite::Actor 3 | expose :call 4 | 5 | def initialize(app) 6 | @app = app 7 | end 8 | 9 | def call(env) 10 | @app.call(env) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /examples/secure/actors/secure.rb: -------------------------------------------------------------------------------- 1 | # Simple agent 2 | class Secure 3 | include Nanite::Actor 4 | expose :echo 5 | 6 | def echo(payload) 7 | "Nanite said #{payload.empty? ? "nothing at all" : payload} @ #{Time.now.to_s} *securely*" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/nanite/daemonize.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | module DaemonizeHelper 3 | def daemonize 4 | exit if fork 5 | Process.setsid 6 | exit if fork 7 | File.umask 0000 8 | STDIN.reopen "/dev/null" 9 | STDOUT.reopen "/dev/null", "a" 10 | STDERR.reopen STDOUT 11 | end 12 | end 13 | end -------------------------------------------------------------------------------- /lib/nanite/identity.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | module Identity 3 | def self.generate 4 | values = [ 5 | rand(0x0010000), 6 | rand(0x0010000), 7 | rand(0x0010000), 8 | rand(0x0010000), 9 | rand(0x0010000), 10 | rand(0x1000000), 11 | rand(0x1000000), 12 | ] 13 | "%04x%04x%04x%04x%04x%06x%06x" % values 14 | end 15 | end 16 | end -------------------------------------------------------------------------------- /examples/secure/README: -------------------------------------------------------------------------------- 1 | This example demonstrates how to setup Nanite so that every packet is encrypted 2 | and signed using X.509 certificates and RSA cryptographic keys. 3 | 4 | To run it: 5 | 1. Make sure you have AMQP setup properly (same setup as simpleagent example) 6 | 2. run a secure agent ('nanite-agent --token secure --format secure') 7 | 3. run ./cli.rb 8 | 9 | This will send a request to the agent telling it to print some output using 10 | the Nanite secure serializer. 11 | -------------------------------------------------------------------------------- /examples/client-agent/README: -------------------------------------------------------------------------------- 1 | This example demonstrates the ability for agents to send requests to other agents. 2 | 3 | To run it: 4 | 1. Make sure you have AMQP setup properly (same setup as simpleagent example) 5 | 2. run a simple agent ('nanite-agent --token fred') in examples/simpleagent 6 | 2. run 'nanite-agent --token client' in this folder 7 | 3. run ./cli.rb 8 | 9 | This will send a request to the client agent telling it to send a request to 10 | the /simple/echo operation exposed by the simple agent. 11 | -------------------------------------------------------------------------------- /examples/secure/init.rb: -------------------------------------------------------------------------------- 1 | # Configure secure serializer 2 | certs_dir = File.join(File.dirname(__FILE__), 'certs') 3 | mapper_cert = Nanite::Certificate.load(File.join(certs_dir, 'mapper_cert.pem')) 4 | store = Nanite::StaticCertificateStore.new(mapper_cert, mapper_cert) 5 | agent_cert = Nanite::Certificate.load(File.join(certs_dir, 'agent_cert.pem')) 6 | agent_key = Nanite::RsaKeyPair.load(File.join(certs_dir, 'agent_key.pem')) 7 | Nanite::SecureSerializer.init("agent", agent_cert, agent_key, store) 8 | 9 | # Register actor 10 | register Secure.new -------------------------------------------------------------------------------- /examples/client-agent/actors/client.rb: -------------------------------------------------------------------------------- 1 | # you can execute this nanite from the cli.rb command line example app 2 | # you should first run a simpleagent 3 | # this example will send a request to the /simple/echo operation 4 | 5 | class Client 6 | include Nanite::Actor 7 | expose :delegate 8 | 9 | def delegate(payload) 10 | nanite, payload = payload 11 | request(nanite, payload) do |res| 12 | p "Got response '#{res.inspect}' from #{nanite}" 13 | end 14 | "sent request to #{nanite} with payload '#{payload}'" 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/encrypted_document_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::EncryptedDocument do 4 | 5 | include SpecHelpers 6 | 7 | before(:all) do 8 | @test_data = "Test Data to Sign" 9 | @cert, @key = issue_cert 10 | @doc = Nanite::EncryptedDocument.new(@test_data, @cert) 11 | end 12 | 13 | it 'should create encrypted data' do 14 | @doc.encrypted_data.should_not be_nil 15 | end 16 | 17 | it 'should decrypt correctly' do 18 | @doc.decrypted_data(@key, @cert).should == @test_data 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /examples/crew.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | agents = %w[tom jay edward yehuda james corey josh ezra womble jeff loren lance sam kevin kirk] 4 | 5 | def process_exists?(str) 6 | result = `ps auwx | grep '#{str}' | grep -v grep | grep -v tail` 7 | !result.empty? 8 | end 9 | 10 | def run_agent(name, num, root) 11 | if !process_exists?(name) 12 | system("#{File.dirname(__FILE__)}/nanite-agent -u #{name} -p testing -t #{name} -n #{root} -j &") 13 | end 14 | end 15 | 16 | agents.each_with_index do |a,idx| 17 | run_agent(a, idx, "/Users/ez/nanite/examples/rack-worker") 18 | end 19 | 20 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $TESTING=true 2 | $:.unshift File.join(File.dirname(__FILE__), '..', 'lib') 3 | 4 | require 'rubygems' 5 | require 'spec' 6 | require 'nanite' 7 | 8 | module SpecHelpers 9 | 10 | # Create test certificate 11 | def issue_cert 12 | test_dn = { 'C' => 'US', 13 | 'ST' => 'California', 14 | 'L' => 'Santa Barbara', 15 | 'O' => 'Nanite', 16 | 'OU' => 'Certification Services', 17 | 'CN' => 'Nanite test' } 18 | dn = Nanite::DistinguishedName.new(test_dn) 19 | key = Nanite::RsaKeyPair.new 20 | [ Nanite::Certificate.new(key, dn, dn), key ] 21 | end 22 | 23 | end -------------------------------------------------------------------------------- /lib/nanite/security/cached_certificate_store_proxy.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Proxy to actual certificate store which caches results in an LRU 4 | # cache. 5 | class CachedCertificateStoreProxy 6 | 7 | # Initialize cache proxy with given certificate store. 8 | def initialize(store) 9 | @signer_cache = CertificateCache.new 10 | @store = store 11 | end 12 | 13 | # Results from 'get_recipients' are not cached 14 | def get_recipients(obj) 15 | @store.get_recipients(obj) 16 | end 17 | 18 | # Check cache for signer certificate 19 | def get_signer(id) 20 | @signer_cache.get(id) { @store.get_signer(id) } 21 | end 22 | 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /nanite.gemspec: -------------------------------------------------------------------------------- 1 | spec = Gem::Specification.new do |s| 2 | s.name = "nanite" 3 | s.version = "0.4.0" 4 | s.platform = Gem::Platform::RUBY 5 | s.has_rdoc = true 6 | s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO'] 7 | s.summary = "self assembling fabric of ruby daemons" 8 | s.description = s.summary 9 | s.author = "Ezra Zygmuntowicz" 10 | s.email = "ezra@engineyard.com" 11 | s.homepage = "http://github.com/ezmobius/nanite" 12 | 13 | s.bindir = "bin" 14 | s.executables = %w( nanite-agent nanite-mapper nanite-admin ) 15 | 16 | s.add_dependency('amqp', '>= 0.6.0') 17 | 18 | s.require_path = 'lib' 19 | s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*") 20 | end 21 | -------------------------------------------------------------------------------- /lib/nanite/actor_registry.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class ActorRegistry 3 | attr_reader :actors 4 | 5 | def initialize 6 | @actors = {} 7 | end 8 | 9 | def register(actor, prefix) 10 | raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor 11 | Nanite::Log.info("Registering #{actor.inspect} with prefix #{prefix.inspect}") 12 | prefix ||= actor.class.default_prefix 13 | actors[prefix.to_s] = actor 14 | end 15 | 16 | def services 17 | actors.map {|prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq 18 | end 19 | 20 | def actor_for(prefix) 21 | actor = actors[prefix] 22 | end 23 | end # ActorRegistry 24 | end # Nanite -------------------------------------------------------------------------------- /spec/certificate_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::Certificate do 4 | 5 | include SpecHelpers 6 | 7 | before(:all) do 8 | @certificate, key = issue_cert 9 | end 10 | 11 | it 'should save' do 12 | filename = File.join(File.dirname(__FILE__), "cert.pem") 13 | @certificate.save(filename) 14 | File.size(filename).should be > 0 15 | File.delete(filename) 16 | end 17 | 18 | it 'should load' do 19 | filename = File.join(File.dirname(__FILE__), "cert.pem") 20 | @certificate.save(filename) 21 | cert = Nanite::Certificate.load(filename) 22 | File.delete(filename) 23 | cert.should_not be_nil 24 | cert.data.should == @certificate.data 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /spec/distinguished_name_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::DistinguishedName do 4 | 5 | before(:all) do 6 | test_dn = { 'C' => 'US', 7 | 'ST' => 'California', 8 | 'L' => 'Santa Barbara', 9 | 'O' => 'RightScale', 10 | 'OU' => 'Certification Services', 11 | 'CN' => 'rightscale.com/emailAddress=cert@rightscale.com' } 12 | @dn = Nanite::DistinguishedName.new(test_dn) 13 | end 14 | 15 | it 'should convert to string and X509 DN' do 16 | @dn.to_s.should_not be_nil 17 | @dn.to_x509.should_not be_nil 18 | end 19 | 20 | it 'should correctly encode' do 21 | @dn.to_s.should == @dn.to_x509.to_s 22 | end 23 | 24 | end 25 | -------------------------------------------------------------------------------- /lib/nanite/local_state.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class LocalState < ::Hash 3 | def initialize(hsh={}) 4 | hsh.each do |k,v| 5 | self[k] = v 6 | end 7 | end 8 | 9 | def all_services 10 | all(:services) 11 | end 12 | 13 | def all_tags 14 | all(:tags) 15 | end 16 | 17 | def nanites_for(service, *tags) 18 | tags = tags.dup.flatten 19 | nanites = select { |name, state| state[:services].include?(service) } 20 | unless tags.empty? 21 | nanites.select { |a| !(a[1][:tags] & tags).empty? } 22 | else 23 | nanites 24 | end 25 | end 26 | 27 | private 28 | 29 | def all(key) 30 | map { |n,s| s[key] }.flatten.uniq.compact 31 | end 32 | 33 | end # LocalState 34 | end # Nanite 35 | -------------------------------------------------------------------------------- /spec/signature_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::Signature do 4 | 5 | include SpecHelpers 6 | 7 | before(:all) do 8 | @test_data = "Test Data" 9 | @cert, @key = issue_cert 10 | @sig = Nanite::Signature.new(@test_data, @cert, @key) 11 | end 12 | 13 | it 'should create signed data' do 14 | @sig.to_s.should_not be_empty 15 | end 16 | 17 | it 'should verify the signature' do 18 | cert2, key2 = issue_cert 19 | 20 | @sig.should be_a_match(@cert) 21 | @sig.should_not be_a_match(cert2) 22 | end 23 | 24 | it 'should load from serialized signature' do 25 | sig2 = Nanite::Signature.from_data(@sig.data) 26 | sig2.should_not be_nil 27 | sig2.should be_a_match(@cert) 28 | end 29 | 30 | end -------------------------------------------------------------------------------- /docs/links.txt: -------------------------------------------------------------------------------- 1 | http://saladwithsteve.com/2007/08/future-is-messaging.html 2 | http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/2008-June/001287.html 3 | http://hopper.squarespace.com/blog/2008/6/22/introducing-shovel-an-amqp-relay.html 4 | http://hopper.squarespace.com/blog/2008/1/12/introducing-the-erlang-amqp-client.html 5 | http://blog.folknology.com/2008/05/31/reactored-my-packet-based-future-finally-emerges/ 6 | http://hopper.squarespace.com/blog/2008/6/21/build-your-own-amqp-client.html 7 | http://www.lshift.net/news.20071116intelrabbit 8 | http://www.acmqueue.com/modules.php?name=Content&pa=showpage&pid=485 9 | http://www.slideshare.net/Georgio_1999/rubymanor-presentation/ 10 | 11 | AMQP 12 | -- 13 | http://hopper.squarespace.com/blog/2008/7/22/simple-amqp-library-for-ruby.html 14 | http://everburning.com/news/ridding-the-rabbit/ -------------------------------------------------------------------------------- /examples/rabbitconf.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | puts `rabbitmqctl add_vhost /nanite` 4 | 5 | # create 'mapper' and 'nanite' users, give them each the password 'testing' 6 | %w[mapper nanite].each do |agent| 7 | puts `rabbitmqctl add_user #{agent} testing` 8 | puts `rabbitmqctl map_user_vhost #{agent} /nanite` 9 | end 10 | 11 | # grant the mapper user the ability to do anything with the /nanite vhost 12 | # the three regex's map to config, write, read permissions respectively 13 | puts `rabbitmqctl set_permissions -p /nanite mapper ".*" ".*" ".*"` 14 | 15 | # grant the nanite user more limited permissions on the /nanite vhost 16 | puts `rabbitmqctl set_permissions -p /nanite nanite "^nanite.*" ".*" ".*"` 17 | 18 | puts `rabbitmqctl list_users` 19 | puts `rabbitmqctl list_vhosts` 20 | puts `rabbitmqctl list_permissions -p /nanite` -------------------------------------------------------------------------------- /spec/rsa_key_pair_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::RsaKeyPair do 4 | 5 | before(:all) do 6 | @pair = Nanite::RsaKeyPair.new 7 | end 8 | 9 | it 'should create a private and a public keys' do 10 | @pair.has_private?.should be_true 11 | end 12 | 13 | it 'should strip out private key in to_public' do 14 | @pair.to_public.has_private?.should be_false 15 | end 16 | 17 | it 'should save' do 18 | filename = File.join(File.dirname(__FILE__), "key.pem") 19 | @pair.save(filename) 20 | File.size(filename).should be > 0 21 | File.delete(filename) 22 | end 23 | 24 | it 'should load' do 25 | filename = File.join(File.dirname(__FILE__), "key.pem") 26 | @pair.save(filename) 27 | key = Nanite::RsaKeyPair.load(filename) 28 | File.delete(filename) 29 | key.should_not be_nil 30 | key.data.should == @pair.data 31 | end 32 | 33 | end -------------------------------------------------------------------------------- /spec/static_certificate_store_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::StaticCertificateStore do 4 | 5 | include SpecHelpers 6 | 7 | before(:all) do 8 | @signer, key = issue_cert 9 | @recipient, key = issue_cert 10 | @cert, @key = issue_cert 11 | @store = Nanite::StaticCertificateStore.new(@signer, @recipient) 12 | end 13 | 14 | it 'should not raise when passed nil objects' do 15 | res = nil 16 | lambda { res = @store.get_signer(nil) }.should_not raise_error 17 | res.should == [ @signer ] 18 | lambda { res = @store.get_recipients(nil) }.should_not raise_error 19 | res.should == [ @recipient ] 20 | end 21 | 22 | it 'should return signer certificates' do 23 | @store.get_signer('anything').should == [ @signer ] 24 | end 25 | 26 | it 'should return recipient certificates' do 27 | @store.get_recipients('anything').should == [ @recipient ] 28 | end 29 | 30 | end -------------------------------------------------------------------------------- /lib/nanite/reaper.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class Reaper 3 | 4 | def initialize(frequency=2) 5 | @timeouts = {} 6 | EM.add_periodic_timer(frequency) { EM.next_tick { reap } } 7 | end 8 | 9 | def timeout(token, seconds, &blk) 10 | @timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk} 11 | end 12 | 13 | def reset_with_autoregister_hack(token,seconds,&blk) 14 | unless @timeouts[token] 15 | timeout(token, seconds, &blk) 16 | end 17 | reset(token) 18 | end 19 | 20 | def reset(token) 21 | @timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds] 22 | end 23 | 24 | private 25 | 26 | def reap 27 | time = Time.now 28 | @timeouts.reject! do |token, data| 29 | if time > data[:timestamp] 30 | data[:callback].call 31 | true 32 | else 33 | false 34 | end 35 | end 36 | end 37 | end 38 | end -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | TODO: 2 | 3 | - The examples/crew.rb file is pointing towards a hard coded user dir. Needs to be 4 | documented as part of a working example. 5 | - examples/async_rack_front/async_rack_front.ru needs to be documented and verified working. 6 | 7 | Ian: 8 | - Sync Mapper/Agent#start with nanite-mapper/agent 9 | - Update docs for Agent#start and Mapper#start 10 | - Update docs in nanite-agent and nanite-mapper 11 | - Ensure file transfer works 12 | - Check secure stuff still works 13 | - Check custom status_proc works 14 | - Check documentation, only document public methods 15 | - ensure the removal of threaded_actors option doesn't cause shit to block 16 | 17 | - Look into using EM deferables for actors dispatch. 18 | - Integration specs that spawn a small cluster of nanites 19 | - Rename Ping to Status 20 | - request/push should take *args for payload? 21 | 22 | Maybe: 23 | - Make mapper queue durable and Results respect :persistent flag on the request 24 | - Add a global result received callback -------------------------------------------------------------------------------- /lib/nanite/security/distinguished_name.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Build X.509 compliant distinguished names 4 | # Distinghuished names are used to desccribe both a certificate issuer and 5 | # subject. 6 | class DistinguishedName 7 | 8 | # Initialize distinguished name from hash 9 | # e.g.: 10 | # { 'C' => 'US', 11 | # 'ST' => 'California', 12 | # 'L' => 'Santa Barbara', 13 | # 'O' => 'RightScale', 14 | # 'OU' => 'Certification Services', 15 | # 'CN' => 'rightscale.com/emailAddress=cert@rightscale.com' } 16 | # 17 | def initialize(hash) 18 | @value = hash 19 | end 20 | 21 | # Conversion to OpenSSL X509 DN 22 | def to_x509 23 | if @value 24 | OpenSSL::X509::Name.new(@value.to_a, OpenSSL::X509::Name::OBJECT_TYPE_TEMPLATE) 25 | end 26 | end 27 | 28 | # Human readable form 29 | def to_s 30 | '/' + @value.to_a.collect { |p| p.join('=') }.join('/') if @value 31 | end 32 | 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /examples/admin_with_handlers.diff: -------------------------------------------------------------------------------- 1 | diff --git a/lib/nanite/admin.rb b/lib/nanite/admin.rb 2 | index d18e2fa..8c1d3c4 100644 3 | --- a/lib/nanite/admin.rb 4 | +++ b/lib/nanite/admin.rb 5 | @@ -25,7 +25,13 @@ module Nanite 6 | options[:target] = @selection 7 | end 8 | 9 | - @mapper.request(cmd, req.params['payload'], options) do |response, responsejob| 10 | + # demonstrate use of handlers by printing to console 11 | + handlers = { 12 | + 'random' => lambda { |key, nan, state| puts "int: RANDOM from #{nan.inspect}: #{state.inspect}" }, 13 | + '*' => lambda { |key, nan, state, job| puts "int: #{key.inspect} nan: #{nan.inspect} state: #{state.inspect}, job: #{job}" } 14 | + } 15 | + 16 | + @mapper.request(cmd, req.params['payload'], options.merge(:intermediate_handler => handlers)) do |response, responsejob| 17 | env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response, responsejob))]] 18 | end 19 | AsyncResponse 20 | -------------------------------------------------------------------------------- /examples/async_rack_front/async_rack_front.ru: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'nanite' 3 | require 'nanite/mapper' 4 | 5 | # you need raggi's patched async version of thin: 6 | # git clone git://github.com/raggi/thin.git 7 | # cd thin 8 | # git branch async 9 | # git checkout async 10 | # git pull origin async_for_rack 11 | # rake install 12 | # thin -R async_rack_front.ru -p 4000 start 13 | 14 | class NaniteApp 15 | 16 | AsyncResponse = [-1, {}, []].freeze 17 | 18 | def call(env) 19 | mapper = Nanite::Mapper.start 20 | def call(env) 21 | env.delete('rack.errors') 22 | input = env.delete('rack.input') 23 | async_callback = env.delete('async.callback') 24 | 25 | mapper.request('/rack/call', env, :selector => :random, :timeout => 15) do |response| 26 | if response 27 | async_callback.call response.values.first 28 | else 29 | async_callback.call [500, {'Content-Type' => 'text/html'}, "Request Timeout"] 30 | end 31 | end 32 | AsyncResponse 33 | end 34 | [200, {'Content-Type' => 'text/html'}, "warmed up nanite mapper"] 35 | end 36 | end 37 | 38 | run NaniteApp.new 39 | -------------------------------------------------------------------------------- /lib/nanite/console.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | module ConsoleHelper 3 | def self.included(base) 4 | @@base = base 5 | end 6 | 7 | def start_console 8 | puts "Starting #{@@base.name.split(":").last.downcase} console (#{self.identity}) (Nanite #{Nanite::VERSION})" 9 | Thread.new do 10 | Console.start(self) 11 | end 12 | end 13 | end 14 | 15 | module Console 16 | class << self; attr_accessor :instance; end 17 | 18 | def self.start(binding) 19 | require 'irb' 20 | old_args = ARGV.dup 21 | ARGV.replace ["--simple-prompt"] 22 | 23 | IRB.setup(nil) 24 | self.instance = IRB::Irb.new(IRB::WorkSpace.new(binding)) 25 | 26 | @CONF = IRB.instance_variable_get(:@CONF) 27 | @CONF[:IRB_RC].call self.instance.context if @CONF[:IRB_RC] 28 | @CONF[:MAIN_CONTEXT] = self.instance.context 29 | 30 | catch(:IRB_EXIT) { self.instance.eval_input } 31 | ensure 32 | ARGV.replace old_args 33 | # Clean up tty settings in some evil, evil cases 34 | begin; catch(:IRB_EXIT) { irb_exit }; rescue Exception; end 35 | # Make nanite exit when irb does 36 | EM.stop if EM.reactor_running? 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/cached_certificate_store_proxy_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::CachedCertificateStoreProxy do 4 | 5 | include SpecHelpers 6 | 7 | before(:all) do 8 | @signer, key = issue_cert 9 | @recipient, key = issue_cert 10 | @store = mock("Store") 11 | @proxy = Nanite::CachedCertificateStoreProxy.new(@store) 12 | end 13 | 14 | it 'should not raise and return nil for non existent certificates' do 15 | res = nil 16 | @store.should_receive(:get_recipients).with(nil).and_return(nil) 17 | lambda { res = @proxy.get_recipients(nil) }.should_not raise_error 18 | res.should == nil 19 | @store.should_receive(:get_signer).with(nil).and_return(nil) 20 | lambda { res = @proxy.get_signer(nil) }.should_not raise_error 21 | res.should == nil 22 | end 23 | 24 | it 'should return recipient certificates' do 25 | @store.should_receive(:get_recipients).with('anything').and_return(@recipient) 26 | @proxy.get_recipients('anything').should == @recipient 27 | end 28 | 29 | it 'should return signer certificates' do 30 | @store.should_receive(:get_signer).with('anything').and_return(@signer) 31 | @proxy.get_signer('anything').should == @signer 32 | end 33 | 34 | end 35 | -------------------------------------------------------------------------------- /spec/secure_serializer_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | module Nanite 4 | 5 | # Add the ability to compare pings for test purposes 6 | class Ping 7 | def ==(other) 8 | @status == other.status && @identity == other.identity 9 | end 10 | end 11 | 12 | end 13 | 14 | describe Nanite::SecureSerializer do 15 | 16 | include SpecHelpers 17 | 18 | before(:all) do 19 | @certificate, @key = issue_cert 20 | @store = Nanite::StaticCertificateStore.new(@certificate, @certificate) 21 | @identity = "id" 22 | @data = Nanite::Ping.new("Test", 0.5) 23 | end 24 | 25 | it 'should raise when not initialized' do 26 | lambda { Nanite::SecureSerializer.dump(@data) }.should raise_error 27 | end 28 | 29 | it 'should deserialize signed data' do 30 | Nanite::SecureSerializer.init(@identity, @certificate, @key, @store, false) 31 | data = Nanite::SecureSerializer.dump(@data) 32 | Nanite::SecureSerializer.load(data).should == @data 33 | end 34 | 35 | it 'should deserialize encrypted data' do 36 | Nanite::SecureSerializer.init(@identity, @certificate, @key, @store, true) 37 | data = Nanite::SecureSerializer.dump(@data) 38 | Nanite::SecureSerializer.load(data).should == @data 39 | end 40 | 41 | end -------------------------------------------------------------------------------- /lib/nanite/log/formatter.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'time' 3 | 4 | module Nanite 5 | class Log 6 | class Formatter < Logger::Formatter 7 | @@show_time = true 8 | 9 | def self.show_time=(show=false) 10 | @@show_time = show 11 | end 12 | 13 | # Prints a log message as '[time] severity: message' if Nanite::Log::Formatter.show_time == true. 14 | # Otherwise, doesn't print the time. 15 | def call(severity, time, progname, msg) 16 | if @@show_time 17 | sprintf("[%s] %s: %s\n", time.rfc2822(), severity, msg2str(msg)) 18 | else 19 | sprintf("%s: %s\n", severity, msg2str(msg)) 20 | end 21 | end 22 | 23 | # Converts some argument to a Logger.severity() call to a string. Regular strings pass through like 24 | # normal, Exceptions get formatted as "message (class)\nbacktrace", and other random stuff gets 25 | # put through "object.inspect" 26 | def msg2str(msg) 27 | case msg 28 | when ::String 29 | msg 30 | when ::Exception 31 | "#{ msg.message } (#{ msg.class })\n" << 32 | (msg.backtrace || []).join("\n") 33 | else 34 | msg.inspect 35 | end 36 | end 37 | end 38 | end 39 | end -------------------------------------------------------------------------------- /lib/nanite/pid_file.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class PidFile 3 | def initialize(identity, options) 4 | @pid_dir = File.expand_path(options[:pid_dir] || options[:root] || Dir.pwd) 5 | @pid_file = File.join(@pid_dir, "nanite.#{identity}.pid") 6 | end 7 | 8 | def check 9 | if pid = read_pid 10 | if process_running? pid 11 | raise "#{@pid_file} already exists (pid: #{pid})" 12 | else 13 | Log.info "removing stale pid file: #{@pid_file}" 14 | remove 15 | end 16 | end 17 | end 18 | 19 | def ensure_dir 20 | FileUtils.mkdir_p @pid_dir 21 | end 22 | 23 | def write 24 | ensure_dir 25 | open(@pid_file,'w') {|f| f.write(Process.pid) } 26 | File.chmod(0644, @pid_file) 27 | end 28 | 29 | def remove 30 | File.delete(@pid_file) if exists? 31 | end 32 | 33 | def read_pid 34 | open(@pid_file,'r') {|f| f.read.to_i } if exists? 35 | end 36 | 37 | def exists? 38 | File.exists? @pid_file 39 | end 40 | 41 | def to_s 42 | @pid_file 43 | end 44 | 45 | private 46 | def process_running?(pid) 47 | Process.getpgid(pid) != -1 48 | rescue Errno::ESRCH 49 | false 50 | end 51 | end 52 | end -------------------------------------------------------------------------------- /spec/certificate_cache_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::CertificateCache do 4 | 5 | before(:each) do 6 | @cache = Nanite::CertificateCache.new(2) 7 | end 8 | 9 | it 'should allow storing and retrieving objects' do 10 | @cache['some_id'].should be_nil 11 | @cache['some_id'] = 'some_value' 12 | @cache['some_id'].should == 'some_value' 13 | end 14 | 15 | it 'should not store more than required' do 16 | @cache[1] = 'oldest' 17 | @cache[2] = 'older' 18 | @cache[1].should == 'oldest' 19 | @cache[2].should == 'older' 20 | 21 | @cache[3] = 'new' 22 | @cache[3].should == 'new' 23 | 24 | @cache[1].should be_nil 25 | @cache[2].should == 'older' 26 | end 27 | 28 | it 'should use LRU to remove entries' do 29 | @cache[1] = 'oldest' 30 | @cache[2] = 'older' 31 | @cache[1].should == 'oldest' 32 | @cache[2].should == 'older' 33 | 34 | @cache[1] = 'new' 35 | @cache[3] = 'newer' 36 | @cache[1].should == 'new' 37 | @cache[3].should == 'newer' 38 | 39 | @cache[2].should be_nil 40 | end 41 | 42 | it 'should store items returned by block' do 43 | @cache[1].should be_nil 44 | item = @cache.get(1) { 'item' } 45 | item.should == 'item' 46 | @cache[1].should == 'item' 47 | end 48 | 49 | end -------------------------------------------------------------------------------- /examples/simpleagent/actors/simple.rb: -------------------------------------------------------------------------------- 1 | # you can execute this nanite from the cli.rb command line example app 2 | 3 | class Simple 4 | include Nanite::Actor 5 | expose :echo, :time, :gems, :yielding, :delayed 6 | 7 | def echo(payload) 8 | "Nanite said #{payload.empty? ? "nothing at all" : payload} @ #{Time.now.to_s}" 9 | end 10 | 11 | def time(payload) 12 | Time.now 13 | end 14 | 15 | def yielding(payload) 16 | 3.times do 17 | yield :random, "%06x%06x" % [ rand(0x1000000), rand(0x1000000) ] 18 | end 19 | [1,2,3].each do |num| 20 | yield :testkey, num 21 | end 22 | ["a","b","c"].each do |val| 23 | yield val 24 | end 25 | "Nanite said #{payload.empty? ? "nothing at all" : payload} @ #{Time.now.to_s}" 26 | end 27 | 28 | def delayed(payload) 29 | 3.times do 30 | yield :random, "%06x%06x" % [ rand(0x1000000), rand(0x1000000) ] 31 | sleep 1 32 | end 33 | [1,2,3].each do |num| 34 | yield :testkey, num 35 | sleep 2 36 | end 37 | ["a","b","c"].each do |val| 38 | yield val 39 | end 40 | "Nanite said #{payload.empty? ? "nothing at all" : payload} @ #{Time.now.to_s}" 41 | end 42 | 43 | def gems(filter) 44 | ::Gem.source_index.refresh!.search(filter).flatten.collect {|gemspec| "#{gemspec.name} #{gemspec.version}"} 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/nanite/security/signature.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Signature that can be validated against certificates 4 | class Signature 5 | 6 | FLAGS = OpenSSL::PKCS7::NOCERTS || OpenSSL::PKCS7::BINARY || OpenSSL::PKCS7::NOATTR || OpenSSL::PKCS7::NOSMIMECAP || OpenSSL::PKCS7::DETACH 7 | 8 | # Create signature using certificate and key pair. 9 | # 10 | # Arguments: 11 | # - 'data': Data to be signed 12 | # - 'cert': Certificate used for signature 13 | # - 'key': RsaKeyPair used for signature 14 | # 15 | def initialize(data, cert, key) 16 | @p7 = OpenSSL::PKCS7.sign(cert.raw_cert, key.raw_key, data, [], FLAGS) 17 | @store = OpenSSL::X509::Store.new 18 | end 19 | 20 | # Load signature previously serialized via 'data' 21 | def self.from_data(data) 22 | sig = Signature.allocate 23 | sig.instance_variable_set(:@p7, OpenSSL::PKCS7::PKCS7.new(data)) 24 | sig.instance_variable_set(:@store, OpenSSL::X509::Store.new) 25 | sig 26 | end 27 | 28 | # 'true' if signature was created using given cert, 'false' otherwise 29 | def match?(cert) 30 | @p7.verify([cert.raw_cert], @store, nil, OpenSSL::PKCS7::NOVERIFY) 31 | end 32 | 33 | # Signature in PEM format 34 | def data 35 | @p7.to_pem 36 | end 37 | alias :to_s :data 38 | 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /examples/secure/certs/agent_cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDfzCCAmegAwIBAgIBATANBgkqhkiG9w0BAQUFADCBgjEUMBIGA1UEAwwLTmFu 3 | aXRlIHRlc3QxFjAUBgNVBAcMDVNhbnRhIEJhcmJhcmExEzARBgNVBAgMCkNhbGlm 4 | b3JuaWExCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZOYW5pdGUxHzAdBgNVBAsMFkNl 5 | cnRpZmljYXRpb24gU2VydmljZXMwHhcNMDkwNDEzMjI0MTU5WhcNMTkwNDExMjI0 6 | MTU5WjCBgjEUMBIGA1UEAwwLTmFuaXRlIHRlc3QxFjAUBgNVBAcMDVNhbnRhIEJh 7 | cmJhcmExEzARBgNVBAgMCkNhbGlmb3JuaWExCzAJBgNVBAYTAlVTMQ8wDQYDVQQK 8 | DAZOYW5pdGUxHzAdBgNVBAsMFkNlcnRpZmljYXRpb24gU2VydmljZXMwggEiMA0G 9 | CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQJMoo+7dKAcm65rhgAf5zQgkHQZAA 10 | mqBvwowbkLftjwSyf0vm6bgGNWJT/Wwt0CyQK9L5YIqxT2HBp34bCTgdUM1N0mWT 11 | OoqXCBQpDhedUzIvoMgDnqp81rt0EEZCoTuIsjJw8mJ4423hjXZ0oqMvazHzAexC 12 | HugdmGd8To2WRFv/Mj2ZXW6BzMPbtACiGj/tUh0Ln5dQdENRuk3jgRhZ5AgrODYC 13 | sVjYodyYHomUuXxNfxArV3WYFKGFBfyr4mf0eb6iZk9JOMj5n0xl3nGfhPMrGdah 14 | dOPT8qXylApNyP6In1FnalL+n18Z12DQWlxzhhYfaXTHj26xRHMxtIplAgMBAAEw 15 | DQYJKoZIhvcNAQEFBQADggEBAIo5FtLirlRMQROIkAQruPbxF9YzzwioenWiM/Ki 16 | A57kA8jCeV48iulD8hf/DHu5DrUQW5Q8rrsKMcDLlZlzJzu2DvBED5xUFXHtk8U0 17 | iid69Ku5uOi/vemDa4PHvnTNSDiG1nNnvtSiQvoG7/d885REiWDUV7Wzkh6uPMWu 18 | SvsDvU3FFMj3avIJSFHMU8q7YO9YmMAeagR00zFn9ZxFZd87LWPWByjEMMw8cZty 19 | IStrCL6GxMPhq53xEMPQQqPElAUExB6fujw7OJTbdQbEkxeOLvCbvmMQiyBiVe6j 20 | ELnSAE9Wjh69LdnXtWyaoT4GUOUdItNh+8qFvXEOepZLF9c= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /examples/secure/certs/mapper_cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDfzCCAmegAwIBAgIBATANBgkqhkiG9w0BAQUFADCBgjEUMBIGA1UEAwwLTmFu 3 | aXRlIHRlc3QxFjAUBgNVBAcMDVNhbnRhIEJhcmJhcmExEzARBgNVBAgMCkNhbGlm 4 | b3JuaWExCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZOYW5pdGUxHzAdBgNVBAsMFkNl 5 | cnRpZmljYXRpb24gU2VydmljZXMwHhcNMDkwNDEzMjI0MTE4WhcNMTkwNDExMjI0 6 | MTE4WjCBgjEUMBIGA1UEAwwLTmFuaXRlIHRlc3QxFjAUBgNVBAcMDVNhbnRhIEJh 7 | cmJhcmExEzARBgNVBAgMCkNhbGlmb3JuaWExCzAJBgNVBAYTAlVTMQ8wDQYDVQQK 8 | DAZOYW5pdGUxHzAdBgNVBAsMFkNlcnRpZmljYXRpb24gU2VydmljZXMwggEiMA0G 9 | CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPZDBDgXxLrn84UleJVKuzRAwFzz4I 10 | EEtxOs4SF75PWJszcRjD1l+1N0pUEwx3iLnrMmTX6OHXiGYVJo2uimrc9ky4B8WM 11 | KUWXw/3q+Rr5/+PO7NXu08Zwzli2hKBI8HOBbzUt1/AuYiHGNVTfN9Lby2F2ZLSg 12 | ivapBr2AZjDcMvUMAQNhBIdOMvlmMFxwmlTbNI4PVyOrxFLaOa8oSV7BYjNiawLS 13 | TfMxfHSsnt1b289t183JEAdaVRodgy9/plsISfBSGlRxbZICwtHMdy6RjQFii9mi 14 | OOjnS84u5hhBsh4SJEZBvNG8Y34R6urtyYg703o3oESpoaPEAdxLf0qpAgMBAAEw 15 | DQYJKoZIhvcNAQEFBQADggEBAM7W455IHIgEUu2LmvILhLolCdr/LeF6Ag4qW1vH 16 | l/383bf8KVWrwnfC169HGULUMUdWv4aSRloyhOCTtrO13cXUQFxA+TsDy3qJQnKo 17 | 0Qcu/TcXh8/z/JvGZXYu8pnBLzG2BSh3LD6ftQ5GZUzKZxloIAkDxYmCFXdpximz 18 | rC5kWLnTrZW2oZPYOO3PazoU/c4Z2Fb8vzQOEIinHlAbBnpisssmV9pW2k/dq4VG 19 | aqkoyZ/IerEgN0eeDNcOI1autUXTZspHt/Q7ZeUJCYwHQ/MrmWE7Lqs4Byw5NvUp 20 | 8f6yguOJG8DzK1dsx/tzLRB9o7MvWqOXHRDrlP1hbIrgmF8= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /lib/nanite/security/static_certificate_store.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Simple certificate store, serves a static set of certificates. 4 | class StaticCertificateStore 5 | 6 | # Initialize store: 7 | # 8 | # - Signer certificates are used when loading data to check the digital 9 | # signature. The signature associated with the serialized data needs 10 | # to match with one of the signer certificates for loading to succeed. 11 | # 12 | # - Recipient certificates are used when serializing data for encryption. 13 | # Loading the data can only be done through serializers that have been 14 | # initialized with a certificate that's in the recipient certificates if 15 | # encryption is enabled. 16 | # 17 | def initialize(signer_certs, recipients_certs) 18 | signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each) 19 | @signer_certs = signer_certs 20 | recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each) 21 | @recipients_certs = recipients_certs 22 | end 23 | 24 | # Retrieve signer certificate for given id 25 | def get_signer(identity) 26 | @signer_certs 27 | end 28 | 29 | # Recipient certificate(s) that will be able to decrypt the serialized data 30 | def get_recipients(obj) 31 | @recipients_certs 32 | end 33 | 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /bin/nanite-mapper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.dirname(__FILE__) + '/../lib/nanite' 4 | require 'optparse' 5 | 6 | include Nanite::CommonConfig 7 | 8 | options = {} 9 | 10 | opts = OptionParser.new do |opts| 11 | opts.banner = "Usage: nanite-mapper [-flags] [argument]" 12 | opts.define_head "Nanite Mapper: clustered head unit for self assembling cluster of ruby processes." 13 | opts.separator '*'*80 14 | 15 | setup_mapper_options(opts, options) 16 | end 17 | 18 | opts.parse! 19 | 20 | if ARGV[0] == 'stop' || ARGV[0] == 'status' 21 | mapper = Nanite::Mapper.new(options) 22 | pid_file = Nanite::PidFile.new(mapper.identity, mapper.options) 23 | unless pid = pid_file.read_pid 24 | puts "#{pid_file} not found" 25 | exit 26 | end 27 | if ARGV[0] == 'stop' 28 | puts "Stopping nanite mapper #{mapper.identity} (pid #{pid})" 29 | begin 30 | Process.kill('TERM', pid) 31 | rescue Errno::ESRCH 32 | puts "Process does not exist (pid #{pid})" 33 | exit 34 | end 35 | puts 'Done.' 36 | else 37 | if Process.getpgid(pid) != -1 38 | psdata = `ps up #{pid}`.split("\n").last.split 39 | memory = (psdata[5].to_i / 1024) 40 | puts "The mapper is alive, using #{memory}MB of memory" 41 | else 42 | puts "The mapper is not running but has a stale pid file at #{pid_file}" 43 | end 44 | end 45 | exit 46 | end 47 | 48 | EM.run do 49 | Nanite.start_mapper(options) 50 | end 51 | -------------------------------------------------------------------------------- /examples/simpleagent/cli.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.dirname(__FILE__) + '/../../lib/nanite' 4 | 5 | # cli.rb 6 | # 7 | # You will need to have run the examples/rabbitconf.rb script at least one time before you 8 | # run this so the expected users and vhosts are in place in RabbitMQ. 9 | # 10 | # You should also have started the 'simpleagent' nanite in a separate shell by running: 11 | # 12 | # cd /examples/simpleagent 13 | # nanite-agent 14 | # 15 | # This test script takes a little more than 16 seconds to run since we start a new 16 | # mapper within, and pause while we wait for it to initialize, receive pings from 17 | # available agents (which have a default ping time of 15 seconds), and register 18 | # those agents and their methods. When this process is presumed complete after 19 | # 16 seconds we can finally send the nanite agent the task to execute. 20 | 21 | EM.run do 22 | # start up a new mapper with a ping time of 15 seconds 23 | Nanite.start_mapper(:host => 'localhost', :user => 'mapper', :pass => 'testing', :vhost => '/nanite', :log_level => 'info') 24 | 25 | # have this run after 16 seconds so we can be pretty sure that the mapper 26 | # has already received pings from running nanites and registered them. 27 | EM.add_timer(16) do 28 | # call our /simple/echo nanite, and pass it a string to echo back 29 | Nanite.request("/simple/echo", "hello world!") do |res| 30 | p res 31 | EM.stop_event_loop 32 | end 33 | end 34 | end 35 | 36 | -------------------------------------------------------------------------------- /lib/nanite/security/rsa_key_pair.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Allows generating RSA key pairs and extracting public key component 4 | # Note: Creating a RSA key pair can take a fair amount of time (seconds) 5 | class RsaKeyPair 6 | 7 | DEFAULT_LENGTH = 2048 8 | 9 | # Underlying OpenSSL keys 10 | attr_reader :raw_key 11 | 12 | # Create new RSA key pair using 'length' bits 13 | def initialize(length = DEFAULT_LENGTH) 14 | @raw_key = OpenSSL::PKey::RSA.generate(length) 15 | end 16 | 17 | # Does key pair include private key? 18 | def has_private? 19 | raw_key.private? 20 | end 21 | 22 | # New RsaKeyPair instance with identical public key but no private key 23 | def to_public 24 | RsaKeyPair.from_data(raw_key.public_key.to_pem) 25 | end 26 | 27 | # Key material in PEM format 28 | def data 29 | raw_key.to_pem 30 | end 31 | alias :to_s :data 32 | 33 | # Load key pair previously serialized via 'data' 34 | def self.from_data(data) 35 | res = RsaKeyPair.allocate 36 | res.instance_variable_set(:@raw_key, OpenSSL::PKey::RSA.new(data)) 37 | res 38 | end 39 | 40 | # Load key from file 41 | def self.load(file) 42 | from_data(File.read(file)) 43 | end 44 | 45 | # Save key to file in PEM format 46 | def save(file) 47 | File.open(file, "w") do |f| 48 | f.write(@raw_key.to_pem) 49 | end 50 | end 51 | 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/nanite/util.rb: -------------------------------------------------------------------------------- 1 | class String 2 | ## 3 | # Convert to snake case. 4 | # 5 | # "FooBar".snake_case #=> "foo_bar" 6 | # "HeadlineCNNNews".snake_case #=> "headline_cnn_news" 7 | # "CNN".snake_case #=> "cnn" 8 | # 9 | # @return [String] Receiver converted to snake case. 10 | # 11 | # @api public 12 | def snake_case 13 | return self.downcase if self =~ /^[A-Z]+$/ 14 | self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/ 15 | return $+.downcase 16 | end 17 | 18 | ## 19 | # Convert a constant name to a path, assuming a conventional structure. 20 | # 21 | # "FooBar::Baz".to_const_path # => "foo_bar/baz" 22 | # 23 | # @return [String] Path to the file containing the constant named by receiver 24 | # (constantized string), assuming a conventional structure. 25 | # 26 | # @api public 27 | def to_const_path 28 | snake_case.gsub(/::/, "/") 29 | end 30 | end 31 | 32 | class Object 33 | module InstanceExecHelper; end 34 | include InstanceExecHelper 35 | def instance_exec(*args, &block) 36 | begin 37 | old_critical, Thread.critical = Thread.critical, true 38 | n = 0 39 | n += 1 while respond_to?(mname="__instance_exec#{n}") 40 | InstanceExecHelper.module_eval{ define_method(mname, &block) } 41 | ensure 42 | Thread.critical = old_critical 43 | end 44 | begin 45 | ret = send(mname, *args) 46 | ensure 47 | InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil 48 | end 49 | ret 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/nanite/security_provider.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | # This class is used to interface the nanite mapper with an external security 3 | # module. 4 | # There are two points of integration: 5 | # 1. When an agent registers with a mapper 6 | # 2. When an agent sends a request to another agent 7 | # 8 | # In both these cases the security module is called back and can deny the 9 | # operation. 10 | # Note: it's the responsability of the module to do any logging or 11 | # notification that is required. 12 | class SecurityProvider 13 | 14 | # Register an external security module 15 | # This module should expose the 'authorize_registration' and 16 | # 'authorize_request' methods. 17 | def self.register(mod) 18 | @security_module = mod 19 | end 20 | 21 | # Used internally by nanite to retrieve the current security module 22 | def self.get 23 | @security_module || default_security_module 24 | end 25 | 26 | # Default security module, authorizes all operations 27 | def self.default_security_module 28 | @default_sec_mod ||= DefaultSecurityModule.new 29 | end 30 | 31 | end 32 | 33 | # Default security module 34 | class DefaultSecurityModule 35 | 36 | # Authorize registration of agent (registration is an instance of Register) 37 | def authorize_registration(registration) 38 | true 39 | end 40 | 41 | # Authorize given inter-agent request (request is an instance of Request) 42 | def authorize_request(request) 43 | true 44 | end 45 | 46 | end 47 | end -------------------------------------------------------------------------------- /lib/nanite/security/certificate_cache.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Implements a simple LRU cache: items that are the least accessed are 4 | # deleted first. 5 | class CertificateCache 6 | 7 | # Max number of items to keep in memory 8 | DEFAULT_CACHE_MAX_COUNT = 100 9 | 10 | # Initialize cache 11 | def initialize(max_count = DEFAULT_CACHE_MAX_COUNT) 12 | @items = {} 13 | @list = [] 14 | @max_count = max_count 15 | end 16 | 17 | # Add item to cache 18 | def put(key, item) 19 | if @items.include?(key) 20 | delete(key) 21 | end 22 | if @list.size == @max_count 23 | delete(@list.first) 24 | end 25 | @items[key] = item 26 | @list.push(key) 27 | item 28 | end 29 | alias :[]= :put 30 | 31 | # Retrieve item from cache 32 | # Store item returned by given block if any 33 | def get(key) 34 | if @items.include?(key) 35 | @list.each_index do |i| 36 | if @list[i] == key 37 | @list.delete_at(i) 38 | break 39 | end 40 | end 41 | @list.push(key) 42 | @items[key] 43 | else 44 | return nil unless block_given? 45 | self[key] = yield 46 | end 47 | end 48 | alias :[] :get 49 | 50 | # Delete item from cache 51 | def delete(key) 52 | c = @items[key] 53 | if c 54 | @items.delete(key) 55 | @list.each_index do |i| 56 | if @list[i] == key 57 | @list.delete_at(i) 58 | break 59 | end 60 | end 61 | c 62 | end 63 | end 64 | 65 | end 66 | end -------------------------------------------------------------------------------- /lib/nanite/actor.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | # This mixin provides Nanite actor functionality. 3 | # 4 | # To use it simply include it your class containing the functionality to be exposed: 5 | # 6 | # class Foo 7 | # include Nanite::Actor 8 | # expose :bar 9 | # 10 | # def bar(payload) 11 | # # ... 12 | # end 13 | # 14 | # end 15 | module Actor 16 | 17 | def self.included(base) 18 | base.class_eval do 19 | include Nanite::Actor::InstanceMethods 20 | extend Nanite::Actor::ClassMethods 21 | end # base.class_eval 22 | end # self.included 23 | 24 | module ClassMethods 25 | def default_prefix 26 | to_s.to_const_path 27 | end 28 | 29 | def expose(*meths) 30 | @exposed ||= [] 31 | meths.each do |meth| 32 | @exposed << meth unless @exposed.include?(meth) 33 | end 34 | end 35 | 36 | def provides_for(prefix) 37 | return [] unless @exposed 38 | @exposed.map {|meth| "/#{prefix}/#{meth}".squeeze('/')} 39 | end 40 | 41 | def on_exception(proc = nil, &blk) 42 | raise 'No callback provided for on_exception' unless proc || blk 43 | @exception_callback = proc || blk 44 | end 45 | 46 | def exception_callback 47 | @exception_callback 48 | end 49 | 50 | end # ClassMethods 51 | 52 | module InstanceMethods 53 | # send nanite request to another agent (through the mapper) 54 | def request(*args, &blk) 55 | MapperProxy.instance.request(*args, &blk) 56 | end 57 | end # InstanceMethods 58 | 59 | end # Actor 60 | end # Nanite -------------------------------------------------------------------------------- /spec/actor_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | class WebDocumentImporter 4 | include Nanite::Actor 5 | expose :import, :cancel 6 | 7 | def import 8 | 1 9 | end 10 | def cancel 11 | 0 12 | end 13 | def continue 14 | 1 15 | end 16 | end 17 | 18 | module Actors 19 | class ComedyActor 20 | include Nanite::Actor 21 | expose :fun_tricks 22 | def fun_tricks 23 | :rabbit_in_the_hat 24 | end 25 | end 26 | end 27 | 28 | describe Nanite::Actor do 29 | 30 | describe ".expose" do 31 | it "should single expose method only once" do 32 | 3.times { WebDocumentImporter.expose(:continue) } 33 | WebDocumentImporter.provides_for("webfiles").should == ["/webfiles/import", "/webfiles/cancel", "/webfiles/continue"] 34 | end 35 | end 36 | 37 | describe ".default_prefix" do 38 | it "is calculated as default prefix as const path of class name" do 39 | Actors::ComedyActor.default_prefix.should == "actors/comedy_actor" 40 | WebDocumentImporter.default_prefix.should == "web_document_importer" 41 | end 42 | end 43 | 44 | describe ".provides_for(prefix)" do 45 | before :each do 46 | @provides = Actors::ComedyActor.provides_for("money") 47 | end 48 | it "returns an array" do 49 | @provides.should be_kind_of(Array) 50 | end 51 | 52 | it "maps exposed service methods to prefix" do 53 | @provides.should == ["/money/fun_tricks"] 54 | wdi_provides = WebDocumentImporter.provides_for("webfiles") 55 | wdi_provides.should include("/webfiles/import") 56 | wdi_provides.should include("/webfiles/cancel") 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/nanite/security/certificate.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # X.509 Certificate management 4 | class Certificate 5 | 6 | # Underlying OpenSSL cert 7 | attr_accessor :raw_cert 8 | 9 | # Generate a signed X.509 certificate 10 | # 11 | # Arguments: 12 | # - key: RsaKeyPair, key pair used to sign certificate 13 | # - issuer: DistinguishedName, certificate issuer 14 | # - subject: DistinguishedName, certificate subject 15 | # - valid_for: Time in seconds before certificate expires (10 years by default) 16 | def initialize(key, issuer, subject, valid_for = 3600*24*365*10) 17 | @raw_cert = OpenSSL::X509::Certificate.new 18 | @raw_cert.version = 2 19 | @raw_cert.serial = 1 20 | @raw_cert.subject = subject.to_x509 21 | @raw_cert.issuer = issuer.to_x509 22 | @raw_cert.public_key = key.to_public.raw_key 23 | @raw_cert.not_before = Time.now 24 | @raw_cert.not_after = Time.now + valid_for 25 | @raw_cert.sign(key.raw_key, OpenSSL::Digest::SHA1.new) 26 | end 27 | 28 | # Load certificate from file 29 | def self.load(file) 30 | from_data(File.new(file)) 31 | end 32 | 33 | # Initialize with raw certificate 34 | def self.from_data(data) 35 | cert = OpenSSL::X509::Certificate.new(data) 36 | res = Certificate.allocate 37 | res.instance_variable_set(:@raw_cert, cert) 38 | res 39 | end 40 | 41 | # Save certificate to file in PEM format 42 | def save(file) 43 | File.open(file, "w") do |f| 44 | f.write(@raw_cert.to_pem) 45 | end 46 | end 47 | 48 | # Certificate data in PEM format 49 | def data 50 | @raw_cert.to_pem 51 | end 52 | alias :to_s :data 53 | 54 | end 55 | end -------------------------------------------------------------------------------- /lib/nanite/security/encrypted_document.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Represents a signed an encrypted document that can be later decrypted using 4 | # the right private key and whose signature can be verified using the right 5 | # cert. 6 | # This class can be used both to encrypt and sign data and to then check the 7 | # signature and decrypt an encrypted document. 8 | class EncryptedDocument 9 | 10 | # Encrypt and sign data using certificate and key pair. 11 | # 12 | # Arguments: 13 | # - 'data': Data to be encrypted 14 | # - 'certs': Recipient certificates (certificates corresponding to private 15 | # keys that may be used to decrypt data) 16 | # - 'cipher': Cipher used for encryption, AES 256 CBC by default 17 | # 18 | def initialize(data, certs, cipher = 'AES-256-CBC') 19 | cipher = OpenSSL::Cipher::Cipher.new(cipher) 20 | certs = [ certs ] unless certs.respond_to?(:collect) 21 | raw_certs = certs.collect { |c| c.raw_cert } 22 | @pkcs7 = OpenSSL::PKCS7.encrypt(raw_certs, data, cipher, OpenSSL::PKCS7::BINARY) 23 | end 24 | 25 | # Initialize from encrypted data. 26 | def self.from_data(encrypted_data) 27 | doc = EncryptedDocument.allocate 28 | doc.instance_variable_set(:@pkcs7, OpenSSL::PKCS7::PKCS7.new(encrypted_data)) 29 | doc 30 | end 31 | 32 | # Encrypted data using DER format 33 | def encrypted_data 34 | @pkcs7.to_pem 35 | end 36 | 37 | # Decrypted data 38 | # 39 | # Arguments: 40 | # - 'key': Key used for decryption 41 | # - 'cert': Certificate to use for decryption 42 | def decrypted_data(key, cert) 43 | @pkcs7.decrypt(key.raw_key, cert.raw_cert) 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/nanite/serializer.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class Serializer 3 | 4 | class SerializationError < StandardError 5 | attr_accessor :action, :packet 6 | def initialize(action, packet, serializers, msg = nil) 7 | @action, @packet = action, packet 8 | msg = ":\n#{msg}" if msg && !msg.empty? 9 | super("Could not #{action} #{packet.inspect} using #{serializers.inspect}#{msg}") 10 | end 11 | end # SerializationError 12 | 13 | # The secure serializer should not be part of the cascading 14 | def initialize(preferred_format = :marshal) 15 | preferred_format ||= :marshal 16 | if preferred_format.to_s == 'secure' 17 | @serializers = [ SecureSerializer ] 18 | else 19 | preferred_serializer = SERIALIZERS[preferred_format.to_sym] 20 | @serializers = SERIALIZERS.values.clone 21 | @serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer 22 | end 23 | end 24 | 25 | def dump(packet) 26 | cascade_serializers(:dump, packet) 27 | end 28 | 29 | def load(packet) 30 | cascade_serializers(:load, packet) 31 | end 32 | 33 | private 34 | 35 | SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze 36 | 37 | def cascade_serializers(action, packet) 38 | errors = [] 39 | @serializers.map do |serializer| 40 | begin 41 | o = serializer.send(action, packet) 42 | rescue Exception => e 43 | o = nil 44 | errors << "#{e.message}\n\t#{e.backtrace[0]}" 45 | end 46 | return o if o 47 | end 48 | raise SerializationError.new(action, packet, @serializers, errors.join("\n")) 49 | end 50 | 51 | end # Serializer 52 | end # Nanite 53 | -------------------------------------------------------------------------------- /examples/secure/certs/agent_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEA0CTKKPu3SgHJuua4YAH+c0IJB0GQAJqgb8KMG5C37Y8Esn9L 3 | 5um4BjViU/1sLdAskCvS+WCKsU9hwad+Gwk4HVDNTdJlkzqKlwgUKQ4XnVMyL6DI 4 | A56qfNa7dBBGQqE7iLIycPJieONt4Y12dKKjL2sx8wHsQh7oHZhnfE6NlkRb/zI9 5 | mV1ugczD27QAoho/7VIdC5+XUHRDUbpN44EYWeQIKzg2ArFY2KHcmB6JlLl8TX8Q 6 | K1d1mBShhQX8q+Jn9Hm+omZPSTjI+Z9MZd5xn4TzKxnWoXTj0/Kl8pQKTcj+iJ9R 7 | Z2pS/p9fGddg0Fpcc4YWH2l0x49usURzMbSKZQIDAQABAoIBAQC5vYrVxcKLfUr6 8 | yXcQ3damHfCWlBAgKdwvv0luM9eeW3kSnHwsRLaee7cSQlzuCQ4rXaA4P8mkbrVK 9 | rd/f2ntD1d5PqwhPuPcBzgjHljbGgv6O+AhlCosTOkAvU3KPdPf+hR50IihzvmhI 10 | 1gc2EAhVgXAibb3aR2BVQLJggGzjAONrBieNtG23HyvovasxWqFVcvC6ojwdv1vU 11 | HQ4G9vsQFt9B3UpL5Dp2rRzH2rdS/bOQcE/XJrj16WB4mvv5ZdGes3QwbUUCr2nE 12 | 88C2Hv4OwIjAOufOA4r184poQByk3xz2xvE1aHk1z756ymKS0nzFNqIdvhh8CuFs 13 | 64gozqFBAoGBAO7Kd4z+6dxHnAbNRpd9Bo9zj5yLVs1Xxk67OktSkWOLWay4Adp3 14 | O7G4N4KD3RIxAkiNa7YqGYMiUIHMVmxMWBTs7fIXhiuLNAwfOTp8rotWY/+AbJtD 15 | X+sW3+hfHLHZVmc5NuC01owaBkq9rfHt3yKsGxQ2H2fGPsZ5YZftraN1AoGBAN8k 16 | 5xUSmcvy6iY9ZwiBlYJvNY0CUDomSy/dH+Mr+jLaHUtVdib5+xKXOz/rl6oHIoBg 17 | AbNvrydFFZAKKMP+VogUoBEKJQu5V3aMC+F2iX+hH7F9uRBO1qfq5bssfcoYVzdE 18 | dwwnJJFg9Dtj8gKSJM0gPcDJtu6Iq2oyVxrtpB0xAoGAQhZzA+jBHKANBKznbqHB 19 | oQ4gnvjJa0dxsPSMa7BMYLp+dRjwGmCJeuQ7KLoKAzDhKudI+j+ok+x5T5MYEB0I 20 | +7GfITN1hLOgEgLKOe+r9xBP2LTYWhhJgazxB0TbkjVOfn8NOx+RiXDicOkSLTTi 21 | Xpw3Wl6oTnZFhXSul/fF6MkCgYEA1JC0yLAlbdEDGFZZthKrN8t0edUAU8Ryzy+X 22 | 1rHn71UznrXUjaCIedcyi9uDaGbPiPbr5+C2O3Rl7AahesLn702mhVTioV53YOhb 23 | 0PU4WipEpLGc/1n6xY7QNRNW6JNzUaSYuyz2er6fKYw12BkbkYPrD6GX73alw84c 24 | h/xk36ECgYEA39Yy0HanluRl7kiVrgexWUU66DoCYv3JBId3swJV/By95Jyrm6cG 25 | UHnzG0IuSG/lHTMsDeHIqzEm/nLovuk51koRB8aU1lURkL97RL+Ke6PcbOfAKCxc 26 | FG05phqw4d3p5WcRuKvoz9Af/Rg/V/P0w4M6qjQ7+cQh69+nklE7e0Y= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /examples/secure/certs/mapper_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAz2QwQ4F8S65/OFJXiVSrs0QMBc8+CBBLcTrOEhe+T1ibM3EY 3 | w9ZftTdKVBMMd4i56zJk1+jh14hmFSaNropq3PZMuAfFjClFl8P96vka+f/jzuzV 4 | 7tPGcM5YtoSgSPBzgW81LdfwLmIhxjVU3zfS28thdmS0oIr2qQa9gGYw3DL1DAED 5 | YQSHTjL5ZjBccJpU2zSOD1cjq8RS2jmvKElewWIzYmsC0k3zMXx0rJ7dW9vPbdfN 6 | yRAHWlUaHYMvf6ZbCEnwUhpUcW2SAsLRzHcukY0BYovZojjo50vOLuYYQbIeEiRG 7 | QbzRvGN+Eerq7cmIO9N6N6BEqaGjxAHcS39KqQIDAQABAoIBAQDMWNzRd0HA1KRY 8 | G5JTJ9Gy5mauRsDJczzSye2ImEqVpinuqw6K0K0nQt6DLu3GzjrbC6GfuMDm/WgQ 9 | iRdrlsmrBSkc39N/wp7m5mBIacFx+fg+yNAW7iaGulthJBdHJ124eJo+nJn1Kg47 10 | XgtNkpS6q5BR9vKfia13aMPKQ2b7X5fUnva1ewaMFnXL3QEZeQMnlaGiqZkh/H6D 11 | Gb5be6sp+S7/eLwuJIo77CADE1C1/oahF5jGRn/3x2AAFoDi9H1GRrk1AclW797T 12 | 2cjOfuWNwgNFKnhGQTp8vBe//dAkzkooOzedVx5Otrsa/wbAfdgHkwNcxOuzI5iU 13 | E7opsU8tAoGBAPE5VdN2Dj2WIf6zr0LT8GZDjF+6o6E44XdTDbJnltn/nMhoViQN 14 | Jtqiry7eBe5OUvrR+d8hDR1X3gGkLt9X8/rJD9MU/RX2qTyCkBaG8cXah52LpXZs 15 | 0m1ERA9/7bEu7xd962LXIUsWrVWBdHIgWi76G2VufT0KMxGcQaf7uaQvAoGBANwY 16 | Uds/rZG2NP4AlmqreGcJMZn5feJZTTAvXyNPNXeuCf9yxjTSTb5CXfwbvTBdZma9 17 | 0SbP8YqM1zN3MMfLWlmTaynh2KOYoDYivETq87fBEo473aoFgCtRj1Uu2OCYUvj6 18 | pgM2aNjBGfBPkLiXX9BroV6XahfrjQS8MymdMdCnAoGBAOMgdCJOFyx/zvRuJUhM 19 | HB993I9KQcWMtCZRKiVgChaq6el87Ba312ynH0GGxZjTYKhPyvD/wsRtNljfSU78 20 | Fnp8lAzQx1fFzyVtXdQILV942pZJpXrzDdLaIydg8eIR6gd5hYekPfHgoUZQRN8s 21 | fR2cWu1PclZl4x3ivaM4jlv7AoGAJkWEVl+oSuIoLNLlVP8kHKaYnXGo1ic5KNvT 22 | 60vmvDzjOCw5Ew5OvnrD5Qbdyv8yw0LWj60xDyqWcTiUnulZeKINzGadSTVNBpzb 23 | qBBC+Pz1wQ3f6aSzONUh22oeBE/4fZr4Ky7W5HoPjJ4JSE/qNU/B67TgmfA3wU8I 24 | hn/zHZkCgYBoLzW9vlwt+uCm1IN/VPTAuIRQc0ZxzSQTSl3RritQwjck7XfjaWt+ 25 | OdKRqQRqtRPJutDNKoBx/qlKNYrhUQ/z6Ow37pfiw1ezoRroFzRpC9INTYlE+srT 26 | cBoLs2xS4Irm9hMaL2gh1jt0y/TN5CWOYIX+qOiv4a79C91CO0U1mg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /examples/client-agent/cli.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.dirname(__FILE__) + '/../../lib/nanite' 4 | 5 | # cli.rb 6 | # 7 | # You will need to have run the examples/rabbitconf.rb script at least one time before you 8 | # run this so the expected users and vhosts are in place in RabbitMQ. 9 | # 10 | # You should also have started the 'simpleagent' nanite in a separate shell by running: 11 | # 12 | # cd /examples/simpleagent 13 | # nanite-agent --token fred 14 | # 15 | # You should also have started the 'client' nanite in another shell by running: 16 | # cd /examples/client-agent 17 | # nanite-agent --token client 18 | # 19 | # This test script takes a little more than 16 seconds to run since we start a new 20 | # mapper within, and pause while we wait for it to initialize, receive pings from 21 | # available agents (which have a default ping time of 15 seconds), and register those 22 | # agents and their methods. When this process is presumed complete after 16 seconds 23 | # we can finally send the nanite agent the task to execute. 24 | 25 | EM.run do 26 | # start up a new mapper with a ping time of 2 seconds 27 | Nanite.start_mapper(:host => 'localhost', :user => 'mapper', :pass => 'testing', :vhost => '/nanite', :log_level => 'debug') 28 | 29 | # have this run after 16 seconds so we can be pretty sure that the mapper 30 | # has already received pings from running nanites and registered them. 31 | EM.add_timer(16) do 32 | # call our /client/delegate nanite and ask it to send a request to the /simple/echo nanite 33 | Nanite.request("/client/delegate", ["/simple/echo", "echo said hello world!"]) do |res| 34 | p res 35 | # don't stop right away so that the echo agent has time to return its result to the client 36 | # agent. The client agent will log the response. 37 | EM.add_timer(3) { EM.stop_event_loop } 38 | end 39 | end 40 | end 41 | 42 | -------------------------------------------------------------------------------- /lib/nanite/amqp.rb: -------------------------------------------------------------------------------- 1 | class MQ 2 | class Queue 3 | # Asks the broker to redeliver all unacknowledged messages on a 4 | # specifieid channel. Zero or more messages may be redelivered. 5 | # 6 | # * requeue (default false) 7 | # If this parameter is false, the message will be redelivered to the original recipient. 8 | # If this flag is true, the server will attempt to requeue the message, potentially then 9 | # delivering it to an alternative subscriber. 10 | # 11 | def recover requeue = false 12 | @mq.callback{ 13 | @mq.send Protocol::Basic::Recover.new({ :requeue => requeue }) 14 | } 15 | self 16 | end 17 | end 18 | end 19 | 20 | # monkey patch to the amqp gem that adds :no_declare => true option for new 21 | # Exchange objects. This allows us to send messeages to exchanges that are 22 | # declared by the mappers and that we have no configuration priviledges on. 23 | # temporary until we get this into amqp proper 24 | MQ::Exchange.class_eval do 25 | def initialize mq, type, name, opts = {} 26 | @mq = mq 27 | @type, @name, @opts = type, name, opts 28 | @mq.exchanges[@name = name] ||= self 29 | @key = opts[:key] 30 | 31 | @mq.callback{ 32 | @mq.send AMQP::Protocol::Exchange::Declare.new({ :exchange => name, 33 | :type => type, 34 | :nowait => true }.merge(opts)) 35 | } unless name == "amq.#{type}" or name == '' or opts[:no_declare] 36 | end 37 | end 38 | 39 | module Nanite 40 | module AMQPHelper 41 | def start_amqp(options) 42 | connection = AMQP.connect(:user => options[:user], :pass => options[:pass], :vhost => options[:vhost], 43 | :host => options[:host], :port => (options[:port] || ::AMQP::PORT).to_i, :insist => options[:insist] || false) 44 | MQ.new(connection) 45 | end 46 | end 47 | end -------------------------------------------------------------------------------- /spec/util_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe String do 4 | 5 | describe ".snake_case" do 6 | 7 | it "should downcase single word" do 8 | ["FOO", "Foo", "foo"].each do |w| 9 | w.snake_case.should == "foo" 10 | end 11 | end 12 | 13 | it "should not separate numbers from end of word" do 14 | ["Foo1234", "foo1234"].each do |w| 15 | w.snake_case.should == "foo1234" 16 | end 17 | end 18 | 19 | it "should separate numbers from word it starts with uppercase letter" do 20 | "1234Foo".snake_case.should == "1234_foo" 21 | end 22 | 23 | it "should not separate numbers from word starts with lowercase letter" do 24 | "1234foo".snake_case.should == "1234foo" 25 | end 26 | 27 | it "should downcase camel-cased words and connect with underscore" do 28 | ["FooBar", "fooBar"].each do |w| 29 | w.snake_case.should == "foo_bar" 30 | end 31 | end 32 | 33 | it "should start new word with uppercase letter before lower case letter" do 34 | ["FooBARBaz", "fooBARBaz"].each do |w| 35 | w.snake_case.should == "foo_bar_baz" 36 | end 37 | end 38 | 39 | end 40 | 41 | describe ".to_const_path" do 42 | 43 | it "should snake-case the string" do 44 | str = "hello" 45 | str.should_receive(:snake_case).and_return("snake-cased hello") 46 | str.to_const_path 47 | end 48 | 49 | it "should leave (snake-cased) string without '::' unchanged" do 50 | "hello".to_const_path.should == "hello" 51 | end 52 | 53 | it "should replace single '::' with '/'" do 54 | "hello::world".to_const_path.should == "hello/world" 55 | end 56 | 57 | it "should replace multiple '::' with '/'" do 58 | "hello::nanite::world".to_const_path.should == "hello/nanite/world" 59 | end 60 | 61 | end 62 | 63 | end # String 64 | -------------------------------------------------------------------------------- /spec/actor_registry_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::ActorRegistry do 4 | 5 | before(:all) do 6 | class WebDocumentImporter 7 | include Nanite::Actor 8 | expose :import, :cancel 9 | 10 | def import 11 | 1 12 | end 13 | def cancel 14 | 0 15 | end 16 | end 17 | 18 | module Actors 19 | class ComedyActor 20 | include Nanite::Actor 21 | expose :fun_tricks 22 | def fun_tricks 23 | :rabbit_in_the_hat 24 | end 25 | end 26 | end 27 | end 28 | 29 | before(:each) do 30 | Nanite::Log.stub!(:info) 31 | @registry = Nanite::ActorRegistry.new 32 | end 33 | 34 | it "should know about all services" do 35 | @registry.register(WebDocumentImporter.new, nil) 36 | @registry.register(Actors::ComedyActor.new, nil) 37 | @registry.services.sort.should == ["/actors/comedy_actor/fun_tricks", "/web_document_importer/cancel", "/web_document_importer/import"] 38 | end 39 | 40 | it "should not register anything except Nanite::Actor" do 41 | lambda { @registry.register(String.new, nil) }.should raise_error(ArgumentError) 42 | end 43 | 44 | it "should register an actor" do 45 | importer = WebDocumentImporter.new 46 | @registry.register(importer, nil) 47 | @registry.actors['web_document_importer'].should == importer 48 | end 49 | 50 | it "should log info message that actor was registered" do 51 | importer = WebDocumentImporter.new 52 | Nanite::Log.should_receive(:info).with("Registering #{importer.inspect} with prefix nil") 53 | @registry.register(importer, nil) 54 | end 55 | 56 | it "should handle actors registered with a custom prefix" do 57 | importer = WebDocumentImporter.new 58 | @registry.register(importer, 'monkey') 59 | @registry.actors['monkey'].should == importer 60 | end 61 | 62 | end # Nanite::ActorRegistry 63 | -------------------------------------------------------------------------------- /lib/nanite.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'amqp' 3 | require 'mq' 4 | require 'json' 5 | require 'logger' 6 | require 'yaml' 7 | require 'openssl' 8 | 9 | $:.unshift File.dirname(__FILE__) 10 | require 'nanite/amqp' 11 | require 'nanite/util' 12 | require 'nanite/config' 13 | require 'nanite/packets' 14 | require 'nanite/identity' 15 | require 'nanite/console' 16 | require 'nanite/daemonize' 17 | require 'nanite/pid_file' 18 | require 'nanite/job' 19 | require 'nanite/mapper' 20 | require 'nanite/actor' 21 | require 'nanite/actor_registry' 22 | require 'nanite/streaming' 23 | require 'nanite/dispatcher' 24 | require 'nanite/agent' 25 | require 'nanite/cluster' 26 | require 'nanite/reaper' 27 | require 'nanite/log' 28 | require 'nanite/mapper_proxy' 29 | require 'nanite/security_provider' 30 | require 'nanite/security/cached_certificate_store_proxy' 31 | require 'nanite/security/certificate' 32 | require 'nanite/security/certificate_cache' 33 | require 'nanite/security/distinguished_name' 34 | require 'nanite/security/encrypted_document' 35 | require 'nanite/security/rsa_key_pair' 36 | require 'nanite/security/secure_serializer' 37 | require 'nanite/security/signature' 38 | require 'nanite/security/static_certificate_store' 39 | require 'nanite/serializer' 40 | 41 | module Nanite 42 | VERSION = '0.4.0' unless defined?(Nanite::VERSION) 43 | 44 | class MapperNotRunning < StandardError; end 45 | 46 | class << self 47 | attr_reader :mapper, :agent 48 | 49 | def start_agent(options = {}) 50 | @agent = Nanite::Agent.start(options) 51 | end 52 | 53 | def start_mapper(options = {}) 54 | @mapper = Nanite::Mapper.start(options) 55 | end 56 | 57 | def request(*args, &blk) 58 | ensure_mapper 59 | @mapper.request(*args, &blk) 60 | end 61 | 62 | def push(*args) 63 | ensure_mapper 64 | @mapper.push(*args) 65 | end 66 | 67 | def ensure_mapper 68 | raise MapperNotRunning.new('A mapper needs to be started via Nanite.start_mapper') unless @mapper 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/nanite/mapper_proxy.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # This class allows sending requests to nanite agents without having 4 | # to run a local mapper. 5 | # It is used by Actor.request which can be used by actors than need 6 | # to send requests to remote agents. 7 | # All requests go through the mapper for security purposes. 8 | class MapperProxy 9 | 10 | $:.push File.dirname(__FILE__) 11 | require 'amqp' 12 | 13 | include AMQPHelper 14 | 15 | attr_accessor :pending_requests, :identity, :options, :amqp, :serializer 16 | 17 | # Accessor for actor 18 | def self.instance 19 | @@instance 20 | end 21 | 22 | def initialize(id, opts) 23 | @options = opts || {} 24 | @identity = id 25 | @pending_requests = {} 26 | @amqp = start_amqp(options) 27 | @serializer = Serializer.new(options[:format]) 28 | @@instance = self 29 | end 30 | 31 | # Send request to given agent through the mapper 32 | def request(type, payload = '', opts = {}, &blk) 33 | raise "Mapper proxy not initialized" unless identity && options 34 | request = Request.new(type, payload, opts) 35 | request.from = identity 36 | request.token = Identity.generate 37 | request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent] 38 | pending_requests[request.token] = 39 | { :intermediate_handler => opts[:intermediate_handler], :result_handler => blk } 40 | amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request)) 41 | end 42 | 43 | # Handle intermediary result 44 | def handle_intermediate_result(res) 45 | handlers = pending_requests[res.token] 46 | handlers[:intermediate_handler].call(res) if handlers && handlers[:intermediate_handler] 47 | end 48 | 49 | # Handle final result 50 | def handle_result(res) 51 | handlers = pending_requests.delete(res.token) 52 | handlers[:result_handler].call(res) if handlers && handlers[:result_handler] 53 | end 54 | 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rake/gempackagetask' 3 | require "spec/rake/spectask" 4 | begin; require 'rubygems'; rescue LoadError; end 5 | begin 6 | require 'hanna/rdoctask' 7 | rescue LoadError 8 | require 'rake/rdoctask' 9 | end 10 | require 'rake/clean' 11 | 12 | GEM = "nanite" 13 | VER = "0.4.1" 14 | AUTHOR = "Ezra Zygmuntowicz" 15 | EMAIL = "ezra@engineyard.com" 16 | HOMEPAGE = "http://github.com/ezmobius/nanite" 17 | SUMMARY = "self assembling fabric of ruby daemons" 18 | 19 | Dir.glob('tasks/*.rake').each { |r| Rake.application.add_import r } 20 | 21 | spec = Gem::Specification.new do |s| 22 | s.name = GEM 23 | s.version = ::VER 24 | s.platform = Gem::Platform::RUBY 25 | s.has_rdoc = true 26 | s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO'] 27 | s.summary = SUMMARY 28 | s.description = s.summary 29 | s.author = AUTHOR 30 | s.email = EMAIL 31 | s.homepage = HOMEPAGE 32 | 33 | s.bindir = "bin" 34 | s.executables = %w( nanite-agent nanite-mapper nanite-admin ) 35 | 36 | s.add_dependency('amqp', '>= 0.6.0') 37 | 38 | s.require_path = 'lib' 39 | s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*") 40 | end 41 | 42 | Rake::GemPackageTask.new(spec) do |pkg| 43 | pkg.gem_spec = spec 44 | end 45 | 46 | task :install => [:package] do 47 | sh %{sudo gem install pkg/#{GEM}-#{VER}} 48 | end 49 | 50 | desc "Run unit specs" 51 | Spec::Rake::SpecTask.new do |t| 52 | t.spec_opts = ["--format", "specdoc", "--colour"] 53 | t.spec_files = FileList["spec/**/*_spec.rb"] 54 | end 55 | 56 | desc 'Generate RDoc documentation' 57 | Rake::RDocTask.new do |rd| 58 | rd.title = spec.name 59 | rd.rdoc_dir = 'rdoc' 60 | rd.main = "README.rdoc" 61 | rd.rdoc_files.include("lib/**/*.rb", *spec.extra_rdoc_files) 62 | end 63 | CLOBBER.include(:clobber_rdoc) 64 | 65 | desc 'Generate and open documentation' 66 | task :docs => :rdoc do 67 | case RUBY_PLATFORM 68 | when /darwin/ ; sh 'open rdoc/index.html' 69 | when /mswin|mingw/ ; sh 'start rdoc\index.html' 70 | else 71 | sh 'firefox rdoc/index.html' 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /tasks/rabbitmq.rake: -------------------------------------------------------------------------------- 1 | # Inspired by rabbitmq.rake the Redbox project at http://github.com/rick/redbox/tree/master 2 | require 'fileutils' 3 | 4 | class RabbitMQ 5 | 6 | def self.basedir 7 | basedir = File.expand_path(File.dirname(__FILE__) + "/../") # ick 8 | end 9 | 10 | def self.rabbitdir 11 | "#{basedir}/vendor/rabbitmq-server-1.5.1" 12 | end 13 | 14 | def self.dtach_socket 15 | "#{basedir}/tmp/rabbitmq.dtach" 16 | end 17 | 18 | # Just check for existance of dtach socket 19 | def self.running? 20 | File.exists? dtach_socket 21 | end 22 | 23 | def self.setup_environment 24 | ENV['MNESIA_BASE'] ||= "#{basedir}/db/mnesia" 25 | ENV['LOG_BASE'] ||= "#{basedir}/log" 26 | 27 | # Kind of a hack around the way rabbitmq-server does args. I need to set 28 | # RABBITMQ_NODE_ONLY to prevent RABBITMQ_START_RABBIT from being set with -noinput. 29 | # Then RABBITMQ_SERVER_START_ARGS passes in the actual '-s rabbit' necessary. 30 | ENV['RABBITMQ_NODE_ONLY'] ||= "0" 31 | ENV['RABBITMQ_SERVER_START_ARGS'] ||= "-s rabbit" 32 | end 33 | 34 | def self.start 35 | setup_environment 36 | exec "dtach -A #{dtach_socket} #{rabbitdir}/scripts/rabbitmq-server" 37 | end 38 | 39 | def self.attach 40 | exec "dtach -a #{dtach_socket}" 41 | end 42 | 43 | def self.stop 44 | system "#{rabbitdir}/scripts/rabbitmqctl stop" 45 | end 46 | 47 | end 48 | 49 | namespace :rabbitmq do 50 | 51 | task :ensure_directories do 52 | FileUtils.mkdir_p("tmp") 53 | FileUtils.mkdir_p("log") 54 | FileUtils.mkdir_p("vendor") 55 | end 56 | 57 | desc "Start RabbitMQ" 58 | task :start => [:ensure_directories, :download] do 59 | RabbitMQ.start 60 | end 61 | 62 | desc "stop" 63 | task :stop do 64 | RabbitMQ.stop 65 | end 66 | 67 | desc "Attach to RabbitMQ dtach socket" 68 | task :attach do 69 | RabbitMQ.attach 70 | end 71 | 72 | desc "Download package" 73 | task :download do 74 | unless File.exists?(RabbitMQ.rabbitdir) 75 | FileUtils.mkdir_p("vendor") 76 | Dir.chdir("vendor") do 77 | system "curl http://www.rabbitmq.com/releases/rabbitmq-server/v1.5.1/rabbitmq-server-1.5.1.tar.gz -O && 78 | tar xvzf rabbitmq-server-1.5.1.tar.gz" 79 | end 80 | end 81 | end 82 | 83 | 84 | end 85 | -------------------------------------------------------------------------------- /bin/nanite-admin: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # To work without being installed as a gem: 4 | libdir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')) 5 | $:.unshift libdir unless $:.include? libdir 6 | 7 | require 'nanite' 8 | require 'nanite/admin' 9 | require 'eventmachine' 10 | require 'thin' 11 | # IMPORTANT! 12 | # You need raggi's patched async version of Thin at the moment to use 13 | # the nanite-admin tool. 14 | # 15 | # raggi's Git repo contains a branch called 'async_for_rack' which contains the 16 | # version of Thin you want to install. raggi has apparently removed the 'master' 17 | # branch of his Git repo so you may see a warning like that shown below. 18 | # 19 | # git clone git://github.com/raggi/thin.git thin-raggi-async 20 | # ... 21 | # warning: remote HEAD refers to nonexistent ref, unable to checkout. <<== IGNORE THIS 22 | # 23 | # cd thin-raggi-async/ 24 | # git checkout --track -b async_for_rack origin/async_for_rack 25 | # warning: You appear to be on a branch yet to be born. <<== IGNORE THIS 26 | # warning: Forcing checkout of origin/async_for_rack. <<== IGNORE THIS 27 | # Branch async_for_rack set up to track remote branch refs/remotes/origin/async_for_rack. 28 | # Switched to a new branch "async_for_rack" 29 | 30 | # run : 'rake install' to build and install the Thin gem 31 | # cd 32 | # ./bin/nanite-admin 33 | 34 | # When you need to update this Thin install you should be able to do a 'git pull' on the 35 | # "async_for_rack" branch. 36 | 37 | require File.dirname(__FILE__) + '/../lib/nanite' 38 | require 'yaml' 39 | require "optparse" 40 | 41 | include Nanite::CommonConfig 42 | 43 | options = {} 44 | 45 | opts = OptionParser.new do |opts| 46 | opts.banner = "Usage: nanite-admin [-flags] [argument]" 47 | opts.define_head "Nanite Admin: a basic control interface for your nanite cluster." 48 | opts.separator '*'*80 49 | 50 | setup_mapper_options(opts, options) 51 | 52 | opts.on("--thin-debug", "Set the equivalent of the '--debug' flag on the Thin webserver.") do 53 | options[:thin_debug] = true 54 | end 55 | end 56 | 57 | opts.parse! 58 | 59 | EM.run do 60 | Nanite.start_mapper(options) 61 | Nanite::Log.info "starting nanite-admin" 62 | Rack::Handler::Thin.run(Nanite::Admin.new(Nanite.mapper), :Port => 4000) do 63 | Thin::Logging.debug = options[:thin_debug] 64 | end 65 | end -------------------------------------------------------------------------------- /bin/nanite-agent: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.dirname(__FILE__) + '/../lib/nanite' 4 | require 'optparse' 5 | 6 | include Nanite::CommonConfig 7 | 8 | options = {} 9 | 10 | opts = OptionParser.new do |opts| 11 | opts.banner = "Usage: nanite-agent [-flag] [argument]" 12 | opts.define_head "Nanite Agent: ruby process that acts upon messages passed to it by a mapper." 13 | opts.separator '*'*80 14 | 15 | setup_common_options(opts, options, 'agent') 16 | 17 | opts.on("-n", "--nanite NANITE_ROOT", "Specify the root of your nanite agent project.") do |nanite| 18 | options[:root] = nanite 19 | end 20 | 21 | opts.on("--ping-time PINGTIME", "Specify how often the agents contacts the mapper") do |ping| 22 | options[:ping_time] = ping 23 | end 24 | 25 | opts.on("--actors-dir DIR", "Path to directory containing actors (NANITE_ROOT/actors by default)") do |dir| 26 | options[:actors_dir] = dir 27 | end 28 | 29 | opts.on("--actors ACTORS", "Comma separated list of actors to load (all ruby files in actors directory by default)") do |a| 30 | options[:actors] = a.split(',') 31 | end 32 | 33 | opts.on("--initrb FILE", "Path to agent initialization file (NANITE_ROOT/init.rb by default)") do |initrb| 34 | options[:initrb] = initrb 35 | end 36 | 37 | opts.on("--single-threaded", "Run all operations in one thread") do 38 | options[:single_threaded] = true 39 | end 40 | end 41 | 42 | opts.parse! 43 | 44 | if ARGV[0] == 'stop' || ARGV[0] == 'status' 45 | agent = Nanite::Agent.new(options) 46 | pid_file = Nanite::PidFile.new(agent.identity, agent.options) 47 | unless pid = pid_file.read_pid 48 | puts "#{pid_file} not found" 49 | exit 50 | end 51 | if ARGV[0] == 'stop' 52 | puts "Stopping nanite agent #{agent.identity} (pid #{pid})" 53 | begin 54 | Process.kill('TERM', pid) 55 | rescue Errno::ESRCH 56 | puts "Process does not exist (pid #{pid})" 57 | exit 58 | end 59 | puts 'Done.' 60 | else 61 | if Process.getpgid(pid) != -1 62 | psdata = `ps up #{pid}`.split("\n").last.split 63 | memory = (psdata[5].to_i / 1024) 64 | puts "The agent is alive, using #{memory}MB of memory" 65 | else 66 | puts "The agent is not running but has a stale pid file at #{pid_file}" 67 | end 68 | end 69 | exit 70 | end 71 | 72 | EM.run do 73 | Nanite.start_agent(options) 74 | end 75 | -------------------------------------------------------------------------------- /examples/secure/cli.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require File.dirname(__FILE__) + '/../../lib/nanite' 4 | 5 | # cli.rb 6 | # 7 | # You will need to have run the examples/rabbitconf.rb script at least one time before you 8 | # run this so the expected users and vhosts are in place in RabbitMQ. 9 | # 10 | # You should have started the 'secure' nanite in another shell by running: 11 | # cd /examples/secure-agent 12 | # nanite-agent --token encrypter 13 | # 14 | # This test script takes a little more than 16 seconds to run since we start a new 15 | # mapper within, and pause while we wait for it to initialize, receive pings from 16 | # available agents (which have a default ping time of 15 seconds), and register those 17 | # agents and their methods. When this process is presumed complete after 16 seconds 18 | # we can finally send the nanite agent the task to execute. 19 | 20 | # Configure secure serializer 21 | certs_dir = File.join(File.dirname(__FILE__), 'certs') 22 | agent_cert = Nanite::Certificate.load(File.join(certs_dir, 'agent_cert.pem')) 23 | store = Nanite::StaticCertificateStore.new(agent_cert, agent_cert) 24 | mapper_cert = Nanite::Certificate.load(File.join(certs_dir, 'mapper_cert.pem')) 25 | mapper_key = Nanite::RsaKeyPair.load(File.join(certs_dir, 'mapper_key.pem')) 26 | Nanite::SecureSerializer.init("mapper", mapper_cert, mapper_key, store) 27 | 28 | 29 | # Monkey patch secure serializer for the purpose of the example 30 | module Nanite 31 | class SecureSerializer 32 | class << self 33 | alias :orig_dump :dump 34 | def dump(obj) 35 | puts "Serializing #{obj}..." 36 | res = orig_dump(obj) 37 | puts res 38 | puts "-----------------------\n\n" 39 | res 40 | end 41 | alias :orig_load :load 42 | def load(json) 43 | puts "Loading #{json}..." 44 | res = orig_load(json) 45 | puts res 46 | puts "-----------------------\n\n" 47 | res 48 | end 49 | end 50 | end 51 | end 52 | 53 | EM.run do 54 | # start up a new mapper 55 | Nanite.start_mapper(:format => :secure, :host => 'localhost', :user => 'mapper', :pass => 'testing', :vhost => '/nanite', :log_level => 'debug') 56 | 57 | EM.add_timer(16) do 58 | puts "Sending request..." 59 | Nanite.request("/secure/echo", "hello secure world!") do |res| 60 | p res 61 | EM.stop_event_loop 62 | puts "Done." 63 | end 64 | end 65 | end 66 | 67 | -------------------------------------------------------------------------------- /lib/nanite/log.rb: -------------------------------------------------------------------------------- 1 | require 'nanite/config' 2 | require 'nanite/log/formatter' 3 | require 'logger' 4 | 5 | module Nanite 6 | class Log 7 | 8 | @logger = nil 9 | 10 | class << self 11 | attr_accessor :logger, :level, :file #:nodoc 12 | 13 | # Use Nanite::Logger.init when you want to set up the logger manually. 14 | # If this method is called with no arguments, it will log to STDOUT at the :info level. 15 | # It also configures the Logger instance it creates to use the custom Nanite::Log::Formatter class. 16 | def init(identity, path = false) 17 | if path 18 | @file = File.join(path, "nanite.#{identity}.log") 19 | else 20 | @file = STDOUT 21 | end 22 | @logger = Logger.new(file) 23 | @logger.formatter = Nanite::Log::Formatter.new 24 | level = @log_level = :info 25 | end 26 | 27 | # Sets the level for the Logger by symbol or by command line argument. 28 | # Throws an ArgumentError if you feed it a bogus log level (that is not 29 | # one of :debug, :info, :warn, :error, :fatal or the corresponding strings) 30 | def level=(loglevel) 31 | init() unless @logger 32 | loglevel = loglevel.intern if loglevel.is_a?(String) 33 | @logger.info("Setting log level to #{loglevel.to_s.upcase}") 34 | case loglevel 35 | when :debug 36 | @logger.level = Logger::DEBUG 37 | when :info 38 | @logger.level = Logger::INFO 39 | when :warn 40 | @logger.level = Logger::WARN 41 | when :error 42 | @logger.level = Logger::ERROR 43 | when :fatal 44 | @logger.level = Logger::FATAL 45 | else 46 | raise ArgumentError, "Log level must be one of :debug, :info, :warn, :error, or :fatal" 47 | end 48 | end 49 | 50 | # Passes any other method calls on directly to the underlying Logger object created with init. If 51 | # this method gets hit before a call to Nanite::Logger.init has been made, it will call 52 | # Nanite::Logger.init() with no arguments. 53 | def method_missing(method_symbol, *args) 54 | init(identity) unless @logger 55 | if args.length > 0 56 | @logger.send(method_symbol, *args) 57 | else 58 | @logger.send(method_symbol) 59 | end 60 | end 61 | 62 | end # class << self 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /lib/nanite/security/secure_serializer.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | # Serializer implementation which secures messages by using 4 | # X.509 certificate sigining. 5 | class SecureSerializer 6 | 7 | # Initialize serializer, must be called prior to using it. 8 | # 9 | # - 'identity': Identity associated with serialized messages 10 | # - 'cert': Certificate used to sign and decrypt serialized messages 11 | # - 'key': Private key corresponding to 'cert' 12 | # - 'store': Certificate store. Exposes certificates used for 13 | # encryption and signature validation. 14 | # - 'encrypt': Whether data should be signed and encrypted ('true') 15 | # or just signed ('false'), 'true' by default. 16 | # 17 | def self.init(identity, cert, key, store, encrypt = true) 18 | @identity = identity 19 | @cert = cert 20 | @key = key 21 | @store = store 22 | @encrypt = encrypt 23 | end 24 | 25 | # Was serializer initialized? 26 | def self.initialized? 27 | @identity && @cert && @key && @store 28 | end 29 | 30 | # Serialize message and sign it using X.509 certificate 31 | def self.dump(obj) 32 | raise "Missing certificate identity" unless @identity 33 | raise "Missing certificate" unless @cert 34 | raise "Missing certificate key" unless @key 35 | raise "Missing certificate store" unless @store || !@encrypt 36 | json = obj.to_json 37 | if @encrypt 38 | certs = @store.get_recipients(obj) 39 | json = EncryptedDocument.new(json, certs).encrypted_data if certs 40 | end 41 | sig = Signature.new(json, @cert, @key) 42 | { 'id' => @identity, 'data' => json, 'signature' => sig.data, 'encrypted' => !certs.nil? }.to_json 43 | end 44 | 45 | # Unserialize data using certificate store 46 | def self.load(json) 47 | begin 48 | raise "Missing certificate store" unless @store 49 | raise "Missing certificate" unless @cert || !@encrypt 50 | raise "Missing certificate key" unless @key || !@encrypt 51 | data = JSON.load(json) 52 | sig = Signature.from_data(data['signature']) 53 | certs = @store.get_signer(data['id']) 54 | certs = [ certs ] unless certs.respond_to?(:each) 55 | jsn = data['data'] if certs.any? { |c| sig.match?(c) } 56 | if jsn && @encrypt && data['encrypted'] 57 | jsn = EncryptedDocument.from_data(jsn).decrypted_data(@key, @cert) 58 | end 59 | JSON.load(jsn) if jsn 60 | rescue Exception => e 61 | Nanite::Log.error("Loading of secure packet failed: #{e.message}\n#{e.backtrace.join("\n")}") 62 | raise 63 | end 64 | end 65 | 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/nanite/dispatcher.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class Dispatcher 3 | attr_reader :registry, :serializer, :identity, :amq, :options 4 | attr_accessor :evmclass 5 | 6 | def initialize(amq, registry, serializer, identity, options) 7 | @amq = amq 8 | @registry = registry 9 | @serializer = serializer 10 | @identity = identity 11 | @options = options 12 | @evmclass = EM 13 | end 14 | 15 | def dispatch(deliverable) 16 | prefix, meth = deliverable.type.split('/')[1..-1] 17 | meth ||= :index 18 | actor = registry.actor_for(prefix) 19 | 20 | operation = lambda do 21 | begin 22 | intermediate_results_proc = lambda { |*args| self.handle_intermediate_results(actor, meth, deliverable, *args) } 23 | args = [ deliverable.payload ] 24 | args.push(deliverable) if actor.method(meth).arity == 2 25 | actor.send(meth, *args, &intermediate_results_proc) 26 | rescue Exception => e 27 | handle_exception(actor, meth, deliverable, e) 28 | end 29 | end 30 | 31 | callback = lambda do |r| 32 | if deliverable.kind_of?(Request) 33 | r = Result.new(deliverable.token, deliverable.reply_to, r, identity) 34 | amq.queue(deliverable.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r)) 35 | end 36 | r # For unit tests 37 | end 38 | 39 | if @options[:single_threaded] 40 | @evmclass.next_tick { callback.call(operation.call) } 41 | else 42 | @evmclass.defer(operation, callback) 43 | end 44 | end 45 | 46 | protected 47 | 48 | def handle_intermediate_results(actor, meth, deliverable, *args) 49 | messagekey = case args.size 50 | when 1 51 | 'defaultkey' 52 | when 2 53 | args.first.to_s 54 | else 55 | raise ArgumentError, "handle_intermediate_results passed unexpected number of arguments (#{args.size})" 56 | end 57 | message = args.last 58 | @evmclass.defer(lambda { 59 | [deliverable.reply_to, IntermediateMessage.new(deliverable.token, deliverable.reply_to, identity, messagekey, message)] 60 | }, lambda { |r| 61 | amq.queue(r.first, :no_declare => options[:secure]).publish(serializer.dump(r.last)) 62 | }) 63 | end 64 | 65 | private 66 | 67 | def describe_error(e) 68 | "#{e.class.name}: #{e.message}\n #{e.backtrace.join("\n ")}" 69 | end 70 | 71 | def handle_exception(actor, meth, deliverable, e) 72 | error = describe_error(e) 73 | Nanite::Log.error(error) 74 | begin 75 | if actor.class.exception_callback 76 | case actor.class.exception_callback 77 | when Symbol, String 78 | actor.send(actor.class.exception_callback, meth.to_sym, deliverable, e) 79 | when Proc 80 | actor.instance_exec(meth.to_sym, deliverable, e, &actor.class.exception_callback) 81 | end 82 | end 83 | rescue Exception => e1 84 | error = describe_error(e1) 85 | Nanite::Log.error(error) 86 | end 87 | error 88 | end 89 | end 90 | end -------------------------------------------------------------------------------- /lib/nanite/job.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class JobWarden 3 | attr_reader :serializer, :jobs 4 | 5 | def initialize(serializer) 6 | @serializer = serializer 7 | @jobs = {} 8 | end 9 | 10 | def new_job(request, targets, inthandler = nil, blk = nil) 11 | job = Job.new(request, targets, inthandler, blk) 12 | jobs[job.token] = job 13 | job 14 | end 15 | 16 | def process(msg) 17 | Nanite::Log.debug("processing message: #{msg.inspect}") 18 | 19 | if job = jobs[msg.token] 20 | job.process(msg) 21 | 22 | if job.intermediate_handler && (job.pending_keys.size > 0) 23 | 24 | unless job.pending_keys.size == 1 25 | raise "IntermediateMessages are currently dispatched as they arrive, shouldn't have more than one key in pending_keys: #{job.pending_keys.inspect}" 26 | end 27 | 28 | key = job.pending_keys.first 29 | handler = job.intermediate_handler_for_key(key) 30 | if handler 31 | case handler.arity 32 | when 3 33 | handler.call(key, msg.from, job.intermediate_state[msg.from][key].last) 34 | when 4 35 | handler.call(key, msg.from, job.intermediate_state[msg.from][key].last, job) 36 | end 37 | end 38 | 39 | job.reset_pending_intermediate_state_keys 40 | end 41 | 42 | if job.completed? 43 | jobs.delete(job.token) 44 | if job.completed 45 | case job.completed.arity 46 | when 1 47 | job.completed.call(job.results) 48 | when 2 49 | job.completed.call(job.results, job) 50 | end 51 | end 52 | end 53 | end 54 | end 55 | end # JobWarden 56 | 57 | class Job 58 | attr_reader :results, :request, :token, :completed, :intermediate_state, :pending_keys, :intermediate_handler 59 | attr_accessor :targets # This can be updated when a request gets picked up from the offline queue 60 | 61 | def initialize(request, targets, inthandler = nil, blk = nil) 62 | @request = request 63 | @targets = targets 64 | @token = @request.token 65 | @results = {} 66 | @intermediate_handler = inthandler 67 | @pending_keys = [] 68 | @completed = blk 69 | @intermediate_state = {} 70 | end 71 | 72 | def process(msg) 73 | case msg 74 | when Result 75 | results[msg.from] = msg.results 76 | targets.delete(msg.from) 77 | when IntermediateMessage 78 | intermediate_state[msg.from] ||= {} 79 | intermediate_state[msg.from][msg.messagekey] ||= [] 80 | intermediate_state[msg.from][msg.messagekey] << msg.message 81 | @pending_keys << msg.messagekey 82 | end 83 | end 84 | 85 | def intermediate_handler_for_key(key) 86 | return nil unless @intermediate_handler 87 | case @intermediate_handler 88 | when Proc 89 | @intermediate_handler 90 | when Hash 91 | @intermediate_handler[key] || @intermediate_handler['*'] 92 | end 93 | end 94 | 95 | def reset_pending_intermediate_state_keys 96 | @pending_keys = [] 97 | end 98 | 99 | def completed? 100 | targets.empty? 101 | end 102 | end # Job 103 | 104 | end # Nanite 105 | -------------------------------------------------------------------------------- /examples/rabbitmqctl/actors/rabbit.rb: -------------------------------------------------------------------------------- 1 | class Rabbit 2 | include Nanite::Actor 3 | 4 | expose :add_user,:delete_user, :change_password, :list_users, 5 | :add_vhost, :delete_vhost, :list_vhosts, 6 | :map_user_vhost, :unmap_user_vhost, :list_user_vhosts, :list_vhost_users, 7 | :list_queues, :list_exchanges, :list_exchanges, :list_connections 8 | 9 | def list_queues(vhost) 10 | parse_list(rabbitmqctl(:list_queues, vhost.empty? ? "" : "-p #{vhost}")).first 11 | end 12 | 13 | def list_exchanges(vhost) 14 | parse_list(rabbitmqctl(:list_exchanges, vhost.empty? ? "" : "-p #{vhost}")).first 15 | end 16 | 17 | def list_bindings(vhost) 18 | parse_list(rabbitmqctl(:list_bindings, vhost.empty? ? "" : "-p #{vhost}")).first 19 | end 20 | 21 | def list_connections(payload) 22 | parse_list(rabbitmqctl(:list_connections)).first 23 | end 24 | 25 | def list_users(payload) 26 | parse_list(rabbitmqctl(:list_users)).first 27 | end 28 | 29 | def list_vhosts(payload) 30 | parse_list(rabbitmqctl(:list_vhosts)).first 31 | end 32 | 33 | def list_vhost_users(vhost) 34 | parse_list(rabbitmqctl(:list_vhost_users, vhost)).first 35 | end 36 | 37 | def list_user_vhosts(user) 38 | parse_list(rabbitmqctl(:list_user_vhosts, user)).first 39 | end 40 | 41 | def map_user_vhost(payload) 42 | if String === payload 43 | payload = JSON.parse(payload) 44 | end 45 | res = parse_list(rabbitmqctl(:map_user_vhost, payload['user'], payload['vhost'])) 46 | if res[1] 47 | "problem mapping user to vhost: #{payload['user']}:#{payload['vhost']} #{res[1]}" 48 | else 49 | "successfully mapped user to vhost: #{payload['user']}:#{payload['vhost']}" 50 | end 51 | end 52 | 53 | def unmap_user_vhost(payload) 54 | if String === payload 55 | payload = JSON.parse(payload) 56 | end 57 | res = parse_list(rabbitmqctl(:unmap_user_vhost, payload['user'], payload['vhost'])) 58 | if res[1] 59 | "problem unmapping user from vhost: #{payload['user']}:#{payload['vhost']} #{res[1]}" 60 | else 61 | "successfully unmapped user from vhost: #{payload['user']}:#{payload['vhost']}" 62 | end 63 | end 64 | 65 | def add_vhost(path) 66 | res = parse_list(rabbitmqctl(:add_vhost, path)) 67 | if res[1] 68 | "problem adding vhost: #{path} #{res[1]}" 69 | else 70 | "successfully added vhost: #{path}" 71 | end 72 | end 73 | 74 | def add_user(payload) 75 | if String === payload 76 | payload = JSON.parse(payload) 77 | end 78 | res = parse_list(rabbitmqctl(:add_user, payload['user'], payload['pass'])) 79 | if res[1] 80 | "problem adding user: #{payload['user']} #{res[1]}" 81 | else 82 | "successfully added user: #{payload['user']}" 83 | end 84 | end 85 | 86 | def change_password(payload) 87 | if String === payload 88 | payload = JSON.parse(payload) 89 | end 90 | res = parse_list(rabbitmqctl(:change_password, payload['user'], payload['pass'])) 91 | if res[1] 92 | "problem with change_password user: #{payload['user']} #{res[1]}" 93 | else 94 | "successfully changed password user: #{payload['user']}" 95 | end 96 | end 97 | 98 | def delete_user(payload) 99 | if String === payload 100 | payload = JSON.parse(payload) 101 | end 102 | res = parse_list(rabbitmqctl(:delete_user, payload['user'])) 103 | if res[1] 104 | "problem deleting user: #{payload['user']} #{res[1]}" 105 | else 106 | "successfully deleted user: #{payload['user']}" 107 | end 108 | end 109 | 110 | def delete_vhost(path) 111 | res = parse_list(rabbitmqctl(:delete_vhost, path)) 112 | if res[1] 113 | "problem deleting vhost: #{path} #{res[1]}" 114 | else 115 | "successfully deleted vhost: #{path}" 116 | end 117 | end 118 | 119 | def rabbitmqctl(*args) 120 | `rabbitmqctl #{args.join(' ')}` 121 | end 122 | 123 | def parse_list(out) 124 | res = [] 125 | error = nil 126 | out.each do |line| 127 | res << line.chomp unless line =~ /\.\.\./ 128 | error = $1 if line =~ /Error: (.*)/ 129 | end 130 | [res, error] 131 | end 132 | end -------------------------------------------------------------------------------- /spec/serializer_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::Serializer do 4 | 5 | describe "Format" do 6 | 7 | it "supports JSON format" do 8 | [ :json, "json" ].each do |format| 9 | serializer = Nanite::Serializer.new(format) 10 | serializer.instance_eval { @serializers.first }.should == JSON 11 | end 12 | end 13 | 14 | it "supports Marshal format" do 15 | [ :marshal, "marshal" ].each do |format| 16 | serializer = Nanite::Serializer.new(format) 17 | serializer.instance_eval { @serializers.first }.should == Marshal 18 | end 19 | end 20 | 21 | it "supports YAML format" do 22 | [ :yaml, "yaml" ].each do |format| 23 | serializer = Nanite::Serializer.new(format) 24 | serializer.instance_eval { @serializers.first }.should == YAML 25 | end 26 | end 27 | 28 | it "should default to Marshal format if not specified" do 29 | serializer = Nanite::Serializer.new 30 | serializer.instance_eval { @serializers.first }.should == Marshal 31 | serializer = Nanite::Serializer.new(nil) 32 | serializer.instance_eval { @serializers.first }.should == Marshal 33 | end 34 | 35 | end # Format 36 | 37 | describe "Serialization of Packet" do 38 | 39 | it "should cascade through available serializers" do 40 | serializer = Nanite::Serializer.new 41 | serializer.should_receive(:cascade_serializers).with(:dump, "hello") 42 | serializer.dump("hello") 43 | end 44 | 45 | it "should try all three supported formats (JSON, Marshal, YAML)" do 46 | JSON.should_receive(:dump).with("hello").and_raise(StandardError) 47 | Marshal.should_receive(:dump).with("hello").and_raise(StandardError) 48 | YAML.should_receive(:dump).with("hello").and_raise(StandardError) 49 | 50 | lambda { Nanite::Serializer.new.dump("hello") }.should raise_error(Nanite::Serializer::SerializationError) 51 | end 52 | 53 | it "should raise SerializationError if packet could not be serialized" do 54 | JSON.should_receive(:dump).with("hello").and_raise(StandardError) 55 | Marshal.should_receive(:dump).with("hello").and_raise(StandardError) 56 | YAML.should_receive(:dump).with("hello").and_raise(StandardError) 57 | 58 | serializer = Nanite::Serializer.new 59 | lambda { serializer.dump("hello") }.should raise_error(Nanite::Serializer::SerializationError) 60 | end 61 | 62 | it "should return serialized packet" do 63 | serialized_packet = mock("Packet") 64 | Marshal.should_receive(:dump).with("hello").and_return(serialized_packet) 65 | 66 | serializer = Nanite::Serializer.new(:marshal) 67 | serializer.dump("hello").should == serialized_packet 68 | end 69 | 70 | end # Serialization of Packet 71 | 72 | describe "De-Serialization of Packet" do 73 | 74 | it "should cascade through available serializers" do 75 | serializer = Nanite::Serializer.new 76 | serializer.should_receive(:cascade_serializers).with(:load, "olleh") 77 | serializer.load("olleh") 78 | end 79 | 80 | it "should try all three supported formats (JSON, Marshal, YAML)" do 81 | JSON.should_receive(:load).with("olleh").and_raise(StandardError) 82 | Marshal.should_receive(:load).with("olleh").and_raise(StandardError) 83 | YAML.should_receive(:load).with("olleh").and_raise(StandardError) 84 | 85 | lambda { Nanite::Serializer.new.load("olleh") }.should raise_error(Nanite::Serializer::SerializationError) 86 | end 87 | 88 | it "should raise SerializationError if packet could not be de-serialized" do 89 | JSON.should_receive(:load).with("olleh").and_raise(StandardError) 90 | Marshal.should_receive(:load).with("olleh").and_raise(StandardError) 91 | YAML.should_receive(:load).with("olleh").and_raise(StandardError) 92 | 93 | serializer = Nanite::Serializer.new 94 | lambda { serializer.load("olleh") }.should raise_error(Nanite::Serializer::SerializationError) 95 | end 96 | 97 | it "should return de-serialized packet" do 98 | deserialized_packet = mock("Packet") 99 | Marshal.should_receive(:load).with("olleh").and_return(deserialized_packet) 100 | 101 | serializer = Nanite::Serializer.new(:marshal) 102 | serializer.load("olleh").should == deserialized_packet 103 | end 104 | 105 | end # De-Serialization of Packet 106 | 107 | end # Nanite::Serializer 108 | -------------------------------------------------------------------------------- /spec/dispatcher_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | class Foo 4 | include Nanite::Actor 5 | expose :bar, :index, :i_kill_you 6 | on_exception :handle_exception 7 | 8 | def index(payload) 9 | bar(payload) 10 | end 11 | 12 | def bar(payload) 13 | ['hello', payload] 14 | end 15 | 16 | def bar2(payload, deliverable) 17 | deliverable 18 | end 19 | 20 | def i_kill_you(payload) 21 | raise RuntimeError.new('I kill you!') 22 | end 23 | 24 | def handle_exception(method, deliverable, error) 25 | end 26 | end 27 | 28 | class Bar 29 | include Nanite::Actor 30 | expose :i_kill_you 31 | on_exception do |method, deliverable, error| 32 | @scope = self 33 | @called_with = [method, deliverable, error] 34 | end 35 | 36 | def i_kill_you(payload) 37 | raise RuntimeError.new('I kill you!') 38 | end 39 | end 40 | 41 | # No specs, simply ensures multiple methods for assigning on_exception callback, 42 | # on_exception raises exception when called with an invalid argument. 43 | class Doomed 44 | include Nanite::Actor 45 | on_exception do 46 | end 47 | on_exception lambda {} 48 | on_exception :doh 49 | end 50 | 51 | # Mock the EventMachine deferrer. 52 | class EMMock 53 | def self.defer(op = nil, callback = nil) 54 | callback.call(op.call) 55 | end 56 | end 57 | 58 | describe "Nanite::Dispatcher" do 59 | 60 | before(:each) do 61 | Nanite::Log.stub!(:info) 62 | Nanite::Log.stub!(:error) 63 | amq = mock('amq', :queue => mock('queue', :publish => nil)) 64 | @actor = Foo.new 65 | @registry = Nanite::ActorRegistry.new 66 | @registry.register(@actor, nil) 67 | @dispatcher = Nanite::Dispatcher.new(amq, @registry, Nanite::Serializer.new(:marshal), '0xfunkymonkey', {}) 68 | @dispatcher.evmclass = EMMock 69 | end 70 | 71 | it "should dispatch a request" do 72 | req = Nanite::Request.new('/foo/bar', 'you') 73 | res = @dispatcher.dispatch(req) 74 | res.should(be_kind_of(Nanite::Result)) 75 | res.token.should == req.token 76 | res.results.should == ['hello', 'you'] 77 | end 78 | 79 | it "should dispatch the deliverable to actions that accept it" do 80 | req = Nanite::Request.new('/foo/bar2', 'you') 81 | res = @dispatcher.dispatch(req) 82 | res.should(be_kind_of(Nanite::Result)) 83 | res.token.should == req.token 84 | res.results.should == req 85 | end 86 | 87 | it "should dispatch a request to the default action" do 88 | req = Nanite::Request.new('/foo', 'you') 89 | res = @dispatcher.dispatch(req) 90 | res.should(be_kind_of(Nanite::Result)) 91 | res.token.should == req.token 92 | res.results.should == ['hello', 'you'] 93 | end 94 | 95 | it "should handle custom prefixes" do 96 | @registry.register(Foo.new, 'umbongo') 97 | req = Nanite::Request.new('/umbongo/bar', 'you') 98 | res = @dispatcher.dispatch(req) 99 | res.should(be_kind_of(Nanite::Result)) 100 | res.token.should == req.token 101 | res.results.should == ['hello', 'you'] 102 | end 103 | 104 | it "should call the on_exception callback if something goes wrong" do 105 | req = Nanite::Request.new('/foo/i_kill_you', nil) 106 | @actor.should_receive(:handle_exception).with(:i_kill_you, req, duck_type(:exception, :backtrace)) 107 | @dispatcher.dispatch(req) 108 | end 109 | 110 | it "should call on_exception Procs defined in a subclass with the correct arguments" do 111 | actor = Bar.new 112 | @registry.register(actor, nil) 113 | req = Nanite::Request.new('/bar/i_kill_you', nil) 114 | @dispatcher.dispatch(req) 115 | called_with = actor.instance_variable_get("@called_with") 116 | called_with[0].should == :i_kill_you 117 | called_with[1].should == req 118 | called_with[2].should be_kind_of(RuntimeError) 119 | called_with[2].message.should == 'I kill you!' 120 | end 121 | 122 | it "should call on_exception Procs defined in a subclass in the scope of the actor" do 123 | actor = Bar.new 124 | @registry.register(actor, nil) 125 | req = Nanite::Request.new('/bar/i_kill_you', nil) 126 | @dispatcher.dispatch(req) 127 | actor.instance_variable_get("@scope").should == actor 128 | end 129 | 130 | it "should log error if something goes wrong" do 131 | Nanite::Log.should_receive(:error) 132 | req = Nanite::Request.new('/foo/i_kill_you', nil) 133 | @dispatcher.dispatch(req) 134 | end 135 | 136 | end # Nanite::Dispatcher 137 | -------------------------------------------------------------------------------- /lib/nanite/config.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | 3 | COMMON_DEFAULT_OPTIONS = {:pass => 'testing', :vhost => '/nanite', :secure => false, :host => '0.0.0.0', 4 | :log_level => :info, :format => :marshal, :daemonize => false, :console => false, :root => Dir.pwd} 5 | 6 | module CommonConfig 7 | def setup_mapper_options(opts, options) 8 | setup_common_options(opts, options, 'mapper') 9 | 10 | opts.on("-a", "--agent-timeout TIMEOUT", "How long to wait before an agent is considered to be offline and thus removed from the list of available agents.") do |timeout| 11 | options[:agent_timeout] = timeout 12 | end 13 | 14 | opts.on("-r", "--offline-redelivery-frequency", "The frequency in seconds that messages stored in the offline queue will be retrieved for attempted redelivery to the nanites. Default is 10 seconds.") do |frequency| 15 | options[:offline_redelivery_frequency] = frequency 16 | end 17 | 18 | opts.on("--persistent", "Instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the broker is restarted. Can be overriden on a per-message basis using the request and push methods.") do 19 | options[:persistent] = true 20 | end 21 | 22 | opts.on("--offline-failsafe", "Store messages in an offline queue when all the nanites are offline. Messages will be redelivered when nanites come online. Can be overriden on a per-message basis using the request methods.") do 23 | options[:offline_failsafe] = true 24 | end 25 | 26 | opts.on("--redis HOST_PORT", "Use redis as the agent state storage in the mapper: --redis 127.0.0.1:6379; missing host and/or port will be filled with defaults if colon is present") do |redis| 27 | redishost, redisport = redis.split(':') 28 | redishost = '127.0.0.1' if (redishost.nil? || redishost.empty?) 29 | redisport = '6379' if (redishost.nil? || redishost.empty?) 30 | redis = "#{redishost}:#{redisport}" 31 | options[:redis] = redis 32 | end 33 | 34 | end 35 | 36 | def setup_common_options(opts, options, type) 37 | opts.version = Nanite::VERSION 38 | 39 | opts.on("-i", "--irb-console", "Start #{type} in irb console mode.") do |console| 40 | options[:console] = 'irb' 41 | end 42 | 43 | opts.on("-u", "--user USER", "Specify the rabbitmq username.") do |user| 44 | options[:user] = user 45 | end 46 | 47 | opts.on("-h", "--host HOST", "Specify the rabbitmq hostname.") do |host| 48 | options[:host] = host 49 | end 50 | 51 | opts.on("-P", "--port PORT", "Specify the rabbitmq PORT, default 5672.") do |port| 52 | options[:port] = port 53 | end 54 | 55 | opts.on("-p", "--pass PASSWORD", "Specify the rabbitmq password") do |pass| 56 | options[:pass] = pass 57 | end 58 | 59 | opts.on("-t", "--token IDENITY", "Specify the #{type} identity.") do |ident| 60 | options[:identity] = ident 61 | end 62 | 63 | opts.on("-v", "--vhost VHOST", "Specify the rabbitmq vhost") do |vhost| 64 | options[:vhost] = vhost 65 | end 66 | 67 | opts.on("-s", "--secure", "Use Security features of rabbitmq to restrict nanites to themselves") do 68 | options[:secure] = true 69 | end 70 | 71 | opts.on("-f", "--format FORMAT", "The serialization type to use for transfering data. Can be marshal, json or yaml. Default is marshal") do |fmt| 72 | options[:format] = fmt.to_sym 73 | end 74 | 75 | opts.on("-d", "--daemonize", "Run #{type} as a daemon") do |d| 76 | options[:daemonize] = true 77 | end 78 | 79 | opts.on("--pid-dir PATH", "Specify the pid path, only used with daemonize") do |dir| 80 | options[:pid_dir] = dir 81 | end 82 | 83 | opts.on("-l", "--log-level LEVEL", "Specify the log level (fatal, error, warn, info, debug). Default is info") do |level| 84 | options[:log_level] = level 85 | end 86 | 87 | opts.on("--log-dir PATH", "Specify the log path") do |dir| 88 | options[:log_dir] = dir 89 | end 90 | 91 | opts.on("--tag TAG", "Specify a tag. Can issue multiple times.") do |tag| 92 | options[:tag] ||= [] 93 | options[:tag] << tag 94 | end 95 | 96 | opts.on("--version", "Show the nanite version number") do |res| 97 | puts "Nanite Version #{opts.version}" 98 | exit 99 | end 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/nanite/streaming.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | # Nanite actors can transfer files to each other. 3 | # 4 | # ==== Options 5 | # 6 | # filename : you guessed it, name of the file! 7 | # domain : part of the routing key used to locate receiver(s) 8 | # destination : is a name of the file as it gonna be stored at the destination 9 | # meta : 10 | # 11 | # File streaming is done in chunks. When file streaming starts, 12 | # Nanite::FileStart packet is sent, followed by one or more (usually more ;)) 13 | # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done, 14 | # Nanite::FileEnd packet is sent. 15 | # 16 | # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write 17 | # more than that in one operation via socket. 18 | # 19 | # ==== Domains 20 | # 21 | # Streaming happens using a topic exchange called 'file broadcast', with keys 22 | # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and 23 | # receivers find each other in the cluster. Default domain is 'global'. 24 | # 25 | # Domains also serve as a way to register a callback Nanite agent executes once file 26 | # streaming is completed. If a callback with name of domain is registered, it is called. 27 | # 28 | # Callbacks are registered by passing a block to subscribe_to_files method. 29 | module FileStreaming 30 | def broadcast_file(filename, options = {}) 31 | if File.exist?(filename) 32 | File.open(filename, 'rb') do |file| 33 | broadcast_data(filename, file, options) 34 | end 35 | else 36 | return "file not found" 37 | end 38 | end 39 | 40 | def broadcast_data(filename, io, options = {}) 41 | domain = options[:domain] || 'global' 42 | filename = File.basename(filename) 43 | dest = options[:destination] || filename 44 | sent = 0 45 | 46 | begin 47 | file_push = Nanite::FileStart.new(filename, dest, Identity.generate) 48 | amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}") 49 | res = Nanite::FileChunk.new(file_push.token) 50 | while chunk = io.read(16384) 51 | res.chunk = chunk 52 | amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}") 53 | sent += chunk.length 54 | end 55 | fend = Nanite::FileEnd.new(file_push.token, options[:meta]) 56 | amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}") 57 | "" 58 | ensure 59 | io.close 60 | end 61 | 62 | sent 63 | end 64 | 65 | # FileState represents a file download in progress. 66 | # It incapsulates the following information: 67 | # 68 | # * unique operation token 69 | # * domain (namespace for file streaming operations) 70 | # * file IO chunks are written to on receiver's side 71 | class FileState 72 | 73 | def initialize(token, dest, domain, write, blk) 74 | @token = token 75 | @cb = blk 76 | @domain = domain 77 | @write = write 78 | 79 | if write 80 | @filename = File.join(Nanite.agent.file_root, dest) 81 | @dest = File.open(@filename, 'wb') 82 | else 83 | @dest = dest 84 | end 85 | 86 | @data = "" 87 | end 88 | 89 | def handle_packet(packet) 90 | case packet 91 | when Nanite::FileChunk 92 | Nanite::Log.debug "written chunk to #{@dest.inspect}" 93 | @data << packet.chunk 94 | 95 | if @write 96 | @dest.write(packet.chunk) 97 | end 98 | when Nanite::FileEnd 99 | Nanite::Log.debug "#{@dest.inspect} receiving is completed" 100 | if @write 101 | @dest.close 102 | end 103 | 104 | @cb.call(@data, @dest, packet.meta) 105 | end 106 | end 107 | 108 | end 109 | 110 | def subscribe_to_files(domain='global', write=false, &blk) 111 | Nanite::Log.info "subscribing to file broadcasts for #{domain}" 112 | @files ||= {} 113 | amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet| 114 | case msg = serializer.load(packet) 115 | when FileStart 116 | @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk) 117 | when FileChunk, FileEnd 118 | if file = @files[msg.token] 119 | file.handle_packet(msg) 120 | end 121 | end 122 | end 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /spec/local_state_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | require 'nanite/local_state' 3 | 4 | describe "Nanite::LocalState: " do 5 | 6 | describe "Class" do 7 | 8 | it "should a Hash" do 9 | Nanite::LocalState.new({}).should be_kind_of(Hash) 10 | end 11 | 12 | it "should create empty hash if no hash passed in" do 13 | Nanite::LocalState.new.should == {} 14 | end 15 | 16 | it "should initialize hash with value passed in" do 17 | state = Nanite::LocalState.new({:a => 1, :b => 2, :c => 3}) 18 | state.should == {:a => 1, :b => 2, :c => 3} 19 | end 20 | 21 | end # Class 22 | 23 | 24 | describe "All services" do 25 | 26 | it "should return empty array if no services are defined" do 27 | state = Nanite::LocalState.new({:f => { :foo => 1 }, :b => { :bar => 2 }}) 28 | state.all_services.should == [] 29 | end 30 | 31 | it "should return all :services values" do 32 | state = Nanite::LocalState.new({:f => { :foo => 1 }, :b => { :services => "b's services" }, :c => { :services => "c's services" }}) 33 | state.all_services.should include("b's services") 34 | state.all_services.should include("c's services") 35 | end 36 | 37 | it "should only return one entry for each service" do 38 | state = Nanite::LocalState.new({:f => { :services => "services" }, :b => { :services => "services" }}) 39 | state.all_services.size == 1 40 | state.all_services.should == ["services"] 41 | end 42 | 43 | end # All services 44 | 45 | 46 | describe "All tags" do 47 | 48 | it "should return empty array if no tags are defined" do 49 | state = Nanite::LocalState.new({:f => { :foo => 1 }, :b => { :bar => 2 }}) 50 | state.all_tags.should == [] 51 | end 52 | 53 | it "should return all :tags values" do 54 | state = Nanite::LocalState.new({:f => { :foo => 1 }, :b => { :tags => ["a", "b"] }, :c => { :tags => ["c", "d"] }}) 55 | state.all_tags.should include("a") 56 | state.all_tags.should include("b") 57 | state.all_tags.should include("c") 58 | state.all_tags.should include("d") 59 | end 60 | 61 | it "should only return one entry for each tag" do 62 | state = Nanite::LocalState.new({:f => { :foo => 1 }, :b => { :tags => ["a", "b"] }, :c => { :tags => ["a", "c"] }}) 63 | state.all_tags.size == 3 64 | state.all_tags.should include("a") 65 | state.all_tags.should include("b") 66 | state.all_tags.should include("c") 67 | end 68 | 69 | end # All tags 70 | 71 | 72 | describe "Nanites lookup" do 73 | 74 | it "should find services matching the service criteria if no tags criteria is specified" do 75 | state = Nanite::LocalState.new({:a => { :services => "a's services" }, :b => { :services => "b's services" }}) 76 | state.nanites_for("b's services").should == [[:b, {:services => "b's services"}]] 77 | end 78 | 79 | it "should find all services matching the service criteria if no tags criteria is specified" do 80 | state = Nanite::LocalState.new({:a => { :services => "services" }, :b => { :services => "services" }, :c => { :services => "other services" }}) 81 | state.nanites_for("services").should include([:a, {:services => "services"}]) 82 | state.nanites_for("services").should include([:b, {:services => "services"}]) 83 | end 84 | 85 | it "should only services matching the service criteria that also match the tags criteria" do 86 | state = Nanite::LocalState.new({:a => { :services => "a's services", :tags => ["a_1", "a_2"] }, :b => { :services => "b's services", :tags => ["b_1", "b_2"] }}) 87 | state.nanites_for("b's services").should == [[:b, {:tags=>["b_1", "b_2"], :services=>"b's services"}]] 88 | end 89 | 90 | it "should also return all tags for services matching the service criteria that also match a single tags criterium" do 91 | state = Nanite::LocalState.new({:a => { :services => "services", :tags => ["t_1", "t_2"] }}) 92 | state.nanites_for("services", ["t_1"]).should == [[:a, {:tags=>["t_1", "t_2"], :services=>"services"}]] 93 | end 94 | 95 | it "should return services matching the service criteria and also match the tags criterium" do 96 | state = Nanite::LocalState.new({:a => { :services => "a's services", :tags => ["a_1", "a_2"] }, :b => { :services => "b's services", :tags => ["b_1", "b_2"] }}) 97 | state.nanites_for("b's services", ["b_1"]).should == [[:b, {:tags=>["b_1", "b_2"], :services=>"b's services"}]] 98 | end 99 | 100 | it "should ignore services matching the service criteria and but not the tags criteria" do 101 | state = Nanite::LocalState.new({:a => { :services => "services", :tags => ["t_1", "t_2"] }, :b => { :services => "services", :tags => ["t_3", "t_4"] }}) 102 | state.nanites_for("services", ["t_1"]).should == [[:a, {:services => "services", :tags => ["t_1", "t_2"]}]] 103 | end 104 | 105 | it "should lookup services matching the service criteria and and any of the tags criteria" do 106 | state = Nanite::LocalState.new({'a' => { :services => "services", :tags => ["t_1", "t_2"] }, 'b' => { :services => "services", :tags => ["t_2", "t_3"] }}) 107 | state.nanites_for("services", ["t_1", "t_3"]).sort.should == [['a', {:services => "services", :tags => ["t_1", "t_2"]}], ['b', {:services => "services", :tags => ["t_2", "t_3"]}]] 108 | end 109 | 110 | end # Nanites lookup 111 | 112 | end # Nanite::LocalState 113 | -------------------------------------------------------------------------------- /lib/nanite/state.rb: -------------------------------------------------------------------------------- 1 | require 'redis' 2 | 3 | module Nanite 4 | class State 5 | include Enumerable 6 | 7 | # this class encapsulates the state of a nanite system using redis as the 8 | # data store. here is the schema, for each agent we store a number of items, 9 | # for a nanite with the identity: nanite-foobar we store the following things: 10 | # 11 | # nanite-foobar: 0.72 # load average or 'status' 12 | # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services 13 | # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent 14 | # t-nanite-foobar: 123456789 # unix timestamp of the last state update 15 | # 16 | # also we do an inverted index for quick lookup of agents providing a certain 17 | # service, so for each service the agent provides, we add the nanite to a SET 18 | # of all the nanites that provide said service: 19 | # 20 | # /gems/list: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET 21 | # 22 | # we do that same thing for tags: 23 | # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET 24 | # 25 | # This way we can do a lookup of what nanites provide a set of services and tags based 26 | # on redis SET intersection: 27 | # 28 | # nanites_for('/gems/list', 'some-tag') 29 | # => returns a nested array of nanites and their state that provide the intersection 30 | # of these two service tags 31 | 32 | def initialize(redis) 33 | Nanite::Log.info("initializing redis state: #{redis}") 34 | host, port = redis.split(':') 35 | host ||= '127.0.0.1' 36 | port ||= '6379' 37 | @redis = Redis.new :host => host, :port => port 38 | end 39 | 40 | def log_redis_error(meth,&blk) 41 | blk.call 42 | rescue RedisError => e 43 | Nanite::Log.info("redis error in method: #{meth}") 44 | raise e 45 | end 46 | 47 | def [](nanite) 48 | log_redis_error("[]") do 49 | status = @redis[nanite] 50 | timestamp = @redis["t-#{nanite}"] 51 | services = @redis.set_members("s-#{nanite}") 52 | tags = @redis.set_members("tg-#{nanite}") 53 | return nil unless status && timestamp && services 54 | {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags} 55 | end 56 | end 57 | 58 | def []=(nanite, hsh) 59 | log_redis_error("[]=") do 60 | update_state(nanite, hsh[:status], hsh[:services], hsh[:tags]) 61 | end 62 | end 63 | 64 | def delete(nanite) 65 | log_redis_error("delete") do 66 | (@redis.set_members("s-#{nanite}")||[]).each do |srv| 67 | @redis.set_delete(srv, nanite) 68 | if @redis.set_count(srv) == 0 69 | @redis.delete(srv) 70 | @redis.set_delete("naniteservices", srv) 71 | end 72 | end 73 | (@redis.set_members("tg-#{nanite}")||[]).each do |tag| 74 | @redis.set_delete(tag, nanite) 75 | if @redis.set_count(tag) == 0 76 | @redis.delete(tag) 77 | @redis.set_delete("nanitetags", tag) 78 | end 79 | end 80 | @redis.delete nanite 81 | @redis.delete "s-#{nanite}" 82 | @redis.delete "t-#{nanite}" 83 | @redis.delete "tg-#{nanite}" 84 | end 85 | end 86 | 87 | def all_services 88 | log_redis_error("all_services") do 89 | @redis.set_members("naniteservices") 90 | end 91 | end 92 | 93 | def all_tags 94 | log_redis_error("all_tags") do 95 | @redis.set_members("nanitetags") 96 | end 97 | end 98 | 99 | def update_state(name, status, services, tags) 100 | old_services = @redis.set_members("s-#{name}") 101 | if old_services 102 | (old_services - services).each do |s| 103 | @redis.set_delete(s, name) 104 | @redis.set_delete("naniteservices", s) 105 | end 106 | end 107 | old_tags = @redis.set_members("tg-#{name}") 108 | if old_tags 109 | (old_tags - tags).each do |t| 110 | @redis.set_delete(t, name) 111 | @redis.set_delete("nanitetags", t) 112 | end 113 | end 114 | @redis.delete("s-#{name}") 115 | services.each do |srv| 116 | @redis.set_add(srv, name) 117 | @redis.set_add("s-#{name}", srv) 118 | @redis.set_add("naniteservices", srv) 119 | end 120 | @redis.delete("tg-#{name}") 121 | tags.each do |tag| 122 | next if tag.nil? 123 | @redis.set_add(tag, name) 124 | @redis.set_add("tg-#{name}", tag) 125 | @redis.set_add("nanitetags", tag) 126 | end 127 | @redis[name] = status 128 | @redis["t-#{name}"] = Time.now.to_i 129 | end 130 | 131 | def list_nanites 132 | log_redis_error("list_nanites") do 133 | @redis.keys("nanite-*") 134 | end 135 | end 136 | 137 | def size 138 | list_nanites.size 139 | end 140 | 141 | def clear_state 142 | log_redis_error("clear_state") do 143 | @redis.keys("*").each {|k| @redis.delete k} 144 | end 145 | end 146 | 147 | def each 148 | list_nanites.each do |nan| 149 | yield nan, self[nan] 150 | end 151 | end 152 | 153 | def nanites_for(service, *tags) 154 | keys = tags.dup << service 155 | log_redis_error("nanites_for") do 156 | res = [] 157 | (@redis.set_intersect(keys)||[]).each do |nan| 158 | res << [nan, self[nan]] 159 | end 160 | res 161 | end 162 | end 163 | end 164 | end -------------------------------------------------------------------------------- /lib/nanite/admin.rb: -------------------------------------------------------------------------------- 1 | require 'rack' 2 | 3 | module Nanite 4 | # This is a Rack app for nanite-admin. You need to have an async capable 5 | # version of Thin installed for this to work. See bin/nanite-admin for install 6 | # instructions. 7 | class Admin 8 | def initialize(mapper) 9 | @mapper = mapper 10 | end 11 | 12 | AsyncResponse = [-1, {}, []].freeze 13 | 14 | def call(env) 15 | req = Rack::Request.new(env) 16 | if cmd = req.params['command'] 17 | @command = cmd 18 | @selection = req.params['type'] if req.params['type'] 19 | 20 | options = {} 21 | case @selection 22 | when 'least_loaded', 'random', 'all', 'rr' 23 | options[:selector] = @selection 24 | else 25 | options[:target] = @selection 26 | end 27 | 28 | @mapper.request(cmd, req.params['payload'], options) do |response, responsejob| 29 | env['async.callback'].call [200, {'Content-Type' => 'text/html'}, [layout(ul(response, responsejob))]] 30 | end 31 | AsyncResponse 32 | else 33 | [200, {'Content-Type' => 'text/html'}, layout] 34 | end 35 | end 36 | 37 | def services 38 | buf = "" 43 | buf 44 | end 45 | 46 | def ul(hash, job) 47 | buf = "
    " 48 | hash.each do |k,v| 49 | buf << "
  • #{k}:
    #{v.inspect}
    " 50 | if job.intermediate_state && job.intermediate_state[k] 51 | buf << "
    intermediate state: #{job.intermediate_state[k].inspect}
    " 52 | end 53 | buf << "
    • " 54 | end 55 | buf << "
" 56 | buf 57 | end 58 | 59 | def layout(content=nil) 60 | %Q{ 61 | 62 | 63 | 64 | 65 | 66 | 67 | Nanite Control Tower 68 | 69 | 70 | 71 | 74 | 75 | 83 | 84 | 101 | 102 | 103 | 104 | 105 | 108 | 109 |

#{@mapper.options[:vhost]}

110 |
111 |
112 | 113 | 114 | 115 | 122 | 123 | 124 | #{services} 125 | 126 | 127 | 128 | 129 | 130 |
131 | 132 | #{"

Responses

" if content} 133 | #{content} 134 |
135 | 136 |

Running nanites

137 |
138 | #{"No nanites online." if @mapper.cluster.nanites.size == 0} 139 |
    140 | #{@mapper.cluster.nanites.map {|k,v| "
  • identity : #{k}
    load : #{v[:status]}
    services : #{v[:services].to_a.inspect}
    • " }.join} 141 |
142 |
143 | 148 | 149 | 150 | } 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/nanite/cluster.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class Cluster 3 | attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper 4 | 5 | def initialize(amq, agent_timeout, identity, serializer, mapper, redis=nil) 6 | @amq = amq 7 | @agent_timeout = agent_timeout 8 | @identity = identity 9 | @serializer = serializer 10 | @mapper = mapper 11 | @redis = redis 12 | @security = SecurityProvider.get 13 | if redis 14 | Nanite::Log.info("using redis for state storage") 15 | require 'nanite/state' 16 | @nanites = ::Nanite::State.new(redis) 17 | else 18 | require 'nanite/local_state' 19 | @nanites = Nanite::LocalState.new 20 | end 21 | @reaper = Reaper.new(agent_timeout) 22 | setup_queues 23 | end 24 | 25 | # determine which nanites should receive the given request 26 | def targets_for(request) 27 | return [request.target] if request.target 28 | __send__(request.selector, request.type, request.tags).collect {|name, state| name } 29 | end 30 | 31 | # adds nanite to nanites map: key is nanite's identity 32 | # and value is a services/status pair implemented 33 | # as a hash 34 | def register(reg) 35 | case reg 36 | when Register 37 | if @security.authorize_registration(reg) 38 | nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags } 39 | reaper.timeout(reg.identity, agent_timeout + 1) { nanites.delete(reg.identity) } 40 | Nanite::Log.info("registered: #{reg.identity}, #{nanites[reg.identity].inspect}") 41 | else 42 | Nanite::Log.warning("registration of #{reg.inspect} not authorized") 43 | end 44 | when UnRegister 45 | nanites.delete(reg.identity) 46 | Nanite::Log.info("un-registering: #{reg.identity}") 47 | end 48 | end 49 | 50 | def route(request, targets) 51 | EM.next_tick { targets.map { |target| publish(request, target) } } 52 | end 53 | 54 | def publish(request, target) 55 | # We need to initialize the 'target' field of the request object so that the serializer has 56 | # access to it. 57 | begin 58 | old_target = request.target 59 | request.target = target unless target == 'mapper-offline' 60 | amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent) 61 | ensure 62 | request.target = old_target 63 | end 64 | end 65 | 66 | protected 67 | 68 | # updates nanite information (last ping timestamps, status) 69 | # when heartbeat message is received 70 | def handle_ping(ping) 71 | if nanite = nanites[ping.identity] 72 | nanite[:status] = ping.status 73 | reaper.reset_with_autoregister_hack(ping.identity, agent_timeout + 1) { nanites.delete(ping.identity) } 74 | else 75 | amq.queue(ping.identity).publish(serializer.dump(Advertise.new)) 76 | end 77 | end 78 | 79 | # forward request coming from agent 80 | def handle_request(request) 81 | if @security.authorize_request(request) 82 | result = Result.new(request.token, request.from, nil, mapper.identity) 83 | intm_handler = lambda do |res| 84 | result.results = res 85 | forward_response(result, request.persistent) 86 | end 87 | ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res| 88 | result.results = res 89 | forward_response(result, request.persistent) 90 | end 91 | if ok == false 92 | forward_response(result, request.persistent) 93 | end 94 | else 95 | Nanite::Log.warning("request #{request.inspect} not authorized") 96 | end 97 | end 98 | 99 | # forward response back to agent that originally made the request 100 | def forward_response(res, persistent) 101 | amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent) 102 | end 103 | 104 | # returns least loaded nanite that provides given service 105 | def least_loaded(service, tags=[]) 106 | candidates = nanites_providing(service,tags) 107 | return [] if candidates.empty? 108 | 109 | [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }] 110 | end 111 | 112 | # returns all nanites that provide given service 113 | def all(service, tags=[]) 114 | nanites_providing(service,tags) 115 | end 116 | 117 | # returns a random nanite 118 | def random(service, tags=[]) 119 | candidates = nanites_providing(service,tags) 120 | return [] if candidates.empty? 121 | 122 | [candidates[rand(candidates.size)]] 123 | end 124 | 125 | # selects next nanite that provides given service 126 | # using round robin rotation 127 | def rr(service, tags=[]) 128 | @last ||= {} 129 | @last[service] ||= 0 130 | candidates = nanites_providing(service,tags) 131 | return [] if candidates.empty? 132 | @last[service] = 0 if @last[service] >= candidates.size 133 | candidate = candidates[@last[service]] 134 | @last[service] += 1 135 | [candidate] 136 | end 137 | 138 | # returns all nanites that provide the given service 139 | def nanites_providing(service, *tags) 140 | nanites.nanites_for(service, *tags) 141 | end 142 | 143 | def setup_queues 144 | setup_heartbeat_queue 145 | setup_registration_queue 146 | setup_request_queue 147 | end 148 | 149 | def setup_heartbeat_queue 150 | handler = lambda do |ping| 151 | begin 152 | ping = serializer.load(ping) 153 | Nanite::Log.debug("got heartbeat from #{ping.identity}") if ping.respond_to?(:identity) 154 | handle_ping(ping) 155 | rescue Exception => e 156 | Nanite::Log.error("Error handling heartbeat: #{e.message}") 157 | end 158 | end 159 | hb_fanout = amq.fanout('heartbeat', :durable => true) 160 | if @redis 161 | amq.queue("heartbeat").bind(hb_fanout).subscribe &handler 162 | else 163 | amq.queue("heartbeat-#{identity}", :exclusive => true).bind(hb_fanout).subscribe &handler 164 | end 165 | end 166 | 167 | def setup_registration_queue 168 | handler = lambda do |msg| 169 | begin 170 | msg = serializer.load(msg) 171 | Nanite::Log.debug("got registration from #{msg.identity}") 172 | register(msg) 173 | rescue Exception => e 174 | Nanite::Log.error("Error handling registration: #{e.message}") 175 | end 176 | end 177 | reg_fanout = amq.fanout('registration', :durable => true) 178 | if @redis 179 | amq.queue("registration").bind(reg_fanout).subscribe &handler 180 | else 181 | amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler 182 | end 183 | end 184 | 185 | def setup_request_queue 186 | handler = lambda do |msg| 187 | begin 188 | msg = serializer.load(msg) 189 | Nanite::Log.debug("got request from #{msg.from} of type #{msg.type}") 190 | handle_request(msg) 191 | rescue Exception => e 192 | Nanite::Log.error("Error handling request: #{e.message}") 193 | end 194 | end 195 | req_fanout = amq.fanout('request', :durable => true) 196 | if @redis 197 | amq.queue("request").bind(req_fanout).subscribe &handler 198 | else 199 | amq.queue("request-#{identity}", :exclusive => true).bind(req_fanout).subscribe &handler 200 | end 201 | end 202 | end 203 | end 204 | -------------------------------------------------------------------------------- /spec/packet_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe "Packet: Base class" do 4 | before(:all) do 5 | class TestPacket < Nanite::Packet 6 | @@cls_attr = "ignore" 7 | def initialize(attr1) 8 | @attr1 = attr1 9 | end 10 | end 11 | end 12 | 13 | it "should be an abstract class" do 14 | lambda { Nanite::Packet.new }.should raise_error(NotImplementedError, "Nanite::Packet is an abstract class.") 15 | end 16 | 17 | it "should know how to dump itself to JSON" do 18 | packet = TestPacket.new(1) 19 | packet.should respond_to(:to_json) 20 | end 21 | 22 | it "should dump the class name in 'json_class' JSON key" do 23 | packet = TestPacket.new(42) 24 | packet.to_json().should =~ /\"json_class\":\"TestPacket\"/ 25 | end 26 | 27 | it "should dump instance variables in 'data' JSON key" do 28 | packet = TestPacket.new(188) 29 | packet.to_json().should =~ /\"data\":\{\"attr1\":188\}/ 30 | end 31 | 32 | it "should not dump class variables" do 33 | packet = TestPacket.new(382) 34 | packet.to_json().should_not =~ /cls_attr/ 35 | end 36 | 37 | it "should store instance variables in 'data' JSON key as JSON object" do 38 | packet = TestPacket.new(382) 39 | packet.to_json().should =~ /\"data\":\{[\w:"]+\}/ 40 | end 41 | 42 | it "should remove '@' from instance variables" do 43 | packet = TestPacket.new(2) 44 | packet.to_json().should_not =~ /@attr1/ 45 | packet.to_json().should =~ /attr1/ 46 | end 47 | end 48 | 49 | 50 | describe "Packet: FileStart" do 51 | it "should dump/load as JSON objects" do 52 | packet = Nanite::FileStart.new('foo.txt', 'somewhere/foo.txt', '0xdeadbeef') 53 | packet2 = JSON.parse(packet.to_json) 54 | packet.filename.should == packet2.filename 55 | packet.dest.should == packet2.dest 56 | packet.token.should == packet2.token 57 | end 58 | 59 | it "should dump/load as Marshalled ruby objects" do 60 | packet = Nanite::FileStart.new('foo.txt', 'somewhere/foo.txt', '0xdeadbeef') 61 | packet2 = Marshal.load(Marshal.dump(packet)) 62 | packet.filename.should == packet2.filename 63 | packet.dest.should == packet2.dest 64 | packet.token.should == packet2.token 65 | end 66 | end 67 | 68 | 69 | describe "Packet: FileEnd" do 70 | it "should dump/load as JSON objects" do 71 | packet = Nanite::FileEnd.new('0xdeadbeef', 'metadata') 72 | packet2 = JSON.parse(packet.to_json) 73 | packet.meta.should == packet2.meta 74 | packet.token.should == packet2.token 75 | end 76 | 77 | it "should dump/load as Marshalled ruby objects" do 78 | packet = Nanite::FileEnd.new('0xdeadbeef', 'metadata') 79 | packet2 = Marshal.load(Marshal.dump(packet)) 80 | packet.meta.should == packet2.meta 81 | packet.token.should == packet2.token 82 | end 83 | end 84 | 85 | 86 | describe "Packet: FileChunk" do 87 | it "should dump/load as JSON objects" do 88 | packet = Nanite::FileChunk.new('chunk','0xdeadbeef') 89 | packet2 = JSON.parse(packet.to_json) 90 | packet.chunk.should == packet2.chunk 91 | packet.token.should == packet2.token 92 | end 93 | 94 | it "should dump/load as Marshalled ruby objects" do 95 | packet = Nanite::FileChunk.new('chunk','0xdeadbeef') 96 | packet2 = Marshal.load(Marshal.dump(packet)) 97 | packet.chunk.should == packet2.chunk 98 | packet.token.should == packet2.token 99 | end 100 | end 101 | 102 | 103 | describe "Packet: Request" do 104 | it "should dump/load as JSON objects" do 105 | packet = Nanite::Request.new('/some/foo', 'payload', :from => 'from', :token => '0xdeadbeef', :reply_to => 'reply_to') 106 | packet2 = JSON.parse(packet.to_json) 107 | packet.type.should == packet2.type 108 | packet.payload.should == packet2.payload 109 | packet.from.should == packet2.from 110 | packet.token.should == packet2.token 111 | packet.reply_to.should == packet2.reply_to 112 | end 113 | 114 | it "should dump/load as Marshalled ruby objects" do 115 | packet = Nanite::Request.new('/some/foo', 'payload', :from => 'from', :token => '0xdeadbeef', :reply_to => 'reply_to') 116 | packet2 = Marshal.load(Marshal.dump(packet)) 117 | packet.type.should == packet2.type 118 | packet.payload.should == packet2.payload 119 | packet.from.should == packet2.from 120 | packet.token.should == packet2.token 121 | packet.reply_to.should == packet2.reply_to 122 | end 123 | end 124 | 125 | 126 | describe "Packet: Result" do 127 | it "should dump/load as JSON objects" do 128 | packet = Nanite::Result.new('0xdeadbeef', 'to', 'results', 'from') 129 | packet2 = JSON.parse(packet.to_json) 130 | packet.token.should == packet2.token 131 | packet.to.should == packet2.to 132 | packet.results.should == packet2.results 133 | packet.from.should == packet2.from 134 | end 135 | 136 | it "should dump/load as Marshalled ruby objects" do 137 | packet = Nanite::Result.new('0xdeadbeef', 'to', 'results', 'from') 138 | packet2 = Marshal.load(Marshal.dump(packet)) 139 | packet.token.should == packet2.token 140 | packet.to.should == packet2.to 141 | packet.results.should == packet2.results 142 | packet.from.should == packet2.from 143 | end 144 | end 145 | 146 | 147 | describe "Packet: IntermediateMessage" do 148 | it "should dump/load as JSON objects" do 149 | packet = Nanite::IntermediateMessage.new('0xdeadbeef', 'to', 'from', 'messagekey', 'message') 150 | packet2 = JSON.parse(packet.to_json) 151 | packet.token.should == packet2.token 152 | packet.to.should == packet2.to 153 | packet.from.should == packet2.from 154 | packet.messagekey.should == packet2.messagekey 155 | packet.message.should == packet2.message 156 | end 157 | 158 | it "should dump/load as Marshalled ruby objects" do 159 | packet = Nanite::IntermediateMessage.new('0xdeadbeef', 'to', 'from', 'messagekey', 'message') 160 | packet2 = Marshal.load(Marshal.dump(packet)) 161 | packet.token.should == packet2.token 162 | packet.to.should == packet2.to 163 | packet.from.should == packet2.from 164 | packet.messagekey.should == packet2.messagekey 165 | packet.message.should == packet2.message 166 | end 167 | end 168 | 169 | 170 | describe "Packet: Register" do 171 | it "should dump/load as JSON objects" do 172 | packet = Nanite::Register.new('0xdeadbeef', ['/foo/bar', '/nik/qux'], 0.8, ['foo']) 173 | packet2 = JSON.parse(packet.to_json) 174 | packet.identity.should == packet2.identity 175 | packet.services.should == packet2.services 176 | packet.status.should == packet2.status 177 | end 178 | 179 | it "should dump/load as Marshalled ruby objects" do 180 | packet = Nanite::Register.new('0xdeadbeef', ['/foo/bar', '/nik/qux'], 0.8, ['foo']) 181 | packet2 = Marshal.load(Marshal.dump(packet)) 182 | packet.identity.should == packet2.identity 183 | packet.services.should == packet2.services 184 | packet.status.should == packet2.status 185 | end 186 | end 187 | 188 | 189 | describe "Packet: UnRegister" do 190 | it "should dump/load as JSON objects" do 191 | packet = Nanite::UnRegister.new('0xdeadbeef') 192 | packet2 = JSON.parse(packet.to_json) 193 | packet.identity.should == packet2.identity 194 | end 195 | 196 | it "should dump/load as Marshalled ruby objects" do 197 | packet = Nanite::UnRegister.new('0xdeadbeef') 198 | packet2 = Marshal.load(Marshal.dump(packet)) 199 | packet.identity.should == packet2.identity 200 | end 201 | end 202 | 203 | 204 | describe "Packet: Ping" do 205 | it "should dump/load as JSON objects" do 206 | packet = Nanite::Ping.new('0xdeadbeef', 0.8) 207 | packet2 = JSON.parse(packet.to_json) 208 | packet.identity.should == packet2.identity 209 | packet.status.should == packet2.status 210 | end 211 | 212 | it "should dump/load as Marshalled ruby objects" do 213 | packet = Nanite::Ping.new('0xdeadbeef', 0.8) 214 | packet2 = Marshal.load(Marshal.dump(packet)) 215 | packet.identity.should == packet2.identity 216 | packet.status.should == packet2.status 217 | end 218 | end 219 | -------------------------------------------------------------------------------- /spec/job_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::JobWarden do 4 | 5 | describe "Creating a new Job" do 6 | 7 | before(:each) do 8 | @serializer = mock("Serializer") 9 | @warden = Nanite::JobWarden.new(@serializer) 10 | 11 | @request = mock("Request") 12 | @targets = mock("Targets") 13 | @job = mock("Job", :token => "3faba24fcc") 14 | end 15 | 16 | it "should instantiate a new Job" do 17 | Nanite::Job.should_receive(:new).with(@request, @targets, nil, nil).and_return(@job) 18 | @warden.new_job(@request, @targets) 19 | end 20 | 21 | it "should add the job to the job list" do 22 | Nanite::Job.should_receive(:new).with(@request, @targets, nil, nil).and_return(@job) 23 | @warden.jobs.size.should == 0 24 | @warden.new_job(@request, @targets) 25 | @warden.jobs.size.should == 1 26 | @warden.jobs["3faba24fcc"].should == @job 27 | end 28 | 29 | it "return the newly crated job" do 30 | Nanite::Job.should_receive(:new).with(@request, @targets, nil, nil).and_return(@job) 31 | @warden.new_job(@request, @targets).should == @job 32 | end 33 | 34 | end # Creating a new Job 35 | 36 | 37 | describe "Processing a Message" do 38 | 39 | before(:each) do 40 | @message = mock("Message", :token => "3faba24fcc") 41 | @warden = Nanite::JobWarden.new(@serializer) 42 | @job = mock("Job", :token => "3faba24fcc", :process => true, :completed? => false, :results => 42, :pending_keys => [], :intermediate_handler => true) 43 | 44 | Nanite::Log.stub!(:debug) 45 | end 46 | 47 | it "should log debug message about message to be processed" do 48 | Nanite::Log.should_receive(:debug) 49 | @warden.process(@message) 50 | end 51 | 52 | it "should hand over processing to job" do 53 | Nanite::Job.stub!(:new).and_return(@job) 54 | @job.should_receive(:process).with(@message) 55 | 56 | @warden.new_job("request", "targets") 57 | @warden.process(@message) 58 | end 59 | 60 | it "should delete job from jobs after completion" do 61 | Nanite::Job.stub!(:new).and_return(@job) 62 | @job.should_receive(:process).with(@message) 63 | @job.should_receive(:completed?).and_return(true) 64 | @job.should_receive(:completed).and_return(nil) 65 | 66 | @warden.jobs["3faba24fcc"].should be_nil 67 | @warden.new_job("request", "targets") 68 | @warden.jobs["3faba24fcc"].should == @job 69 | @warden.process(@message) 70 | @warden.jobs["3faba24fcc"].should be_nil 71 | end 72 | 73 | it "should call completed block after completion" do 74 | completed_block = mock("Completed", :arity => 1, :call => true) 75 | 76 | Nanite::Job.stub!(:new).and_return(@job) 77 | @job.should_receive(:process).with(@message) 78 | @job.should_receive(:completed?).and_return(true) 79 | @job.should_receive(:completed).exactly(3).times.and_return(completed_block) 80 | 81 | @warden.new_job("request", "targets") 82 | @warden.process(@message) 83 | end 84 | 85 | it "should pass in job result if arity of completed block is one" do 86 | completed_block = mock("Completed") 87 | 88 | Nanite::Job.stub!(:new).and_return(@job) 89 | @job.should_receive(:process).with(@message) 90 | @job.should_receive(:completed?).and_return(true) 91 | @job.should_receive(:completed).exactly(3).times.and_return(completed_block) 92 | @job.should_receive(:results).and_return("the job result") 93 | completed_block.should_receive(:arity).and_return(1) 94 | completed_block.should_receive(:call).with("the job result") 95 | 96 | @warden.new_job("request", "targets") 97 | @warden.process(@message) 98 | end 99 | 100 | it "should pass in job result and job if arity of completed block is two" do 101 | completed_block = mock("Completed") 102 | 103 | Nanite::Job.stub!(:new).and_return(@job) 104 | @job.should_receive(:process).with(@message) 105 | @job.should_receive(:completed?).and_return(true) 106 | @job.should_receive(:completed).exactly(3).times.and_return(completed_block) 107 | @job.should_receive(:results).and_return("the job result") 108 | completed_block.should_receive(:arity).and_return(2) 109 | completed_block.should_receive(:call).with("the job result", @job) 110 | 111 | @warden.new_job("request", "targets") 112 | @warden.process(@message) 113 | end 114 | 115 | end # Processing a Message 116 | 117 | end # Nanite::JobWarden 118 | 119 | 120 | describe Nanite::Job do 121 | 122 | describe "Creating a Job" do 123 | 124 | before(:each) do 125 | @request = mock("Request", :token => "af534ceaaacdcd") 126 | end 127 | 128 | it "should initialize the request" do 129 | job = Nanite::Job.new(@request, nil, nil) 130 | job.request.should == @request 131 | end 132 | 133 | it "should initialize the targets" do 134 | job = Nanite::Job.new(@request, "targets", nil) 135 | job.targets.should == "targets" 136 | end 137 | 138 | it "should initialize the job token to the request token" do 139 | job = Nanite::Job.new(@request, nil, nil) 140 | job.token.should == "af534ceaaacdcd" 141 | end 142 | 143 | it "should initialize the results to an empty hash" do 144 | job = Nanite::Job.new(@request, nil, nil) 145 | job.results.should == {} 146 | end 147 | 148 | it "should initialize the intermediate state to an empty hash" do 149 | job = Nanite::Job.new(@request, nil, nil) 150 | job.intermediate_state.should == {} 151 | end 152 | 153 | it "should initialize the job block" do 154 | job = Nanite::Job.new(@request, nil, nil, "my block") 155 | job.completed.should == "my block" 156 | end 157 | 158 | end # Creating a new Job 159 | 160 | 161 | describe "Processing a Message" do 162 | 163 | before(:each) do 164 | @request = mock("Request", :token => "feeefe132") 165 | end 166 | 167 | it "should set the job result (for sender) to the message result for 'final' status messages" do 168 | job = Nanite::Job.new(@request, [], nil) 169 | message = Nanite::Result.new('token', 'to', 'results', 'from') 170 | job.results.should == {} 171 | job.process(message) 172 | job.results.should == { 'from' => 'results' } 173 | end 174 | 175 | it "should delete the message sender from the targets for 'final' status messages" do 176 | job = Nanite::Job.new(@request, ['from'], nil) 177 | message = Nanite::Result.new('token', 'to', 'results', 'from') 178 | job.targets.should == ['from'] 179 | job.process(message) 180 | job.targets.should == [] 181 | end 182 | 183 | it "should set the job result (for sender) to the message result for 'intermediate' status messages" do 184 | job = Nanite::Job.new(@request, ['from'], nil) 185 | message = Nanite::IntermediateMessage.new('token', 'to', 'from', 'messagekey', 'message') 186 | job.process(message) 187 | job.intermediate_state.should == { 'from' => { 'messagekey' => ['message'] } } 188 | end 189 | 190 | it "should not delete the message sender from the targets for 'intermediate' status messages" do 191 | job = Nanite::Job.new(@request, ['from'], nil) 192 | message = Nanite::IntermediateMessage.new('token', 'to', 'from', 'messagekey', 'message') 193 | job.targets.should == ['from'] 194 | job.process(message) 195 | job.targets.should == ['from'] 196 | end 197 | 198 | end # Processing a Message 199 | 200 | 201 | describe "Completion" do 202 | 203 | before(:each) do 204 | @request = mock("Request", :token => "af534ceaaacdcd") 205 | end 206 | 207 | it "should be true is targets are empty" do 208 | job = Nanite::Job.new(@request, {}, nil) 209 | job.completed?.should == true 210 | end 211 | 212 | it "should be false is targets are not empty" do 213 | job = Nanite::Job.new(@request, { :a => 1 }, nil) 214 | job.completed?.should == false 215 | end 216 | 217 | end # Completion 218 | 219 | end # Nanite::Job 220 | -------------------------------------------------------------------------------- /lib/nanite/packets.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | # Base class for all Nanite packets, 3 | # knows how to dump itself to JSON 4 | class Packet 5 | def initialize 6 | raise NotImplementedError.new("#{self.class.name} is an abstract class.") 7 | end 8 | def to_json(*a) 9 | { 10 | 'json_class' => self.class.name, 11 | 'data' => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m } 12 | }.to_json(*a) 13 | end 14 | end 15 | 16 | # packet that means start of a file transfer 17 | # operation 18 | class FileStart < Packet 19 | attr_accessor :filename, :token, :dest 20 | def initialize(filename, dest, token) 21 | @filename = filename 22 | @dest = dest 23 | @token = token 24 | end 25 | 26 | def self.json_create(o) 27 | i = o['data'] 28 | new(i['filename'], i['dest'], i['token']) 29 | end 30 | end 31 | 32 | # packet that means end of a file transfer 33 | # operation 34 | class FileEnd < Packet 35 | attr_accessor :token, :meta 36 | def initialize(token, meta) 37 | @token = token 38 | @meta = meta 39 | end 40 | 41 | def self.json_create(o) 42 | i = o['data'] 43 | new(i['token'], i['meta']) 44 | end 45 | end 46 | 47 | # packet that carries data chunks during a file transfer 48 | class FileChunk < Packet 49 | attr_accessor :chunk, :token 50 | def initialize(token, chunk=nil) 51 | @chunk = chunk 52 | @token = token 53 | end 54 | def self.json_create(o) 55 | i = o['data'] 56 | new(i['token'], i['chunk']) 57 | end 58 | end 59 | 60 | # packet that means a work request from mapper 61 | # to actor node 62 | # 63 | # type is a service name 64 | # payload is arbitrary data that is transferred from mapper to actor 65 | # 66 | # Options: 67 | # from is sender identity 68 | # token is a generated request id that mapper uses to identify replies 69 | # reply_to is identity of the node actor replies to, usually a mapper itself 70 | # selector is the selector used to route the request 71 | # target is the target nanite for the request 72 | # persistent signifies if this request should be saved to persistent storage by the AMQP broker 73 | class Request < Packet 74 | attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags 75 | DEFAULT_OPTIONS = {:selector => :least_loaded} 76 | def initialize(type, payload, opts={}) 77 | opts = DEFAULT_OPTIONS.merge(opts) 78 | @type = type 79 | @payload = payload 80 | @from = opts[:from] 81 | @token = opts[:token] 82 | @reply_to = opts[:reply_to] 83 | @selector = opts[:selector] 84 | @target = opts[:target] 85 | @persistent = opts[:persistent] 86 | @tags = opts[:tags] || [] 87 | end 88 | def self.json_create(o) 89 | i = o['data'] 90 | new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :reply_to => i['reply_to'], :selector => i['selector'], 91 | :target => i['target'], :persistent => i['persistent'], :tags => i['tags']}) 92 | end 93 | end 94 | 95 | # packet that means a work push from mapper 96 | # to actor node 97 | # 98 | # type is a service name 99 | # payload is arbitrary data that is transferred from mapper to actor 100 | # 101 | # Options: 102 | # from is sender identity 103 | # token is a generated request id that mapper uses to identify replies 104 | # selector is the selector used to route the request 105 | # target is the target nanite for the request 106 | # persistent signifies if this request should be saved to persistent storage by the AMQP broker 107 | class Push < Packet 108 | attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags 109 | DEFAULT_OPTIONS = {:selector => :least_loaded} 110 | def initialize(type, payload, opts={}) 111 | opts = DEFAULT_OPTIONS.merge(opts) 112 | @type = type 113 | @payload = payload 114 | @from = opts[:from] 115 | @token = opts[:token] 116 | @selector = opts[:selector] 117 | @target = opts[:target] 118 | @persistent = opts[:persistent] 119 | @tags = opts[:tags] || [] 120 | end 121 | def self.json_create(o) 122 | i = o['data'] 123 | new(i['type'], i['payload'], {:from => i['from'], :token => i['token'], :selector => i['selector'], 124 | :target => i['target'], :persistent => i['persistent'], :tags => i['tags']}) 125 | end 126 | end 127 | 128 | # packet that means a work result notification sent from actor to mapper 129 | # 130 | # from is sender identity 131 | # results is arbitrary data that is transferred from actor, a result of actor's work 132 | # token is a generated request id that mapper uses to identify replies 133 | # to is identity of the node result should be delivered to 134 | class Result < Packet 135 | attr_accessor :token, :results, :to, :from 136 | def initialize(token, to, results, from) 137 | @token = token 138 | @to = to 139 | @from = from 140 | @results = results 141 | end 142 | def self.json_create(o) 143 | i = o['data'] 144 | new(i['token'], i['to'], i['results'], i['from']) 145 | end 146 | end 147 | 148 | # packet that means an intermediate status notification sent from actor to mapper. is appended to a list of messages matching messagekey. 149 | # 150 | # from is sender identity 151 | # messagekey is a string that can become part of a redis key, which identifies the name under which the message is stored 152 | # message is arbitrary data that is transferred from actor, an intermediate result of actor's work 153 | # token is a generated request id that mapper uses to identify replies 154 | # to is identity of the node result should be delivered to 155 | class IntermediateMessage < Packet 156 | attr_accessor :token, :messagekey, :message, :to, :from 157 | def initialize(token, to, from, messagekey, message) 158 | @token = token 159 | @to = to 160 | @from = from 161 | @messagekey = messagekey 162 | @message = message 163 | end 164 | def self.json_create(o) 165 | i = o['data'] 166 | new(i['token'], i['to'], i['from'], i['messagekey'], i['message']) 167 | end 168 | end 169 | 170 | # packet that means an availability notification sent from actor to mapper 171 | # 172 | # from is sender identity 173 | # services is a list of services provided by the node 174 | # status is a load of the node by default, but may be any criteria 175 | # agent may use to report it's availability, load, etc 176 | class Register < Packet 177 | attr_accessor :identity, :services, :status, :tags 178 | def initialize(identity, services, status, tags) 179 | @status = status 180 | @tags = tags 181 | @identity = identity 182 | @services = services 183 | end 184 | def self.json_create(o) 185 | i = o['data'] 186 | new(i['identity'], i['services'], i['status'], i['tags']) 187 | end 188 | end 189 | 190 | # packet that means deregister an agent from the mappers 191 | # 192 | # from is sender identity 193 | class UnRegister < Packet 194 | attr_accessor :identity 195 | def initialize(identity) 196 | @identity = identity 197 | end 198 | def self.json_create(o) 199 | i = o['data'] 200 | new(i['identity']) 201 | end 202 | end 203 | 204 | # heartbeat packet 205 | # 206 | # identity is sender's identity 207 | # status is sender's status (see Register packet documentation) 208 | class Ping < Packet 209 | attr_accessor :identity, :status 210 | def initialize(identity, status) 211 | @status = status 212 | @identity = identity 213 | end 214 | def self.json_create(o) 215 | i = o['data'] 216 | new(i['identity'], i['status']) 217 | end 218 | end 219 | 220 | # packet that is sent by workers to the mapper 221 | # when worker initially comes online to advertise 222 | # it's services 223 | class Advertise < Packet 224 | def initialize 225 | end 226 | def self.json_create(o) 227 | new 228 | end 229 | end 230 | end 231 | 232 | -------------------------------------------------------------------------------- /spec/agent_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe "Agent:" do 4 | 5 | describe "Default Option" do 6 | 7 | before(:all) do 8 | EM.stub!(:add_periodic_timer) 9 | AMQP.stub!(:connect) 10 | @amq = mock("AMQueue", :queue => mock("queue", :subscribe => {}), :fanout => mock("fanout", :publish => nil)) 11 | MQ.stub!(:new).and_return(@amq) 12 | @agent = Nanite::Agent.start 13 | end 14 | 15 | it "for daemonize is false" do 16 | @agent.options.should include(:daemonize) 17 | @agent.options[:daemonize].should == false 18 | end 19 | 20 | it "for format is marshal" do 21 | @agent.options.should include(:format) 22 | @agent.options[:format].should == :marshal 23 | end 24 | 25 | it "for console is false" do 26 | @agent.options.should include(:console) 27 | @agent.options[:console].should == false 28 | end 29 | 30 | it "for user is nanite" do 31 | @agent.options.should include(:user) 32 | @agent.options[:user].should == "nanite" 33 | end 34 | 35 | it "for pass(word) is testing" do 36 | @agent.options.should include(:pass) 37 | @agent.options[:pass].should == "testing" 38 | end 39 | 40 | it "for secure is false" do 41 | @agent.options.should include(:secure) 42 | @agent.options[:secure].should == false 43 | end 44 | 45 | it "for host is 0.0.0.0" do 46 | @agent.options.should include(:host) 47 | @agent.options[:host].should == "0.0.0.0" 48 | end 49 | 50 | it "for log_level is info" do 51 | @agent.options.should include(:log_level) 52 | @agent.options[:log_level].should == :info 53 | end 54 | 55 | it "for vhost is /nanite" do 56 | @agent.options.should include(:vhost) 57 | @agent.options[:vhost].should == "/nanite" 58 | end 59 | 60 | it "for ping_time is 15" do 61 | @agent.options.should include(:ping_time) 62 | @agent.options[:ping_time].should == 15 63 | end 64 | 65 | it "for default_services is []" do 66 | @agent.options.should include(:default_services) 67 | @agent.options[:default_services].should == [] 68 | end 69 | 70 | it "for root is #{File.expand_path(File.join(File.dirname(__FILE__), '..'))}" do 71 | @agent.options.should include(:root) 72 | @agent.options[:root].should == File.expand_path(File.join(File.dirname(__FILE__), '..')) 73 | end 74 | 75 | it "for file_root is #{File.expand_path(File.join(File.dirname(__FILE__), '..', 'files'))}" do 76 | @agent.options.should include(:file_root) 77 | @agent.options[:file_root].should == File.expand_path(File.join(File.dirname(__FILE__), '..', 'files')) 78 | end 79 | 80 | end 81 | 82 | describe "Options from config.yml" do 83 | 84 | before(:all) do 85 | @agent = Nanite::Agent.start 86 | end 87 | 88 | end 89 | 90 | describe "Passed in Options" do 91 | 92 | before(:each) do 93 | EM.stub!(:add_periodic_timer) 94 | AMQP.stub!(:connect) 95 | @amq = mock("AMQueue", :queue => mock("queue", :subscribe => {}), :fanout => mock("fanout", :publish => nil)) 96 | MQ.stub!(:new).and_return(@amq) 97 | end 98 | 99 | # TODO figure out how to stub call to daemonize 100 | # it "for daemonize should override default (false)" do 101 | # agent = Nanite::Agent.start(:daemonize => true) 102 | # agent.options.should include(:daemonize) 103 | # agent.options[:daemonize].should == true 104 | # end 105 | 106 | it "for format should override default (marshal)" do 107 | agent = Nanite::Agent.start(:format => :json) 108 | agent.options.should include(:format) 109 | agent.options[:format].should == :json 110 | end 111 | 112 | # TODO figure out how to avoid console output 113 | # it "for console should override default (false)" do 114 | # agent = Nanite::Agent.start(:console => true) 115 | # agent.options.should include(:console) 116 | # agent.options[:console].should == true 117 | # end 118 | 119 | it "for user should override default (nanite)" do 120 | agent = Nanite::Agent.start(:user => "me") 121 | agent.options.should include(:user) 122 | agent.options[:user].should == "me" 123 | end 124 | 125 | it "for pass(word) should override default (testing)" do 126 | agent = Nanite::Agent.start(:pass => "secret") 127 | agent.options.should include(:pass) 128 | agent.options[:pass].should == "secret" 129 | end 130 | 131 | it "for secure should override default (false)" do 132 | agent = Nanite::Agent.start(:secure => true) 133 | agent.options.should include(:secure) 134 | agent.options[:secure].should == true 135 | end 136 | 137 | it "for host should override default (0.0.0.0)" do 138 | agent = Nanite::Agent.start(:host => "127.0.0.1") 139 | agent.options.should include(:host) 140 | agent.options[:host].should == "127.0.0.1" 141 | end 142 | 143 | it "for log_level should override default (info)" do 144 | agent = Nanite::Agent.start(:log_level => :debug) 145 | agent.options.should include(:log_level) 146 | agent.options[:log_level].should == :debug 147 | end 148 | 149 | it "for vhost should override default (/nanite)" do 150 | agent = Nanite::Agent.start(:vhost => "/virtual_host") 151 | agent.options.should include(:vhost) 152 | agent.options[:vhost].should == "/virtual_host" 153 | end 154 | 155 | it "for ping_time should override default (15)" do 156 | agent = Nanite::Agent.start(:ping_time => 5) 157 | agent.options.should include(:ping_time) 158 | agent.options[:ping_time].should == 5 159 | end 160 | 161 | it "for default_services should override default ([])" do 162 | agent = Nanite::Agent.start(:default_services => [:test]) 163 | agent.options.should include(:default_services) 164 | agent.options[:default_services].should == [:test] 165 | end 166 | 167 | it "for root should override default (#{File.expand_path(File.join(File.dirname(__FILE__), '..'))})" do 168 | agent = Nanite::Agent.start(:root => File.expand_path(File.dirname(__FILE__))) 169 | agent.options.should include(:root) 170 | agent.options[:root].should == File.expand_path(File.dirname(__FILE__)) 171 | end 172 | 173 | it "for file_root should override default (#{File.expand_path(File.join(File.dirname(__FILE__), '..', 'files'))})" do 174 | agent = Nanite::Agent.start(:file_root => File.expand_path(File.dirname(__FILE__))) 175 | agent.options.should include(:file_root) 176 | agent.options[:file_root].should == File.expand_path(File.dirname(__FILE__)) 177 | end 178 | 179 | it "for a single tag should result in the agent's tags being set" do 180 | agent = Nanite::Agent.start(:tag => "sample_tag") 181 | agent.tags.should include("sample_tag") 182 | end 183 | 184 | it "for multiple tags should result in the agent's tags being set" do 185 | agent = Nanite::Agent.start(:tag => ["sample_tag_1", "sample_tag_2"]) 186 | agent.tags.should include("sample_tag_1") 187 | agent.tags.should include("sample_tag_2") 188 | end 189 | 190 | end 191 | 192 | describe "Security" do 193 | 194 | before(:each) do 195 | EM.stub!(:add_periodic_timer) 196 | AMQP.stub!(:connect) 197 | @amq = mock("AMQueue", :queue => mock("queue", :subscribe => {}, :publish => {}), :fanout => mock("fanout", :publish => nil)) 198 | MQ.stub!(:new).and_return(@amq) 199 | serializer = Nanite::Serializer.new 200 | @request = Nanite::Request.new('/foo/bar', '') 201 | @push = Nanite::Push.new('/foo/bar', '') 202 | @agent = Nanite::Agent.start 203 | end 204 | 205 | it 'should correctly deny requests' do 206 | security = mock("Security") 207 | @agent.register_security(security) 208 | 209 | security.should_receive(:authorize).twice.and_return(false) 210 | @agent.dispatcher.should_not_receive(:dispatch) 211 | @agent.__send__(:receive, @request) 212 | @agent.__send__(:receive, @push) 213 | end 214 | 215 | it 'should correctly authorize requests' do 216 | security = mock("Security") 217 | @agent.register_security(security) 218 | 219 | security.should_receive(:authorize).twice.and_return(true) 220 | @agent.dispatcher.stub!(:dispatch) 221 | @agent.dispatcher.should_receive(:dispatch).twice 222 | @agent.__send__(:receive, @request) 223 | @agent.__send__(:receive, @push) 224 | end 225 | 226 | it 'should be ignored when not specified' do 227 | @agent.dispatcher.stub!(:dispatch) 228 | @agent.dispatcher.should_receive(:dispatch).twice 229 | @agent.__send__(:receive, @request) 230 | @agent.__send__(:receive, @push) 231 | end 232 | 233 | end 234 | 235 | end 236 | -------------------------------------------------------------------------------- /lib/nanite/agent.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | class Agent 3 | include AMQPHelper 4 | include FileStreaming 5 | include ConsoleHelper 6 | include DaemonizeHelper 7 | 8 | attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags 9 | attr_accessor :status_proc 10 | 11 | DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({:user => 'nanite', :ping_time => 15, 12 | :default_services => []}) unless defined?(DEFAULT_OPTIONS) 13 | 14 | # Initializes a new agent and establishes AMQP connection. 15 | # This must be used inside EM.run block or if EventMachine reactor 16 | # is already started, for instance, by a Thin server that your Merb/Rails 17 | # application runs on. 18 | # 19 | # Agent options: 20 | # 21 | # identity : identity of this agent, may be any string 22 | # 23 | # status_proc : a callable object that returns agent load as a string, 24 | # defaults to load averages string extracted from `uptime` 25 | # format : format to use for packets serialization. One of the three: 26 | # :marshall, :json, or :yaml. Defaults to 27 | # Ruby's Marshall format. For interoperability with 28 | # AMQP clients implemented in other languages, use JSON. 29 | # 30 | # Note that Nanite uses JSON gem, 31 | # and ActiveSupport's JSON encoder may cause clashes 32 | # if ActiveSupport is loaded after JSON gem. 33 | # 34 | # root : application root for this agent, defaults to Dir.pwd 35 | # 36 | # log_dir : path to directory where agent stores it's log file 37 | # if not given, app_root is used. 38 | # 39 | # file_root : path to directory to files this agent provides 40 | # defaults to app_root/files 41 | # 42 | # ping_time : time interval in seconds between two subsequent heartbeat messages 43 | # this agent broadcasts. Default value is 15. 44 | # 45 | # console : true tells Nanite to start interactive console 46 | # 47 | # daemonize : true tells Nanite to daemonize 48 | # 49 | # pid_dir : path to the directory where the agent stores its pid file (only if daemonized) 50 | # defaults to the root or the current working directory. 51 | # 52 | # services : list of services provided by this agent, by default 53 | # all methods exposed by actors are listed 54 | # 55 | # single_threaded: Run all operations in one thread 56 | # 57 | # Connection options: 58 | # 59 | # vhost : AMQP broker vhost that should be used 60 | # 61 | # user : AMQP broker user 62 | # 63 | # pass : AMQP broker password 64 | # 65 | # host : host AMQP broker (or node of interest) runs on, 66 | # defaults to 0.0.0.0 67 | # 68 | # port : port AMQP broker (or node of interest) runs on, 69 | # this defaults to 5672, port used by some widely 70 | # used AMQP brokers (RabbitMQ and ZeroMQ) 71 | # 72 | # On start Nanite reads config.yml, so it is common to specify 73 | # options in the YAML file. However, when both Ruby code options 74 | # and YAML file specify option, Ruby code options take precedence. 75 | def self.start(options = {}) 76 | agent = new(options) 77 | agent.run 78 | agent 79 | end 80 | 81 | def initialize(opts) 82 | set_configuration(opts) 83 | @tags = [] 84 | @tags << opts[:tag] 85 | @tags.flatten! 86 | @options.freeze 87 | end 88 | 89 | def run 90 | log_path = false 91 | if @options[:daemonize] 92 | log_path = (@options[:log_dir] || @options[:root] || Dir.pwd) 93 | end 94 | Log.init(@identity, log_path) 95 | Log.level = @options[:log_level] if @options[:log_level] 96 | @serializer = Serializer.new(@options[:format]) 97 | @status_proc = lambda { parse_uptime(`uptime`) rescue 'no status' } 98 | pid_file = PidFile.new(@identity, @options) 99 | pid_file.check 100 | if @options[:daemonize] 101 | daemonize 102 | pid_file.write 103 | at_exit { pid_file.remove } 104 | end 105 | @amq = start_amqp(@options) 106 | @registry = ActorRegistry.new 107 | @dispatcher = Dispatcher.new(@amq, @registry, @serializer, @identity, @options) 108 | setup_mapper_proxy 109 | load_actors 110 | setup_traps 111 | setup_queue 112 | advertise_services 113 | setup_heartbeat 114 | at_exit { un_register } unless $TESTING 115 | start_console if @options[:console] && !@options[:daemonize] 116 | end 117 | 118 | def register(actor, prefix = nil) 119 | registry.register(actor, prefix) 120 | end 121 | 122 | # Can be used in agent's initialization file to register a security module 123 | # This security module 'authorize' method will be called back whenever the 124 | # agent receives a request and will be given the corresponding deliverable. 125 | # It should return 'true' for the request to proceed. 126 | # Requests will return 'deny_token' or the string "Denied" by default when 127 | # 'authorize' does not return 'true'. 128 | def register_security(security, deny_token = "Denied") 129 | @security = security 130 | @deny_token = deny_token 131 | end 132 | 133 | protected 134 | 135 | def set_configuration(opts) 136 | @options = DEFAULT_OPTIONS.clone 137 | root = opts[:root] || @options[:root] 138 | custom_config = if root 139 | file = File.expand_path(File.join(root, 'config.yml')) 140 | File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {} 141 | else 142 | {} 143 | end 144 | opts.delete(:identity) unless opts[:identity] 145 | @options.update(custom_config.merge(opts)) 146 | @options[:file_root] ||= File.join(@options[:root], 'files') 147 | return @identity = "nanite-#{@options[:identity]}" if @options[:identity] 148 | token = Identity.generate 149 | @identity = "nanite-#{token}" 150 | File.open(File.expand_path(File.join(@options[:root], 'config.yml')), 'w') do |fd| 151 | fd.write(YAML.dump(custom_config.merge(:identity => token))) 152 | end 153 | end 154 | 155 | def load_actors 156 | return unless options[:root] 157 | actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors" 158 | actors = @options[:actors] 159 | Dir["#{actors_dir}/*.rb"].each do |actor| 160 | next if actors && !actors.include?(File.basename(actor, ".rb")) 161 | Nanite::Log.info("loading actor: #{actor}") 162 | require actor 163 | end 164 | init_path = @options[:initrb] || File.join(options[:root], 'init.rb') 165 | instance_eval(File.read(init_path), init_path) if File.exist?(init_path) 166 | end 167 | 168 | def receive(packet) 169 | case packet 170 | when Advertise 171 | Nanite::Log.debug("handling Advertise: #{packet.inspect}") 172 | advertise_services 173 | when Request, Push 174 | Nanite::Log.debug("handling Request: #{packet.inspect}") 175 | if @security && !@security.authorize(packet) 176 | if packet.kind_of?(Request) 177 | r = Result.new(packet.token, packet.reply_to, @deny_token, identity) 178 | amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r)) 179 | end 180 | else 181 | dispatcher.dispatch(packet) 182 | end 183 | when Result 184 | Nanite::Log.debug("handling Result: #{packet.inspect}") 185 | @mapper_proxy.handle_result(packet) 186 | when IntermediateMessage 187 | Nanite::Log.debug("handling Intermediate Result: #{packet.inspect}") 188 | @mapper_proxy.handle_intermediate_result(packet) 189 | end 190 | end 191 | 192 | def tag(*tags) 193 | tags.each {|t| @tags << t} 194 | @tags.uniq! 195 | end 196 | 197 | def setup_queue 198 | amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg| 199 | begin 200 | info.ack 201 | packet = serializer.load(msg) 202 | receive(packet) 203 | rescue Exception => e 204 | Nanite::Log.error("Error handling packet: #{e.message}") 205 | end 206 | end 207 | end 208 | 209 | def setup_heartbeat 210 | EM.add_periodic_timer(options[:ping_time]) do 211 | amq.fanout('heartbeat', :no_declare => options[:secure]).publish(serializer.dump(Ping.new(identity, status_proc.call))) 212 | end 213 | end 214 | 215 | def setup_mapper_proxy 216 | @mapper_proxy = MapperProxy.new(identity, options) 217 | end 218 | 219 | def setup_traps 220 | ['INT', 'TERM'].each do |sig| 221 | old = trap(sig) do 222 | un_register 223 | amq.instance_variable_get('@connection').close do 224 | EM.stop 225 | old.call if old.is_a? Proc 226 | end 227 | end 228 | end 229 | end 230 | 231 | def un_register 232 | unless @unregistered 233 | @unregistered = true 234 | amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(UnRegister.new(identity))) 235 | end 236 | end 237 | 238 | def advertise_services 239 | Nanite::Log.debug("advertise_services: #{registry.services.inspect}") 240 | amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(Register.new(identity, registry.services, status_proc.call, self.tags))) 241 | end 242 | 243 | def parse_uptime(up) 244 | if up =~ /load averages?: (.*)/ 245 | a,b,c = $1.split(/\s+|,\s+/) 246 | (a.to_f + b.to_f + c.to_f) / 3 247 | end 248 | end 249 | end 250 | end 251 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /lib/nanite/mapper.rb: -------------------------------------------------------------------------------- 1 | module Nanite 2 | # Mappers are control nodes in nanite clusters. Nanite clusters 3 | # can follow peer-to-peer model of communication as well as client-server, 4 | # and mappers are nodes that know who to send work requests to agents. 5 | # 6 | # Mappers can reside inside a front end web application written in Merb/Rails 7 | # and distribute heavy lifting to actors that register with the mapper as soon 8 | # as they go online. 9 | # 10 | # Each mapper tracks nanites registered with it. It periodically checks 11 | # when the last time a certain nanite sent a heartbeat notification, 12 | # and removes those that have timed out from the list of available workers. 13 | # As soon as a worker goes back online again it re-registers itself 14 | # and the mapper adds it to the list and makes it available to 15 | # be called again. 16 | # 17 | # This makes Nanite clusters self-healing and immune to individual node 18 | # failures. 19 | class Mapper 20 | include AMQPHelper 21 | include ConsoleHelper 22 | include DaemonizeHelper 23 | 24 | attr_reader :cluster, :identity, :job_warden, :options, :serializer, :amq 25 | 26 | DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({:user => 'mapper', :identity => Identity.generate, :agent_timeout => 15, 27 | :offline_redelivery_frequency => 10, :persistent => false, :offline_failsafe => false}) unless defined?(DEFAULT_OPTIONS) 28 | 29 | # Initializes a new mapper and establishes 30 | # AMQP connection. This must be used inside EM.run block or if EventMachine reactor 31 | # is already started, for instance, by a Thin server that your Merb/Rails 32 | # application runs on. 33 | # 34 | # Mapper options: 35 | # 36 | # identity : identity of this mapper, may be any string 37 | # 38 | # format : format to use for packets serialization. Can be :marshal, :json or :yaml or :secure. 39 | # Defaults to Ruby's Marshall format. For interoperability with 40 | # AMQP clients implemented in other languages, use JSON. 41 | # 42 | # Note that Nanite uses JSON gem, 43 | # and ActiveSupport's JSON encoder may cause clashes 44 | # if ActiveSupport is loaded after JSON gem. 45 | # 46 | # Also using the secure format requires prior initialization of the serializer, see 47 | # SecureSerializer.init 48 | # 49 | # log_level : the verbosity of logging, can be debug, info, warn, error or fatal. 50 | # 51 | # agent_timeout : how long to wait before an agent is considered to be offline 52 | # and thus removed from the list of available agents. 53 | # 54 | # log_dir : log file path, defaults to the current working directory. 55 | # 56 | # console : true tells mapper to start interactive console 57 | # 58 | # daemonize : true tells mapper to daemonize 59 | # 60 | # pid_dir : path to the directory where the agent stores its pid file (only if daemonized) 61 | # defaults to the root or the current working directory. 62 | # 63 | # offline_redelivery_frequency : The frequency in seconds that messages stored in the offline queue will be retrieved 64 | # for attempted redelivery to the nanites. Default is 10 seconds. 65 | # 66 | # persistent : true instructs the AMQP broker to save messages to persistent storage so that they aren't lost when the 67 | # broker is restarted. Default is false. Can be overriden on a per-message basis using the request and push methods. 68 | # 69 | # secure : use Security features of rabbitmq to restrict nanites to themselves 70 | # 71 | # Connection options: 72 | # 73 | # vhost : AMQP broker vhost that should be used 74 | # 75 | # user : AMQP broker user 76 | # 77 | # pass : AMQP broker password 78 | # 79 | # host : host AMQP broker (or node of interest) runs on, 80 | # defaults to 0.0.0.0 81 | # 82 | # port : port AMQP broker (or node of interest) runs on, 83 | # this defaults to 5672, port used by some widely 84 | # used AMQP brokers (RabbitMQ and ZeroMQ) 85 | # 86 | # @api :public: 87 | def self.start(options = {}) 88 | mapper = new(options) 89 | mapper.run 90 | mapper 91 | end 92 | 93 | def initialize(options) 94 | @options = DEFAULT_OPTIONS.clone.merge(options) 95 | root = options[:root] || @options[:root] 96 | custom_config = if root 97 | file = File.expand_path(File.join(root, 'config.yml')) 98 | File.exists?(file) ? (YAML.load(IO.read(file)) || {}) : {} 99 | else 100 | {} 101 | end 102 | options.delete(:identity) unless options[:identity] 103 | @options.update(custom_config.merge(options)) 104 | @identity = "mapper-#{@options[:identity]}" 105 | @options[:file_root] ||= File.join(@options[:root], 'files') 106 | @options.freeze 107 | end 108 | 109 | def run 110 | log_path = false 111 | if @options[:daemonize] 112 | log_path = (@options[:log_dir] || @options[:root] || Dir.pwd) 113 | end 114 | Nanite::Log.init(@identity, log_path) 115 | Nanite::Log.level = @options[:log_level] if @options[:log_level] 116 | @serializer = Serializer.new(@options[:format]) 117 | pid_file = PidFile.new(@identity, @options) 118 | pid_file.check 119 | if @options[:daemonize] 120 | daemonize 121 | pid_file.write 122 | at_exit { pid_file.remove } 123 | end 124 | @amq = start_amqp(@options) 125 | @job_warden = JobWarden.new(@serializer) 126 | @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis]) 127 | Nanite::Log.info('starting mapper') 128 | setup_queues 129 | start_console if @options[:console] && !@options[:daemonize] 130 | end 131 | 132 | # Make a nanite request which expects a response. 133 | # 134 | # ==== Parameters 135 | # type:: The dispatch route for the request 136 | # payload:: Payload to send. This will get marshalled en route 137 | # 138 | # ==== Options 139 | # :selector:: Method for selecting an actor. Default is :least_loaded. 140 | # :least_loaded:: Pick the nanite which has the lowest load. 141 | # :all:: Send the request to all nanites which respond to the service. 142 | # :random:: Randomly pick a nanite. 143 | # :rr: Select a nanite according to round robin ordering. 144 | # :target:: Select a specific nanite via identity, rather than using 145 | # a selector. 146 | # :offline_failsafe:: Store messages in an offline queue when all 147 | # the nanites are offline. Messages will be redelivered when nanites come online. 148 | # Default is false unless the mapper was started with the --offline-failsafe flag. 149 | # :persistent:: Instructs the AMQP broker to save the message to persistent 150 | # storage so that it isnt lost when the broker is restarted. 151 | # Default is false unless the mapper was started with the --persistent flag. 152 | # :intermediate_handler:: Takes a lambda to call when an IntermediateMessage 153 | # event arrives from a nanite. If passed a Hash, hash keys should correspond to 154 | # the IntermediateMessage keys provided by the nanite, and each should have a value 155 | # that is a lambda/proc taking the parameters specified here. Can supply a key '*' 156 | # as a catch-all for unmatched keys. 157 | # 158 | # ==== Block Parameters for intermediate_handler 159 | # key:: array of unique keys for which intermediate state has been received 160 | # since the last call to this block. 161 | # nanite:: nanite which sent the message. 162 | # state:: most recently delivered intermediate state for the key provided. 163 | # job:: (optional) -- if provided, this parameter gets the whole job object, if there's 164 | # a reason to do more complex work with the job. 165 | # 166 | # ==== Block Parameters 167 | # :results:: The returned value from the nanite actor. 168 | # 169 | # @api :public: 170 | def request(type, payload = '', opts = {}, &blk) 171 | request = build_deliverable(Request, type, payload, opts) 172 | send_request(request, opts, &blk) 173 | end 174 | 175 | # Send request with pre-built request instance 176 | def send_request(request, opts = {}, &blk) 177 | request.reply_to = identity 178 | intm_handler = opts.delete(:intermediate_handler) 179 | targets = cluster.targets_for(request) 180 | if !targets.empty? 181 | job = job_warden.new_job(request, targets, intm_handler, blk) 182 | cluster.route(request, job.targets) 183 | job 184 | elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe] 185 | job_warden.new_job(request, [], intm_handler, blk) 186 | cluster.publish(request, 'mapper-offline') 187 | :offline 188 | else 189 | false 190 | end 191 | end 192 | 193 | # Make a nanite request which does not expect a response. 194 | # 195 | # ==== Parameters 196 | # type:: The dispatch route for the request 197 | # payload:: Payload to send. This will get marshalled en route 198 | # 199 | # ==== Options 200 | # :selector:: Method for selecting an actor. Default is :least_loaded. 201 | # :least_loaded:: Pick the nanite which has the lowest load. 202 | # :all:: Send the request to all nanites which respond to the service. 203 | # :random:: Randomly pick a nanite. 204 | # :rr: Select a nanite according to round robin ordering. 205 | # :offline_failsafe:: Store messages in an offline queue when all 206 | # the nanites are offline. Messages will be redelivered when nanites come online. 207 | # Default is false unless the mapper was started with the --offline-failsafe flag. 208 | # :persistent:: Instructs the AMQP broker to save the message to persistent 209 | # storage so that it isnt lost when the broker is restarted. 210 | # Default is false unless the mapper was started with the --persistent flag. 211 | # 212 | # @api :public: 213 | def push(type, payload = '', opts = {}) 214 | push = build_deliverable(Push, type, payload, opts) 215 | targets = cluster.targets_for(push) 216 | if !targets.empty? 217 | cluster.route(push, targets) 218 | true 219 | elsif opts.key?(:offline_failsafe) ? opts[:offline_failsafe] : options[:offline_failsafe] 220 | cluster.publish(push, 'mapper-offline') 221 | :offline 222 | else 223 | false 224 | end 225 | end 226 | 227 | private 228 | 229 | def build_deliverable(deliverable_type, type, payload, opts) 230 | deliverable = deliverable_type.new(type, payload, opts) 231 | deliverable.from = identity 232 | deliverable.token = Identity.generate 233 | deliverable.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent] 234 | deliverable 235 | end 236 | 237 | def setup_queues 238 | setup_offline_queue 239 | setup_message_queue 240 | end 241 | 242 | def setup_offline_queue 243 | offline_queue = amq.queue('mapper-offline', :durable => true) 244 | offline_queue.subscribe(:ack => true) do |info, deliverable| 245 | deliverable = serializer.load(deliverable) 246 | targets = cluster.targets_for(deliverable) 247 | unless targets.empty? 248 | info.ack 249 | if deliverable.kind_of?(Request) 250 | if job = job_warden.jobs[deliverable.token] 251 | job.targets = targets 252 | else 253 | deliverable.reply_to = identity 254 | job_warden.new_job(deliverable, targets) 255 | end 256 | end 257 | cluster.route(deliverable, targets) 258 | end 259 | end 260 | 261 | EM.add_periodic_timer(options[:offline_redelivery_frequency]) { offline_queue.recover } 262 | end 263 | 264 | def setup_message_queue 265 | amq.queue(identity, :exclusive => true).bind(amq.fanout(identity)).subscribe do |msg| 266 | begin 267 | msg = serializer.load(msg) 268 | Nanite::Log.debug("got result from #{msg.from}: #{msg.results.inspect}") 269 | job_warden.process(msg) 270 | rescue Exception => e 271 | Nanite::Log.error("Error handling result: #{e.message}") 272 | end 273 | end 274 | end 275 | end 276 | end 277 | 278 | -------------------------------------------------------------------------------- /spec/cluster_spec.rb: -------------------------------------------------------------------------------- 1 | require File.join(File.dirname(__FILE__), 'spec_helper') 2 | 3 | describe Nanite::Cluster do 4 | 5 | describe "Intialization" do 6 | 7 | before(:each) do 8 | @fanout = mock("fanout") 9 | @binding = mock("binding", :subscribe => true) 10 | @queue = mock("queue", :bind => @binding) 11 | @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout) 12 | @serializer = mock("Serializer") 13 | @reaper = mock("Reaper") 14 | @mapper = mock("Mapper") 15 | Nanite::Reaper.stub!(:new).and_return(@reaper) 16 | end 17 | 18 | describe "of Heartbeat (Queue)" do 19 | 20 | it "should setup the heartbeat (queue) for id" do 21 | @amq.should_receive(:queue).with("heartbeat-the_identity", anything()).and_return(@queue) 22 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 23 | end 24 | 25 | it "should make the heartbeat (queue) exclusive" do 26 | @amq.should_receive(:queue).with("heartbeat-the_identity", { :exclusive => true }).and_return(@queue) 27 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 28 | end 29 | 30 | it "should bind the heartbeat (queue) to 'heartbeat' fanout" do 31 | @amq.should_receive(:fanout).with("heartbeat", { :durable => true }).and_return(@fanout) 32 | @queue.should_receive(:bind).with(@fanout).and_return(@binding) 33 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 34 | end 35 | 36 | end # of Heartbeat (Queue) 37 | 38 | 39 | describe "of Registration (Queue)" do 40 | 41 | it "should setup the registration (queue) for id" do 42 | @amq.should_receive(:queue).with("registration-the_identity", anything()).and_return(@queue) 43 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 44 | end 45 | 46 | it "should make the registration (queue) exclusive" do 47 | @amq.should_receive(:queue).with("registration-the_identity", { :exclusive => true }).and_return(@queue) 48 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 49 | end 50 | 51 | it "should bind the registration (queue) to 'registration' fanout" do 52 | @amq.should_receive(:fanout).with("registration", { :durable => true }).and_return(@fanout) 53 | @queue.should_receive(:bind).with(@fanout).and_return(@binding) 54 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 55 | end 56 | 57 | end # of Registration (Queue) 58 | 59 | describe "of Request (Queue)" do 60 | 61 | it "should setup the request (queue) for id" do 62 | @amq.should_receive(:queue).with("request-the_identity", anything()).and_return(@queue) 63 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 64 | end 65 | 66 | it "should make the request (queue) exclusive" do 67 | @amq.should_receive(:queue).with("request-the_identity", { :exclusive => true }).and_return(@queue) 68 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 69 | end 70 | 71 | it "should bind the request (queue) to 'request' fanout" do 72 | @amq.should_receive(:fanout).with("request", { :durable => true }).and_return(@fanout) 73 | @queue.should_receive(:bind).with(@fanout).and_return(@binding) 74 | cluster = Nanite::Cluster.new(@amq, 10, "the_identity", @serializer, @mapper) 75 | end 76 | 77 | end # of Request (Queue) 78 | 79 | 80 | describe "Reaper" do 81 | 82 | it "should be created" do 83 | Nanite::Reaper.should_receive(:new).with(anything()).and_return(@reaper) 84 | cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper) 85 | end 86 | 87 | it "should use the agent timeout" do 88 | Nanite::Reaper.should_receive(:new).with(443).and_return(@reaper) 89 | cluster = Nanite::Cluster.new(@amq, 443, "the_identity", @serializer, @mapper) 90 | end 91 | 92 | end # Reaper 93 | 94 | end # Intialization 95 | 96 | 97 | describe "Target Selection" do 98 | 99 | before(:each) do 100 | @fanout = mock("fanout") 101 | @binding = mock("binding", :subscribe => true) 102 | @queue = mock("queue", :bind => @binding) 103 | @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout) 104 | @serializer = mock("Serializer") 105 | @reaper = mock("Reaper") 106 | Nanite::Reaper.stub!(:new).and_return(@reaper) 107 | @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper) 108 | end 109 | 110 | it "should return array containing targets for request" do 111 | target = mock("Supplied Target") 112 | request = mock("Request", :target => target) 113 | @cluster.targets_for(request).should be_instance_of(Array) 114 | end 115 | 116 | it "should use target from request" do 117 | target = mock("Supplied Target") 118 | request = mock("Request", :target => target) 119 | @cluster.targets_for(request).should == [target] 120 | end 121 | 122 | it "should use targets choosen by least loaded selector (:least_loaded)" do 123 | targets = { "target 3" => 3 } 124 | request = mock("Request", :target => nil, :selector => :least_loaded, :type => "service", :tags => []) 125 | @cluster.should_receive(:least_loaded).with("service", []).and_return(targets) 126 | @cluster.targets_for(request).should == ["target 3"] 127 | end 128 | 129 | it "should use targets choosen by all selector (:all)" do 130 | targets = { "target 1" => 1, "target 2" => 2, "target 3" => 3 } 131 | request = mock("Request", :target => nil, :selector => :all, :type => "service", :tags => []) 132 | @cluster.should_receive(:all).with("service", []).and_return(targets) 133 | @cluster.targets_for(request).should == ["target 1", "target 2", "target 3"] 134 | end 135 | 136 | it "should use targets choosen by random selector (:random)" do 137 | targets = { "target 3" => 3 } 138 | request = mock("Request", :target => nil, :selector => :random, :type => "service", :tags => []) 139 | @cluster.should_receive(:random).with("service", []).and_return(targets) 140 | @cluster.targets_for(request).should == ["target 3"] 141 | end 142 | 143 | it "should use targets choosen by round-robin selector (:rr)" do 144 | targets = { "target 2" => 2 } 145 | request = mock("Request", :target => nil, :selector => :rr, :type => "service", :tags => []) 146 | @cluster.should_receive(:rr).with("service", []).and_return(targets) 147 | @cluster.targets_for(request).should == ["target 2"] 148 | end 149 | 150 | end # Target Selection 151 | 152 | 153 | describe "Nanite Registration" do 154 | 155 | before(:each) do 156 | @fanout = mock("fanout") 157 | @binding = mock("binding", :subscribe => true) 158 | @queue = mock("queue", :bind => @binding) 159 | @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout) 160 | @serializer = mock("Serializer") 161 | @reaper = mock("Reaper", :timeout => true) 162 | Nanite::Log.stub!(:info) 163 | Nanite::Reaper.stub!(:new).and_return(@reaper) 164 | @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper) 165 | @register_packet = Nanite::Register.new("nanite_id", ["the_nanite_services"], "nanite_status",[]) 166 | end 167 | 168 | it "should add the Nanite to the nanites map" do 169 | @cluster.register(@register_packet) 170 | @cluster.nanites['nanite_id'].should_not be_nil 171 | end 172 | 173 | it "should use hash of the Nanite's services and status as value" do 174 | @cluster.register(@register_packet) 175 | @cluster.nanites['nanite_id'].keys.size == 2 176 | @cluster.nanites['nanite_id'].keys.should include(:services) 177 | @cluster.nanites['nanite_id'].keys.should include(:status) 178 | @cluster.nanites['nanite_id'][:services].should == ["the_nanite_services"] 179 | @cluster.nanites['nanite_id'][:status].should == "nanite_status" 180 | end 181 | 182 | it "should add nanite to reaper" do 183 | @reaper.should_receive(:timeout).with('nanite_id', 33) 184 | @cluster.register(@register_packet) 185 | end 186 | 187 | it "should log info message that nanite was registered" do 188 | Nanite::Log.should_receive(:info) 189 | @cluster.register(@register_packet) 190 | end 191 | 192 | end # Nanite Registration 193 | 194 | 195 | describe "Route" do 196 | 197 | before(:each) do 198 | @fanout = mock("fanout") 199 | @binding = mock("binding", :subscribe => true) 200 | @queue = mock("queue", :bind => @binding) 201 | @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout) 202 | @serializer = mock("Serializer") 203 | @reaper = mock("Reaper") 204 | Nanite::Reaper.stub!(:new).and_return(@reaper) 205 | @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper) 206 | @request = mock("Request") 207 | end 208 | 209 | it "should publish request to all targets" do 210 | target1 = mock("Target 1") 211 | target2 = mock("Target 2") 212 | @cluster.should_receive(:publish).with(@request, target1) 213 | @cluster.should_receive(:publish).with(@request, target2) 214 | EM.run { 215 | @cluster.route(@request, [target1, target2]) 216 | EM.stop 217 | } 218 | end 219 | 220 | end # Route 221 | 222 | 223 | describe "Publish" do 224 | 225 | before(:each) do 226 | @fanout = mock("fanout") 227 | @binding = mock("binding", :subscribe => true) 228 | @queue = mock("queue", :bind => @binding, :publish => true) 229 | @amq = mock("AMQueue", :queue => @queue, :fanout => @fanout) 230 | @serializer = mock("Serializer", :dump => "dumped_value") 231 | @reaper = mock("Reaper") 232 | Nanite::Reaper.stub!(:new).and_return(@reaper) 233 | @cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper) 234 | @request = mock("Request", :persistent => true) 235 | @target = mock("Target of Request") 236 | end 237 | 238 | it "should serialize request before publishing it" do 239 | @request.should_receive(:target=).with(@target) 240 | @request.should_receive(:target=) 241 | @request.should_receive(:target) 242 | @serializer.should_receive(:dump).with(@request).and_return("serialized_request") 243 | @cluster.publish(@request, @target) 244 | end 245 | 246 | it "should publish request to target queue" do 247 | @request.should_receive(:target=).with(@target) 248 | @request.should_receive(:target=) 249 | @request.should_receive(:target) 250 | @queue.should_receive(:publish).with("dumped_value", anything()) 251 | @cluster.publish(@request, @target) 252 | end 253 | 254 | it "should persist request based on request setting" do 255 | @request.should_receive(:target=).with(@target) 256 | @request.should_receive(:target=) 257 | @request.should_receive(:target) 258 | @request.should_receive(:persistent).and_return(false) 259 | @queue.should_receive(:publish).with(anything(), { :persistent => false }) 260 | @cluster.publish(@request, @target) 261 | end 262 | 263 | end # Publish 264 | 265 | describe "Agent Request Handling" do 266 | 267 | before(:each) do 268 | @fanout = mock("fanout") 269 | @binding = mock("binding", :subscribe => true) 270 | @queue = mock("queue", :bind => @binding, :publish => true) 271 | @amq = mock("AMPQueue", :queue => @queue, :fanout => @fanout) 272 | @serializer = mock("Serializer", :dump => "dumped_value") 273 | @target = mock("Target of Request") 274 | @reaper = mock("Reaper") 275 | Nanite::Reaper.stub!(:new).and_return(@reaper) 276 | @request_without_target = mock("Request", :target => nil, :token => "Token", 277 | :reply_to => "Reply To", :from => "From", :persistent => true, :identity => "Identity") 278 | @request_with_target = mock("Request", :target => "Target", :token => "Token", 279 | :reply_to => "Reply To", :from => "From", :persistent => true) 280 | @mapper_with_target = mock("Mapper", :identity => "id") 281 | @mapper_without_target = mock("Mapper", :request => false, :identity => @request_without_target.identity) 282 | @cluster_with_target = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper_with_target) 283 | @cluster_without_target = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper_without_target) 284 | Nanite::Cluster.stub!(:mapper).and_return(@mapper) 285 | end 286 | 287 | it "should forward requests with targets" do 288 | @mapper_with_target.should_receive(:send_request).with(@request_with_target, anything()) 289 | @cluster_with_target.__send__(:handle_request, @request_with_target) 290 | end 291 | 292 | it "should reply back with nil results for requests with no target when offline queue is disabled" do 293 | @mapper_without_target.should_receive(:send_request).with(@request_without_target, anything()) 294 | Nanite::Result.should_receive(:new).with(@request_without_target.token, @request_without_target.from, nil, @request_without_target.identity) 295 | @cluster_without_target.__send__(:handle_request, @request_without_target) 296 | end 297 | 298 | end # Agent Request Handling 299 | 300 | end # Nanite::Cluster 301 | --------------------------------------------------------------------------------