├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE ├── README.md ├── krakow.gemspec ├── lib ├── krakow.rb └── krakow │ ├── command.rb │ ├── command │ ├── auth.rb │ ├── cls.rb │ ├── fin.rb │ ├── identify.rb │ ├── mpub.rb │ ├── nop.rb │ ├── pub.rb │ ├── rdy.rb │ ├── req.rb │ ├── sub.rb │ └── touch.rb │ ├── connection.rb │ ├── connection_features.rb │ ├── connection_features │ ├── deflate.rb │ ├── snappy_frames.rb │ └── ssl.rb │ ├── consumer.rb │ ├── consumer │ └── queue.rb │ ├── discovery.rb │ ├── distribution.rb │ ├── distribution │ └── default.rb │ ├── exceptions.rb │ ├── frame_type.rb │ ├── frame_type │ ├── error.rb │ ├── message.rb │ └── response.rb │ ├── ksocket.rb │ ├── producer.rb │ ├── producer │ └── http.rb │ ├── utils.rb │ ├── utils │ ├── lazy.rb │ └── logging.rb │ └── version.rb └── test ├── helpers ├── nsqd.rb └── spec_helper.rb ├── run.rb └── specs ├── application ├── consumer_reconnect_spec.rb ├── high_volume_multi_prod_con_single_chan_spec.rb ├── high_volume_producer_spec.rb ├── high_volume_single_spec.rb └── producer_reconnect_spec.rb ├── connection_spec.rb ├── consumer_spec.rb └── producer_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.gitignore 3 | Gemfile.lock 4 | *.gem -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v0.4.2 2 | * Update command subclass loading to be explicit (remove glob loading) 3 | 4 | ## v0.4.0 5 | * Prevent duplicate connection consumption loops (#23) (thanks @phopkins) 6 | * Refactor connection implementation to prevent crashing blocks (#22) (thanks @phopkins) 7 | * Properly calculate message sizes using byte length, not string length (#24) (thanks @phopkins) 8 | * Clear responses prior to message transmission (#20) (thanks @i2amsam) 9 | * Rebuild testing specs (not fully covered, but a good start) 10 | * Consumer and Producer provide better connection failure recovery 11 | * Fix in-flight issues within distribution on connection failures 12 | 13 | _NOTE: Large portions of `Krakow::Connection` has been refactored_ 14 | 15 | ## v0.3.12 16 | * Update Consumer#confirm and Consumer#touch to rescue out lookups and abort 17 | 18 | ## v0.3.10 19 | * Remove exclusive from Connection#init! 20 | 21 | ## v0.3.8 22 | * Remove locks and move logic to connection access 23 | * Check for result within response prior to access (prevent slaying actor) 24 | 25 | ## v0.3.6 26 | * Allow `:options` key within `Producer` to set low level connection settings 27 | * Make snappy an optional dependency 28 | * Add initial support for authentication 29 | * Update allowed types for optional notifier 30 | 31 | ## v0.3.4 32 | * Explicitly require version file (#11 and #12) 33 | 34 | ## v0.3.2 35 | * Fix return value from Connection#wait_time_for (#9) (thanks @AlphaB and @davidpelaez) 36 | 37 | ## v0.3.0 38 | * Include jitter to discovery interval lookups 39 | * Typecast to String on PUB and MPUB 40 | * Update exception types used for not implemented methods 41 | * Add #confirm, #requeue, and #touch helpers to FrameType::Message instances 42 | * Update Utils::Lazy implementation to be faster and clearer 43 | * Add #safe_socket method on Connection to add stability 44 | * Rebuild connections on error to prevent consumer teardown 45 | * Reference connections without requirement of connection instance being alive 46 | * Use #read over #recv on underlying socket to ensure proper number of bytes (thanks @thomas-holmes) 47 | * Expand spec testing 48 | 49 | A big thanks to @bschwartz for a large contribution in this changeset 50 | including expanded spec coverage, message proxy helper methods, and 51 | isolation of instability around Connection interactions. 52 | 53 | ## v0.2.2 54 | * Fix `nsqlookupd` attribute in `Consumer` and `Discovery` 55 | 56 | ## v0.2.0 57 | * Fix the rest of the namespacing issues 58 | * Start adding some tests 59 | * Use better exception types (NotImplementedError instead of NoMethodError) 60 | * Be smart about responses within connections 61 | * Add snappy support 62 | * Add deflate support 63 | * Add TLS support 64 | * Prevent division by zero in distribution 65 | * Add query methods to lazy helper (`attribute_name`?) 66 | 67 | ## v0.1.2 68 | * Include backoff support 69 | * Remove `method_missing` magic 70 | * Force message redistribution when connection removed 71 | * Make discovery interval configurable 72 | * Add support for HTTP producer 73 | * Include namespace for custom exceptions #1 (thanks @copiousfreetime) 74 | * Fix timeout method access in req command #1 (thanks @copiousfreetime) 75 | 76 | ## v0.1.0 77 | * Add logging support 78 | * Include valid responses within commands 79 | * Segregate responses from messages 80 | * Manage connections in consumer (closed/reconnect) 81 | * Add message distribution support 82 | 83 | ## v0.0.1 84 | * Initial release 85 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Branches 4 | 5 | ### `master` branch 6 | 7 | The master branch is the current stable released version. 8 | 9 | ### `develop` branch 10 | 11 | The develop branch is the current edge of development. 12 | 13 | ## Pull requests 14 | 15 | * https://github.com/chrisroberts/krakow/pulls 16 | 17 | Please base all pull requests of the `develop` branch. Merges to 18 | `master` only occur through the `develop` branch. Pull requests 19 | based on `master` will likely be cherry picked. 20 | 21 | ## Issues 22 | 23 | Need to report an issue? Use the github issues: 24 | 25 | * https://github.com/chrisroberts/krakow/issues 26 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2014 Chris Roberts 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Krakow 2 | 3 | "KRAKOW! KRAKOW! Two direct hits!" 4 | 5 | ## Spiff 6 | 7 | ```ruby 8 | require 'krakow' 9 | 10 | producer = Krakow::Producer.new( 11 | :host => 'HOST', 12 | :port => 'PORT', 13 | :topic => 'target' 14 | ) 15 | producer.write('KRAKOW!', 'KRAKOW!') 16 | ``` 17 | 18 | ## Zargons 19 | 20 | ```ruby 21 | require 'krakow' 22 | 23 | consumer = Krakow::Consumer.new( 24 | :nsqlookupd => 'http://HOST:PORT', 25 | :topic => 'target', 26 | :channel => 'ship' 27 | ) 28 | 29 | consumer.queue.size # => 2 30 | 2.times do 31 | msg = consumer.queue.pop 32 | puts "Received: #{msg}" 33 | consumer.confirm(msg.message_id) 34 | end 35 | ``` 36 | 37 | ## What is this? 38 | 39 | It's a Ruby library for [NSQ][1] using [Celluloid][2] under the hood. 40 | 41 | ## Information and FAQ that I totally made up 42 | 43 | ### Max in flight for consumers is 1, regardless of number of producers 44 | 45 | Yep, that's right. Just one lowly message at a time. And that's probably not what 46 | you want, so adjust it when you create your consumer instance. 47 | 48 | ```ruby 49 | require 'krakow' 50 | 51 | consumer = Krakow::Consumer.new( 52 | :nsqlookupd => 'http://HOST:PORT', 53 | :topic => 'target', 54 | :channel => 'ship', 55 | :max_in_flight => 30 56 | ) 57 | ``` 58 | 59 | ### Clean up after yourself 60 | 61 | Since [Celluloid][2] is in use under the hood, and the main interaction points are 62 | Actors (`Consumer` and `Producer`) you'll need to be sure you clean up. This simply 63 | means terminating the instance (since falling out of scope will not cause it to be 64 | garbage collected). 65 | 66 | ```ruby 67 | consumer = Krakow::Consumer.new( 68 | :nsqlookupd => 'http://HOST:PORT', 69 | :topic => 'target', 70 | :channel => 'ship', 71 | :max_in_flight => 30 72 | ) 73 | 74 | # do stuff 75 | 76 | consumer.terminate 77 | ``` 78 | 79 | ### Please make it shutup! 80 | 81 | Sure: 82 | 83 | ```ruby 84 | Krakow::Utils::Logging.level = :warn # :debug / :info / :warn / :error / :fatal 85 | ``` 86 | 87 | ### Why is it forcing something called an "unready state"? 88 | 89 | Because forcing starvation is mean. We don't want to be mean, so we'll ensure we 90 | are consuming from all registered connections. 91 | 92 | ### I just want to connect to a producer, not a lookup service 93 | 94 | Fine! 95 | 96 | ```ruby 97 | consumer = Krakow::Consumer.new( 98 | :host => 'HOST', 99 | :port => 'PORT', 100 | :topic => 'target', 101 | :channel => 'ship', 102 | :max_in_flight => 30 103 | ) 104 | ``` 105 | Great for testing, but you really should use the lookup service in the "real world" 106 | 107 | ### Backoff support 108 | 109 | NSQ has this backoff notion. It's pretty swell. Basically, if messages from a specific 110 | producer get re-queued (fail), then message consumption from that producer is halted, 111 | and slowly ramped back up. It gives time for downstream issues to work themselves out, 112 | if possible, instead of just keeping the firehose of gasoline on. Neat. 113 | 114 | By default backoff support is disabled. It can be enabled by setting the `:backoff_interval` 115 | when constructing the `Consumer`. The interval is in seconds (and yes, floats are allowed 116 | for sub-second intervals): 117 | 118 | ```ruby 119 | consumer = Krakow::Consumer.new( 120 | :nsqlookupd => 'http://HOST:PORT', 121 | :topic => 'target', 122 | :channel => 'ship', 123 | :max_in_flight => 30, 124 | :backoff_interval => 1 125 | ) 126 | ``` 127 | 128 | ### I need TLS! 129 | 130 | OK! 131 | 132 | ```ruby 133 | consumer = Krakow::Consumer.new( 134 | :nsqlookupd => 'http://HOST:PORT', 135 | :topic => 'target', 136 | :channel => 'ship', 137 | :connection_options => { 138 | :features => { 139 | :tls_v1 => true 140 | } 141 | } 142 | ) 143 | ``` 144 | 145 | ### I need Snappy compression! 146 | 147 | OK! 148 | 149 | ```ruby 150 | consumer = Krakow::Consumer.new( 151 | :nsqlookupd => 'http://HOST:PORT', 152 | :topic => 'target', 153 | :channel => 'ship', 154 | :connection_options => { 155 | :features => { 156 | :snappy => true 157 | } 158 | } 159 | ) 160 | ``` 161 | 162 | *NOTE*: snappy support requires the snappy 163 | gem and is not provided by default, so you 164 | will need to ensure it is installed either 165 | on the system, or within the bundle. 166 | 167 | ### I need Deflate compression! 168 | 169 | OK! 170 | 171 | ```ruby 172 | consumer = Krakow::Consumer.new( 173 | :nsqlookupd => 'http://HOST:PORT', 174 | :topic => 'target', 175 | :channel => 'ship', 176 | :connection_options => { 177 | :features => { 178 | :deflate => true 179 | } 180 | } 181 | ) 182 | ``` 183 | 184 | ### I want to use TLS based authentication! 185 | 186 | OK! 187 | 188 | ```ruby 189 | consumer = Krakow::Consumer.new( 190 | :nsqlookupd => 'http://HOST:PORT', 191 | :topic => 'target', 192 | :channel => 'ship', 193 | :connection_options => { 194 | :features => { 195 | :tls_v1 => true 196 | }, 197 | :config => { 198 | :ssl_context => { 199 | :certificate => '/path/to/cert', 200 | :key => '/path/to/key' 201 | } 202 | } 203 | } 204 | ) 205 | ``` 206 | 207 | ### Running the tests 208 | 209 | Run them all! 210 | 211 | ``` 212 | bundle exec ruby test/run.rb 213 | ``` 214 | 215 | Or, run part of them: 216 | 217 | ``` 218 | bundle exec ruby test/specs/consumer_spec.rb 219 | ``` 220 | 221 | *NOTE*: the specs expect that `nsqd` and `nsqlookupd` are available in `$PATH` 222 | 223 | ### It doesn't work 224 | 225 | Create an issue on the github repository 226 | 227 | * https://github.com/chrisroberts/krakow/issues 228 | 229 | ### It doesn't do `x` 230 | 231 | Create an issue, or even better, send a PR. 232 | 233 | * https://github.com/chrisroberts/krakow/pulls 234 | 235 | # Info 236 | * Repo: https://github.com/chrisroberts/krakow 237 | * Docs: http://code.chrisroberts.org/krakow 238 | * IRC: Freenode @ spox 239 | 240 | [1]: http://bitly.github.io/nsq/ "NSQ: a realtime distributed messaging platform" 241 | [2]: http://celluloid.io "Celluloid: Actor-based concurrent object framework for Ruby" 242 | 243 | # Contributors 244 | 245 | * Pete Hopkins (@phopkins) 246 | * Sam Phillips (@i2amsam) 247 | * Brendan Schwartz (@bschwartz) 248 | * Thomas Holmes (@thomas-holmes) 249 | * Jeremy Hinegardner (@copiousfreetime) -------------------------------------------------------------------------------- /krakow.gemspec: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/' 2 | require 'krakow/version' 3 | Gem::Specification.new do |s| 4 | s.name = 'krakow' 5 | s.version = Krakow::VERSION.version 6 | s.summary = 'NSQ library' 7 | s.author = 'Chris Roberts' 8 | s.email = 'code@chrisroberts.org' 9 | s.homepage = 'http://github.com/chrisroberts/krakow' 10 | s.description = 'NSQ ruby library' 11 | s.license = 'Apache 2.0' 12 | s.require_path = 'lib' 13 | s.add_runtime_dependency 'celluloid' 14 | s.add_runtime_dependency 'http' 15 | s.add_runtime_dependency 'multi_json' 16 | s.add_runtime_dependency 'digest-crc' 17 | s.add_development_dependency 'childprocess' 18 | s.add_development_dependency 'snappy' 19 | s.add_development_dependency 'minitest' 20 | s.files = Dir['lib/**/*'] + %w(krakow.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE) 21 | s.extra_rdoc_files = %w(CHANGELOG.md CONTRIBUTING.md LICENSE) 22 | end 23 | -------------------------------------------------------------------------------- /lib/krakow.rb: -------------------------------------------------------------------------------- 1 | require 'celluloid' 2 | 3 | if(ENV['DEBUG']) 4 | Celluloid.task_class = Celluloid::TaskThread 5 | end 6 | 7 | require 'celluloid/autostart' 8 | require 'multi_json' 9 | 10 | # NSQ client and producer library 11 | module Krakow 12 | autoload :Command, 'krakow/command' 13 | autoload :Connection, 'krakow/connection' 14 | autoload :ConnectionFeatures, 'krakow/connection_features' 15 | autoload :Consumer, 'krakow/consumer' 16 | autoload :Discovery, 'krakow/discovery' 17 | autoload :Distribution, 'krakow/distribution' 18 | autoload :Error, 'krakow/exceptions' 19 | autoload :FrameType, 'krakow/frame_type' 20 | autoload :Ksocket, 'krakow/ksocket' 21 | autoload :Producer, 'krakow/producer' 22 | autoload :Utils, 'krakow/utils' 23 | end 24 | 25 | require 'krakow/version' 26 | -------------------------------------------------------------------------------- /lib/krakow/command.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Messages for sending to remote server 5 | class Command 6 | 7 | include Utils::Lazy 8 | # @!parse include Utils::Lazy::InstanceMethods 9 | # @!parse extend Utils::Lazy::ClassMethods 10 | 11 | autoload :Auth, 'krakow/command/auth' 12 | autoload :Cls, 'krakow/command/cls' 13 | autoload :Fin, 'krakow/command/fin' 14 | autoload :Identify, 'krakow/command/identify' 15 | autoload :Mpub, 'krakow/command/mpub' 16 | autoload :Nop, 'krakow/command/nop' 17 | autoload :Pub, 'krakow/command/pub' 18 | autoload :Rdy, 'krakow/command/rdy' 19 | autoload :Req, 'krakow/command/req' 20 | autoload :Sub, 'krakow/command/sub' 21 | autoload :Touch, 'krakow/command/touch' 22 | 23 | class << self 24 | 25 | # Allowed OK return values 26 | # 27 | # @return [Array] 28 | def ok 29 | [] 30 | end 31 | 32 | # Allowed ERROR return values 33 | # 34 | # @return [Array] 35 | def error 36 | [] 37 | end 38 | 39 | # Response type expected 40 | # 41 | # @param message [Krakow::Message] message to check 42 | # @return [Symbol] response expected (:none, :error_only, :required) 43 | def response_for(message) 44 | if(message.class.ok.empty?) 45 | if(message.class.error.empty?) 46 | :none 47 | else 48 | :error_only 49 | end 50 | else 51 | :required 52 | end 53 | end 54 | 55 | end 56 | 57 | # @return [Krakow::FrameType] response to command 58 | attr_accessor :response 59 | 60 | # @return [String] name of command 61 | def name 62 | self.class.name.split('::').last.upcase 63 | end 64 | 65 | # Convert to line output 66 | # 67 | # @return [String] socket ready string 68 | def to_line(*args) 69 | raise NotImplementedError.new 'No line conversion method defined!' 70 | end 71 | 72 | # Is response OK 73 | # 74 | # @return [TrueClass, FalseClass] 75 | def ok?(response) 76 | response = response.content if response.is_a?(FrameType) 77 | self.class.ok.include?(response) 78 | end 79 | 80 | # Is response ERROR 81 | # 82 | # @return [TrueClass, FalseClass] 83 | def error?(response) 84 | response = response.content if response.is_a?(FrameType) 85 | self.class.error.include?(response) 86 | end 87 | 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/krakow/command/auth.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Publish single message 6 | class Auth < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :secret, String, :required => true 16 | 17 | # @!endgroup 18 | 19 | def to_line 20 | scrt = secret.to_s 21 | [name, "\n", scrt.bytesize, scrt].pack('a*a*a*a*l>a*') 22 | end 23 | 24 | class << self 25 | def ok 26 | %w(OK) 27 | end 28 | 29 | def error 30 | %w(E_AUTH_FAILED E_UNAUTHORIZED) 31 | end 32 | end 33 | 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/krakow/command/cls.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Close connection 6 | class Cls < Command 7 | 8 | def to_line 9 | "#{name}\n" 10 | end 11 | 12 | class << self 13 | def ok 14 | %w(CLOSE_WAIT) 15 | end 16 | 17 | def error 18 | %w(E_INVALID) 19 | end 20 | end 21 | 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/krakow/command/fin.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Finish a message 6 | class Fin < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :message_id, String, :required => true 16 | 17 | # @!endgroup 18 | 19 | def to_line 20 | "#{name} #{message_id}\n" 21 | end 22 | 23 | class << self 24 | def error 25 | %w(E_INVALID E_FIN_FAILED) 26 | end 27 | end 28 | 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/krakow/command/identify.rb: -------------------------------------------------------------------------------- 1 | require 'multi_json' 2 | require 'krakow' 3 | 4 | module Krakow 5 | class Command 6 | # Update client metadata on server / negotiate features 7 | class Identify < Command 8 | 9 | # @!group Attributes 10 | 11 | # @!macro [attach] attribute 12 | # @!method $1 13 | # @return [$2] the $1 $0 14 | # @!method $1? 15 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 16 | attribute :short_id, [String, Numeric], :required => true 17 | attribute :long_id, [String, Numeric], :required => true 18 | attribute :feature_negotiation, [TrueClass, FalseClass] 19 | attribute :heartbeat_interval, Numeric 20 | attribute :output_buffer_size, Integer 21 | attribute :output_buffer_timeout, Integer 22 | attribute :tls_v1, [TrueClass, FalseClass] 23 | attribute :snappy, [TrueClass, FalseClass] 24 | attribute :deflate, [TrueClass, FalseClass] 25 | attribute :deflate_level, Integer 26 | attribute :sample_rate, Integer 27 | 28 | # @!endgroup 29 | 30 | def to_line 31 | filtered = Hash[* 32 | arguments.map do |key, value| 33 | unless(value.nil?) 34 | [key, value] 35 | end 36 | end.compact.flatten 37 | ] 38 | payload = MultiJson.dump(filtered) 39 | [name, "\n", payload.bytesize, payload].pack('a*a*l>a*') 40 | end 41 | 42 | class << self 43 | def ok 44 | %w(OK) 45 | end 46 | 47 | def error 48 | %w(E_INVALID E_BAD_BODY) 49 | end 50 | 51 | end 52 | 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/krakow/command/mpub.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Publish multiple messages 6 | class Mpub < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :topic_name, String, :required => true 16 | attribute :messages, Array, :required => true 17 | 18 | # @!endgroup 19 | def to_line 20 | formatted_messages = messages.map do |message| 21 | message = message.to_s 22 | [message.bytesize, message].pack('l>a*') 23 | end.join 24 | [name, ' ', topic_name, "\n", formatted_messages.bytesize, messages.size, formatted_messages].pack('a*a*a*a*l>l>a*') 25 | end 26 | 27 | class << self 28 | def ok 29 | %w(OK) 30 | end 31 | 32 | def error 33 | %w(E_INVALID E_BAD_TOPIC E_BAD_BODY E_BAD_MESSAGE E_MPUB_FAILED) 34 | end 35 | end 36 | 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/krakow/command/nop.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # No-op 6 | class Nop < Command 7 | 8 | def to_line 9 | "#{name}\n" 10 | end 11 | 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/krakow/command/pub.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Publish single message 6 | class Pub < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :topic_name, String, :required => true 16 | attribute :message, Object, :required => true 17 | 18 | # @!endgroup 19 | 20 | def to_line 21 | msg = message.to_s 22 | [name, ' ', topic_name, "\n", msg.bytesize, msg].pack('a*a*a*a*l>a*') 23 | end 24 | 25 | class << self 26 | def ok 27 | %w(OK) 28 | end 29 | 30 | def error 31 | %w(E_INVALID E_BAD_TOPIC E_BAD_MESSAGE E_PUB_FAILED) 32 | end 33 | end 34 | 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/krakow/command/rdy.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Update RDY state 6 | class Rdy < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :count, Integer, :required => true 16 | 17 | # @!endgroup 18 | 19 | def to_line 20 | "#{name} #{count}\n" 21 | end 22 | 23 | class << self 24 | def error 25 | %w(E_INVALID) 26 | end 27 | end 28 | 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/krakow/command/req.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Re-queue a message 6 | class Req < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :message_id, String, :required => true 16 | attribute :timeout, Integer, :required => true 17 | 18 | # @!endgroup 19 | 20 | def to_line 21 | "#{name} #{message_id} #{self.timeout}\n" 22 | end 23 | 24 | class << self 25 | def error 26 | %w(E_INVALID E_REQ_FAILED) 27 | end 28 | end 29 | 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/krakow/command/sub.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Subscribe to topic/channel 6 | class Sub < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :topic_name, String, :required => true 16 | attribute :channel_name, String, :required => true 17 | 18 | # @!endgroup 19 | 20 | def to_line 21 | "#{name} #{topic_name} #{channel_name}\n" 22 | end 23 | 24 | class << self 25 | def ok 26 | %w(OK) 27 | end 28 | 29 | def error 30 | %w(E_INVALID E_BAD_TOPIC E_BAD_CHANNEL) 31 | end 32 | end 33 | 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/krakow/command/touch.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Command 5 | # Reset timeout for in-flight message 6 | class Touch < Command 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :message_id, String, :required => true 16 | 17 | # @!endgroup 18 | 19 | def to_line 20 | "#{name} #{message_id}\n" 21 | end 22 | 23 | class << self 24 | def error 25 | %w(E_INVALID E_TOUCH_FAILED) 26 | end 27 | end 28 | 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/krakow/connection.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | 5 | # Provides TCP connection to NSQD 6 | class Connection 7 | 8 | # Generate identifier for connection 9 | # 10 | # @param host [String] 11 | # @param port [String, Integer] 12 | # @param topic [String] 13 | # @param channel [String] 14 | # @return [String] 15 | def self.identifier(host, port, topic, channel) 16 | [host, port, topic, channel].compact.join('__') 17 | end 18 | 19 | include Utils::Lazy 20 | # @!parse include Krakow::Utils::Lazy::InstanceMethods 21 | # @!parse extend Krakow::Utils::Lazy::ClassMethods 22 | 23 | include Celluloid 24 | 25 | # Available connection features 26 | FEATURES = [ 27 | :max_rdy_count, 28 | :max_msg_timeout, 29 | :msg_timeout, 30 | :tls_v1, 31 | :deflate, 32 | :deflate_level, 33 | :max_deflate_level, 34 | :snappy, 35 | :sample_rate, 36 | :auth_required 37 | ] 38 | 39 | # List of features that may not be enabled together 40 | EXCLUSIVE_FEATURES = [[:snappy, :deflate]] 41 | 42 | # List of features that may be enabled by the client 43 | ENABLEABLE_FEATURES = [:tls_v1, :snappy, :deflate, :auth_required] 44 | 45 | finalizer :connection_cleanup 46 | 47 | # @return [Hash] current configuration for endpoint 48 | attr_reader :endpoint_settings 49 | # @return [Ksocket] underlying socket like instance 50 | attr_reader :socket 51 | # @return [TrueClass, FalseClass] 52 | attr_reader :running 53 | 54 | # @!group Attributes 55 | 56 | # @!macro [attach] attribute 57 | # @!method $1 58 | # @return [$2] the $1 $0 59 | # @!method $1? 60 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 61 | attribute :host, String, :required => true 62 | attribute :port, [String,Integer], :required => true 63 | attribute :topic, String 64 | attribute :channel, String 65 | attribute :version, String, :default => 'v2' 66 | attribute :queue, [Queue, Consumer::Queue], :default => ->{ Queue.new } 67 | attribute :callbacks, Hash, :default => ->{ Hash.new } 68 | attribute :responses, Queue, :default => ->{ Queue.new } 69 | attribute :notifier, [Celluloid::Signals, Celluloid::Condition, Celluloid::Actor] 70 | attribute :features, Hash, :default => ->{ Hash.new } 71 | attribute :response_wait, Numeric, :default => 1.0 72 | attribute :response_interval, Numeric, :default => 0.03 73 | attribute :error_wait, Numeric, :default => 0 74 | attribute :enforce_features, [TrueClass,FalseClass], :default => true 75 | attribute :features_args, Hash, :default => ->{ Hash.new } 76 | 77 | # @!endgroup 78 | 79 | # Create new instance 80 | # 81 | # @param args [Hash] 82 | # @option args [String] :host (required) server host 83 | # @option args [String, Numeric] :port (required) server port 84 | # @option args [String] :version 85 | # @option args [Queue] :queue received message queue 86 | # @option args [Hash] :callbacks 87 | # @option args [Queue] :responses received responses queue 88 | # @option args [Celluloid::Actor] :notifier actor to notify on new message 89 | # @option args [Hash] :features features to enable 90 | # @option args [Numeric] :response_wait time to wait for response 91 | # @option args [Numeric] :response_interval sleep interval for wait loop 92 | # @option args [Numeric] :error_wait time to wait for error response 93 | # @option args [TrueClass, FalseClass] :enforce_features fail if features are unavailable 94 | # @option args [Hash] :feature_args options for connection features 95 | def initialize(args={}) 96 | super 97 | @endpoint_settings = {} 98 | @running = false 99 | end 100 | 101 | # @return [String] identifier for this connection 102 | def identifier 103 | self.class.identifier(host, port, topic, channel) 104 | end 105 | 106 | # @return [String] stringify object 107 | def to_s 108 | "<#{self.class.name}:#{object_id} {#{host}:#{port}}>" 109 | end 110 | 111 | # Initialize the connection 112 | # 113 | # @return [nil] 114 | def init! 115 | connect! 116 | async.process_to_queue! 117 | nil 118 | end 119 | 120 | # Send message to remote server 121 | # 122 | # @param message [Krakow::Message] message to send 123 | # @return [TrueClass, Krakow::FrameType] response if expected or true 124 | def transmit(message) 125 | unless(message.respond_to?(:to_line)) 126 | abort TypeError.new("Expecting type `Krakow::FrameType` but received `#{message.class}`") 127 | end 128 | output = message.to_line 129 | response_wait = wait_time_for(message) 130 | if(response_wait > 0) 131 | transmit_with_response(message, response_wait) 132 | else 133 | debug ">>> #{output}" 134 | socket.put(output) 135 | true 136 | end 137 | end 138 | 139 | # Sends message and waits for response 140 | # 141 | # @param message [Krakow::Message] message to send 142 | # @return [Krakow::FrameType] response 143 | def transmit_with_response(message, wait_time) 144 | responses.clear 145 | socket.put(message.to_line) 146 | response = nil 147 | (wait_time / response_interval).to_i.times do |i| 148 | response = responses.pop unless responses.empty? 149 | break if response 150 | sleep(response_interval) 151 | end 152 | if(response) 153 | message.response = response 154 | if(message.error?(response)) 155 | res = Error::BadResponse.new "Message transmission failed #{message}" 156 | res.result = response 157 | abort res 158 | end 159 | response 160 | else 161 | unless(Command.response_for(message) == :error_only) 162 | abort Error::BadResponse::NoResponse.new "No response provided for message #{message}" 163 | end 164 | end 165 | end 166 | 167 | # Destructor method for cleanup 168 | # 169 | # @return [nil] 170 | def connection_cleanup 171 | debug 'Tearing down connection' 172 | @running = false 173 | if(connected?) 174 | socket.terminate 175 | end 176 | @socket = nil 177 | info 'Connection torn down' 178 | nil 179 | end 180 | 181 | # Receive from server 182 | # 183 | # @return [Krakow::FrameType, nil] message or nothing if read was empty 184 | # @raise [Error::ConnectionUnavailable] socket is closed 185 | def receive 186 | debug 'Read wait for frame start' 187 | buf = socket.get(8) 188 | if(buf) 189 | @receiving = true 190 | debug "<<< #{buf.inspect}" 191 | struct = FrameType.decode(buf) 192 | debug "Decoded structure: #{struct.inspect}" 193 | struct[:data] = socket.get(struct[:size]) 194 | debug "<<< #{struct[:data].inspect}" 195 | @receiving = false 196 | frame = FrameType.build(struct) 197 | debug "Struct: #{struct.inspect} Frame: #{frame.inspect}" 198 | frame 199 | else 200 | nil 201 | end 202 | end 203 | 204 | # @return [TrueClass, FalseClass] is connection currently receiving a message 205 | def receiving? 206 | !!@receiving 207 | end 208 | 209 | # Receive messages and place into queue 210 | # 211 | # @return [nil] 212 | def process_to_queue! 213 | unless(@running) 214 | @running = true 215 | while(@running) 216 | message = handle(receive) 217 | if(message) 218 | debug "Adding message to queue #{message}" 219 | queue << message 220 | if(notifier) 221 | warn "Sending new message notification: #{notifier} - #{message}" 222 | notifier.broadcast(message) 223 | end 224 | else 225 | debug 'Received `nil` message. Ignoring.' 226 | end 227 | end 228 | end 229 | nil 230 | end 231 | 232 | # Handle non-message type Krakow::FrameType 233 | # 234 | # @param message [Krakow::FrameType] received message 235 | # @return [Krakow::FrameType, nil] 236 | def handle(message) 237 | # Grab heartbeats upfront 238 | if(message.is_a?(FrameType::Response) && message.response == '_heartbeat_') 239 | debug 'Responding to heartbeat' 240 | transmit Command::Nop.new 241 | nil 242 | else 243 | message = callback_for(:handle, message) 244 | if(!message.is_a?(FrameType::Message)) 245 | debug "Captured non-message type response: #{message}" 246 | responses << message 247 | nil 248 | else 249 | message 250 | end 251 | end 252 | end 253 | 254 | # Execute callback for given type 255 | # 256 | # @overload callback_for(type, arg, connection) 257 | # @param type [Symbol] type of callback 258 | # @param arg [Object] argument for callback (can be multiple) 259 | # @param connection [Krakow::Connection] current connection 260 | # @return [Object] result of callback 261 | def callback_for(type, *args) 262 | callback = callbacks[type] 263 | if(callback) 264 | debug "Processing connection callback for #{type.inspect} (#{callback.inspect})" 265 | if(callback[:actor].alive?) 266 | callback[:actor].send(callback[:method], *(args + [current_actor])) 267 | else 268 | error "Expected actor for callback processing is not alive! (type: `#{type.inspect}`)" 269 | end 270 | else 271 | debug "No connection callback defined for #{type.inspect}" 272 | args.size == 1 ? args.first : args 273 | end 274 | end 275 | 276 | # Returns configured wait time for given message type 277 | # 278 | # @param message [Krakow::Command] 279 | # @return [Numeric] seconds to wait 280 | def wait_time_for(message) 281 | case Command.response_for(message) 282 | when :required 283 | response_wait 284 | when :error_only 285 | error_wait 286 | else 287 | 0 288 | end 289 | end 290 | 291 | # @return [Hash] default settings for IDENTIFY 292 | def identify_defaults 293 | unless(@identify_defaults) 294 | @identify_defaults = { 295 | :short_id => Socket.gethostname, 296 | :long_id => Socket.gethostbyname(Socket.gethostname).flatten.compact.first, 297 | :user_agent => "krakow/#{Krakow::VERSION}", 298 | :feature_negotiation => true 299 | } 300 | end 301 | @identify_defaults 302 | end 303 | 304 | # IDENTIFY with server and negotiate features 305 | # 306 | # @return [TrueClass] 307 | def identify_and_negotiate 308 | expected_features = identify_defaults.merge(features) 309 | ident = Command::Identify.new( 310 | expected_features 311 | ) 312 | socket.put(ident.to_line) 313 | response = receive 314 | if(expected_features[:feature_negotiation]) 315 | begin 316 | @endpoint_settings = MultiJson.load(response.content, :symbolize_keys => true) 317 | info "Connection settings: #{endpoint_settings.inspect}" 318 | # Enable things we need to enable 319 | ENABLEABLE_FEATURES.each do |key| 320 | if(endpoint_settings[key]) 321 | send(key) 322 | elsif(enforce_features && expected_features[key]) 323 | abort Error::ConnectionFeatureFailure.new("Failed to enable #{key} feature on connection!") 324 | end 325 | end 326 | rescue MultiJson::LoadError => e 327 | error "Failed to parse response from Identify request: #{e} - #{response}" 328 | abort e 329 | end 330 | else 331 | @endpoint_settings = {} 332 | end 333 | true 334 | end 335 | 336 | # Send authentication request for connection 337 | # 338 | # @return [TrueClass] 339 | def auth_required 340 | info 'Authentication required for this connection' 341 | if(feature_args[:auth]) 342 | transmit(Command::Auth.new(:secret => feature_args[:auth])) 343 | response = receive 344 | true 345 | else 346 | error 'No authentication information provided for connection!' 347 | abort 'Authentication failure. No authentication secret provided' 348 | end 349 | end 350 | 351 | # Enable snappy feature on underlying socket 352 | # 353 | # @return [TrueClass] 354 | def snappy 355 | info 'Loading support for snappy compression and converting connection' 356 | @socket = ConnectionFeatures::SnappyFrames::Io.new(socket, features_args) 357 | response = receive 358 | info "Snappy connection conversion complete. Response: #{response.inspect}" 359 | true 360 | end 361 | 362 | # Enable deflate feature on underlying socket 363 | # 364 | # @return [TrueClass] 365 | def deflate 366 | debug 'Loading support for deflate compression and converting connection' 367 | @socket = ConnectionFeatures::Deflate::Io.new(socket, features_args) 368 | response = receive 369 | info "Deflate connection conversion complete. Response: #{response.inspect}" 370 | true 371 | end 372 | 373 | # Enable TLS feature on underlying socket 374 | # 375 | # @return [TrueClass] 376 | def tls_v1 377 | info 'Enabling TLS for connection' 378 | @socket = ConnectionFeatures::Ssl::Io.new(socket, features_args) 379 | response = receive 380 | info "TLS enable complete. Response: #{response.inspect}" 381 | true 382 | end 383 | 384 | # @return [TrueClass, FalseClass] underlying socket is connected 385 | def connected? 386 | begin 387 | !!(socket && socket.alive?) 388 | rescue Celluloid::DeadActorError 389 | false 390 | end 391 | end 392 | 393 | protected 394 | 395 | # Connect the underlying socket 396 | # 397 | # @return [nil] 398 | def connect! 399 | debug 'Initializing connection' 400 | unless(@connecting) 401 | @connecting = true 402 | if(socket && socket.alive?) 403 | socket.terminate 404 | @socket = nil 405 | end 406 | @socket = Ksocket.new(:host => host, :port => port) 407 | self.link socket 408 | socket.put version.rjust(4).upcase 409 | identify_and_negotiate 410 | info 'Connection initialized' 411 | @connecting = false 412 | end 413 | nil 414 | end 415 | 416 | end 417 | end 418 | -------------------------------------------------------------------------------- /lib/krakow/connection_features.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Features that wrap the connection 5 | module ConnectionFeatures 6 | autoload :SnappyFrames, 'krakow/connection_features/snappy_frames' 7 | autoload :Deflate, 'krakow/connection_features/deflate' 8 | autoload :Ssl, 'krakow/connection_features/ssl' 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/krakow/connection_features/deflate.rb: -------------------------------------------------------------------------------- 1 | require 'zlib' 2 | require 'krakow' 3 | 4 | module Krakow 5 | module ConnectionFeatures 6 | # Deflate functionality 7 | module Deflate 8 | # Deflatable IO 9 | class Io 10 | 11 | attr_reader :io, :buffer, :headers, :inflator, :deflator 12 | 13 | # Create new deflatable IO 14 | # 15 | # @param io [IO] IO to wrap 16 | # @return [Io] 17 | def initialize(io, args={}) 18 | @io = io 19 | @buffer = '' 20 | @inflator = Zlib::Inflate.new(-Zlib::MAX_WBITS) 21 | @deflator = Zlib::Deflate.new(nil, -Zlib::MAX_WBITS) 22 | end 23 | 24 | # Proxy to underlying socket 25 | # 26 | # @param args [Object] 27 | # @return [Object] 28 | def method_missing(*args) 29 | io.__send__(*args) 30 | end 31 | 32 | # Receive bytes from the IO 33 | # 34 | # @param n [Integer] nuber of bytes 35 | # @return [String] 36 | def recv(n) 37 | until(buffer.length >= n) 38 | read_stream 39 | sleep(0.1) unless buffer.length >= n 40 | end 41 | buffer.slice!(0, n) 42 | end 43 | alias_method :read, :recv 44 | 45 | # Read contents from stream 46 | # 47 | # @return [String] 48 | def read_stream 49 | str = io.read 50 | unless(str.empty?) 51 | buffer << inflator.inflate(str) 52 | end 53 | end 54 | 55 | # Write string to IO 56 | # 57 | # @param string [String] 58 | # @return [Integer] number of bytes written 59 | def write(string) 60 | unless(string.empty?) 61 | output = deflator.deflate(string) 62 | output << deflator.flush 63 | io.write(output) 64 | else 65 | 0 66 | end 67 | end 68 | 69 | # Close the IO 70 | # 71 | # @return [TrueClass] 72 | def close(*args) 73 | super 74 | deflator.deflate(nil, Zlib::FINISH) 75 | deflator.close 76 | true 77 | end 78 | 79 | end 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /lib/krakow/connection_features/snappy_frames.rb: -------------------------------------------------------------------------------- 1 | begin 2 | require 'snappy' 3 | rescue LoadError 4 | $stderr.puts 'ERROR: Failed to locate `snappy` gem. Install `snappy` gem into system or bundle.' 5 | raise 6 | end 7 | require 'digest/crc' 8 | require 'krakow' 9 | 10 | module Krakow 11 | module ConnectionFeatures 12 | # Snappy functionality 13 | # @todo Add support for max size + chunks 14 | # @todo Include support for remaining types 15 | module SnappyFrames 16 | # Snappy-able IO 17 | class Io 18 | 19 | # Header identifier 20 | IDENTIFIER = "\x73\x4e\x61\x50\x70\x59".force_encoding('ASCII-8BIT') 21 | ident_size = [IDENTIFIER.size].pack('L<') 22 | ident_size.slice!(-1,1) 23 | # Size of identifier 24 | IDENTIFIER_SIZE = ident_size 25 | 26 | # Mapping of types 27 | CHUNK_TYPE = { 28 | "\xff".force_encoding('ASCII-8BIT') => :identifier, 29 | "\x00".force_encoding('ASCII-8BIT') => :compressed, 30 | "\x01".force_encoding('ASCII-8BIT') => :uncompressed 31 | } 32 | 33 | attr_reader :io, :buffer 34 | 35 | # Create new snappy-able IO 36 | # 37 | # @param io [IO] IO to wrap 38 | # @return [Io] 39 | def initialize(io, args={}) 40 | @io = io 41 | @snappy_write_ident = false 42 | @buffer = '' 43 | end 44 | 45 | # Proxy to underlying socket 46 | # 47 | # @param args [Object] 48 | # @return [Object] 49 | def method_missing(*args) 50 | io.__send__(*args) 51 | end 52 | 53 | # Mask the checksum 54 | # 55 | # @param checksum [String] 56 | # @return [String] 57 | def checksum_mask(checksum) 58 | (((checksum >> 15) | (checksum << 17)) + 0xa282ead8) & 0xffffffff 59 | end 60 | 61 | # Receive bytes from the IO 62 | # 63 | # @param n [Integer] nuber of bytes 64 | # @return [String] 65 | def recv(n) 66 | read_stream unless buffer.size >= n 67 | result = buffer.slice!(0,n) 68 | result.empty? ? nil : result 69 | end 70 | alias_method :read, :recv 71 | 72 | # Read contents from stream 73 | # 74 | # @return [String] 75 | def read_stream 76 | header = io.recv(4) 77 | ident = CHUNK_TYPE[header.slice!(0)] 78 | size = (header << CHUNK_TYPE.key(:compressed)).unpack('L<').first 79 | content = io.recv(size) 80 | case ident 81 | when :identifier 82 | unless(content == IDENTIFIER) 83 | raise "Invalid stream identification encountered (content: #{content.inspect})" 84 | end 85 | read_stream 86 | when :compressed 87 | checksum = content.slice!(0, 4).unpack('L<').first 88 | deflated = Snappy.inflate(content) 89 | digest = Digest::CRC32c.new 90 | digest << deflated 91 | unless(checksum == checksum_mask(digest.checksum)) 92 | raise 'Checksum mismatch!' 93 | end 94 | buffer << deflated 95 | when :uncompressed 96 | buffer << content 97 | end 98 | end 99 | 100 | # Write string to IO 101 | # 102 | # @param string [String] 103 | # @return [Integer] number of bytes written 104 | def write(string) 105 | unless(@snappy_writer_ident) 106 | send_snappy_identifier 107 | end 108 | digest = Digest::CRC32c.new 109 | digest << string 110 | content = Snappy.deflate(string) 111 | size = content.length + 4 112 | size = [size].pack('L<') 113 | size.slice!(-1,1) 114 | checksum = [checksum_mask(digest.checksum)].pack('L<') 115 | output = [CHUNK_TYPE.key(:compressed), size, checksum, content].pack('a*a*a*a*') 116 | io.write output 117 | end 118 | 119 | # Send the identifier for snappy content 120 | # 121 | # @return [Integer] bytes written 122 | def send_snappy_identifier 123 | io.write [CHUNK_TYPE.key(:identifier), IDENTIFIER_SIZE, IDENTIFIER].pack('a*a*a*') 124 | end 125 | 126 | end 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/krakow/connection_features/ssl.rb: -------------------------------------------------------------------------------- 1 | require 'openssl' 2 | require 'krakow' 3 | 4 | module Krakow 5 | module ConnectionFeatures 6 | # SSL functionality 7 | module Ssl 8 | # SSL-able IO 9 | class Io 10 | 11 | attr_reader :_socket 12 | 13 | # Create new SSL-able IO 14 | # 15 | # @param io [IO] IO to wrap 16 | # @param args [Hash] 17 | # @option args [Hash] :ssl_context 18 | # @return [Io] 19 | def initialize(io, args={}) 20 | ssl_socket_arguments = [io] 21 | if(args[:ssl_context]) 22 | validate_ssl_args!(args[:ssl_context]) 23 | context = OpenSSL::SSL::SSLContext.new 24 | context.cert = OpenSSL::X509::Certificate.new(File.open(args[:ssl_context][:certificate])) 25 | context.key = OpenSSL::PKey::RSA.new(File.open(args[:ssl_context][:key])) 26 | ssl_socket_arguments << context 27 | end 28 | @_socket = Celluloid::IO::SSLSocket.new(*ssl_socket_arguments) 29 | _socket.sync = true 30 | _socket.connect 31 | end 32 | 33 | # Proxy to underlying socket 34 | # 35 | # @param args [Object] 36 | # @return [Object] 37 | def method_missing(*args) 38 | _socket.send(*args) 39 | end 40 | 41 | # Receive bytes from the IO 42 | # 43 | # @param len [Integer] nuber of bytes 44 | # @return [String] 45 | def recv(len) 46 | str = readpartial(len) 47 | if(len > str.length) 48 | str << sysread(len - str.length) 49 | end 50 | str 51 | end 52 | 53 | private 54 | 55 | # Validate the SSL configuration provided 56 | # 57 | # @param args [Hash] 58 | # @option args [String] :certificate path to certificate 59 | # @option args [String] :key path to key 60 | # @raise [ArgumentError, LoadError] 61 | def validate_ssl_args!(args) 62 | [:key, :certificate].each do |arg_key| 63 | unless(args.has_key?(arg_key)) 64 | raise ArgumentError.new "The `:ssl_context` option requires `#{arg_key.inspect}` to be set" 65 | end 66 | unless(File.readable?(args[arg_key])) 67 | raise LoadError.new "Unable to read the `#{arg_key.inspect}` file from the `:ssl_context` arguments" 68 | end 69 | end 70 | end 71 | 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/krakow/consumer.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Consume messages from a server 5 | class Consumer 6 | 7 | autoload :Queue, 'krakow/consumer/queue' 8 | 9 | include Utils::Lazy 10 | # @!parse include Krakow::Utils::Lazy::InstanceMethods 11 | # @!parse extend Krakow::Utils::Lazy::ClassMethods 12 | 13 | include Celluloid 14 | 15 | trap_exit :connection_failure 16 | finalizer :consumer_cleanup 17 | 18 | attr_reader :connections, :discovery, :distribution, :queue 19 | 20 | # @!group Attributes 21 | 22 | # @!macro [attach] attribute 23 | # @!method $1 24 | # @return [$2] the $1 $0 25 | # @!method $1? 26 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 27 | attribute :topic, String, :required => true 28 | attribute :channel, String, :required => true 29 | attribute :host, String 30 | attribute :port, [String, Integer] 31 | attribute :nsqlookupd, [Array, String] 32 | attribute :max_in_flight, Integer, :default => 1 33 | attribute :backoff_interval, Numeric 34 | attribute :discovery_interval, Numeric, :default => 30 35 | attribute :discovery_jitter, Numeric, :default => 10.0 36 | attribute :notifier, [Celluloid::Signals, Celluloid::Condition, Celluloid::Actor] 37 | attribute :connection_options, Hash, :default => ->{ Hash.new } 38 | 39 | # @!endgroup 40 | 41 | def initialize(args={}) 42 | super 43 | arguments[:connection_options] = {:features => {}, :config => {}}.merge( 44 | arguments[:connection_options] || {} 45 | ) 46 | @connections = {} 47 | @queue = Queue.new( 48 | current_actor, 49 | :removal_callback => :remove_message 50 | ) 51 | @distribution = Distribution::Default.new( 52 | :max_in_flight => max_in_flight, 53 | :backoff_interval => backoff_interval, 54 | :consumer => current_actor 55 | ) 56 | if(nsqlookupd) 57 | debug "Connections will be established via lookup #{nsqlookupd.inspect}" 58 | @discovery = Discovery.new(:nsqlookupd => nsqlookupd) 59 | discover 60 | elsif(host && port) 61 | direct_connect 62 | else 63 | abort Error::ConfigurationError.new('No connection information provided!') 64 | end 65 | end 66 | 67 | # @return [TrueClass, FalseClass] currently connected to at least 68 | # one nsqd 69 | def connected? 70 | !!connections.values.any? do |con| 71 | begin 72 | con.connected? 73 | rescue Celluloid::DeadActorError 74 | false 75 | end 76 | end 77 | end 78 | 79 | # Connect to nsqd instance directly 80 | # 81 | # @return [Connection] 82 | def direct_connect 83 | debug "Connection will be established via direct connection #{host}:#{port}" 84 | connection = build_connection(host, port, queue) 85 | if(register(connection)) 86 | info "Registered new connection #{connection}" 87 | distribution.redistribute! 88 | else 89 | abort Error::ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}") 90 | end 91 | connection 92 | end 93 | 94 | # Returns [Krakow::Connection] associated to key 95 | # 96 | # @param key [Object] identifier 97 | # @return [Krakow::Connection] associated connection 98 | def connection(key) 99 | @connections[key] 100 | end 101 | 102 | # @return [String] stringify object 103 | def to_s 104 | "<#{self.class.name}:#{object_id} T:#{topic} C:#{channel}>" 105 | end 106 | 107 | # Instance destructor 108 | # 109 | # @return [nil] 110 | def consumer_cleanup 111 | debug 'Tearing down consumer' 112 | if(distribution && distribution.alive?) 113 | distribution.terminate 114 | end 115 | if(queue && queue.alive?) 116 | queue.terminate 117 | end 118 | connections.values.each do |con| 119 | con.terminate if con.alive? 120 | end 121 | info 'Consumer torn down' 122 | nil 123 | end 124 | 125 | # Build a new [Krakow::Connection] 126 | # 127 | # @param host [String] remote host 128 | # @param port [String, Integer] remote port 129 | # @param queue [Queue] queue for messages 130 | # @return [Krakow::Connection, nil] new connection or nil 131 | def build_connection(host, port, queue) 132 | begin 133 | connection = Connection.new( 134 | :host => host, 135 | :port => port, 136 | :queue => queue, 137 | :topic => topic, 138 | :channel => channel, 139 | :notifier => notifier, 140 | :features => connection_options[:features], 141 | :features_args => connection_options[:config], 142 | :callbacks => { 143 | :handle => { 144 | :actor => current_actor, 145 | :method => :process_message 146 | } 147 | } 148 | ) 149 | queue.register_connection(connection) 150 | connection 151 | rescue => e 152 | error "Failed to build connection (host: #{host} port: #{port} queue: #{queue}) - #{e.class}: #{e}" 153 | debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}" 154 | nil 155 | end 156 | end 157 | 158 | # Process a given message if required 159 | # 160 | # @param message [Krakow::FrameType] 161 | # @param connection [Krakow::Connection] 162 | # @return [Krakow::FrameType] 163 | # @note If we receive a message that is already in flight, attempt 164 | # to scrub message from wait queue. If message is found, retry 165 | # distribution registration. If message is not found, assume it 166 | # is currently being processed and do not allow new message to 167 | # be queued 168 | def process_message(message, connection) 169 | discard = false 170 | if(message.is_a?(FrameType::Message)) 171 | message.origin = current_actor 172 | message.connection = connection 173 | retried = false 174 | begin 175 | distribution.register_message(message, connection.identifier) 176 | rescue KeyError => e 177 | if(!retried && queue.scrub_duplicate_message(message)) 178 | retried = true 179 | retry 180 | else 181 | error "Received message is currently in flight and not in wait queue. Discarding! (#{message})" 182 | discard = true 183 | end 184 | end 185 | end 186 | discard ? nil : message 187 | end 188 | 189 | # Send RDY for connection based on distribution rules 190 | # 191 | # @param connection [Krakow::Connection] 192 | # @return [nil] 193 | def update_ready!(connection) 194 | distribution.set_ready_for(connection) 195 | nil 196 | end 197 | 198 | # Initialize the consumer by starting lookup and adding connections 199 | # 200 | # @return [nil] 201 | def init! 202 | debug 'Running consumer `init!` connection builds' 203 | found = discovery.lookup(topic) 204 | debug "Discovery results: #{found.inspect}" 205 | connection = nil 206 | found.each do |node| 207 | debug "Processing discovery result: #{node.inspect}" 208 | key = Connection.identifier(node[:broadcast_address], node[:tcp_port], topic, channel) 209 | unless(connections[key]) 210 | connection = build_connection(node[:broadcast_address], node[:tcp_port], queue) 211 | info "Registered new connection #{connection}" if register(connection) 212 | else 213 | debug "Discovery result already registered: #{node.inspect}" 214 | end 215 | end 216 | distribution.redistribute! if connection 217 | nil 218 | end 219 | 220 | # Start the discovery interval lookup 221 | # 222 | # @return [nil] 223 | def discover 224 | init! 225 | after(discovery_interval + (discovery_jitter * rand)){ discover } 226 | end 227 | 228 | # Register connection with distribution 229 | # 230 | # @param connection [Krakow::Connection] 231 | # @return [TrueClass, FalseClass] true if subscription was successful 232 | def register(connection) 233 | begin 234 | connection.init! 235 | connection.transmit(Command::Sub.new(:topic_name => topic, :channel_name => channel)) 236 | self.link connection 237 | connections[connection.identifier] = connection 238 | distribution.add_connection(connection) 239 | true 240 | rescue Error::BadResponse => e 241 | debug "Failed to establish connection: #{e.result ? e.result.error : ''}" 242 | connection.terminate 243 | false 244 | end 245 | end 246 | 247 | # Remove connection references when connection is terminated 248 | # 249 | # @param actor [Object] terminated actor 250 | # @param reason [Exception] reason for termination 251 | # @return [nil] 252 | def connection_failure(actor, reason) 253 | if(reason && key = connections.key(actor)) 254 | warn "Connection failure detected. Removing connection: #{key} - #{reason}" 255 | connections.delete(key) 256 | begin 257 | distribution.remove_connection(key) 258 | rescue Error::ConnectionUnavailable, Error::ConnectionFailure 259 | warn 'Caught connection unavailability' 260 | end 261 | queue.deregister_connection(key) 262 | distribution.redistribute! 263 | direct_connect unless discovery 264 | end 265 | nil 266 | end 267 | 268 | # Remove message 269 | # 270 | # @param messages [Array] 271 | # @return [NilClass] 272 | # @note used mainly for queue callback 273 | def remove_message(messages) 274 | [messages].flatten.compact.each do |msg| 275 | distribution.unregister_message(msg.message_id) 276 | update_ready!(msg.connection) 277 | end 278 | nil 279 | end 280 | 281 | # Confirm message has been processed 282 | # 283 | # @param message_id [String, Krakow::FrameType::Message] 284 | # @return [TrueClass] 285 | # @raise [KeyError] connection not found 286 | def confirm(message_id) 287 | message_id = message_id.message_id if message_id.respond_to?(:message_id) 288 | begin 289 | begin 290 | connection = distribution.in_flight_lookup(message_id) 291 | connection.transmit(Command::Fin.new(:message_id => message_id)) 292 | distribution.success(connection.identifier) 293 | rescue => e 294 | abort e 295 | end 296 | true 297 | rescue KeyError => e 298 | error "Message confirmation failed: #{e}" 299 | abort e 300 | rescue Error::LookupFailed => e 301 | error "Lookup of message for confirmation failed! " 302 | abort e 303 | rescue Error::ConnectionUnavailable => e 304 | abort e 305 | rescue Celluloid::DeadActorError 306 | abort Error::ConnectionUnavailable.new 307 | ensure 308 | con = distribution.unregister_message(message_id) 309 | update_ready!(con) if con 310 | end 311 | end 312 | alias_method :finish, :confirm 313 | 314 | # Requeue message (generally due to processing failure) 315 | # 316 | # @param message_id [String, Krakow::FrameType::Message] 317 | # @param timeout [Numeric] 318 | # @return [TrueClass] 319 | def requeue(message_id, timeout=0) 320 | message_id = message_id.message_id if message_id.respond_to?(:message_id) 321 | distribution.in_flight_lookup(message_id) do |connection| 322 | distribution.unregister_message(message_id) 323 | connection.transmit( 324 | Command::Req.new( 325 | :message_id => message_id, 326 | :timeout => timeout 327 | ) 328 | ) 329 | distribution.failure(connection.identifier) 330 | update_ready!(connection) 331 | end 332 | true 333 | end 334 | 335 | # Touch message (to extend timeout) 336 | # 337 | # @param message_id [String, Krakow::FrameType::Message] 338 | # @return [TrueClass] 339 | def touch(message_id) 340 | message_id = message_id.message_id if message_id.respond_to?(:message_id) 341 | begin 342 | distribution.in_flight_lookup(message_id) do |connection| 343 | connection.transmit( 344 | Command::Touch.new(:message_id => message_id) 345 | ) 346 | end 347 | true 348 | rescue Error::LookupFailed => e 349 | error "Lookup of message for touch failed! " 350 | abort e 351 | end 352 | end 353 | 354 | end 355 | end 356 | -------------------------------------------------------------------------------- /lib/krakow/consumer/queue.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Consume messages from a server 5 | class Consumer 6 | 7 | class Queue 8 | 9 | include Celluloid 10 | include Utils::Lazy 11 | 12 | # @return [Consumer] 13 | attr_reader :consumer 14 | # @return [Array] order of message removal 15 | attr_reader :pop_order 16 | # @return [Symbol] callback method name 17 | attr_reader :removal_callback 18 | 19 | # Create new consumer queue instance 20 | # 21 | # @param consumer [Consumer] 22 | # @return [self] 23 | def initialize(consumer, *args) 24 | opts = args.detect{|x| x.is_a?(Hash)} 25 | @consumer = consumer 26 | @removal_callback = opts[:removal_callback] 27 | @messages = {} 28 | @pop_order = [] 29 | @cleaner = nil 30 | end 31 | 32 | # Message container 33 | # 34 | # @yieldparam [Hash] messages 35 | # @return [Hash] messages or block result 36 | def messages 37 | if(block_given?) 38 | yield @messages 39 | else 40 | @messages 41 | end 42 | end 43 | 44 | # Register a new connection 45 | # 46 | # @param connection [Connection] 47 | # @return [TrueClass] 48 | def register_connection(connection) 49 | messages do |collection| 50 | collection[connection.identifier] = [] 51 | end 52 | true 53 | end 54 | 55 | # Remove connection registration and remove all messages 56 | # 57 | # @param identifier [String] connection identifier 58 | # @return [Array] messages queued for deregistered connection 59 | def deregister_connection(identifier) 60 | messages do |collection| 61 | removed = collection.delete(identifier) 62 | pop_order.delete(identifier) 63 | removed 64 | end 65 | end 66 | 67 | # Push new message into queue 68 | # 69 | # @param message [FrameType::Message] 70 | # @return [self] 71 | def push(message) 72 | unless(message.is_a?(FrameType::Message)) 73 | abort TypeError.new "Expecting `FrameType::Message` but received `#{message.class}`!" 74 | end 75 | messages do |collection| 76 | begin 77 | collection[message.connection.identifier] << message 78 | pop_order << message.connection.identifier 79 | rescue Celluloid::DeadActorError 80 | abort Error::ConnectionUnavailable.new 81 | end 82 | end 83 | signal(:new_message) 84 | current_actor 85 | end 86 | alias_method :<<, :push 87 | alias_method :enq, :push 88 | 89 | # Pop first item off the queue 90 | # 91 | # @return [Object] 92 | def pop 93 | message = nil 94 | until(message) 95 | wait(:new_message) if pop_order.empty? 96 | messages do |collection| 97 | key = pop_order.shift 98 | if(key) 99 | message = collection[key].shift 100 | message = validate_message(message) 101 | end 102 | end 103 | end 104 | message 105 | end 106 | alias_method :deq, :pop 107 | 108 | # @return [Integer] number of queued messages 109 | def size 110 | messages do |collection| 111 | collection.values.map(&:size).inject(&:+) 112 | end 113 | end 114 | 115 | # Remove duplicate message from queue if possible 116 | # 117 | # @param message [FrameType::Message] 118 | # @return [TrueClass, FalseClass] 119 | def scrub_duplicate_message(message) 120 | messages do |collection| 121 | idx = collection[message.connection.identifier].index do |msg| 122 | msg.message_id == message.message_id 123 | end 124 | if(idx) 125 | msg = collection[message.connection.identifier].delete_at(idx) 126 | if(removal_callback) 127 | consumer.send(removal_callback, [message]) 128 | end 129 | true 130 | else 131 | false 132 | end 133 | end 134 | end 135 | 136 | # Validate message 137 | def validate_message(message) 138 | if(message.instance_stamp > message.instance_stamp + (message.connection.endpoint_settings[:msg_timeout] / 1000.0)) 139 | warn "Message exceeded timeout! Discarding. (#{message})" 140 | if(removal_callback) 141 | consumer.send(removal_callback, [message]) 142 | end 143 | nil 144 | else 145 | message 146 | end 147 | end 148 | 149 | end 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /lib/krakow/discovery.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'http' 3 | require 'multi_json' 4 | require 'krakow' 5 | 6 | module Krakow 7 | 8 | # Provides queue topic discovery 9 | class Discovery 10 | 11 | include Utils::Lazy 12 | 13 | # @!group Attributes 14 | 15 | # @!macro [attach] attribute 16 | # @!method $1 17 | # @return [$2] the $1 $0 18 | # @!method $1? 19 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 20 | attribute :nsqlookupd, [Array, String], :required => true 21 | 22 | # @!endgroup 23 | 24 | # Get list of end points with given topic name available 25 | # 26 | # @param topic [String] topic name 27 | # @return [Array] 28 | def lookup(topic) 29 | result = [nsqlookupd].flatten.map do |location| 30 | uri = URI.parse(location) 31 | uri.path = '/lookup' 32 | uri.query = "topic=#{topic}&ts=#{Time.now.to_i}" 33 | begin 34 | debug "Requesting lookup for topic #{topic} - #{uri}" 35 | content = HTTP.with(:accept => 'application/octet-stream').get(uri.to_s) 36 | unless(content.respond_to?(:to_hash)) 37 | data = MultiJson.load(content.to_s) 38 | else 39 | data = content.to_hash 40 | end 41 | debug "Lookup response (#{uri.to_s}): #{data.inspect}" 42 | if(data['data'] && data['data']['producers']) 43 | data['data']['producers'].map do |producer| 44 | Hash[*producer.map{|k,v| [k.to_sym, v]}.flatten] 45 | end 46 | end 47 | rescue => e 48 | warn "Lookup exception encountered: #{e.class.name} - #{e}" 49 | nil 50 | end 51 | end.compact.flatten(1).uniq 52 | debug "Discovery lookup result: #{result.inspect}" 53 | result 54 | end 55 | 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/krakow/distribution.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Message distribution 5 | # @abstract 6 | class Distribution 7 | 8 | autoload :Default, 'krakow/distribution/default' 9 | # autoload :ProducerWeighted, 'krakow/distribution/producer_weighted' 10 | # autoload :ConsumerWeighted, 'krakow/distribution/consumer_weighted' 11 | 12 | include Celluloid 13 | include Utils::Lazy 14 | # @!parse include Krakow::Utils::Lazy::InstanceMethods 15 | # @!parse extend Krakow::Utils::Lazy::ClassMethods 16 | 17 | attr_accessor :ideal, :flight_record, :registry 18 | 19 | # @!group Attributes 20 | 21 | # @!macro [attach] attribute 22 | # @!method $1 23 | # @return [$2] the $1 $0 24 | # @!method $1? 25 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 26 | attribute :consumer, Krakow::Consumer, :required => true 27 | attribute :watch_dog_interval, Numeric, :default => 1.0 28 | attribute :backoff_interval, Numeric 29 | attribute :max_in_flight, Integer, :default => 1 30 | 31 | # @!endgroup 32 | 33 | def initialize(args={}) 34 | super 35 | @ideal = 0 36 | @flight_record = {} 37 | @registry = {} 38 | end 39 | 40 | # [Abstract] Reset flight distributions 41 | def redistribute! 42 | raise NotImplementedError.new 'Custom `#redistrubute!` method must be provided!' 43 | end 44 | 45 | # [Abstract] Determine RDY value for given connection 46 | # @param connection_identifier [String] 47 | # @return [Integer] 48 | def calculate_ready!(connection_identifier) 49 | raise NotImplementedError.new 'Custom `#calculate_ready!` method must be provided!' 50 | end 51 | 52 | # Remove message metadata from registry 53 | # 54 | # @param message [Krakow::FrameType::Message, String] message or ID 55 | # @return [Krakow::Connection, NilClass] 56 | def unregister_message(message) 57 | msg_id = message.respond_to?(:message_id) ? message.message_id : message.to_s 58 | connection = connection_lookup(flight_record[msg_id]) 59 | flight_record.delete(msg_id) 60 | if(connection) 61 | begin 62 | ident = connection.identifier 63 | registry_info = registry_lookup(ident) 64 | registry_info[:in_flight] -= 1 65 | calculate_ready!(ident) 66 | connection 67 | rescue Celluloid::DeadActorError 68 | warn 'Connection is dead. No recalculation applied on ready.' 69 | end 70 | else 71 | warn 'No connection associated to message via lookup. No recalculation applied on ready.' 72 | end 73 | end 74 | 75 | # Return the currently configured RDY value for given connnection 76 | # 77 | # @param connection_identifier [String] 78 | # @return [Integer] 79 | def ready_for(connection_identifier) 80 | registry_lookup(connection_identifier)[:ready] 81 | end 82 | 83 | 84 | # Send RDY for given connection 85 | # 86 | # @param connection [Krakow::Connection] 87 | # @return [Krakow::FrameType::Error,nil] 88 | def set_ready_for(connection, *_) 89 | connection.transmit( 90 | Command::Rdy.new( 91 | :count => ready_for(connection.identifier) 92 | ) 93 | ) 94 | end 95 | 96 | # Initial ready value used for new connections 97 | # 98 | # @return [Integer] 99 | def initial_ready 100 | ideal > 0 ? 1 : 0 101 | end 102 | 103 | # Registers message into registry and configures for distribution 104 | # 105 | # @param message [FrameType::Message] 106 | # @param connection_identifier [String] 107 | # @return [Integer] 108 | def register_message(message, connection_identifier) 109 | if(flight_record[message.message_id]) 110 | abort KeyError.new "Message is already registered in flight record! (#{message.message_id})" 111 | else 112 | registry_info = registry_lookup(connection_identifier) 113 | registry_info[:in_flight] += 1 114 | flight_record[message.message_id] = connection_identifier 115 | calculate_ready!(connection_identifier) 116 | end 117 | end 118 | 119 | # Add connection to make available for RDY distribution 120 | # 121 | # @param connection [Krakow::Connection] 122 | # @return [TrueClass] 123 | def add_connection(connection) 124 | unless(registry[connection.identifier]) 125 | registry[connection.identifier] = { 126 | :ready => initial_ready, 127 | :in_flight => 0, 128 | :failures => 0, 129 | :backoff_until => 0 130 | } 131 | end 132 | true 133 | end 134 | 135 | # Remove connection from RDY distribution 136 | # 137 | # @param connection_identifier [String] 138 | # @return [TrueClass] 139 | def remove_connection(connection_identifier, *args) 140 | # remove connection from registry 141 | registry.delete(connection_identifier) 142 | # remove any in flight messages 143 | flight_record.delete_if do |k,v| 144 | if(v == connection_identifier) 145 | warn "Removing in flight reference due to failed connection: #{v}" 146 | true 147 | end 148 | end 149 | true 150 | end 151 | 152 | # Return connection associated with given registry key 153 | # 154 | # @param identifier [String] connection identifier 155 | # @return [Krakow::Connection, nil] 156 | def connection_lookup(identifier) 157 | consumer.connection(identifier) 158 | end 159 | 160 | # Return source connection for given message ID 161 | # 162 | # @param msg_id [String] 163 | # @yield execute with connection 164 | # @yieldparam connection [Krakow::Connection] 165 | # @return [Krakow::Connection, Object] 166 | def in_flight_lookup(msg_id) 167 | connection = connection_lookup(flight_record[msg_id]) 168 | unless(connection) 169 | abort Krakow::Error::LookupFailed.new("Failed to locate in flight message (ID: #{msg_id})") 170 | end 171 | if(block_given?) 172 | begin 173 | yield connection 174 | rescue => e 175 | abort e 176 | end 177 | else 178 | connection 179 | end 180 | end 181 | 182 | # Return registry information for given connection 183 | # @param connection_identifier [String] 184 | # @return [Hash] registry information 185 | # @raise [Krakow::Error::LookupFailed] 186 | def registry_lookup(connection_identifier) 187 | registry[connection_identifier] || 188 | abort(Krakow::Error::LookupFailed.new("Failed to locate connection information in registry (#{connection_identifier})")) 189 | end 190 | 191 | # @return [Array] connections in registry 192 | def connections 193 | registry.keys.map do |identifier| 194 | connection_lookup(identifier) 195 | end.compact 196 | end 197 | 198 | # Log failure of processed message 199 | # 200 | # @param connection_identifier [String] 201 | # @return [TrueClass] 202 | def failure(connection_identifier) 203 | if(backoff_interval) 204 | registry_info = registry_lookup(connection_identifier) 205 | registry_info[:failures] += 1 206 | registry_info[:backoff_until] = Time.now.to_i + (registry_info[:failures] * backoff_interval) 207 | end 208 | true 209 | end 210 | 211 | # Log success of processed message 212 | # 213 | # @param connection_identifier [String] 214 | # @return [TrueClass] 215 | def success(connection_identifier) 216 | if(backoff_interval) 217 | registry_info = registry_lookup(connection_identifier) 218 | if(registry_info[:failures] > 1) 219 | registry_info[:failures] -= 1 220 | registry_info[:backoff_until] = Time.now.to_i + (registry_info[:failures] * backoff_interval) 221 | else 222 | registry_info[:failures] = 0 223 | end 224 | end 225 | true 226 | end 227 | 228 | end 229 | end 230 | -------------------------------------------------------------------------------- /lib/krakow/distribution/default.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class Distribution 5 | # Default distribution implementation. This uses a round-robin 6 | # approach for less than ideal states. 7 | class Default < Distribution 8 | 9 | attr_reader :less_than_ideal_stack, :watch_dog 10 | 11 | # recalculate `ideal` and update RDY on connections 12 | def redistribute! 13 | @ideal = registry.size < 1 ? 0 : max_in_flight / registry.size 14 | debug "Distribution calculated ideal: #{ideal}" 15 | if(less_than_ideal?) 16 | registry.each do |connection_id, reg_info| 17 | reg_info[:ready] = 0 18 | end 19 | max_in_flight.times do 20 | less_than_ideal_ready! 21 | end 22 | connections.each do |connection| 23 | set_ready_for(connection, :force) 24 | end 25 | watch_dog.cancel if watch_dog 26 | @watch_dog = every(watch_dog_interval) do 27 | force_unready 28 | end 29 | else 30 | if(watch_dog) 31 | watch_dog.cancel 32 | @watch_dog = nil 33 | end 34 | connections.each do |connection| 35 | current_ready = ready_for(connection.identifier) 36 | calculate_ready!(connection.identifier) 37 | unless(current_ready == ready_for(connection.identifier)) 38 | debug "Redistribution ready setting update for connection #{connection}" 39 | set_ready_for(connection) 40 | end 41 | end 42 | end 43 | end 44 | 45 | # Is ideal less than 1 46 | # 47 | # @return [TrueClass, FalseClass] 48 | def less_than_ideal? 49 | ideal < 1 50 | end 51 | 52 | # Find next connection to receive RDY count 53 | # 54 | # @return [Krakow::Connection, nil] 55 | def less_than_ideal_ready! 56 | admit_defeat = false 57 | connection = nil 58 | until(connection || (admit_defeat && less_than_ideal_stack.empty?)) 59 | if(less_than_ideal_stack.nil? || less_than_ideal_stack.empty?) 60 | @less_than_ideal_stack = waiting_connections 61 | admit_defeat = true 62 | end 63 | con = less_than_ideal_stack.pop 64 | if(con) 65 | unless(registry_lookup(con.identifier)[:backoff_until] > Time.now.to_i) 66 | connection = con 67 | end 68 | end 69 | end 70 | if(connection) 71 | registry_lookup(connection.identifier)[:ready] = 1 72 | connection 73 | end 74 | end 75 | 76 | # Adds extra functionality to provide round robin RDY setting 77 | # when in less than ideal state 78 | # 79 | # @param connection [Krakow::Connection] 80 | # @param args [Symbol] 81 | # @return [Krakow::FrameType::Error, nil] 82 | def set_ready_for(connection, *args) 83 | super connection 84 | if(less_than_ideal? && !args.include?(:force)) 85 | debug "RDY set ignored due to less than ideal state (con: #{connection})" 86 | con = less_than_ideal_ready! 87 | if(con) 88 | watch_dog.reset if watch_dog 89 | super con 90 | else 91 | warn 'Failed to set RDY state while less than ideal. Connection stack is empty!' 92 | end 93 | end 94 | end 95 | 96 | # Update connection ready count 97 | # @param connection_identifier [String] 98 | # @return [Integer, nil] 99 | def calculate_ready!(connection_identifier) 100 | begin 101 | registry_info = registry_lookup(connection_identifier) 102 | unless(less_than_ideal?) 103 | registry_info[:ready] = ideal - registry_info[:in_flight] 104 | if(registry_info[:ready] < 0 || registry_info[:backoff_until] > Time.now.to_i) 105 | registry_info[:ready] = 0 106 | registry_info[:backoff_timer].cancel if registry[:backoff_timer] 107 | registry_info[:backoff_timer] = after(registry_info[:backoff_until] - Time.now.to_i) do 108 | calculate_ready!(connection_identifier) 109 | set_ready_for(connection_lookup(connection_identifier)) unless less_than_ideal? 110 | end 111 | end 112 | registry_info[:ready] 113 | else 114 | registry_info[:ready] = 0 115 | end 116 | rescue Error::ConnectionFailure 117 | warn 'Failed connection encountered!' 118 | rescue Error::ConnectionUnavailable 119 | warn 'Unavailable connection encountered!' 120 | end 121 | end 122 | 123 | # All connections without RDY state 124 | # 125 | # @return [Array] 126 | def waiting_connections 127 | registry.find_all do |conn_id, info| 128 | info[:ready] < 1 && info[:in_flight] < 1 && info[:backoff_until] < Time.now.to_i 129 | end.map{|conn_id, info| connection_lookup(conn_id) }.compact 130 | end 131 | 132 | # All connections with RDY state 133 | # 134 | # @return [Array] 135 | def rdy_connections 136 | registry.find_all do |conn_id, info| 137 | info[:ready] > 0 138 | end.map{|conn_id, info| connection_lookup(conn_id) }.compact 139 | end 140 | 141 | # Force a connection to give up RDY state so next in stack can receive 142 | # 143 | # @return [nil] 144 | def force_unready 145 | debug 'Forcing a connection into an unready state due to less than ideal state' 146 | connection = rdy_connections.shuffle.first 147 | if(connection) 148 | debug "Stripping RDY state from connection: #{connection}" 149 | calculate_ready!(connection.identifier) 150 | set_ready_for(connection) 151 | else 152 | warn "Failed to locate available connection for RDY aquisition!" 153 | end 154 | nil 155 | end 156 | 157 | end 158 | end 159 | end 160 | -------------------------------------------------------------------------------- /lib/krakow/exceptions.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Base error type 5 | class Error < StandardError 6 | 7 | # Failed to enable required feature on connection 8 | class ConnectionFeatureFailure < Error; end 9 | # Failed to perform lookup (not found) 10 | class LookupFailed < Error; end 11 | # Connection has failed 12 | class ConnectionFailure < Error; end 13 | # Configuration is not in valid state 14 | class ConfigurationError < Error; end 15 | # Connection is temporarily unavailable 16 | class ConnectionUnavailable < Error; end 17 | # Consumer was not set 18 | class OriginNotFound < Error; end 19 | 20 | # Invalid response 21 | class BadResponse < Error 22 | # @return [Response] error response 23 | attr_accessor :result 24 | # No response received 25 | class NoResponse < BadResponse 26 | end 27 | end 28 | 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/krakow/frame_type.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Received message 5 | # @abstract 6 | class FrameType 7 | 8 | autoload :Error, 'krakow/frame_type/error' 9 | autoload :Message, 'krakow/frame_type/message' 10 | autoload :Response, 'krakow/frame_type/response' 11 | 12 | include Utils::Lazy 13 | # @!parse include Krakow::Utils::Lazy::InstanceMethods 14 | # @!parse extend Krakow::Utils::Lazy::ClassMethods 15 | 16 | # Registered frame types 17 | FRAME_TYPE_MAP = [ 18 | FrameType::Response, 19 | FrameType::Error, 20 | FrameType::Message 21 | ] 22 | # Size bytes 23 | SIZE_BYTES = 4 24 | 25 | class << self 26 | 27 | # Information about incoming frame 28 | # @param bytes [String] 29 | # @return [Hash] 30 | def decode(bytes) 31 | size, type = bytes.unpack('l>l>') 32 | {:size => size - SIZE_BYTES, :type => type} 33 | end 34 | 35 | # Build proper FrameType instance based on args 36 | # @param args [Hash] 37 | # @option args [FrameType] :type class of frame 38 | # @option args [String] :data 39 | # @option args [Integer] :size 40 | # @return [FrameType] 41 | def build(args={}) 42 | klass = FRAME_TYPE_MAP[args[:type].to_i] 43 | if(klass == FrameType::Response) 44 | klass.new(:response => args[:data]) 45 | elsif(klass == FrameType::Error) 46 | klass.new(:error => args[:data]) 47 | elsif(klass == FrameType::Message) 48 | unpacked = args[:data].unpack("Q>s>a16a#{args[:size]}") 49 | klass.new( 50 | Hash[*([:timestamp, :attempts, :message_id, :message].zip(unpacked).flatten)] 51 | ) 52 | else 53 | raise TypeError.new "Unknown frame type received: #{args[:type].inspect} - #{klass.inspect}" 54 | end 55 | end 56 | end 57 | 58 | # Content of message 59 | # 60 | # @return [String] 61 | def content 62 | raise NotImplementedError.new 'Content method not properly defined!' 63 | end 64 | 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/krakow/frame_type/error.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class FrameType 5 | # Error from server 6 | class Error < FrameType 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :error, String, :required => true 16 | 17 | # @!endgroup 18 | 19 | # @return [String] content of error 20 | def content 21 | error 22 | end 23 | 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/krakow/frame_type/message.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class FrameType 5 | # Message received from server 6 | class Message < FrameType 7 | 8 | # @return [Float] time of message instance creation 9 | attr_reader :instance_stamp 10 | attr_accessor :origin, :connection 11 | 12 | # @!group Attributes 13 | 14 | # @!macro [attach] attribute 15 | # @!method $1 16 | # @return [$2] the $1 $0 17 | # @!method $1? 18 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 19 | attribute :attempts, Integer, :required => true 20 | attribute :timestamp, Integer, :required => true 21 | attribute :message_id, String, :required => true 22 | attribute :message, String, :required => true 23 | 24 | # @!endgroup 25 | 26 | def initialize(*args) 27 | super 28 | @instance_stamp = Time.now.to_f 29 | end 30 | 31 | # Message content 32 | # 33 | # @return [String] 34 | def content 35 | message 36 | end 37 | 38 | # @return [Krakow::Consumer] 39 | def origin 40 | unless(@origin) 41 | error 'No origin has been specified for this message' 42 | abort Krakow::Error::OriginNotFound.new('No origin specified for this message') 43 | end 44 | @origin 45 | end 46 | 47 | # @return [Krakow::Connection] 48 | def connection 49 | unless(@connection) 50 | error 'No origin connection has been specified for this message' 51 | abort Krakow::Error::ConnectionNotFound.new('No connection specified for this message') 52 | end 53 | @connection 54 | end 55 | 56 | # Proxy to [Krakow::Consumer#confirm] 57 | def confirm(*args) 58 | origin.confirm(*[self, *args].compact) 59 | end 60 | alias_method :finish, :confirm 61 | 62 | # Proxy to [Krakow::Consumer#requeue] 63 | def requeue(*args) 64 | origin.requeue(*[self, *args].compact) 65 | end 66 | 67 | # Proxy to [Krakow::Consumer#touch] 68 | def touch(*args) 69 | origin.touch(*[self, *args].compact) 70 | end 71 | 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/krakow/frame_type/response.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | class FrameType 5 | # Response from server 6 | class Response < FrameType 7 | 8 | # @!group Attributes 9 | 10 | # @!macro [attach] attribute 11 | # @!method $1 12 | # @return [$2] the $1 $0 13 | # @!method $1? 14 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 15 | attribute :response, String, :required => true 16 | 17 | # @!endgroup 18 | 19 | # @return [String] content of response 20 | def content 21 | response 22 | end 23 | 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/krakow/ksocket.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | require 'socket' 3 | 4 | module Krakow 5 | class Ksocket 6 | 7 | include Utils::Lazy 8 | include Celluloid 9 | 10 | # @return [String] 11 | attr_reader :buffer 12 | 13 | finalizer :closedown_socket 14 | 15 | # Teardown helper 16 | def closedown_socket 17 | @writing = @reading = false 18 | if(socket && !socket.closed?) 19 | socket.close 20 | end 21 | end 22 | 23 | # Create new socket wrapper 24 | # 25 | # @param args [Hash] 26 | # @option args [Socket-ish] :socket 27 | # @option args [String] :host 28 | # @option args [Integer] :port 29 | # @return [self] 30 | def initialize(args={}) 31 | if(args[:socket]) 32 | @socket = args[:socket] 33 | else 34 | unless([:host, :port].all?{|k| args.include?(k)}) 35 | raise ArgumentError.new 'Missing required arguments. Expecting `:socket` or `:host` and `:port`.' 36 | end 37 | @socket = TCPSocket.new(args[:host], args[:port]) 38 | end 39 | @buffer = '' 40 | async.read_loop 41 | end 42 | 43 | # @return [TrueClass, FalseClass] read loop enabled 44 | def reading? 45 | !!@reading 46 | end 47 | 48 | # Read from socket and push into local Queue 49 | def read_loop 50 | unless(reading?) 51 | @reading = true 52 | while(reading?) 53 | res = defer do 54 | Kernel.select([socket], nil, nil, nil) 55 | socket{|s| s.readpartial(1024)} 56 | end 57 | if(res) 58 | debug "Received content from socket: #{res.inspect}" 59 | buffer << res 60 | signal(:content_read) 61 | else 62 | debug 'No content received from socket read. Ignoring.' 63 | end 64 | end 65 | end 66 | end 67 | 68 | # Fetch bytes from socket 69 | # 70 | # @param n [Integer] 71 | # @return [String] 72 | def get(n) 73 | until(buffer.length >= n) 74 | wait(:content_read) 75 | end 76 | buffer.slice!(0, n) 77 | end 78 | alias_method :recv, :get 79 | alias_method :read, :get 80 | alias_method :sysread, :get 81 | alias_method :readpartial, :get 82 | 83 | # Push bytes to socket 84 | # 85 | # @param line [String] 86 | # @return [Integer] 87 | def put(line) 88 | socket{|s| s.write(line)} 89 | end 90 | alias_method :write, :put 91 | 92 | # @return [Socket] 93 | def socket 94 | if(block_given?) 95 | yield @socket 96 | else 97 | @socket 98 | end 99 | end 100 | 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/krakow/producer.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | 5 | # TCP based producer 6 | class Producer 7 | 8 | autoload :Http, 'krakow/producer/http' 9 | 10 | include Utils::Lazy 11 | # @!parse include Utils::Lazy::InstanceMethods 12 | # @!parse extend Utils::Lazy::ClassMethods 13 | 14 | include Celluloid 15 | 16 | trap_exit :connection_failure 17 | finalizer :producer_cleanup 18 | 19 | # set exclusive methods 20 | exclusive :write 21 | 22 | attr_reader :connection 23 | attr_reader :notifier 24 | 25 | # @!group Attributes 26 | 27 | # @!macro [attach] attribute 28 | # @!method $1 29 | # @return [$2] the $1 $0 30 | # @!method $1? 31 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 32 | attribute :host, String, :required => true 33 | attribute :port, [String, Integer], :required => true 34 | attribute :topic, String, :required => true 35 | attribute :reconnect_retries, Integer, :default => 10 36 | attribute :reconnect_interval, Integer, :default => 5 37 | attribute :connection_options, Hash, :default => ->{ Hash.new } 38 | 39 | # @!endgroup 40 | 41 | def initialize(args={}) 42 | super 43 | arguments[:connection_options] = {:features => {}, :config => {}, :options => {}}.merge( 44 | arguments.fetch(:connection_options, {}) 45 | ) 46 | connect 47 | end 48 | 49 | # Establish connection to configured `host` and `port` 50 | # 51 | # @return nil 52 | def connect 53 | @connecting = true 54 | info "Establishing connection to: #{host}:#{port}" 55 | begin 56 | con_args = connection_options[:options].dup.tap do |args| 57 | args[:host] = host 58 | args[:port] = port 59 | if(connection_options[:features]) 60 | args[:features] = connection_options[:features] 61 | end 62 | if(connection_options[:config]) 63 | args[:features_args] = connection_options[:config] 64 | end 65 | end 66 | @connection = Connection.new(con_args) 67 | @connection.init! 68 | self.link @connection 69 | info "Connection established: #{@connection}" 70 | nil 71 | rescue => e 72 | abort e 73 | end 74 | @connecting = false 75 | end 76 | 77 | # @return [String] stringify object 78 | def to_s 79 | "<#{self.class.name}:#{object_id} {#{host}:#{port}} T:#{topic}>" 80 | end 81 | 82 | # @return [TrueClass, FalseClass] currently connected to server 83 | def connected? 84 | begin 85 | !!(!@connecting && 86 | connection && 87 | connection.alive? && 88 | connection.connected?) 89 | rescue Celluloid::DeadActorError 90 | false 91 | end 92 | end 93 | 94 | # Process connection failure and attempt reconnection 95 | # 96 | # @return [TrueClass] 97 | def connection_failure(obj, reason) 98 | if(obj == connection && !reason.nil?) 99 | begin 100 | @connection = nil 101 | warn "Connection failure detected for #{host}:#{port} - #{reason}" 102 | obj.terminate if obj.alive? 103 | connect 104 | rescue => reason 105 | warn "Failed to establish connection to #{host}:#{port}. Pausing #{reconnect_interval} before retry" 106 | sleep reconnect_interval 107 | retry 108 | end 109 | end 110 | true 111 | end 112 | 113 | # Instance destructor 114 | # @return nil 115 | def producer_cleanup 116 | debug 'Tearing down producer' 117 | if(connection && connection.alive?) 118 | connection.terminate 119 | end 120 | @connection = nil 121 | info 'Producer torn down' 122 | nil 123 | end 124 | 125 | # Write message to server 126 | # 127 | # @param message [String] message to write 128 | # @return [Krakow::FrameType, TrueClass] 129 | # @note if connection response wait is set to 0, writes will 130 | # return a `true` value on completion 131 | # @raise [Krakow::Error::ConnectionUnavailable] 132 | def write(*message) 133 | if(message.empty?) 134 | abort ArgumentError.new 'Expecting one or more messages to send. None provided.' 135 | end 136 | begin 137 | if(message.size > 1) 138 | debug 'Multiple message publish' 139 | connection.transmit( 140 | Command::Mpub.new( 141 | :topic_name => topic, 142 | :messages => message 143 | ) 144 | ) 145 | else 146 | debug 'Single message publish' 147 | connection.transmit( 148 | Command::Pub.new( 149 | :message => message.first, 150 | :topic_name => topic 151 | ) 152 | ) 153 | end 154 | rescue Celluloid::Task::TerminatedError 155 | abort Error::ConnectionUnavailable.new 'Connection is currently unavailable' 156 | rescue => e 157 | abort e 158 | end 159 | end 160 | 161 | end 162 | end 163 | -------------------------------------------------------------------------------- /lib/krakow/producer/http.rb: -------------------------------------------------------------------------------- 1 | require 'http' 2 | require 'uri' 3 | require 'ostruct' 4 | 5 | require 'cgi' 6 | # NOTE: Prevents weird "first" run behavior 7 | begin 8 | require 'json' 9 | rescue LoadError 10 | # ignore (maybe log?) 11 | end 12 | 13 | require 'krakow' 14 | 15 | module Krakow 16 | class Producer 17 | 18 | # HTTP based producer 19 | class Http 20 | 21 | include Utils::Lazy 22 | # @!parse include Krakow::Utils::Lazy::InstanceMethods 23 | # @!parse extend Krakow::Utils::Lazy::ClassMethods 24 | 25 | # Wrapper for HTTP response hash 26 | class Response < OpenStruct 27 | end 28 | 29 | attr_reader :uri 30 | 31 | # @!group Attributes 32 | 33 | # @!macro [attach] attribute 34 | # @!method $1 35 | # @return [$2] the $1 $0 36 | # @!method $1? 37 | # @return [TrueClass, FalseClass] truthiness of the $1 $0 38 | attribute :endpoint, String, :required => true 39 | attribute :topic, String, :required => true 40 | attribute :config, Hash, :default => ->{ Hash.new } 41 | attribute :ssl_context, Hash 42 | 43 | # @!endgroup 44 | 45 | def initialize(args={}) 46 | super 47 | build_ssl_context if ssl_context 48 | @uri = URI.parse(endpoint) 49 | end 50 | 51 | # Create a new SSL context 52 | # 53 | # @return [OpenSSL::SSL::SSLContext] 54 | def build_ssl_context 55 | require 'openssl' 56 | context = OpenSSL::SSL::SSLContext.new 57 | context.cert = OpenSSL::X509::Certificate.new(File.open(ssl_context[:certificate])) 58 | context.key = OpenSSL::PKey::RSA.new(File.open(ssl_context[:key])) 59 | config[:ssl_context] = context 60 | end 61 | 62 | # Send a message via HTTP 63 | # 64 | # @param method [String, Symbol] HTTP method to use (:get, :put, etc) 65 | # @param path [String] URI path 66 | # @param args [Hash] payload hash 67 | # @return [Response] 68 | def send_message(method, path, args={}) 69 | build = uri.dup 70 | build.path = "/#{path}" 71 | response = HTTP.send(method, build.to_s, args.merge(config)) 72 | begin 73 | response = MultiJson.load(response.body.to_s) 74 | rescue MultiJson::LoadError 75 | response = { 76 | 'status_code' => response.code, 77 | 'status_txt' => response.body.to_s, 78 | 'response' => response.body.to_s, 79 | 'data' => nil, 80 | } 81 | end 82 | Response.new(response) 83 | end 84 | 85 | # Send messages 86 | # 87 | # @param payload [String] message 88 | # @return [Response] 89 | def write(*payload) 90 | if(payload.size == 1) 91 | payload = payload.first 92 | send_message(:post, :pub, 93 | :body => payload, 94 | :params => {:topic => topic} 95 | ) 96 | else 97 | send_message(:post, :mpub, 98 | :body => payload.join("\n"), 99 | :params => {:topic => topic} 100 | ) 101 | end 102 | end 103 | 104 | # Create the topic 105 | # 106 | # @return [Response] 107 | def create_topic 108 | send_message(:post, :create_topic, 109 | :params => {:topic => topic} 110 | ) 111 | end 112 | 113 | # Delete the topic 114 | # 115 | # @return [Response] 116 | def delete_topic 117 | send_message(:post, :delete_topic, 118 | :params => {:topic => topic} 119 | ) 120 | end 121 | 122 | # Create channel on topic 123 | # 124 | # @param chan [String] channel name 125 | # @return [Response] 126 | def create_channel(chan) 127 | send_message(:post, :create_channel, 128 | :params => { 129 | :topic => topic, 130 | :channel => chan 131 | } 132 | ) 133 | end 134 | 135 | # Delete channel on topic 136 | # 137 | # @param chan [String] channel name 138 | # @return [Response] 139 | def delete_channel(chan) 140 | send_message(:post, :delete_channel, 141 | :params => { 142 | :topic => topic, 143 | :channel => chan 144 | } 145 | ) 146 | end 147 | 148 | # Remove all messages from topic 149 | # 150 | # @return [Response] 151 | def empty_topic 152 | send_message(:post, :empty_topic, 153 | :params => {:topic => topic} 154 | ) 155 | end 156 | 157 | # Remove all messages from given channel on topic 158 | # 159 | # @param chan [String] channel name 160 | # @return [Response] 161 | def empty_channel(chan) 162 | send_message(:post, :empty_channel, 163 | :params => { 164 | :topic => topic, 165 | :channel => chan 166 | } 167 | ) 168 | end 169 | 170 | # Pause messages on given channel 171 | # 172 | # @param chan [String] channel name 173 | # @return [Response] 174 | def pause_channel(chan) 175 | send_message(:post, :pause_channel, 176 | :params => { 177 | :topic => topic, 178 | :channel => chan 179 | } 180 | ) 181 | end 182 | 183 | # Resume messages on a given channel 184 | # 185 | # @param chan [String] channel name 186 | # @return [Response] 187 | def unpause_channel(chan) 188 | send_message(:post, :unpause_channel, 189 | :params => { 190 | :topic => topic, 191 | :channel => chan 192 | } 193 | ) 194 | end 195 | 196 | # Server stats 197 | # 198 | # @param format [String] format of data 199 | # @return [Response] 200 | def stats(format='json') 201 | send_message(:get, :stats, 202 | :params => { 203 | :format => format 204 | } 205 | ) 206 | end 207 | 208 | # Ping the server 209 | # 210 | # @return [Response] 211 | def ping 212 | send_message(:get, :ping) 213 | end 214 | 215 | # Server information 216 | # 217 | # @return [Response] 218 | def info 219 | send_message(:get, :info) 220 | end 221 | 222 | end 223 | end 224 | end 225 | -------------------------------------------------------------------------------- /lib/krakow/utils.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | # Helper utilities 5 | module Utils 6 | autoload :Lazy, 'krakow/utils/lazy' 7 | autoload :Logging, 'krakow/utils/logging' 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/krakow/utils/lazy.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | module Utils 5 | # Adds functionality to facilitate laziness 6 | module Lazy 7 | 8 | include Utils::Logging 9 | 10 | # Instance methods for laziness 11 | module InstanceMethods 12 | 13 | # @return [Hash] argument hash 14 | attr_reader :arguments 15 | 16 | # Create new instance 17 | # 18 | # @param args [Hash] 19 | # @return [Object] 20 | def initialize(args={}) 21 | @arguments = {}.tap do |hash| 22 | self.class.attributes.each do |name, options| 23 | val = args[name] 24 | if(options[:required] && !args.has_key?(name)) 25 | raise ArgumentError.new("Missing required option: `#{name}`") 26 | end 27 | if(val && options[:type] && !(valid = [options[:type]].flatten.compact).detect{|k| val.is_a?(k)}) 28 | raise TypeError.new("Invalid type for option `#{name}` (#{val} <#{val.class}>). Valid - #{valid.map(&:to_s).join(',')}") 29 | end 30 | if(val.nil? && options[:default] && !args.has_key?(name)) 31 | val = options[:default].respond_to?(:call) ? options[:default].call : options[:default] 32 | end 33 | hash[name] = val 34 | end 35 | end 36 | end 37 | alias_method :super_init, :initialize 38 | 39 | # @return [String] 40 | def to_s 41 | "<#{self.class.name}:#{object_id}>" 42 | end 43 | 44 | # @return [String] 45 | def inspect 46 | "<#{self.class.name}:#{object_id} [#{arguments.inspect}]>" 47 | end 48 | 49 | end 50 | 51 | # Class methods for laziness 52 | module ClassMethods 53 | 54 | # Add new attributes to class 55 | # 56 | # @param name [String] 57 | # @param type [Class, Array] 58 | # @param options [Hash] 59 | # @option options [true, false] :required must be provided on initialization 60 | # @option options [Object, Proc] :default default value 61 | # @return [nil] 62 | def attribute(name, type, options={}) 63 | name = name.to_sym 64 | attributes[name] = {:type => type}.merge(options) 65 | define_method(name) do 66 | arguments[name.to_sym] 67 | end 68 | define_method("#{name}?") do 69 | !!arguments[name.to_sym] 70 | end 71 | nil 72 | end 73 | 74 | # Return attributes 75 | # 76 | # @param args [Symbol] :required or :optional 77 | # @return [Array] 78 | def attributes(*args) 79 | @attributes ||= {} 80 | if(args.include?(:required)) 81 | Hash[@attributes.find_all{|k,v| v[:required]}] 82 | elsif(args.include?(:optional)) 83 | Hash[@attributes.find_all{|k,v| !v[:required]}] 84 | else 85 | @attributes 86 | end 87 | end 88 | 89 | # Directly set attribute hash 90 | # 91 | # @param attrs [Hash] 92 | # @return [TrueClass] 93 | # @todo need deep dup here 94 | def set_attributes(attrs) 95 | @attributes = attrs.dup 96 | true 97 | end 98 | 99 | end 100 | 101 | class << self 102 | 103 | # Injects laziness into class 104 | # 105 | # @param klass [Class] 106 | def included(klass) 107 | klass.class_eval do 108 | include InstanceMethods 109 | extend ClassMethods 110 | 111 | class << self 112 | 113 | def inherited(klass) 114 | klass.set_attributes(self.attributes) 115 | end 116 | 117 | end 118 | end 119 | end 120 | 121 | end 122 | 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/krakow/utils/logging.rb: -------------------------------------------------------------------------------- 1 | require 'krakow' 2 | 3 | module Krakow 4 | module Utils 5 | # Logging helpers 6 | module Logging 7 | 8 | # Define base logging types 9 | %w(debug info warn error).each do |key| 10 | define_method(key) do |string| 11 | log(key, string) 12 | end 13 | end 14 | 15 | # Log message 16 | # 17 | # @param args [Array, nil] 18 | # @return [Logger, nil] 19 | def log(*args) 20 | if(args.empty?) 21 | Celluloid::Logger 22 | else 23 | severity, string = args 24 | Celluloid::Logger.send(severity.to_sym, "#{self}: #{string}") 25 | nil 26 | end 27 | end 28 | 29 | class << self 30 | # Set the logging output level 31 | # 32 | # @param level [Integer] 33 | # @return [Integer, nil] 34 | def level=(level) 35 | if(Celluloid.logger.class == Logger) 36 | Celluloid.logger.level = Logger.const_get(level.to_s.upcase.to_sym) 37 | end 38 | end 39 | end 40 | 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/krakow/version.rb: -------------------------------------------------------------------------------- 1 | module Krakow 2 | # Current version 3 | VERSION = Gem::Version.new('0.4.2') 4 | end 5 | -------------------------------------------------------------------------------- /test/helpers/nsqd.rb: -------------------------------------------------------------------------------- 1 | require 'childprocess' 2 | 3 | ChildProcess.posix_spawn = true 4 | 5 | require 'tmpdir' 6 | require 'fileutils' 7 | require_relative 'spec_helper' 8 | 9 | module Krakow 10 | class Nsqd 11 | 12 | def self.ports(n=1) 13 | @port ||= 6677 14 | result = n.times.inject([]) do |m,k| 15 | m.push(@port += 1) 16 | end 17 | n == 1 ? result.first : result 18 | end 19 | 20 | attr_reader :args 21 | attr_reader :nsqds 22 | attr_reader :lookupds 23 | 24 | def initialize(args={}) 25 | @args = args 26 | @nsqds = [] 27 | @lookupds = [] 28 | @dir = Dir.mktmpdir('krakow') 29 | end 30 | 31 | def run! 32 | @lookupds = args.fetch(:lookupds, 1).times.map do 33 | build_lookupd 34 | end 35 | @nsqds = args.fetch(:nsqds, 1).times.map do 36 | build_nsqd 37 | end 38 | nsqd_tcp_addresses.each do |addr| 39 | begin 40 | TCPSocket.new(*addr.split(':')) 41 | rescue Errno::ECONNREFUSED 42 | retry 43 | end 44 | end 45 | end 46 | 47 | def halt! 48 | @nsqds.map(&:first).map(&:stop) 49 | @lookupds.map(&:first).map(&:stop) 50 | @nsqds.map(&:first).map(&:wait) 51 | @lookupds.map(&:first).map(&:wait) 52 | FileUtils.rm_rf(@dir) 53 | end 54 | 55 | def build_nsqd 56 | ports = self.class.ports(2) 57 | options = { 58 | 'data-path' => @dir, 59 | 'tcp-address' => "0.0.0.0:#{ports.first}", 60 | 'http-address' => "0.0.0.0:#{ports.last}" 61 | }.merge(args.fetch(:nsqd_options, {})) 62 | cmd = ['nsqd'] + options.map{|k,v| ["-#{k}", v]} 63 | unless(@lookupds.empty?) 64 | lookupd_tcp_addresses.each do |laddr| 65 | cmd.push('-lookupd-tcp-address').push(laddr) 66 | end 67 | end 68 | [start_process(ChildProcess.build(*cmd.flatten)), options] 69 | end 70 | 71 | def build_lookupd 72 | ports = self.class.ports(2) 73 | options = { 74 | 'tcp-address' => "0.0.0.0:#{ports.first}", 75 | 'http-address' => "0.0.0.0:#{ports.last}" 76 | }.merge(args.fetch(:lookupd_options, {})) 77 | cmd = ['nsqlookupd'] + options.map{|k,v| ["-#{k}", v]} 78 | [start_process(ChildProcess.build(*cmd.flatten)), options] 79 | end 80 | 81 | def nsqd_tcp_addresses 82 | @nsqds.map(&:last).map do |nopts| 83 | nopts['tcp-address'] 84 | end 85 | end 86 | 87 | def nsqd_http_addresses 88 | @nsqds.map(&:last).map do |nopts| 89 | "http://#{nopts['http-address']}" 90 | end 91 | end 92 | 93 | def lookupd_http_addresses 94 | @lookupds.map(&:last).map do |lopts| 95 | "http://#{lopts['http-address']}" 96 | end 97 | end 98 | 99 | def lookupd_tcp_addresses 100 | @lookupds.map(&:last).map do |lopts| 101 | lopts['tcp-address'] 102 | end 103 | end 104 | 105 | def start_process(process) 106 | if(ENV['DEBUG']) 107 | process.io.inherit! 108 | end 109 | process.cwd = @dir 110 | process.start 111 | process 112 | end 113 | 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /test/helpers/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler.setup 3 | 4 | require 'timeout' 5 | require 'krakow' 6 | require 'minitest/autorun' 7 | require_relative 'nsqd' 8 | 9 | TOPIC_NAME = ('a'..'z').to_a + ('A'..'Z').to_a 10 | 11 | class MiniTest::Test 12 | 13 | # Block execution until a condition is met 14 | # Times out after 5 seconds by default 15 | # 16 | # example: 17 | # wait_for { @consumer.queue.length > 0 } 18 | # 19 | def wait_for(timeout = 5) 20 | raise ArgumentError.new 'Block must be provided!' unless block_given? 21 | Timeout::timeout(timeout) do 22 | loop do 23 | break if yield 24 | sleep(0.1) 25 | end 26 | end 27 | end 28 | 29 | # Assert that something must take longer than a certain amount of time to complete 30 | def must_take_longer_than(time, roof=false, &block) 31 | start_time = Time.now.to_f 32 | yield 33 | end_time = Time.now.to_f 34 | 35 | result = (end_time - start_time) 36 | result = result.ceil if roof 37 | result.must_be :>, time 38 | end 39 | 40 | def must_take_less_than(time, roof=false, &block) 41 | start_time = Time.now.to_f 42 | yield 43 | end_time = Time.now.to_f 44 | 45 | result = (end_time - start_time) 46 | result = result.ceil if roof 47 | result.must_be :<, time 48 | end 49 | end 50 | 51 | Celluloid.logger.level = ENV['DEBUG'] ? 0 : 4 52 | -------------------------------------------------------------------------------- /test/run.rb: -------------------------------------------------------------------------------- 1 | # Run all the *_spec.rb files in the specs directory 2 | Dir.glob(File.join(File.dirname(__FILE__), 'specs', '**', '*_spec.rb')).each do |path| 3 | require File.expand_path(path) 4 | end 5 | -------------------------------------------------------------------------------- /test/specs/application/consumer_reconnect_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new( 7 | :nsqd_options => { 8 | 'msg-timeout' => '5s' 9 | } 10 | ) 11 | @nsqd.run! 12 | end 13 | 14 | after do 15 | @nsqd.halt! 16 | end 17 | 18 | describe 'Consumer with socket failures' do 19 | 20 | before do 21 | @topic = TOPIC_NAME.shuffle.join 22 | @producer = @nsqd.nsqd_tcp_addresses.map do |addr| 23 | host, port = addr.split(':') 24 | Krakow::Producer.new( 25 | :host => host, 26 | :port => port, 27 | :topic => @topic 28 | ) 29 | end.first 30 | wait_for{ @producer.connected? } 31 | 100.times do |i| 32 | @producer.write(i.to_s) 33 | end 34 | @consumer = Krakow::Consumer.new( 35 | :nsqlookupd => @nsqd.lookupd_http_addresses.first, 36 | :topic => @topic, 37 | :channel => 'default', 38 | :max_in_flight => 20, 39 | :discovery_interval => 5 40 | ) 41 | wait_for{ @consumer.connected? } 42 | sleep(1) 43 | end 44 | 45 | after do 46 | @producer.terminate 47 | @consumer.terminate 48 | end 49 | 50 | it 'should confirm all messages' do 51 | retries = 0 52 | result = [] 53 | 100.times do |i| 54 | begin 55 | wait_for(15){ result << @consumer.queue.pop.confirm } 56 | if(((i + 1) % 50) == 0) 57 | @consumer.connections.values.first.socket.socket.close 58 | end 59 | rescue => e 60 | wait_for(30){ @consumer.connected? } 61 | retries += 1 62 | raise if retries > 5 63 | retry 64 | end 65 | end 66 | result.size.must_equal 100 67 | end 68 | 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /test/specs/application/high_volume_multi_prod_con_single_chan_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new(:nsqds => 20) 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe 'High Volume - Multi Producer / Multi Consumer / Single Channel' do 15 | 16 | before do 17 | @topic = TOPIC_NAME.shuffle.join 18 | @producers = @nsqd.nsqd_tcp_addresses.map do |addr| 19 | host, port = addr.split(':') 20 | Krakow::Producer.new( 21 | :host => host, 22 | :port => port, 23 | :topic => @topic, 24 | :connection_options => { 25 | :options => { 26 | :response_wait => 0 27 | } 28 | } 29 | ) 30 | end.map do |producer| 31 | producer.write('seed') 32 | producer 33 | end 34 | @consumers = 15.times.map do 35 | Krakow::Consumer.new( 36 | :nsqlookupd => @nsqd.lookupd_http_addresses.first, 37 | :topic => @topic, 38 | :channel => 'default', 39 | :max_in_flight => 20 40 | ) 41 | end 42 | wait_for(10){ @consumers.all?{|consumer| !consumer.connections.empty?} } 43 | @consumers.each do |consumer| 44 | if(consumer.queue.size > 0) 45 | consumer.queue.pop.confirm 46 | end 47 | end 48 | end 49 | 50 | after do 51 | @producers.map(&:terminate) 52 | @consumers.map(&:terminate) 53 | end 54 | 55 | it 'should consume all generated messages' do 56 | @generated = [] 57 | @received = Queue.new 58 | generator = lambda do 59 | 1000.times do |i| 60 | @generated << i.to_s 61 | @producers.shuffle.first.write(i.to_s) 62 | end 63 | end 64 | consumers = @consumers.map do |consumer| 65 | Thread.new do 66 | loop do 67 | msg = consumer.queue.pop 68 | consumer.confirm(msg) 69 | @received.push(msg.message) 70 | end 71 | end 72 | end 73 | Thread.new{ generator.call } 74 | collector = Thread.new do 75 | loop do 76 | @generated.delete(@received.pop) 77 | end 78 | end 79 | sleep(3) 80 | wait_for(200){@received.size == 0 && @generated.empty?} 81 | @received.must_be :empty? 82 | @generated.must_be :empty? 83 | end 84 | 85 | end 86 | 87 | end 88 | -------------------------------------------------------------------------------- /test/specs/application/high_volume_producer_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe 'High Volume - Single Producer / Large payloads' do 15 | 16 | describe 'With delivery confirmation' do 17 | 18 | before do 19 | @topic = TOPIC_NAME.shuffle.join 20 | @producer = @nsqd.nsqd_tcp_addresses.map do |addr| 21 | host, port = addr.split(':') 22 | Krakow::Producer.new(:host => host, :port => port, :topic => @topic) 23 | end.first 24 | wait_for{ @producer.connected? } 25 | end 26 | 27 | after do 28 | @producer.terminate 29 | end 30 | 31 | it 'should produce all payloads' do 32 | 100.times do 33 | output = 2000.times.map{TOPIC_NAME.shuffle.join}.join 34 | @producer.write(output).response.must_equal 'OK' 35 | end 36 | end 37 | 38 | end 39 | 40 | describe 'Without delivery confirmation' do 41 | 42 | before do 43 | @topic = TOPIC_NAME.shuffle.join 44 | @producer = @nsqd.nsqd_tcp_addresses.map do |addr| 45 | host, port = addr.split(':') 46 | Krakow::Producer.new(:host => host, :port => port, :topic => @topic, :connection_options => {:options => {:response_wait => 0}}) 47 | end.first 48 | wait_for{ @producer.connected? } 49 | end 50 | 51 | after do 52 | @producer.terminate 53 | end 54 | 55 | it 'should produce all payloads' do 56 | 10000.times do 57 | output = 2000.times.map{TOPIC_NAME.shuffle.join}.join 58 | @producer.write(output).must_equal true 59 | end 60 | end 61 | 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /test/specs/application/high_volume_single_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe 'High Volume - Single Producer / Single Consumer / Single Channel' do 15 | 16 | before do 17 | @topic = TOPIC_NAME.shuffle.join 18 | @producer = @nsqd.nsqd_tcp_addresses.map do |addr| 19 | host, port = addr.split(':') 20 | Krakow::Producer.new(:host => host, :port => port, :topic => @topic) 21 | end.map do |producer| 22 | producer.write('seed') 23 | producer 24 | end.first 25 | sleep(1) 26 | @consumer = Krakow::Consumer.new( 27 | :nsqlookupd => @nsqd.lookupd_http_addresses.first, 28 | :topic => @topic, :channel => 'default', :max_in_flight => 100 29 | ) 30 | wait_for(10){ !@consumer.connections.empty? } 31 | @consumer.queue.pop.confirm 32 | end 33 | 34 | after do 35 | @producer.terminate 36 | @consumer.terminate 37 | end 38 | 39 | it 'should consume all generated messages' do 40 | @generated = [] 41 | @received = Queue.new 42 | generator = lambda do 43 | 1000.times do |i| 44 | @generated << i.to_s 45 | @producer.write(i.to_s) 46 | end 47 | end 48 | Thread.new do 49 | loop do 50 | msg = @consumer.queue.pop 51 | @consumer.confirm(msg) 52 | @received.push(msg.message) 53 | end 54 | end 55 | Thread.new{ generator.call } 56 | collector = Thread.new do 57 | loop do 58 | @generated.delete(@received.pop) 59 | end 60 | end 61 | sleep(3) 62 | start = Time.now.to_i 63 | wait_for(120){@received.size == 0 && @generated.empty?} 64 | @received.must_be :empty? 65 | @generated.must_be :empty? 66 | end 67 | 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /test/specs/application/producer_reconnect_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe 'Producer with socket failures' do 15 | 16 | before do 17 | @topic = TOPIC_NAME.shuffle.join 18 | @producer = @nsqd.nsqd_tcp_addresses.map do |addr| 19 | host, port = addr.split(':') 20 | Krakow::Producer.new( 21 | :host => host, 22 | :port => port, 23 | :topic => @topic, 24 | :connection_options => { 25 | :options => { 26 | :response_wait => 0 27 | } 28 | } 29 | ) 30 | end.first 31 | wait_for{ @producer.connected? } 32 | end 33 | 34 | after do 35 | @producer.terminate 36 | end 37 | 38 | it 'should send all messages' do 39 | result = [] 40 | 100.times do |i| 41 | begin 42 | result << @producer.write('test') 43 | if(i % 20 == 0) 44 | @producer.connection.socket.socket.close 45 | end 46 | rescue 47 | wait_for{ @producer.connected? } 48 | retry 49 | end 50 | end 51 | result.size.must_equal 100 52 | end 53 | 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /test/specs/connection_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe Krakow::Connection do 15 | 16 | before do 17 | host, port = @nsqd.nsqd_tcp_addresses.first.split(':') 18 | @connection = Krakow::Connection.new( 19 | :host => host, 20 | :port => port 21 | ) 22 | @connection.init! 23 | wait_for{ @connection.connected? } 24 | end 25 | 26 | after do 27 | @connection.terminate 28 | end 29 | 30 | it 'should be connected to nsqd' do 31 | @connection.connected?.must_equal true 32 | end 33 | 34 | it 'should have an identifier based on configuration' do 35 | @connection.identifier.must_equal [@connection.host, @connection.port].join('__') 36 | end 37 | 38 | it 'should have a settings provided by nsqd' do 39 | @connection.endpoint_settings.wont_be :empty? 40 | end 41 | 42 | it 'should not transmit non-frametype messages' do 43 | ->{ @connection.transmit('ohai') }.must_raise TypeError 44 | end 45 | 46 | end 47 | 48 | end 49 | -------------------------------------------------------------------------------- /test/specs/consumer_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe Krakow::Consumer do 15 | 16 | describe 'Direct connection' do 17 | 18 | before do 19 | @topic = TOPIC_NAME.shuffle.join 20 | host, port = @nsqd.nsqd_tcp_addresses.first.split(':') 21 | @producer = Krakow::Producer.new( 22 | :host => host, 23 | :port => port, 24 | :topic => @topic 25 | ) 26 | @consumer = Krakow::Consumer.new( 27 | :host => host, 28 | :port => port, 29 | :topic => @topic, 30 | :channel => '_default' 31 | ) 32 | wait_for{ @producer.connected? && !@consumer.connections.empty? } 33 | end 34 | 35 | after do 36 | sleep(0.2) # cooldown 37 | @producer.terminate 38 | @consumer.terminate 39 | end 40 | 41 | it 'should have a queue with a size' do 42 | @consumer.queue.size.must_equal 0 43 | end 44 | 45 | it 'should wait for messages' do 46 | ->{ wait_for(0.2){ @consumer.queue.pop } }.must_raise Timeout::Error 47 | end 48 | 49 | it 'should receive messages' do 50 | @producer.write('krakow-test').response.must_equal 'OK' 51 | @consumer.queue.pop.message.must_equal 'krakow-test' 52 | end 53 | 54 | it 'should receive stream of messages' do 55 | msgs = 100.times.map{ TOPIC_NAME.shuffle.join } 56 | @producer.write(*msgs).response.must_equal 'OK' 57 | msgs.each do |msg| 58 | result = @consumer.queue.pop 59 | result.message.must_equal msg 60 | result.confirm 61 | end 62 | end 63 | 64 | end 65 | 66 | describe 'Discovery connection' do 67 | 68 | before do 69 | @topic = TOPIC_NAME.shuffle.join 70 | host, port = @nsqd.nsqd_tcp_addresses.first.split(':') 71 | @producer = Krakow::Producer.new( 72 | :host => host, 73 | :port => port, 74 | :topic => @topic 75 | ) 76 | wait_for{ @producer.connected? } 77 | @producer.write('seed') 78 | sleep(1) 79 | @consumer = Krakow::Consumer.new( 80 | :nsqlookupd => @nsqd.lookupd_http_addresses.first, 81 | :topic => @topic, 82 | :channel => '_default' 83 | ) 84 | wait_for{ @consumer.connections.values.all?{|c| c.connected?} } 85 | end 86 | 87 | after do 88 | sleep(0.2) # cooldown 89 | @producer.terminate 90 | @consumer.terminate 91 | end 92 | 93 | it 'should have an nsqd connection' do 94 | @consumer.connections.wont_be :empty? 95 | @consumer.connections.values.first.connected?.must_equal true 96 | end 97 | 98 | it 'should receive messages' do 99 | msg = @consumer.queue.pop 100 | msg.message.must_equal 'seed' 101 | @consumer.confirm(msg) 102 | @producer.write('krakow-test').response.must_equal 'OK' 103 | @consumer.queue.pop.message.must_equal 'krakow-test' 104 | end 105 | 106 | end 107 | end 108 | 109 | end 110 | -------------------------------------------------------------------------------- /test/specs/producer_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers/spec_helper' 2 | 3 | describe Krakow do 4 | 5 | before do 6 | @nsqd = Krakow::Nsqd.new 7 | @nsqd.run! 8 | end 9 | 10 | after do 11 | @nsqd.halt! 12 | end 13 | 14 | describe Krakow::Producer do 15 | 16 | describe 'State' do 17 | it 'should teardown all resources on terminate' do 18 | topic = TOPIC_NAME.shuffle.join 19 | host, port = @nsqd.nsqd_tcp_addresses.first.split(':') 20 | producer = Krakow::Producer.new( 21 | :host => host, 22 | :port => port, 23 | :topic => @topic 24 | ) 25 | producer.connected?.must_equal true 26 | connection = producer.connection 27 | socket = connection.socket 28 | producer.terminate 29 | wait_for{ !producer.alive? } 30 | producer.alive?.must_equal false 31 | connection.alive?.must_equal false 32 | socket.alive?.must_equal false 33 | end 34 | end 35 | 36 | describe 'Usage' do 37 | before do 38 | @topic = TOPIC_NAME.shuffle.join 39 | host, port = @nsqd.nsqd_tcp_addresses.first.split(':') 40 | @producer = Krakow::Producer.new( 41 | :host => host, 42 | :port => port, 43 | :topic => @topic 44 | ) 45 | end 46 | 47 | after do 48 | @producer.terminate 49 | end 50 | 51 | it 'should have an active connection' do 52 | @producer.connected?.must_equal true 53 | end 54 | 55 | it 'should write a single message' do 56 | @producer.write('testing').response.must_equal 'OK' 57 | end 58 | 59 | it 'should write multiple messages' do 60 | @producer.write('testing1', 'testing2', 'testing3').response.must_equal 'OK' 61 | end 62 | 63 | it 'should automatically reconnect a failed connection' do 64 | @producer.connection.socket.socket.close 65 | result = nil 66 | begin 67 | @producer.write('testing') 68 | rescue => result 69 | end 70 | [IOError, Krakow::Error::ConnectionUnavailable].must_include result.class 71 | wait_for{ @producer.connected? } 72 | @producer.write('testing').response.must_equal 'OK' 73 | end 74 | 75 | end 76 | 77 | end 78 | 79 | describe Krakow::Producer::Http do 80 | before do 81 | @producer = Krakow::Producer::Http.new( 82 | :endpoint => @nsqd.nsqd_http_addresses.first, 83 | :topic => TOPIC_NAME.shuffle.join 84 | ) 85 | end 86 | 87 | it 'should write a single message' do 88 | result = @producer.write('testing') 89 | result.status_txt.must_equal 'OK' 90 | result.response.must_equal 'OK' 91 | end 92 | 93 | it 'should write multiple messages' do 94 | result = @producer.write('testing1', 'testing2', 'testing3') 95 | result.status_txt.must_equal 'OK' 96 | result.response.must_equal 'OK' 97 | end 98 | end 99 | 100 | end 101 | --------------------------------------------------------------------------------