├── .codeclimate.yml ├── .dockerignore ├── .env.sample ├── .gitignore ├── .rspec ├── .rubocop.yml ├── .ruby-version ├── .travis.yml ├── CHANGELOG.md ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── PROTOCOL.md ├── Procfile ├── Procfile.dev ├── README.md ├── Rakefile ├── bin └── firehose ├── coffeelint.json ├── config └── rainbows.rb ├── docker-compose.yml ├── firehose.gemspec ├── lib ├── firehose.rb └── firehose │ ├── cli.rb │ ├── client.rb │ ├── client │ ├── consumer.rb │ └── producer.rb │ ├── logging.rb │ ├── rack.rb │ ├── rack │ ├── app.rb │ ├── consumer.rb │ ├── consumer │ │ ├── http_long_poll.rb │ │ └── web_socket.rb │ ├── metrics_api.rb │ ├── ping.rb │ └── publisher.rb │ ├── rails.rb │ ├── server.rb │ ├── server │ ├── app.rb │ ├── channel_subscription.rb │ ├── configuration.rb │ ├── message.rb │ ├── message_buffer.rb │ ├── message_filter.rb │ ├── metrics.rb │ ├── metrics_collector.rb │ ├── publisher.rb │ ├── redis.rb │ └── subscriber.rb │ └── version.rb ├── package.json └── spec ├── integrations ├── integration_test_helper.rb ├── rainbows_spec.rb ├── shared_examples.rb └── thin_spec.rb ├── lib ├── client │ ├── consumer_spec.rb │ └── producer_spec.rb ├── firehose_spec.rb ├── rack │ ├── consumer │ │ └── http_long_poll_spec.rb │ ├── consumer_spec.rb │ ├── ping_spec.rb │ └── publisher_spec.rb └── server │ ├── app_spec.rb │ ├── channel_subscription_spec.rb │ ├── configuration_spec.rb │ ├── message_buffer_spec.rb │ ├── message_filter_spec.rb │ ├── metrics_spec.rb │ ├── publisher_spec.rb │ ├── redis_spec.rb │ └── subscriber_spec.rb └── spec_helper.rb /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | coffeelint: 4 | enabled: true 5 | duplication: 6 | enabled: true 7 | config: 8 | languages: 9 | - ruby 10 | - javascript 11 | - python 12 | - php 13 | fixme: 14 | enabled: true 15 | rubocop: 16 | enabled: true 17 | ratings: 18 | paths: 19 | - "**.coffee" 20 | - "**.inc" 21 | - "**.js" 22 | - "**.jsx" 23 | - "**.module" 24 | - "**.php" 25 | - "**.py" 26 | - "**.rb" 27 | exclude_paths: 28 | - config/ 29 | - spec/ 30 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | docker-compose.yml 3 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | # This is picked up by em-hiredis 2 | REDIS_URL=redis://127.0.0.1:6379/0 3 | 4 | # Configure the verbosity of the logger. 5 | LOG_LEVEL=debug 6 | 7 | # Firehose port binding. 8 | PORT=7474 9 | 10 | # Firehose IP address binding. 11 | HOST=127.0.0.1 12 | 13 | # Can be rainbows or thin 14 | SERVER=rainbows 15 | 16 | # Configure a production or development environment for Rainbows! or Thin. 17 | RACK_ENV=development -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | .DS_Store 4 | *.swp 5 | .env 6 | log/* 7 | pkg/* 8 | tmp 9 | *.sublime-project 10 | *.sublime-workspace 11 | public/javascripts 12 | node_modules 13 | vendor 14 | lib/assets/javascripts/firehose/version.js.coffee 15 | coverage 16 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --colour --backtrace -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | DisabledByDefault: true 3 | 4 | #################### Lint ################################ 5 | 6 | Lint/AmbiguousOperator: 7 | Description: >- 8 | Checks for ambiguous operators in the first argument of a 9 | method invocation without parentheses. 10 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#parens-as-args' 11 | Enabled: true 12 | 13 | Lint/AmbiguousRegexpLiteral: 14 | Description: >- 15 | Checks for ambiguous regexp literals in the first argument of 16 | a method invocation without parenthesis. 17 | Enabled: true 18 | 19 | Lint/AssignmentInCondition: 20 | Description: "Don't use assignment in conditions." 21 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#safe-assignment-in-condition' 22 | Enabled: true 23 | 24 | Lint/BlockAlignment: 25 | Description: 'Align block ends correctly.' 26 | Enabled: true 27 | 28 | Lint/CircularArgumentReference: 29 | Description: "Don't refer to the keyword argument in the default value." 30 | Enabled: true 31 | 32 | Lint/ConditionPosition: 33 | Description: >- 34 | Checks for condition placed in a confusing position relative to 35 | the keyword. 36 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#same-line-condition' 37 | Enabled: true 38 | 39 | Lint/Debugger: 40 | Description: 'Check for debugger calls.' 41 | Enabled: true 42 | 43 | Lint/DefEndAlignment: 44 | Description: 'Align ends corresponding to defs correctly.' 45 | Enabled: true 46 | 47 | Lint/DeprecatedClassMethods: 48 | Description: 'Check for deprecated class method calls.' 49 | Enabled: true 50 | 51 | Lint/DuplicateMethods: 52 | Description: 'Check for duplicate methods calls.' 53 | Enabled: true 54 | 55 | Lint/EachWithObjectArgument: 56 | Description: 'Check for immutable argument given to each_with_object.' 57 | Enabled: true 58 | 59 | Lint/ElseLayout: 60 | Description: 'Check for odd code arrangement in an else block.' 61 | Enabled: true 62 | 63 | Lint/EmptyEnsure: 64 | Description: 'Checks for empty ensure block.' 65 | Enabled: true 66 | 67 | Lint/EmptyInterpolation: 68 | Description: 'Checks for empty string interpolation.' 69 | Enabled: true 70 | 71 | Lint/EndAlignment: 72 | Description: 'Align ends correctly.' 73 | Enabled: true 74 | 75 | Lint/EndInMethod: 76 | Description: 'END blocks should not be placed inside method definitions.' 77 | Enabled: true 78 | 79 | Lint/EnsureReturn: 80 | Description: 'Do not use return in an ensure block.' 81 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-return-ensure' 82 | Enabled: true 83 | 84 | Lint/Eval: 85 | Description: 'The use of eval represents a serious security risk.' 86 | Enabled: true 87 | 88 | Lint/FormatParameterMismatch: 89 | Description: 'The number of parameters to format/sprint must match the fields.' 90 | Enabled: true 91 | 92 | Lint/HandleExceptions: 93 | Description: "Don't suppress exception." 94 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#dont-hide-exceptions' 95 | Enabled: true 96 | 97 | Lint/InvalidCharacterLiteral: 98 | Description: >- 99 | Checks for invalid character literals with a non-escaped 100 | whitespace character. 101 | Enabled: true 102 | 103 | Lint/LiteralInCondition: 104 | Description: 'Checks of literals used in conditions.' 105 | Enabled: true 106 | 107 | Lint/LiteralInInterpolation: 108 | Description: 'Checks for literals used in interpolation.' 109 | Enabled: true 110 | 111 | Lint/Loop: 112 | Description: >- 113 | Use Kernel#loop with break rather than begin/end/until or 114 | begin/end/while for post-loop tests. 115 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#loop-with-break' 116 | Enabled: true 117 | 118 | Lint/NestedMethodDefinition: 119 | Description: 'Do not use nested method definitions.' 120 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-nested-methods' 121 | Enabled: true 122 | 123 | Lint/NonLocalExitFromIterator: 124 | Description: 'Do not use return in iterator to cause non-local exit.' 125 | Enabled: true 126 | 127 | Lint/ParenthesesAsGroupedExpression: 128 | Description: >- 129 | Checks for method calls with a space before the opening 130 | parenthesis. 131 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#parens-no-spaces' 132 | Enabled: true 133 | 134 | Lint/RequireParentheses: 135 | Description: >- 136 | Use parentheses in the method call to avoid confusion 137 | about precedence. 138 | Enabled: true 139 | 140 | Lint/RescueException: 141 | Description: 'Avoid rescuing the Exception class.' 142 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-blind-rescues' 143 | Enabled: true 144 | 145 | Lint/ShadowingOuterLocalVariable: 146 | Description: >- 147 | Do not use the same name as outer local variable 148 | for block arguments or block local variables. 149 | Enabled: true 150 | 151 | Lint/StringConversionInInterpolation: 152 | Description: 'Checks for Object#to_s usage in string interpolation.' 153 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-to-s' 154 | Enabled: true 155 | 156 | Lint/UnderscorePrefixedVariableName: 157 | Description: 'Do not use prefix `_` for a variable that is used.' 158 | Enabled: true 159 | 160 | Lint/UnneededDisable: 161 | Description: >- 162 | Checks for rubocop:disable comments that can be removed. 163 | Note: this cop is not disabled when disabling all cops. 164 | It must be explicitly disabled. 165 | Enabled: true 166 | 167 | Lint/UnusedBlockArgument: 168 | Description: 'Checks for unused block arguments.' 169 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#underscore-unused-vars' 170 | Enabled: true 171 | 172 | Lint/UnusedMethodArgument: 173 | Description: 'Checks for unused method arguments.' 174 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#underscore-unused-vars' 175 | Enabled: true 176 | 177 | Lint/UnreachableCode: 178 | Description: 'Unreachable code.' 179 | Enabled: true 180 | 181 | Lint/UselessAccessModifier: 182 | Description: 'Checks for useless access modifiers.' 183 | Enabled: true 184 | 185 | Lint/UselessAssignment: 186 | Description: 'Checks for useless assignment to a local variable.' 187 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#underscore-unused-vars' 188 | Enabled: true 189 | 190 | Lint/UselessComparison: 191 | Description: 'Checks for comparison of something with itself.' 192 | Enabled: true 193 | 194 | Lint/UselessElseWithoutRescue: 195 | Description: 'Checks for useless `else` in `begin..end` without `rescue`.' 196 | Enabled: true 197 | 198 | Lint/UselessSetterCall: 199 | Description: 'Checks for useless setter call to a local variable.' 200 | Enabled: true 201 | 202 | Lint/Void: 203 | Description: 'Possible use of operator/literal/variable in void context.' 204 | Enabled: true 205 | 206 | ###################### Metrics #################################### 207 | 208 | Metrics/AbcSize: 209 | Description: >- 210 | A calculated magnitude based on number of assignments, 211 | branches, and conditions. 212 | Reference: 'http://c2.com/cgi/wiki?AbcMetric' 213 | Enabled: false 214 | Max: 20 215 | 216 | Metrics/BlockNesting: 217 | Description: 'Avoid excessive block nesting' 218 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#three-is-the-number-thou-shalt-count' 219 | Enabled: true 220 | Max: 4 221 | 222 | Metrics/ClassLength: 223 | Description: 'Avoid classes longer than 250 lines of code.' 224 | Enabled: true 225 | Max: 250 226 | 227 | Metrics/CyclomaticComplexity: 228 | Description: >- 229 | A complexity metric that is strongly correlated to the number 230 | of test cases needed to validate a method. 231 | Enabled: true 232 | 233 | Metrics/LineLength: 234 | Description: 'Limit lines to 80 characters.' 235 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#80-character-limits' 236 | Enabled: false 237 | 238 | Metrics/MethodLength: 239 | Description: 'Avoid methods longer than 30 lines of code.' 240 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#short-methods' 241 | Enabled: true 242 | Max: 30 243 | 244 | Metrics/ModuleLength: 245 | Description: 'Avoid modules longer than 250 lines of code.' 246 | Enabled: true 247 | Max: 250 248 | 249 | Metrics/ParameterLists: 250 | Description: 'Avoid parameter lists longer than three or four parameters.' 251 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#too-many-params' 252 | Enabled: true 253 | 254 | Metrics/PerceivedComplexity: 255 | Description: >- 256 | A complexity metric geared towards measuring complexity for a 257 | human reader. 258 | Enabled: false 259 | 260 | ##################### Performance ############################# 261 | 262 | Performance/Count: 263 | Description: >- 264 | Use `count` instead of `select...size`, `reject...size`, 265 | `select...count`, `reject...count`, `select...length`, 266 | and `reject...length`. 267 | Enabled: true 268 | 269 | Performance/Detect: 270 | Description: >- 271 | Use `detect` instead of `select.first`, `find_all.first`, 272 | `select.last`, and `find_all.last`. 273 | Reference: 'https://github.com/JuanitoFatas/fast-ruby#enumerabledetect-vs-enumerableselectfirst-code' 274 | Enabled: true 275 | 276 | Performance/FlatMap: 277 | Description: >- 278 | Use `Enumerable#flat_map` 279 | instead of `Enumerable#map...Array#flatten(1)` 280 | or `Enumberable#collect..Array#flatten(1)` 281 | Reference: 'https://github.com/JuanitoFatas/fast-ruby#enumerablemaparrayflatten-vs-enumerableflat_map-code' 282 | Enabled: true 283 | EnabledForFlattenWithoutParams: false 284 | # If enabled, this cop will warn about usages of 285 | # `flatten` being called without any parameters. 286 | # This can be dangerous since `flat_map` will only flatten 1 level, and 287 | # `flatten` without any parameters can flatten multiple levels. 288 | 289 | Performance/ReverseEach: 290 | Description: 'Use `reverse_each` instead of `reverse.each`.' 291 | Reference: 'https://github.com/JuanitoFatas/fast-ruby#enumerablereverseeach-vs-enumerablereverse_each-code' 292 | Enabled: true 293 | 294 | Performance/Sample: 295 | Description: >- 296 | Use `sample` instead of `shuffle.first`, 297 | `shuffle.last`, and `shuffle[Fixnum]`. 298 | Reference: 'https://github.com/JuanitoFatas/fast-ruby#arrayshufflefirst-vs-arraysample-code' 299 | Enabled: true 300 | 301 | Performance/Size: 302 | Description: >- 303 | Use `size` instead of `count` for counting 304 | the number of elements in `Array` and `Hash`. 305 | Reference: 'https://github.com/JuanitoFatas/fast-ruby#arraycount-vs-arraysize-code' 306 | Enabled: true 307 | 308 | Performance/StringReplacement: 309 | Description: >- 310 | Use `tr` instead of `gsub` when you are replacing the same 311 | number of characters. Use `delete` instead of `gsub` when 312 | you are deleting characters. 313 | Reference: 'https://github.com/JuanitoFatas/fast-ruby#stringgsub-vs-stringtr-code' 314 | Enabled: true 315 | 316 | ##################### Rails ################################## 317 | 318 | Rails/ActionFilter: 319 | Description: 'Enforces consistent use of action filter methods.' 320 | Enabled: false 321 | 322 | Rails/Date: 323 | Description: >- 324 | Checks the correct usage of date aware methods, 325 | such as Date.today, Date.current etc. 326 | Enabled: false 327 | 328 | Rails/Delegate: 329 | Description: 'Prefer delegate method for delegations.' 330 | Enabled: false 331 | 332 | Rails/FindBy: 333 | Description: 'Prefer find_by over where.first.' 334 | Enabled: false 335 | 336 | Rails/FindEach: 337 | Description: 'Prefer all.find_each over all.find.' 338 | Enabled: false 339 | 340 | Rails/HasAndBelongsToMany: 341 | Description: 'Prefer has_many :through to has_and_belongs_to_many.' 342 | Enabled: false 343 | 344 | Rails/Output: 345 | Description: 'Checks for calls to puts, print, etc.' 346 | Enabled: false 347 | 348 | Rails/ReadWriteAttribute: 349 | Description: >- 350 | Checks for read_attribute(:attr) and 351 | write_attribute(:attr, val). 352 | Enabled: false 353 | 354 | Rails/ScopeArgs: 355 | Description: 'Checks the arguments of ActiveRecord scopes.' 356 | Enabled: false 357 | 358 | Rails/TimeZone: 359 | Description: 'Checks the correct usage of time zone aware methods.' 360 | StyleGuide: 'https://github.com/bbatsov/rails-style-guide#time' 361 | Reference: 'http://danilenko.org/2012/7/6/rails_timezones' 362 | Enabled: false 363 | 364 | Rails/Validation: 365 | Description: 'Use validates :attribute, hash of validations.' 366 | Enabled: false 367 | 368 | ################## Style ################################# 369 | 370 | Style/AccessModifierIndentation: 371 | Description: Check indentation of private/protected visibility modifiers. 372 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#indent-public-private-protected' 373 | Enabled: false 374 | 375 | Style/AccessorMethodName: 376 | Description: Check the naming of accessor methods for get_/set_. 377 | Enabled: false 378 | 379 | Style/Alias: 380 | Description: 'Use alias_method instead of alias.' 381 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#alias-method' 382 | Enabled: false 383 | 384 | Style/AlignArray: 385 | Description: >- 386 | Align the elements of an array literal if they span more than 387 | one line. 388 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#align-multiline-arrays' 389 | Enabled: false 390 | 391 | Style/AlignHash: 392 | Description: >- 393 | Align the elements of a hash literal if they span more than 394 | one line. 395 | Enabled: false 396 | 397 | Style/AlignParameters: 398 | Description: >- 399 | Align the parameters of a method call if they span more 400 | than one line. 401 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-double-indent' 402 | Enabled: false 403 | 404 | Style/AndOr: 405 | Description: 'Use &&/|| instead of and/or.' 406 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-and-or-or' 407 | Enabled: false 408 | 409 | Style/ArrayJoin: 410 | Description: 'Use Array#join instead of Array#*.' 411 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#array-join' 412 | Enabled: false 413 | 414 | Style/AsciiComments: 415 | Description: 'Use only ascii symbols in comments.' 416 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#english-comments' 417 | Enabled: false 418 | 419 | Style/AsciiIdentifiers: 420 | Description: 'Use only ascii symbols in identifiers.' 421 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#english-identifiers' 422 | Enabled: false 423 | 424 | Style/Attr: 425 | Description: 'Checks for uses of Module#attr.' 426 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#attr' 427 | Enabled: false 428 | 429 | Style/BeginBlock: 430 | Description: 'Avoid the use of BEGIN blocks.' 431 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-BEGIN-blocks' 432 | Enabled: false 433 | 434 | Style/BarePercentLiterals: 435 | Description: 'Checks if usage of %() or %Q() matches configuration.' 436 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-q-shorthand' 437 | Enabled: false 438 | 439 | Style/BlockComments: 440 | Description: 'Do not use block comments.' 441 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-block-comments' 442 | Enabled: false 443 | 444 | Style/BlockEndNewline: 445 | Description: 'Put end statement of multiline block on its own line.' 446 | Enabled: false 447 | 448 | Style/BlockDelimiters: 449 | Description: >- 450 | Avoid using {...} for multi-line blocks (multiline chaining is 451 | always ugly). 452 | Prefer {...} over do...end for single-line blocks. 453 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#single-line-blocks' 454 | Enabled: false 455 | 456 | Style/BracesAroundHashParameters: 457 | Description: 'Enforce braces style around hash parameters.' 458 | Enabled: false 459 | 460 | Style/CaseEquality: 461 | Description: 'Avoid explicit use of the case equality operator(===).' 462 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-case-equality' 463 | Enabled: false 464 | 465 | Style/CaseIndentation: 466 | Description: 'Indentation of when in a case/when/[else/]end.' 467 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#indent-when-to-case' 468 | Enabled: false 469 | 470 | Style/CharacterLiteral: 471 | Description: 'Checks for uses of character literals.' 472 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-character-literals' 473 | Enabled: false 474 | 475 | Style/ClassAndModuleCamelCase: 476 | Description: 'Use CamelCase for classes and modules.' 477 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#camelcase-classes' 478 | Enabled: false 479 | 480 | Style/ClassAndModuleChildren: 481 | Description: 'Checks style of children classes and modules.' 482 | Enabled: false 483 | 484 | Style/ClassCheck: 485 | Description: 'Enforces consistent use of `Object#is_a?` or `Object#kind_of?`.' 486 | Enabled: false 487 | 488 | Style/ClassMethods: 489 | Description: 'Use self when defining module/class methods.' 490 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#def-self-class-methods' 491 | Enabled: false 492 | 493 | Style/ClassVars: 494 | Description: 'Avoid the use of class variables.' 495 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-class-vars' 496 | Enabled: false 497 | 498 | Style/ClosingParenthesisIndentation: 499 | Description: 'Checks the indentation of hanging closing parentheses.' 500 | Enabled: false 501 | 502 | Style/ColonMethodCall: 503 | Description: 'Do not use :: for method call.' 504 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#double-colons' 505 | Enabled: false 506 | 507 | Style/CommandLiteral: 508 | Description: 'Use `` or %x around command literals.' 509 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-x' 510 | Enabled: false 511 | 512 | Style/CommentAnnotation: 513 | Description: 'Checks formatting of annotation comments.' 514 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#annotate-keywords' 515 | Enabled: false 516 | 517 | Style/CommentIndentation: 518 | Description: 'Indentation of comments.' 519 | Enabled: false 520 | 521 | Style/ConstantName: 522 | Description: 'Constants should use SCREAMING_SNAKE_CASE.' 523 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#screaming-snake-case' 524 | Enabled: false 525 | 526 | Style/DefWithParentheses: 527 | Description: 'Use def with parentheses when there are arguments.' 528 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#method-parens' 529 | Enabled: false 530 | 531 | Style/DeprecatedHashMethods: 532 | Description: 'Checks for use of deprecated Hash methods.' 533 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#hash-key' 534 | Enabled: false 535 | 536 | Style/Documentation: 537 | Description: 'Document classes and non-namespace modules.' 538 | Enabled: false 539 | 540 | Style/DotPosition: 541 | Description: 'Checks the position of the dot in multi-line method calls.' 542 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#consistent-multi-line-chains' 543 | Enabled: false 544 | 545 | Style/DoubleNegation: 546 | Description: 'Checks for uses of double negation (!!).' 547 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-bang-bang' 548 | Enabled: false 549 | 550 | Style/EachWithObject: 551 | Description: 'Prefer `each_with_object` over `inject` or `reduce`.' 552 | Enabled: false 553 | 554 | Style/ElseAlignment: 555 | Description: 'Align elses and elsifs correctly.' 556 | Enabled: false 557 | 558 | Style/EmptyElse: 559 | Description: 'Avoid empty else-clauses.' 560 | Enabled: false 561 | 562 | Style/EmptyLineBetweenDefs: 563 | Description: 'Use empty lines between defs.' 564 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#empty-lines-between-methods' 565 | Enabled: false 566 | 567 | Style/EmptyLines: 568 | Description: "Don't use several empty lines in a row." 569 | Enabled: false 570 | 571 | Style/EmptyLinesAroundAccessModifier: 572 | Description: "Keep blank lines around access modifiers." 573 | Enabled: false 574 | 575 | Style/EmptyLinesAroundBlockBody: 576 | Description: "Keeps track of empty lines around block bodies." 577 | Enabled: false 578 | 579 | Style/EmptyLinesAroundClassBody: 580 | Description: "Keeps track of empty lines around class bodies." 581 | Enabled: false 582 | 583 | Style/EmptyLinesAroundModuleBody: 584 | Description: "Keeps track of empty lines around module bodies." 585 | Enabled: false 586 | 587 | Style/EmptyLinesAroundMethodBody: 588 | Description: "Keeps track of empty lines around method bodies." 589 | Enabled: false 590 | 591 | Style/EmptyLiteral: 592 | Description: 'Prefer literals to Array.new/Hash.new/String.new.' 593 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#literal-array-hash' 594 | Enabled: false 595 | 596 | Style/EndBlock: 597 | Description: 'Avoid the use of END blocks.' 598 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-END-blocks' 599 | Enabled: false 600 | 601 | Style/EndOfLine: 602 | Description: 'Use Unix-style line endings.' 603 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#crlf' 604 | Enabled: false 605 | 606 | Style/EvenOdd: 607 | Description: 'Favor the use of Fixnum#even? && Fixnum#odd?' 608 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#predicate-methods' 609 | Enabled: false 610 | 611 | Style/ExtraSpacing: 612 | Description: 'Do not use unnecessary spacing.' 613 | Enabled: false 614 | 615 | Style/FileName: 616 | Description: 'Use snake_case for source file names.' 617 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#snake-case-files' 618 | Enabled: false 619 | 620 | Style/InitialIndentation: 621 | Description: >- 622 | Checks the indentation of the first non-blank non-comment line in a file. 623 | Enabled: false 624 | 625 | Style/FirstParameterIndentation: 626 | Description: 'Checks the indentation of the first parameter in a method call.' 627 | Enabled: false 628 | 629 | Style/FlipFlop: 630 | Description: 'Checks for flip flops' 631 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-flip-flops' 632 | Enabled: false 633 | 634 | Style/For: 635 | Description: 'Checks use of for or each in multiline loops.' 636 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-for-loops' 637 | Enabled: false 638 | 639 | Style/FormatString: 640 | Description: 'Enforce the use of Kernel#sprintf, Kernel#format or String#%.' 641 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#sprintf' 642 | Enabled: false 643 | 644 | Style/GlobalVars: 645 | Description: 'Do not introduce global variables.' 646 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#instance-vars' 647 | Reference: 'http://www.zenspider.com/Languages/Ruby/QuickRef.html' 648 | Enabled: false 649 | 650 | Style/GuardClause: 651 | Description: 'Check for conditionals that can be replaced with guard clauses' 652 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-nested-conditionals' 653 | Enabled: false 654 | 655 | Style/HashSyntax: 656 | Description: >- 657 | Prefer Ruby 1.9 hash syntax { a: 1, b: 2 } over 1.8 syntax 658 | { :a => 1, :b => 2 }. 659 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#hash-literals' 660 | Enabled: false 661 | 662 | Style/IfUnlessModifier: 663 | Description: >- 664 | Favor modifier if/unless usage when you have a 665 | single-line body. 666 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#if-as-a-modifier' 667 | Enabled: false 668 | 669 | Style/IfWithSemicolon: 670 | Description: 'Do not use if x; .... Use the ternary operator instead.' 671 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-semicolon-ifs' 672 | Enabled: false 673 | 674 | Style/IndentationConsistency: 675 | Description: 'Keep indentation straight.' 676 | Enabled: false 677 | 678 | Style/IndentationWidth: 679 | Description: 'Use 2 spaces for indentation.' 680 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-indentation' 681 | Enabled: false 682 | 683 | Style/IndentArray: 684 | Description: >- 685 | Checks the indentation of the first element in an array 686 | literal. 687 | Enabled: false 688 | 689 | Style/IndentHash: 690 | Description: 'Checks the indentation of the first key in a hash literal.' 691 | Enabled: false 692 | 693 | Style/InfiniteLoop: 694 | Description: 'Use Kernel#loop for infinite loops.' 695 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#infinite-loop' 696 | Enabled: false 697 | 698 | Style/Lambda: 699 | Description: 'Use the new lambda literal syntax for single-line blocks.' 700 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#lambda-multi-line' 701 | Enabled: false 702 | 703 | Style/LambdaCall: 704 | Description: 'Use lambda.call(...) instead of lambda.(...).' 705 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#proc-call' 706 | Enabled: false 707 | 708 | Style/LeadingCommentSpace: 709 | Description: 'Comments should start with a space.' 710 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#hash-space' 711 | Enabled: false 712 | 713 | Style/LineEndConcatenation: 714 | Description: >- 715 | Use \ instead of + or << to concatenate two string literals at 716 | line end. 717 | Enabled: false 718 | 719 | Style/MethodCallParentheses: 720 | Description: 'Do not use parentheses for method calls with no arguments.' 721 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-args-no-parens' 722 | Enabled: false 723 | 724 | Style/MethodDefParentheses: 725 | Description: >- 726 | Checks if the method definitions have or don't have 727 | parentheses. 728 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#method-parens' 729 | Enabled: false 730 | 731 | Style/MethodName: 732 | Description: 'Use the configured style when naming methods.' 733 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#snake-case-symbols-methods-vars' 734 | Enabled: false 735 | 736 | Style/ModuleFunction: 737 | Description: 'Checks for usage of `extend self` in modules.' 738 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#module-function' 739 | Enabled: false 740 | 741 | Style/MultilineBlockChain: 742 | Description: 'Avoid multi-line chains of blocks.' 743 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#single-line-blocks' 744 | Enabled: false 745 | 746 | Style/MultilineBlockLayout: 747 | Description: 'Ensures newlines after multiline block do statements.' 748 | Enabled: false 749 | 750 | Style/MultilineIfThen: 751 | Description: 'Do not use then for multi-line if/unless.' 752 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-then' 753 | Enabled: false 754 | 755 | Style/MultilineOperationIndentation: 756 | Description: >- 757 | Checks indentation of binary operations that span more than 758 | one line. 759 | Enabled: false 760 | 761 | Style/MultilineTernaryOperator: 762 | Description: >- 763 | Avoid multi-line ?: (the ternary operator); 764 | use if/unless instead. 765 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-multiline-ternary' 766 | Enabled: false 767 | 768 | Style/NegatedIf: 769 | Description: >- 770 | Favor unless over if for negative conditions 771 | (or control flow or). 772 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#unless-for-negatives' 773 | Enabled: false 774 | 775 | Style/NegatedWhile: 776 | Description: 'Favor until over while for negative conditions.' 777 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#until-for-negatives' 778 | Enabled: false 779 | 780 | Style/NestedTernaryOperator: 781 | Description: 'Use one expression per branch in a ternary operator.' 782 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-nested-ternary' 783 | Enabled: false 784 | 785 | Style/Next: 786 | Description: 'Use `next` to skip iteration instead of a condition at the end.' 787 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-nested-conditionals' 788 | Enabled: false 789 | 790 | Style/NilComparison: 791 | Description: 'Prefer x.nil? to x == nil.' 792 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#predicate-methods' 793 | Enabled: false 794 | 795 | Style/NonNilCheck: 796 | Description: 'Checks for redundant nil checks.' 797 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-non-nil-checks' 798 | Enabled: false 799 | 800 | Style/Not: 801 | Description: 'Use ! instead of not.' 802 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#bang-not-not' 803 | Enabled: false 804 | 805 | Style/NumericLiterals: 806 | Description: >- 807 | Add underscores to large numeric literals to improve their 808 | readability. 809 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#underscores-in-numerics' 810 | Enabled: false 811 | 812 | Style/OneLineConditional: 813 | Description: >- 814 | Favor the ternary operator(?:) over 815 | if/then/else/end constructs. 816 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#ternary-operator' 817 | Enabled: false 818 | 819 | Style/OpMethod: 820 | Description: 'When defining binary operators, name the argument other.' 821 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#other-arg' 822 | Enabled: false 823 | 824 | Style/OptionalArguments: 825 | Description: >- 826 | Checks for optional arguments that do not appear at the end 827 | of the argument list 828 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#optional-arguments' 829 | Enabled: false 830 | 831 | Style/ParallelAssignment: 832 | Description: >- 833 | Check for simple usages of parallel assignment. 834 | It will only warn when the number of variables 835 | matches on both sides of the assignment. 836 | This also provides performance benefits 837 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#parallel-assignment' 838 | Enabled: false 839 | 840 | Style/ParenthesesAroundCondition: 841 | Description: >- 842 | Don't use parentheses around the condition of an 843 | if/unless/while. 844 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-parens-if' 845 | Enabled: false 846 | 847 | Style/PercentLiteralDelimiters: 848 | Description: 'Use `%`-literal delimiters consistently' 849 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-literal-braces' 850 | Enabled: false 851 | 852 | Style/PercentQLiterals: 853 | Description: 'Checks if uses of %Q/%q match the configured preference.' 854 | Enabled: false 855 | 856 | Style/PerlBackrefs: 857 | Description: 'Avoid Perl-style regex back references.' 858 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-perl-regexp-last-matchers' 859 | Enabled: false 860 | 861 | Style/PredicateName: 862 | Description: 'Check the names of predicate methods.' 863 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#bool-methods-qmark' 864 | Enabled: false 865 | 866 | Style/Proc: 867 | Description: 'Use proc instead of Proc.new.' 868 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#proc' 869 | Enabled: false 870 | 871 | Style/RaiseArgs: 872 | Description: 'Checks the arguments passed to raise/fail.' 873 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#exception-class-messages' 874 | Enabled: false 875 | 876 | Style/RedundantBegin: 877 | Description: "Don't use begin blocks when they are not needed." 878 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#begin-implicit' 879 | Enabled: false 880 | 881 | Style/RedundantException: 882 | Description: "Checks for an obsolete RuntimeException argument in raise/fail." 883 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-explicit-runtimeerror' 884 | Enabled: false 885 | 886 | Style/RedundantReturn: 887 | Description: "Don't use return where it's not required." 888 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-explicit-return' 889 | Enabled: false 890 | 891 | Style/RedundantSelf: 892 | Description: "Don't use self where it's not needed." 893 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-self-unless-required' 894 | Enabled: false 895 | 896 | Style/RegexpLiteral: 897 | Description: 'Use / or %r around regular expressions.' 898 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-r' 899 | Enabled: false 900 | 901 | Style/RescueEnsureAlignment: 902 | Description: 'Align rescues and ensures correctly.' 903 | Enabled: false 904 | 905 | Style/RescueModifier: 906 | Description: 'Avoid using rescue in its modifier form.' 907 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-rescue-modifiers' 908 | Enabled: false 909 | 910 | Style/SelfAssignment: 911 | Description: >- 912 | Checks for places where self-assignment shorthand should have 913 | been used. 914 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#self-assignment' 915 | Enabled: false 916 | 917 | Style/Semicolon: 918 | Description: "Don't use semicolons to terminate expressions." 919 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-semicolon' 920 | Enabled: false 921 | 922 | Style/SignalException: 923 | Description: 'Checks for proper usage of fail and raise.' 924 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#fail-method' 925 | Enabled: false 926 | 927 | Style/SingleLineBlockParams: 928 | Description: 'Enforces the names of some block params.' 929 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#reduce-blocks' 930 | Enabled: false 931 | 932 | Style/SingleLineMethods: 933 | Description: 'Avoid single-line methods.' 934 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-single-line-methods' 935 | Enabled: false 936 | 937 | Style/SpaceBeforeFirstArg: 938 | Description: >- 939 | Checks that exactly one space is used between a method name 940 | and the first argument for method calls without parentheses. 941 | Enabled: true 942 | 943 | Style/SpaceAfterColon: 944 | Description: 'Use spaces after colons.' 945 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-operators' 946 | Enabled: false 947 | 948 | Style/SpaceAfterComma: 949 | Description: 'Use spaces after commas.' 950 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-operators' 951 | Enabled: false 952 | 953 | Style/SpaceAroundKeyword: 954 | Description: 'Use spaces around keywords.' 955 | Enabled: false 956 | 957 | Style/SpaceAfterMethodName: 958 | Description: >- 959 | Do not put a space between a method name and the opening 960 | parenthesis in a method definition. 961 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#parens-no-spaces' 962 | Enabled: false 963 | 964 | Style/SpaceAfterNot: 965 | Description: Tracks redundant space after the ! operator. 966 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-space-bang' 967 | Enabled: false 968 | 969 | Style/SpaceAfterSemicolon: 970 | Description: 'Use spaces after semicolons.' 971 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-operators' 972 | Enabled: false 973 | 974 | Style/SpaceBeforeBlockBraces: 975 | Description: >- 976 | Checks that the left block brace has or doesn't have space 977 | before it. 978 | Enabled: false 979 | 980 | Style/SpaceBeforeComma: 981 | Description: 'No spaces before commas.' 982 | Enabled: false 983 | 984 | Style/SpaceBeforeComment: 985 | Description: >- 986 | Checks for missing space between code and a comment on the 987 | same line. 988 | Enabled: false 989 | 990 | Style/SpaceBeforeSemicolon: 991 | Description: 'No spaces before semicolons.' 992 | Enabled: false 993 | 994 | Style/SpaceInsideBlockBraces: 995 | Description: >- 996 | Checks that block braces have or don't have surrounding space. 997 | For blocks taking parameters, checks that the left brace has 998 | or doesn't have trailing space. 999 | Enabled: false 1000 | 1001 | Style/SpaceAroundBlockParameters: 1002 | Description: 'Checks the spacing inside and after block parameters pipes.' 1003 | Enabled: false 1004 | 1005 | Style/SpaceAroundEqualsInParameterDefault: 1006 | Description: >- 1007 | Checks that the equals signs in parameter default assignments 1008 | have or don't have surrounding space depending on 1009 | configuration. 1010 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-around-equals' 1011 | Enabled: false 1012 | 1013 | Style/SpaceAroundOperators: 1014 | Description: 'Use a single space around operators.' 1015 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-operators' 1016 | Enabled: false 1017 | 1018 | Style/SpaceInsideBrackets: 1019 | Description: 'No spaces after [ or before ].' 1020 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-spaces-braces' 1021 | Enabled: false 1022 | 1023 | Style/SpaceInsideHashLiteralBraces: 1024 | Description: "Use spaces inside hash literal braces - or don't." 1025 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-operators' 1026 | Enabled: false 1027 | 1028 | Style/SpaceInsideParens: 1029 | Description: 'No spaces after ( or before ).' 1030 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-spaces-braces' 1031 | Enabled: false 1032 | 1033 | Style/SpaceInsideRangeLiteral: 1034 | Description: 'No spaces inside range literals.' 1035 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-space-inside-range-literals' 1036 | Enabled: false 1037 | 1038 | Style/SpaceInsideStringInterpolation: 1039 | Description: 'Checks for padding/surrounding spaces inside string interpolation.' 1040 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#string-interpolation' 1041 | Enabled: false 1042 | 1043 | Style/SpecialGlobalVars: 1044 | Description: 'Avoid Perl-style global variables.' 1045 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-cryptic-perlisms' 1046 | Enabled: false 1047 | 1048 | Style/StringLiterals: 1049 | Description: 'Checks if uses of quotes match the configured preference.' 1050 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#consistent-string-literals' 1051 | Enabled: false 1052 | 1053 | Style/StringLiteralsInInterpolation: 1054 | Description: >- 1055 | Checks if uses of quotes inside expressions in interpolated 1056 | strings match the configured preference. 1057 | Enabled: false 1058 | 1059 | Style/StructInheritance: 1060 | Description: 'Checks for inheritance from Struct.new.' 1061 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-extend-struct-new' 1062 | Enabled: false 1063 | 1064 | Style/SymbolLiteral: 1065 | Description: 'Use plain symbols instead of string symbols when possible.' 1066 | Enabled: false 1067 | 1068 | Style/SymbolProc: 1069 | Description: 'Use symbols as procs instead of blocks when possible.' 1070 | Enabled: false 1071 | 1072 | Style/Tab: 1073 | Description: 'No hard tabs.' 1074 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#spaces-indentation' 1075 | Enabled: false 1076 | 1077 | Style/TrailingBlankLines: 1078 | Description: 'Checks trailing blank lines and final newline.' 1079 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#newline-eof' 1080 | Enabled: false 1081 | 1082 | Style/TrailingCommaInArguments: 1083 | Description: 'Checks for trailing comma in parameter lists.' 1084 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-trailing-params-comma' 1085 | Enabled: false 1086 | 1087 | Style/TrailingCommaInLiteral: 1088 | Description: 'Checks for trailing comma in literals.' 1089 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-trailing-array-commas' 1090 | Enabled: false 1091 | 1092 | Style/TrailingWhitespace: 1093 | Description: 'Avoid trailing whitespace.' 1094 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-trailing-whitespace' 1095 | Enabled: false 1096 | 1097 | Style/TrivialAccessors: 1098 | Description: 'Prefer attr_* methods to trivial readers/writers.' 1099 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#attr_family' 1100 | Enabled: false 1101 | 1102 | Style/UnlessElse: 1103 | Description: >- 1104 | Do not use unless with else. Rewrite these with the positive 1105 | case first. 1106 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-else-with-unless' 1107 | Enabled: false 1108 | 1109 | Style/UnneededCapitalW: 1110 | Description: 'Checks for %W when interpolation is not needed.' 1111 | Enabled: false 1112 | 1113 | Style/UnneededPercentQ: 1114 | Description: 'Checks for %q/%Q when single quotes or double quotes would do.' 1115 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-q' 1116 | Enabled: false 1117 | 1118 | Style/TrailingUnderscoreVariable: 1119 | Description: >- 1120 | Checks for the usage of unneeded trailing underscores at the 1121 | end of parallel variable assignment. 1122 | Enabled: false 1123 | 1124 | Style/VariableInterpolation: 1125 | Description: >- 1126 | Don't interpolate global, instance and class variables 1127 | directly in strings. 1128 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#curlies-interpolate' 1129 | Enabled: false 1130 | 1131 | Style/VariableName: 1132 | Description: 'Use the configured style when naming variables.' 1133 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#snake-case-symbols-methods-vars' 1134 | Enabled: false 1135 | 1136 | Style/WhenThen: 1137 | Description: 'Use when x then ... for one-line cases.' 1138 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#one-line-cases' 1139 | Enabled: false 1140 | 1141 | Style/WhileUntilDo: 1142 | Description: 'Checks for redundant do after while or until.' 1143 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-multiline-while-do' 1144 | Enabled: false 1145 | 1146 | Style/WhileUntilModifier: 1147 | Description: >- 1148 | Favor modifier while/until usage when you have a 1149 | single-line body. 1150 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#while-as-a-modifier' 1151 | Enabled: false 1152 | 1153 | Style/WordArray: 1154 | Description: 'Use %w or %W for arrays of words.' 1155 | StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-w' 1156 | Enabled: false 1157 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.4.0 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | 3 | rvm: 4 | - 2.3.1 5 | - 2.4.0 6 | - 2.5.0 7 | - 2.6.0 8 | 9 | services: 10 | - redis-server 11 | 12 | script: 13 | - bundle exec rake spec && bundle exec codeclimate-test-reporter 14 | 15 | sudo: required 16 | dist: trusty 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## master 2 | 3 | ## 1.4.6 4 | - Update some dependencies (minor version upgrades) 5 | - Remove old JS client files (the current JS client can be found @ https://github.com/firehoseio/js_client) 6 | - Keep track of some client information in Firehose::Server::ChannelSubscription::ClientInfo 7 | - Improve logging 8 | - Use String#byte_size instead of String#size for Content-Length response header 9 | 10 | ## 1.4.3 11 | - Track errors and timeouts in metrics (type of error/timeout per channel and global error/timeout count) 12 | 13 | ## 1.4.2 14 | - Ensure firehose server port is an integer and parsed correctly 15 | 16 | ## 1.4.1 17 | - Fixed duplicate call to Rack async.callback which caused the server to send a PUT request response twice 18 | 19 | ## 1.4.0 20 | - Update faraday gem (old version had a bug that could cause our specs to fail) 21 | - Added metrics for channel & global events & expose them via HTTP as JSON at /metrics@firehose 22 | - Lock down dependency version to prevent build failure & fix code climate reporter to work with latest version in travis build. 23 | - Add support for defining & detecting deprecated channels that cause subscriptions and publishes to those channels to be logged. 24 | - Add support for persisting channels & messages. 25 | 26 | ## 1.3.9 27 | - Bugfix: Prevent channel sequence of 0 when there's messages buffered. 28 | 29 | ## 1.3.8 30 | - Fix params parsing for multiplexed subscriptions 31 | - Fix http long poll spec related to params parsing 32 | 33 | ## 1.3.7 34 | - Configuration of Redis via the `Firehose::Server.redis` object. 35 | - Less rubygem dependencies: JSON gem removed because its included in Ruby 2.x 36 | - DEPRECATED: `firehose javascript` command will be removed. Those have been moved to https://github.com/firehoseio/js_client. 37 | - Add support for per-subscriber message filtering via `Firehose::Server::MessageFilter` interface 38 | - Add `Firehose::Server::ChannelSubscription::Failed` Exception which can be raised by custom message filters to abort a channel subscription and return an error to the client. 39 | 40 | ## 1.3.6 41 | 42 | - DEPRECATION WARNING: `firehose javascript` has been removed from this project. 43 | - BUG: The null message bug fix in 1.3.5 introduce different behavior into the messages delivered when a client reconnects with a last_sequence. This has been corrected and 1.3.5 will be yanked. 44 | 45 | ## 1.3.5 46 | 47 | - Fix bug where null messages are sent over WebSockets to client. Issue https://github.com/firehoseio/firehose/issues/51. 48 | - Drop support for Ruby 1.9.3. Documented official Ruby support policy in README.md 49 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.4.10-buster 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | qt5-default libqt5webkit5-dev gstreamer1.0-plugins-base gstreamer1.0-tools gstreamer1.0-x 5 | 6 | WORKDIR /firehose/ 7 | COPY . /firehose/ 8 | RUN gem update --system 3.2.3 9 | RUN gem install bundler:2.3.6 10 | RUN bundle 11 | 12 | EXPOSE 7474 13 | CMD bundle exec firehose server 14 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "codeclimate-test-reporter", group: :test, require: nil 4 | 5 | gem "eventmachine", ">= 1.0.0" 6 | gem "em-hiredis", ">= 0.2.0" 7 | gem "thor" 8 | gem "faye-websocket" 9 | gem "em-http-request", ">= 1.0.0" 10 | gem "rack" 11 | 12 | # Specify your gem's dependencies in firehose.gemspec 13 | gemspec 14 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | firehose (1.4.6) 5 | em-hiredis (>= 0.2.0) 6 | em-http-request (>= 1.0.0) 7 | eventmachine (>= 1.0.0) 8 | faraday 9 | faye-websocket 10 | rack 11 | thor 12 | 13 | GEM 14 | remote: https://rubygems.org/ 15 | specs: 16 | addressable (2.8.0) 17 | public_suffix (>= 2.0.2, < 5.0) 18 | async_rack_test (0.0.5) 19 | capybara (2.12.1) 20 | addressable 21 | mime-types (>= 1.16) 22 | nokogiri (>= 1.3.3) 23 | rack (>= 1.0.0) 24 | rack-test (>= 0.5.4) 25 | xpath (~> 2.0) 26 | capybara-webkit (1.12.0) 27 | capybara (>= 2.3.0, < 2.13.0) 28 | json 29 | codeclimate-test-reporter (1.0.6) 30 | simplecov 31 | coffee-script (2.4.1) 32 | coffee-script-source 33 | execjs 34 | coffee-script-source (1.12.2) 35 | concurrent-ruby (1.0.5) 36 | cookiejar (0.3.3) 37 | crack (0.4.3) 38 | safe_yaml (~> 1.0.0) 39 | daemons (1.4.1) 40 | diff-lcs (1.3) 41 | docile (1.1.5) 42 | em-hiredis (0.3.1) 43 | eventmachine (~> 1.0) 44 | hiredis (~> 0.6.0) 45 | em-http-request (1.1.5) 46 | addressable (>= 2.3.4) 47 | cookiejar (!= 0.3.1) 48 | em-socksify (>= 0.3) 49 | eventmachine (>= 1.0.3) 50 | http_parser.rb (>= 0.6.0) 51 | em-socksify (0.3.1) 52 | eventmachine (>= 1.0.0.beta.4) 53 | eventmachine (1.2.7) 54 | execjs (2.7.0) 55 | faraday (1.9.3) 56 | faraday-em_http (~> 1.0) 57 | faraday-em_synchrony (~> 1.0) 58 | faraday-excon (~> 1.1) 59 | faraday-httpclient (~> 1.0) 60 | faraday-multipart (~> 1.0) 61 | faraday-net_http (~> 1.0) 62 | faraday-net_http_persistent (~> 1.0) 63 | faraday-patron (~> 1.0) 64 | faraday-rack (~> 1.0) 65 | faraday-retry (~> 1.0) 66 | ruby2_keywords (>= 0.0.4) 67 | faraday-em_http (1.0.0) 68 | faraday-em_synchrony (1.0.0) 69 | faraday-excon (1.1.0) 70 | faraday-httpclient (1.0.1) 71 | faraday-multipart (1.0.3) 72 | multipart-post (>= 1.2, < 3) 73 | faraday-net_http (1.0.1) 74 | faraday-net_http_persistent (1.2.0) 75 | faraday-patron (1.0.0) 76 | faraday-rack (1.0.0) 77 | faraday-retry (1.0.3) 78 | faye-websocket (0.10.7) 79 | eventmachine (>= 0.12.0) 80 | websocket-driver (>= 0.5.1) 81 | foreman (0.83.0) 82 | thor (~> 0.19.1) 83 | hashdiff (0.3.2) 84 | hiredis (0.6.1) 85 | http_parser.rb (0.6.0) 86 | json (2.6.1) 87 | kgio (2.11.0) 88 | libv8-node (15.14.0.1) 89 | mime-types (3.1) 90 | mime-types-data (~> 3.2015) 91 | mime-types-data (3.2016.0521) 92 | mini_portile2 (2.4.0) 93 | mini_racer (0.4.0) 94 | libv8-node (~> 15.14.0.0) 95 | multipart-post (2.1.1) 96 | nokogiri (1.10.3) 97 | mini_portile2 (~> 2.4.0) 98 | public_suffix (4.0.6) 99 | rack (1.6.13) 100 | rack-test (0.6.3) 101 | rack (>= 1.0) 102 | rainbows (4.4.3) 103 | kgio (~> 2.5) 104 | rack (~> 1.1) 105 | unicorn (~> 4.1) 106 | raindrops (0.17.0) 107 | rake (12.0.0) 108 | rspec (2.99.0) 109 | rspec-core (~> 2.99.0) 110 | rspec-expectations (~> 2.99.0) 111 | rspec-mocks (~> 2.99.0) 112 | rspec-core (2.99.2) 113 | rspec-expectations (2.99.2) 114 | diff-lcs (>= 1.1.3, < 2.0) 115 | rspec-mocks (2.99.4) 116 | ruby2_keywords (0.0.5) 117 | safe_yaml (1.0.4) 118 | simplecov (0.13.0) 119 | docile (~> 1.1.0) 120 | json (>= 1.8, < 3) 121 | simplecov-html (~> 0.10.0) 122 | simplecov-html (0.10.0) 123 | sprockets (3.7.2) 124 | concurrent-ruby (~> 1.0) 125 | rack (> 1, < 3) 126 | thin (1.8.1) 127 | daemons (~> 1.0, >= 1.0.9) 128 | eventmachine (~> 1.0, >= 1.0.4) 129 | rack (>= 1, < 3) 130 | thor (0.19.4) 131 | unicorn (4.9.0) 132 | kgio (~> 2.6) 133 | rack 134 | raindrops (~> 0.7) 135 | webmock (2.3.2) 136 | addressable (>= 2.3.6) 137 | crack (>= 0.3.2) 138 | hashdiff 139 | websocket-driver (0.6.5) 140 | websocket-extensions (>= 0.1.0) 141 | websocket-extensions (0.1.2) 142 | xpath (2.0.0) 143 | nokogiri (~> 1.3) 144 | 145 | PLATFORMS 146 | ruby 147 | 148 | DEPENDENCIES 149 | async_rack_test 150 | capybara-webkit 151 | codeclimate-test-reporter 152 | coffee-script 153 | em-hiredis (>= 0.2.0) 154 | em-http-request (>= 1.0.0) 155 | eventmachine (>= 1.0.0) 156 | faye-websocket 157 | firehose! 158 | foreman 159 | mini_racer 160 | rack 161 | rack-test 162 | rainbows (~> 4.4.3) 163 | rake 164 | rspec (~> 2) 165 | sprockets 166 | thin 167 | thor 168 | webmock 169 | 170 | BUNDLED WITH 171 | 2.3.6 172 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2011-2013 Brad Gessler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /PROTOCOL.md: -------------------------------------------------------------------------------- 1 | ## Protocol 2 | 3 | Firehose is designed to publish messages to flakey clients. Each message is published to the client with a sequence number. If the client gets disconnected it reconnects to Firehose with the last sequence number and tries to obtain the remaining messages. 4 | 5 | ### HTTP Long Polling 6 | 7 | For clients that don't support web sockets, HTTP long polling may be utilized to receive messages from Firehose. 8 | 9 | #### Single 10 | 11 | A single HTTP long polling connection subscribes to a channel via an HTTP request: 12 | 13 | ``` 14 | GET /my/channel/name?last_message_sequence=1&whisky=tango 15 | ``` 16 | 17 | The `last_message_sequence` parameter is used by Firehose reply messages to the client that may have been published while the client was disconnected. 18 | 19 | All other query parameters are passed into the channel as channel params. 20 | 21 | #### Multiplexing 22 | 23 | A client may listen for messages from multiple Firehose channels over one HTTP connection. The client initiates the connection via: 24 | 25 | ``` 26 | POST /channels@firehose 27 | 28 | { 29 | "/my/channel": "1", 30 | "/another/channel": "2" 31 | } 32 | ``` 33 | 34 | To subscribe to multiple channels with params over long polling, the following format may be utilized: 35 | 36 | ``` 37 | POST /channels@firehose 38 | 39 | { 40 | "/my/channel": { 41 | "last_message_sequence": "1", 42 | "whisky": "tango" 43 | }, 44 | "/another/channel": { 45 | "last_message_sequence": "2", 46 | "hotel": "foxtrot" 47 | } 48 | } 49 | ``` 50 | 51 | The key is the channel name and the value is the current message sequence of the message. 52 | 53 | 54 | ### WebSockets 55 | 56 | #### Single Channel Subscription 57 | 58 | A single WebSocket connection subscribes to a channel via the following JSON subscription message: 59 | 60 | ``` 61 | { 62 | last_message_sequence: 1, 63 | params: { 64 | whisky: "tango" 65 | } 66 | } 67 | ``` 68 | 69 | The `message_sequence` parameter is used by Firehose reply messages to the client that may have been published while the client was disconnected. 70 | 71 | Additional parameters stored under the `params` key are passed into the channel as channel params. 72 | 73 | #### Multiplexing 74 | 75 | A client may listen for messages from multiple Firehose channels over one WebSocket connection. The client initiates the multiplexed subscriptions by opening a WebSocket connection to the `channels@firehose` special endpoint, then sending the following JSON subscription message: 76 | 77 | ``` 78 | { 79 | multiplex_subscribe: [ 80 | { 81 | channel: "/my/channel", 82 | last_message_sequence: 1, 83 | params: { 84 | "whisky": "tango" 85 | } 86 | }, 87 | { 88 | channel: "/another/channel", 89 | last_message_sequence: 2, 90 | params: { 91 | "hotel": "foxtrot" 92 | } 93 | } 94 | ] 95 | } 96 | ``` 97 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | firehose: ./bin/firehose server -p $PORT 2 | -------------------------------------------------------------------------------- /Procfile.dev: -------------------------------------------------------------------------------- 1 | firehose: ./bin/firehose server -p $PORT 2 | redis: redis-server 3 | # guard: guard -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | __ _ _ 2 | / _(_) | | 3 | | |_ _ _ __ ___| |__ ___ ___ ___ 4 | | _| | '__/ _ \ '_ \ / _ \/ __|/ _ \ 5 | | | | | | | __/ | | | (_) \__ \ __/ 6 | |_| |_|_| \___|_| |_|\___/|___/\___| 7 | 8 | Build realtime web applications in Ruby and JS 9 | 10 | [![Build Status](https://travis-ci.org/firehoseio/firehose.svg?branch=master)](https://travis-ci.org/firehoseio/firehose) [![Code Climate](https://codeclimate.com/github/firehoseio/firehose/badges/gpa.svg)](https://codeclimate.com/github/firehoseio/firehose) [![Test Coverage](https://codeclimate.com/github/firehoseio/firehose/badges/coverage.svg)](https://codeclimate.com/github/firehoseio/firehose/coverage) 11 | 12 | # What is Firehose? 13 | 14 | Firehose is both a Rack application and JavaScript library that makes building real-time web applications possible. 15 | 16 | # Getting Started 17 | 18 | First, you'll need to [install and run Redis 2.6](http://redis.io/download). 19 | Version 2.6 is required because Firehose uses [Lua/EVAL](http://redis.io/commands/eval) for its transactions, which is not available in earlier versions of Redis. 20 | 21 | Then install the gem. 22 | 23 | ```sh 24 | $ gem install firehose 25 | ``` 26 | 27 | ## The Server 28 | 29 | Now fire up the server. 30 | 31 | ``` 32 | $ firehose server 33 | >> Thin web server (v1.3.1 codename Triple Espresso) 34 | >> Maximum connections set to 1024 35 | >> Listening on 127.0.0.1:7474, CTRL+C to stop 36 | ``` 37 | 38 | or 39 | 40 | ``` 41 | docker-compose build 42 | docker-compose up 43 | ``` 44 | 45 | In case you're wondering, the Firehose application server runs the Rack app `Firehose::Rack::App.new` inside of Thin or Rainbows! `Firehose::Rack::App` consists of a bunch of smaller apps and a middleware, which is useful for hacking. 46 | 47 | ## Publish a message to a bunch of subscribers 48 | 49 | Lets test it out! Open two terminal windows. In one window, curl: 50 | 51 | ```sh 52 | $ curl "http://localhost:7474/hello" 53 | ``` 54 | 55 | Then run the following in the other terminal: 56 | 57 | ```sh 58 | $ curl -X PUT -d "Greetings fellow human being..." "http://localhost:7474/hello" 59 | ``` 60 | 61 | and you should see the message in the other terminal. 62 | 63 | ```sh 64 | Greetings fellow human being... 65 | ``` 66 | 67 | ## Run the tests 68 | 69 | ```sh 70 | docker-compose run firehose bundle exec rspec spec 71 | ``` 72 | 73 | ## Yeah, so? 74 | 75 | You have a dirt simple HTTP pub-sub feed. You could setup an `after_commit` hook on ActiveRecord to push JSON to an end-point. On the other side, you could have a Backbone.js application that picks up the changes and updates the client-side UI. 76 | 77 | Holy mackerel! Its a nice, clean, RESTful way to build real-time web applications. 78 | 79 | # The JavaScript Consumer 80 | 81 | Firehose doesn't just stop at curl; it has a full-featured JavaScript client that lets you subscribe to channels for live updates. 82 | 83 | Still have the server running? Copy and paste the code below into Firebug or the WebKit console. 84 | 85 | ```javascript 86 | 87 | new Firehose.Consumer({ 88 | message: function(msg){ 89 | console.log(msg); 90 | }, 91 | connected: function(){ 92 | console.log("Great Scotts!! We're connected!"); 93 | }, 94 | disconnected: function(){ 95 | console.log("Well shucks, we're not connected anymore"); 96 | }, 97 | error: function(){ 98 | console.log("Well then, something went horribly wrong."); 99 | }, 100 | // Note that we do NOT specify a protocol here because we don't 101 | // know that yet. 102 | uri: '//localhost:7474/hello' 103 | }).connect(); 104 | ``` 105 | 106 | There's also a Consumer that uses channel multiplexing. 107 | The multiplexed consumer is useful for scenarios where you want to subscribe 108 | to messages from many channels at once, without having to use one connection 109 | per channel. You can specify a list of channels to subscribe to, including a 110 | handler function per channel that gets called with all messages coming from that 111 | channel. 112 | 113 | Example: 114 | 115 | ```javascript 116 | new Firehose.MultiplexedConsumer({ 117 | connected: function(){ 118 | console.log("Great Scotts!! We're connected!"); 119 | }, 120 | disconnected: function(){ 121 | console.log("Well shucks, we're not connected anymore"); 122 | }, 123 | error: function(){ 124 | console.log("Well then, something went horribly wrong."); 125 | }, 126 | // Note that we don't specify a general message handler function 127 | // but instead define one per channel below 128 | 129 | // Note that we do NOT specify a protocol here because we don't 130 | // know that yet. We also don't specify a specific channel name as part of 131 | // the URI but instead pass in a list of subscriptions below 132 | uri: '//localhost:7474/', 133 | 134 | // List of channel subscriptions: 135 | channels: { 136 | "/my/channel/1": { 137 | last_sequence: 10, // defaults to 0 and can be ommitted 138 | message: function(msg) { 139 | console.log("got message on channel 1:"); 140 | console.log(msg); 141 | } 142 | }, 143 | "/my/channel/2": { 144 | message: function(msg) { 145 | console.log("got message on channel 2:"); 146 | console.log(msg); 147 | } 148 | } 149 | } 150 | }).connect(); 151 | ``` 152 | 153 | Then publish another message. 154 | 155 | ```sh 156 | $ curl -X PUT -d "\"This is almost magical\"" "http://localhost:7474/hello" 157 | ``` 158 | 159 | # How is it different from socket.io? 160 | 161 | socket.io attempts to store connection state per node instance. Firehose makes no attempt to store connection state. 162 | 163 | Also, socket.io attempts to abstract a low-latency full-duplex port. Firehose assumes that its impossible to simulate this in older web browsers that don't support WebSockets. As such, Firehose focuses on low-latency server-to-client connections and encourages the use of existing HTTP transports, like POST and PUT, for client-to-server communications. 164 | 165 | # The Ruby Publisher 166 | 167 | While you can certainly make your own PUT requests when publishing messages, Firehose includes a Ruby client for easy publishing. 168 | 169 | ```ruby 170 | require 'firehose' 171 | require 'json' 172 | json = {'hello'=> 'world'}.to_json 173 | firehose = Firehose::Client::Producer::Http.new('//127.0.0.1:7474') 174 | firehose.publish(json).to("/my/messages/path") 175 | ``` 176 | 177 | ## Publishing Options 178 | You can pass additional options to the publisher that set specific custom 179 | configuration http headers. The options available are: 180 | - TTL (how long should the message be buffered for) 181 | - Buffer size (how many messages for the channel should be kept in the buffer) 182 | - Deprecated (if marked as deprecated, any publications or subscriptions to the channel will be logged with a deprecation warning) 183 | - Persist (persisting causes the channel & message to not be expired after a given or the default TTL) 184 | 185 | The corresponding HTTP headers and allowed values are: 186 | - `Cache-Control: int` 187 | - `X-Firehose-Buffer-Size: int` 188 | - `X-Firehose-Deprecated: true | false` 189 | - `X-Firehose-Persist: true | false` 190 | 191 | ```ruby 192 | firehose = Firehose::Client::Producer::Http.new('//127.0.0.1:7474') 193 | # mark channel as deprecated 194 | firehose.publish(json).to("/my/messages/path", deprecated: true) 195 | # expire after 120 seconds 196 | firehose.publish(json).to("/my/messages/path", ttl: 120) 197 | # only keep last item 198 | firehose.publish(json).to("/my/messages/path", buffer_size: 1) 199 | # persist channel & message forever (or until a new message for this channel declares a new TTL and persist != true) 200 | firehose.publish(json).to("/my/messages/path", persist: true) 201 | ``` 202 | 203 | These options can be of course be combined within a single request. 204 | 205 | # Configuration 206 | 207 | Firehose can be configured via environmental variables. Take a look at the [`.env.sample`](./.env.sample) file for more info. 208 | 209 | ## Server Configuration 210 | 211 | The Firehose server may be configured via the `Firehose::Server.configuration` object as follows: 212 | 213 | ```ruby 214 | require "firehose" 215 | 216 | # Implement a custom message handler. 217 | class MyFilter < Firehose::Server::MessageFilter 218 | def process(message) 219 | # SHOUT AT ALL THE SUBSCRIBERS! 220 | name = params["name"] 221 | message.payload = "HEY #{name}!, #{message.payload.upcase}!" 222 | end 223 | end 224 | 225 | Firehose::Server.configuration do |config| 226 | # Custom message filter. This is useful if you want to implement 227 | # authorization per-message for Firehose. 228 | config.message_filter = MyFilter 229 | 230 | # Configure redis connection. 231 | config.redis.url = ENV.fetch "FIREHOSE_REDIS_URL", "redis://redis:6379/10" 232 | end 233 | ``` 234 | 235 | ## Custom MessageFilters 236 | 237 | As mentioned above you can define custom MessageFilters which allow you to 238 | add custom logic for things like authentication & filtering of content. 239 | By default, the `Firehose::Server::MessageFilter` base class is used, which does 240 | nothing to the messages being published. 241 | You can override the following methods in your own implementations: 242 | 243 | ```ruby 244 | class MyFilter < Firehose::Server::MessageFilter 245 | # Optional override if you need to do any other setup operation. 246 | # Make sure to call super(channel). 247 | # - channel: name of the channel (String) 248 | def initialize(channel) 249 | super(channel) 250 | MyLogger.info "Subscribing to channel: #{channel}" 251 | end 252 | 253 | # Optional, called once before process(). 254 | # - params: Hash of params of the subscription message the client sent 255 | def on_subscribe(params) 256 | @my_param = params["my-param"].to_i 257 | 258 | # You can also optionally raise an instance of 259 | # Firehose::Server::ChannelSubscription::Failed 260 | # this will cause the client to receive an error message of the form: 261 | # { error: "Subscription failed", reason: error_reason } 262 | # and the client will call its `subscriptionFailed` callback (if configured) 263 | end 264 | 265 | # Custom logic for a message to be published to client. 266 | # - message: Firehose::Server::Message instance 267 | def process(message) 268 | if @my_param > 10 269 | message.payload += "My-Param: #{@my_param}" 270 | end 271 | end 272 | 273 | # optional cleanup logic 274 | def on_unsubscribe 275 | end 276 | end 277 | ``` 278 | 279 | ## Deprecation logging for channels 280 | 281 | ## Client publishing option 282 | You can mark a message as deprecated (to be logged by Firehose) by passing 283 | `deprecated: true` as an option to `Firehose::Client::Producer::HTTP#put`. 284 | 285 | ```ruby 286 | firehose = Firehose::Client::Producer::Http.new('//127.0.0.1:7474') 287 | firehose.publish("{'hello': 'world'}").to("/my/messages/path", deprecated: true) 288 | ``` 289 | 290 | ## Server side config 291 | You can specify a list of channels that are marked as deprecated and will cause subscription and publish events on any of those channels to be logged with a special deprecation message. 292 | 293 | Example config: 294 | 295 | ```ruby 296 | Firehose::Server.configuration do |config| 297 | # set a static list of deprecated channels: 298 | config.deprecated_channels = ["/foo/bar.json", "/foo/bar/baz.json"] 299 | # provide a block to determine if a channel is deprecated via custom logic: 300 | config.deprecated_channel do |channel| 301 | channel =~ /^\/foo\/*\.json$/ 302 | end 303 | end 304 | ``` 305 | 306 | ## Rack Configuration 307 | 308 | There are two rack applications that are included with Firehose: `Firehose::Rack::Producer` which a client can `PUT` HTTP request with message payloads to publish information on Firehose and the `Firehose::Rack::Consumer` application which a client connects to via HTTP long polling or WebSockets to consume a message. 309 | 310 | ### Consumer Configuration 311 | 312 | ```ruby 313 | # Kitchen-sink rack configuration file example 314 | require 'firehose' 315 | 316 | consumer = Firehose::Rack::Consumer.new do |app| 317 | # Configure how long the server should wait before send the client a 204 318 | # with a request to reconnect. Typically browsers time-out the client connection 319 | # after 30 seconds, so we set the `Firehose.Consumer` JS client to 25, and the 320 | # server to 20 to make sure latency or timing doesn't cause any problems. 321 | app.http_long_poll.timeout = 20 322 | end 323 | 324 | run consumer 325 | ``` 326 | 327 | ### Publisher Configuration 328 | 329 | ```ruby 330 | # Kitchen-sink rack configuration file example 331 | require 'firehose' 332 | 333 | # There's nothing to configure with the Publisher, but its possible that 334 | # you might include rack middleware authorization mechanisms here to control 335 | # who can publish to Firehose. 336 | 337 | run Firehose::Rack::Publisher.new 338 | ``` 339 | 340 | ## Sprockets 341 | 342 | Using Sprockets is the recommended method of including the included client-side assets in a web page. 343 | 344 | 1. Add the firehose gem in your app's Gemfile. 345 | 346 | 2. Append the firehose gem's assets to the sprockets path. In a Rails app, this is usually done in an initializer. 347 | 348 | ```ruby 349 | # Add firehose to a custom sprockets configuration. 350 | my_sprockets_env = Sprockets::Environment.new 351 | Firehose::Assets::Sprockets.configure my_sprockets_env 352 | ``` 353 | 354 | 3. Require your config file and the firehose gem. This would look something like this: 355 | 356 | ```ruby 357 | #= require some/other/js/file 358 | #= require lib/firehose_config 359 | #= require firehose 360 | #= require some/more/js/files 361 | ``` 362 | 363 | It is important that your firehose config file comes first. 364 | 365 | ### Not using sprockets? 366 | 367 | If you don't intend to use the Firehose JavaScript client in a Ruby stack where Sprockets is available, you can grab the unminified source by running: 368 | 369 | ```sh 370 | $ firehose javascript > firehose.js 371 | ``` 372 | 373 | Copy the firehose.js where needed in your project. 374 | 375 | # Web Server 376 | 377 | Firehose currently supports Thin and Rainbows! (which is the default). Neither is listed as a dependency in the gemspec so that you don't need to install whichever one you aren't using. You can set which server to use via the `.env` file (recommended) or with the `-s` option to `bin/firehose`. 378 | 379 | # Exception Notification 380 | 381 | If you'd like to be notified of exceptions, add something like this in your custom config.ru file. 382 | 383 | ```ruby 384 | # Use exceptional to handle anything missed by Rack::Exceptional 385 | if exceptional_key = ENV['EXCEPTIONAL_KEY'] 386 | require 'exceptional' 387 | EM.error_handler do |e| 388 | Firehose.logger.error "Unhandled exception: #{e.class} #{e.message}\n#{e.backtrace.join "\n"}" 389 | ::Exceptional.handle(e) 390 | end 391 | end 392 | ``` 393 | 394 | # Deployment 395 | 396 | The recommended method of deploying Firehose is to deploy it separately from your main app. 397 | 398 | 1. Create a new project with a Gemfile such as 399 | 400 | ``` 401 | gem "firehose" 402 | gem "airbrake" 403 | gem "rainbows", :require => false 404 | gem "rack", "~> 1.4.0" # if you're using Rainbows. See https://github.com/firehoseio/firehose/commit/dfe55fff 405 | gem "foreman", :require => false 406 | gem "capistrano", :require => false 407 | ``` 408 | 409 | Of course, you could use `exceptional` instead of `airbrake` and `thin` instead of `rainbows`. 410 | 411 | 2. Set up `config/deploy.rb` to your liking. You can follow most directions for using Capistrano and Foreman to deploy Rack apps, such as https://gist.github.com/1027117 412 | 413 | 3. Set up `config/rainbows.rb` (if you are using Rainbows!). The gem includes example configurations scripts to get you started. There's also an example at https://gist.github.com/bradgessler/f2416efdbb1771e983b3. 414 | 415 | # New releases & version bump 416 | 417 | For a new release of Firehose, bump the version number in `lib/firehose/version.rb` as well as `package.json`. 418 | Make sure, they have the same version number. 419 | 420 | # Support 421 | 422 | ## Ruby version 423 | 424 | Firehose will support the latest minor 2.x revisions of Ruby that are officially supported by the Ruby community. More details at https://www.ruby-lang.org/. 425 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "rake/testtask" 2 | require 'bundler/gem_tasks' 3 | require 'rspec/core/rake_task' 4 | 5 | task :default => :spec 6 | task :debug => ["spec:debug"] 7 | task :ci => ["spec:ci"] 8 | 9 | task :spec => ["spec:all"] 10 | 11 | namespace :spec do 12 | desc "Run all specs" 13 | task :all => [:ruby] 14 | 15 | desc 'run Rspec specs' 16 | task :ruby do 17 | sh 'rspec spec' 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /bin/firehose: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | 5 | $:.unshift File.expand_path('../../lib', __FILE__) 6 | 7 | require 'firehose' 8 | 9 | Firehose::CLI.start -------------------------------------------------------------------------------- /coffeelint.json: -------------------------------------------------------------------------------- 1 | { 2 | "arrow_spacing": { 3 | "level": "ignore" 4 | }, 5 | "braces_spacing": { 6 | "level": "ignore", 7 | "spaces": 0, 8 | "empty_object_spaces": 0 9 | }, 10 | "camel_case_classes": { 11 | "level": "error" 12 | }, 13 | "coffeescript_error": { 14 | "level": "error" 15 | }, 16 | "colon_assignment_spacing": { 17 | "level": "ignore", 18 | "spacing": { 19 | "left": 0, 20 | "right": 0 21 | } 22 | }, 23 | "cyclomatic_complexity": { 24 | "value": 10, 25 | "level": "ignore" 26 | }, 27 | "duplicate_key": { 28 | "level": "error" 29 | }, 30 | "empty_constructor_needs_parens": { 31 | "level": "ignore" 32 | }, 33 | "ensure_comprehensions": { 34 | "level": "warn" 35 | }, 36 | "eol_last": { 37 | "level": "ignore" 38 | }, 39 | "indentation": { 40 | "value": 2, 41 | "level": "error" 42 | }, 43 | "line_endings": { 44 | "level": "ignore", 45 | "value": "unix" 46 | }, 47 | "max_line_length": { 48 | "value": 80, 49 | "level": "error", 50 | "limitComments": true 51 | }, 52 | "missing_fat_arrows": { 53 | "level": "ignore", 54 | "is_strict": false 55 | }, 56 | "newlines_after_classes": { 57 | "value": 3, 58 | "level": "ignore" 59 | }, 60 | "no_backticks": { 61 | "level": "error" 62 | }, 63 | "no_debugger": { 64 | "level": "warn", 65 | "console": false 66 | }, 67 | "no_empty_functions": { 68 | "level": "ignore" 69 | }, 70 | "no_empty_param_list": { 71 | "level": "ignore" 72 | }, 73 | "no_implicit_braces": { 74 | "level": "ignore", 75 | "strict": true 76 | }, 77 | "no_implicit_parens": { 78 | "strict": true, 79 | "level": "ignore" 80 | }, 81 | "no_interpolation_in_single_quotes": { 82 | "level": "ignore" 83 | }, 84 | "no_plusplus": { 85 | "level": "ignore" 86 | }, 87 | "no_stand_alone_at": { 88 | "level": "ignore" 89 | }, 90 | "no_tabs": { 91 | "level": "error" 92 | }, 93 | "no_this": { 94 | "level": "ignore" 95 | }, 96 | "no_throwing_strings": { 97 | "level": "error" 98 | }, 99 | "no_trailing_semicolons": { 100 | "level": "error" 101 | }, 102 | "no_trailing_whitespace": { 103 | "level": "error", 104 | "allowed_in_comments": false, 105 | "allowed_in_empty_lines": true 106 | }, 107 | "no_unnecessary_double_quotes": { 108 | "level": "ignore" 109 | }, 110 | "no_unnecessary_fat_arrows": { 111 | "level": "warn" 112 | }, 113 | "non_empty_constructor_needs_parens": { 114 | "level": "ignore" 115 | }, 116 | "prefer_english_operator": { 117 | "level": "ignore", 118 | "doubleNotLevel": "ignore" 119 | }, 120 | "space_operators": { 121 | "level": "ignore" 122 | }, 123 | "spacing_after_comma": { 124 | "level": "ignore" 125 | }, 126 | "transform_messes_up_line_numbers": { 127 | "level": "warn" 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /config/rainbows.rb: -------------------------------------------------------------------------------- 1 | # Configuration from http://rainbows.rubyforge.org/Rainbows/Configurator.html. Don't juse 2 | # blindly copy and paste this configuration! Be sure you have read and understand the Rainbows 3 | # configuration documentation for your environment. 4 | 5 | Rainbows! do 6 | use :EventMachine # concurrency model 7 | worker_connections 400 8 | keepalive_timeout 0 # disables keepalives 9 | keepalive_requests 666 # default:100 10 | client_max_body_size 5 * 1024 * 1024 # 5 megabytes 11 | client_header_buffer_size 2 * 1024 # 2 kilobytes 12 | end 13 | 14 | # the rest of the Unicorn configuration... 15 | worker_processes [ENV['WORKER_PROCESSES'].to_i, 1].max # Default to 1 16 | working_directory ENV['WORKING_DIRECTORY'] if ENV['WORKING_DIRECTORY'] 17 | logger Firehose.logger -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Isolated development and test environment. 2 | version: '2' 3 | services: 4 | firehose: 5 | build: . 6 | depends_on: 7 | - redis 8 | environment: 9 | REDIS_URL: redis://redis:6379/0 10 | ports: 11 | - 7474:7474 12 | volumes: 13 | - .:/firehose/ 14 | redis: 15 | image: redis 16 | expose: 17 | - 6379 18 | -------------------------------------------------------------------------------- /firehose.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "firehose/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "firehose" 7 | s.version = Firehose::VERSION 8 | s.authors = ["Brad Gessler", "Steel Fu", "Paul Cortens", "Zach Zolton", "Christopher Bertels"] 9 | s.email = ["brad@polleverywhere.com", "steel@polleverywhere.com", "paul@polleverywhere.com", "zach@polleverywhere.com", "christopher@polleverywhere.com"] 10 | s.homepage = "http://firehose.io/" 11 | s.summary = %q{Build realtime Ruby web applications} 12 | s.description = %q{Firehose is a realtime web application toolkit for building realtime Ruby web applications.} 13 | s.license = "MIT" 14 | 15 | s.files = `git ls-files`.split("\n") 16 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 17 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 18 | s.require_paths = ["lib"] 19 | 20 | # specify any dependencies here; for example: 21 | s.add_runtime_dependency "eventmachine", ">= 1.0.0" 22 | s.add_runtime_dependency "em-hiredis", ">= 0.2.0" 23 | s.add_runtime_dependency "thor" 24 | s.add_runtime_dependency "faraday" 25 | s.add_runtime_dependency "faye-websocket" 26 | s.add_runtime_dependency "em-http-request", ">= 1.0.0" 27 | s.add_runtime_dependency "rack" 28 | 29 | s.add_development_dependency "rspec", "~> 2" 30 | s.add_development_dependency "webmock" 31 | s.add_development_dependency "coffee-script" 32 | s.add_development_dependency "rainbows", "~> 4.4.3" 33 | s.add_development_dependency "thin" 34 | s.add_development_dependency "rack-test" 35 | s.add_development_dependency "async_rack_test" 36 | s.add_development_dependency "foreman" 37 | s.add_development_dependency "sprockets" 38 | s.add_development_dependency "rake" 39 | s.add_development_dependency "capybara-webkit" 40 | s.add_development_dependency "mini_racer" 41 | end 42 | -------------------------------------------------------------------------------- /lib/firehose.rb: -------------------------------------------------------------------------------- 1 | ENV['RACK_ENV'] ||= 'development' # TODO - Lets not rock out envs like its 1999. 2 | 3 | require 'uri' 4 | require 'firehose/version' 5 | require 'firehose/logging' 6 | require 'json' 7 | 8 | # TODO - Figure if we need to have an if/else for Rails::Engine loading and Firehose::Assets::Sprockets.auto_detect 9 | require 'firehose/rails' if defined?(::Rails::Engine) 10 | 11 | module Firehose 12 | autoload :Server, 'firehose/server' 13 | autoload :Client, 'firehose/client' 14 | autoload :Assets, 'firehose/assets' 15 | autoload :Rack, 'firehose/rack' 16 | autoload :CLI, 'firehose/cli' 17 | 18 | # Default URI for the Firehose server. Consider the port "well-known" and bindable from other apps. 19 | URI = URI.parse("//0.0.0.0:7474").freeze 20 | end 21 | -------------------------------------------------------------------------------- /lib/firehose/cli.rb: -------------------------------------------------------------------------------- 1 | require 'thor' 2 | require 'eventmachine' 3 | require 'uri' 4 | 5 | # Enable native async-io libs 6 | EM.kqueue if EM.kqueue? 7 | EM.epoll if EM.epoll? 8 | 9 | module Firehose 10 | class CLI < Thor 11 | def initialize(*args) 12 | super 13 | # Disable buffering to $stdio for Firehose.logger 14 | $stdout.sync = true 15 | end 16 | 17 | desc "javascript", "Compile the Firehose JavaScript." 18 | def javascript 19 | $stderr.puts "DEPRECATION WARNING: Firehose JS assets have been moved to https://github.com/firehoseio/js_client" 20 | $stdout.puts Firehose::Assets::Sprockets.javascript 21 | end 22 | 23 | desc "version", "Display the current version." 24 | def version 25 | puts %[Firehose #{Firehose::VERSION} "#{Firehose::CODENAME}"] 26 | end 27 | 28 | desc "server", "Start an instance of a server." 29 | method_option :port, :type => :numeric, :default => (ENV['PORT'] || Firehose::URI.port).to_i, :required => false, :aliases => '-p' 30 | method_option :host, :type => :string, :default => ENV['HOST'] || Firehose::URI.host, :required => false, :aliases => '-h' 31 | method_option :server, :type => :string, :default => ENV['SERVER'] ||'rainbows', :required => false, :aliases => '-s' 32 | def server 33 | begin 34 | Firehose::Server::App.new(options).start 35 | rescue => e 36 | Firehose.logger.error "#{e.message}: #{e.backtrace}" 37 | raise e 38 | end 39 | end 40 | 41 | desc "consume URI", "Consume messages from a resource." 42 | method_option :concurrency, :type => :numeric, :default => 1, :aliases => '-c' 43 | def consume(uri) 44 | EM.run do 45 | options[:concurrency].times { Firehose::Client::Consumer.parse(uri).request } 46 | end 47 | end 48 | 49 | desc "publish URI [PAYLOAD]", "Publish messages to a resource." 50 | method_option :ttl, :type => :numeric, :aliases => '-t' 51 | method_option :times, :type => :numeric, :aliases => '-n', :default => 1 52 | method_option :interval, :type => :numeric, :aliases => '-i' 53 | def publish(uri, payload=nil) 54 | payload ||= $stdin.read 55 | client = Firehose::Client::Producer::Http.new(uri) 56 | path = ::URI.parse(uri).path 57 | times = options[:times] 58 | ttl = options[:ttl] 59 | 60 | EM.run do 61 | # TODO I think this can be cleaned up so the top-level if/else can be ditched. 62 | if interval = options[:interval] 63 | # Publish messages at a forced interval. 64 | EM.add_periodic_timer interval do 65 | client.publish(payload).to(path, :ttl => ttl) 66 | EM.stop if times && (times-=1).zero? 67 | end 68 | else 69 | # Publish messages as soon as the last message was published. 70 | worker = Proc.new do 71 | client.publish(payload).to(path, :ttl => ttl) 72 | times && (times-=1).zero? ? EM.stop : worker.call 73 | end 74 | worker.call 75 | end 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/firehose/client.rb: -------------------------------------------------------------------------------- 1 | require 'em-http' 2 | require 'faye/websocket' 3 | 4 | module Firehose 5 | # Ruby clients that connect to Firehose to either publish or consume messages. 6 | module Client 7 | autoload :Consumer, 'firehose/client/consumer' 8 | autoload :Producer, 'firehose/client/producer' 9 | end 10 | end -------------------------------------------------------------------------------- /lib/firehose/client/consumer.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | # TODO - Spec this thing out. Unfortunately its not tested at all, mostly because the JSON client 4 | # is more important (and tested). Still, this should be tested, at least used by the specs 5 | # to test HTTP functionality of the server. 6 | module Firehose 7 | module Client 8 | module Consumer 9 | # TODO - Test this libs. I had to throw these quickly into our app so that we could get 10 | # some stress testing out of the way. 11 | # TODO - Replace the integration test clients with these guys. You'll want to refactor each 12 | # transport to use on(:message), on(:conncect), and on(:disconnect) callbacks. 13 | TransportNotSupportedError = Class.new(RuntimeError) 14 | 15 | # Build up a benchmark client based on a given URI. Accepts ws:// and http:// for now. 16 | def self.parse(uri) 17 | case transport = ::URI.parse(uri).scheme 18 | # TODO - Fix ws:// transport! See class WebSocket below to udnerstand 19 | # why this doesn't work and support is dropped from the CLI. 20 | # when 'ws' 21 | # Consumer::WebSocket.new(uri) 22 | when 'http' 23 | Consumer::HttpLongPoll.new(uri) 24 | else 25 | raise TransportNotSupportedError.new("Transport #{transport.inspect} not supported.") 26 | end 27 | end 28 | 29 | # TODO - This won't even work. Dropping ws:// above until this is tested. This thing 30 | # should be sending message sequences to Firehose. 31 | # Connect to Firehose via WebSockets and consume messages. 32 | class WebSocket 33 | attr_reader :url, :logger 34 | 35 | def initialize(url, logger = Firehose.logger) 36 | @url, @logger = url, logger 37 | end 38 | 39 | def request 40 | ws = Faye::WebSocket::Client.new(url) 41 | ws.onmessage = lambda do |event| 42 | logger.info "WS | #{event.data[0...40].inspect}" 43 | end 44 | ws.onclose = lambda do |event| 45 | logger.info "WS | Closed" 46 | end 47 | ws.onerror do 48 | logger.error "WS | Failed" 49 | end 50 | end 51 | end 52 | 53 | # Connect to Firehose via HTTP Long Polling and consume messages. 54 | class HttpLongPoll 55 | JITTER = 0.003 56 | 57 | attr_reader :url, :logger 58 | 59 | def initialize(url, logger = Firehose.logger) 60 | @url, @logger = url, logger 61 | end 62 | 63 | def request(last_sequence=0) 64 | http = EM::HttpRequest.new(url, :inactivity_timeout => 0).get(:query => {'last_message_sequence' => last_sequence}) 65 | http.callback do 66 | case status = http.response_header.status 67 | when 200 68 | json = JSON.parse(http.response) 69 | next_sequence = json['last_sequence'].to_i 70 | message = json['message'] 71 | 72 | logger.info "HTTP 200 | Next Sequence: #{next_sequence} - #{message[0...40].inspect}" 73 | EM::add_timer(jitter) { request next_sequence } 74 | when 204 75 | logger.info "HTTP 204 | Last Sequence #{last_sequence}" 76 | EM::add_timer(jitter) { request last_sequence } 77 | else 78 | logger.error "HTTP #{status} | Failed" 79 | end 80 | end 81 | http.errback do 82 | logger.error "Connection Failed" 83 | end 84 | end 85 | 86 | private 87 | # Random jitter between long poll requests. 88 | def jitter 89 | rand*JITTER 90 | end 91 | end 92 | end 93 | end 94 | end -------------------------------------------------------------------------------- /lib/firehose/client/producer.rb: -------------------------------------------------------------------------------- 1 | require "faraday" 2 | require "uri" 3 | 4 | module Firehose 5 | module Client 6 | module Producer 7 | # Publish messages to Firehose via an HTTP interface. 8 | class Http 9 | # Exception gets raised when a 202 is _not_ received from the server after a message is published. 10 | PublishError = Class.new(RuntimeError) 11 | TimeoutError = Class.new(Faraday::TimeoutError) 12 | DEFAULT_TIMEOUT = 1 # How many seconds should we wait for a publish to take? 13 | DEFAULT_ERROR_HANDLER = ->(e) { raise e } 14 | 15 | # A DSL for publishing requests. This doesn't so much, but lets us call 16 | # Firehose::Client::Producer::Http#publish('message').to('channel'). Slick eh? If you don't like it, 17 | # just all Firehose::Client::Producer::Http#put('message', 'channel') 18 | class Builder 19 | def initialize(producer, message) 20 | @producer, @message = producer, message 21 | self 22 | end 23 | 24 | def to(channel, opts={}, &callback) 25 | @producer.put(@message, channel, opts, &callback) 26 | end 27 | end 28 | 29 | # URI for the Firehose server. This URI does not include the path of the channel. 30 | attr_reader :uri, :timeout 31 | 32 | def initialize(uri = Firehose::URI, timeout=DEFAULT_TIMEOUT) 33 | @uri = ::URI.parse(uri.to_s) 34 | @uri.scheme ||= 'http' 35 | @timeout = timeout 36 | end 37 | 38 | # A DSL for publishing messages. 39 | def publish(message) 40 | Builder.new(self, message) 41 | end 42 | 43 | # Publish the message via HTTP. 44 | def put(message, channel, opts, &block) 45 | ttl = opts[:ttl] 46 | timeout = opts[:timeout] || @timeout || DEFAULT_TIMEOUT 47 | buffer_size = opts[:buffer_size] 48 | deprecated = opts[:deprecated] 49 | persist = opts[:persist] 50 | 51 | response = conn.put do |req| 52 | req.options[:timeout] = timeout 53 | if conn.path_prefix.nil? || conn.path_prefix == '/' 54 | # This avoids a double / if the channel starts with a / too (which is expected). 55 | req.path = channel 56 | else 57 | if conn.path_prefix =~ /\/\Z/ || channel =~ /\A\// 58 | req.path = [conn.path_prefix, channel].compact.join 59 | else 60 | # Add a / so the prefix and channel aren't just rammed together. 61 | req.path = [conn.path_prefix, channel].compact.join('/') 62 | end 63 | end 64 | req.body = message 65 | req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl 66 | req.headers["X-Firehose-Buffer-Size"] = buffer_size.to_s if buffer_size 67 | req.headers["X-Firehose-Deprecated"] = (!!deprecated).to_s if opts.include?(:deprecated) 68 | req.headers["X-Firehose-Persist"] = (!!persist).to_s if opts.include?(:persist) 69 | end 70 | response.on_complete do 71 | case response.status 72 | when 202 # Fire off the callback if everything worked out OK. 73 | block.call(response) if block 74 | else 75 | # don't pass along basic auth header, if present 76 | response_data = response.inspect.gsub(/"Authorization"=>"Basic \S+"/, '"Authorization" => "Basic [HIDDEN]"') 77 | endpoint = "#{uri}/#{channel}".gsub(/:\/\/\S+@/, "://") 78 | error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{endpoint}': #{response_data}") 79 | end 80 | end 81 | 82 | # Hide Faraday with this Timeout exception, and through the error handler. 83 | rescue Faraday::TimeoutError, Faraday::ConnectionFailed => e 84 | error_handler.call TimeoutError.new(e) 85 | end 86 | 87 | # Handle errors that could happen while publishing a message. 88 | def on_error(&block) 89 | @error_handler = block 90 | end 91 | 92 | # Raise an exception if an error occurs when connecting to the Firehose. 93 | def error_handler 94 | @error_handler || DEFAULT_ERROR_HANDLER 95 | end 96 | 97 | # What adapter should Firehose use to PUT the message? List of adapters is 98 | # available at https://github.com/technoweenie/faraday. 99 | def self.adapter=(adapter) 100 | @adapter = adapter 101 | end 102 | 103 | # Use :net_http for the default Faraday adapter. 104 | def self.adapter 105 | @adapter ||= Faraday.default_adapter 106 | end 107 | 108 | private 109 | # Build out a Faraday connection 110 | def conn 111 | @conn ||= Faraday.new(:url => uri.to_s) do |builder| 112 | builder.adapter self.class.adapter 113 | end 114 | end 115 | end 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/firehose/logging.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | 3 | module Firehose 4 | # Sets up logging 5 | def self.logger 6 | @logger ||= Logger.new($stdout) 7 | end 8 | 9 | def self.logger=(logger) 10 | @logger = logger 11 | end 12 | 13 | self.logger.level = if ENV['LOG_LEVEL'] 14 | Logger.const_get(ENV['LOG_LEVEL'].upcase) 15 | else 16 | case ENV['RACK_ENV'] 17 | when 'test' then Logger::ERROR 18 | when 'development' then Logger::DEBUG 19 | else Logger::INFO 20 | end 21 | end 22 | 23 | # TODO: Provide some way to allow this to be configured via an ENV variable. 24 | self.logger.formatter = lambda do |severity, time, name, msg| 25 | out_time = time.utc.strftime "%Y-%m-%d %H:%M:%S.%L" 26 | "[#{out_time} ##$$] #{severity} : #{msg}\n" 27 | end 28 | end 29 | 30 | # stdout gets "lost" in Foreman if this isn't here 31 | # https://github.com/ddollar/foreman/wiki/Missing-Output 32 | $stdout.sync = true if ENV['RACK_ENV'] == 'development' || ENV['SYNC_LOGGING'] -------------------------------------------------------------------------------- /lib/firehose/rack.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Rack 3 | autoload :Consumer, 'firehose/rack/consumer' 4 | autoload :Publisher, 'firehose/rack/publisher' 5 | autoload :Ping, 'firehose/rack/ping' 6 | autoload :App, 'firehose/rack/app' 7 | autoload :MetricsAPI, 'firehose/rack/metrics_api' 8 | 9 | # Evented web servers recognize the -1 HTTP code as a response deferral, which 10 | # is needed to stream responses via WebSockets or HTTP long polling. 11 | ASYNC_RESPONSE = [-1, {}, []].freeze 12 | 13 | # Normally we'd want to use a custom header to reduce the likelihood of some 14 | # HTTP middleware clobbering the value. But Safari seems to ignore our CORS 15 | # header instructions, so we are using 'pragma' because it is always allowed. 16 | LAST_MESSAGE_SEQUENCE_HEADER = 'Pragma' 17 | 18 | # Rack wants the header to start with HTTP, so we deal with that here. 19 | RACK_LAST_MESSAGE_SEQUENCE_HEADER = "HTTP_#{LAST_MESSAGE_SEQUENCE_HEADER.upcase.gsub('-', '_')}" 20 | 21 | # Disable CORS preflight caches for requests in development mode. 22 | CORS_OPTIONS_MAX_AGE = ENV['RACK_ENV'] == 'development' ? '1' : '1728000' 23 | 24 | module Helpers 25 | # Calculates the content of a message body for the response so that HTTP Keep-Alive 26 | # connections work. 27 | def response(status, body='', headers={}) 28 | headers = {'Content-Length' => body.bytesize.to_s}.merge(headers) 29 | [status, headers, [body]] 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/firehose/rack/app.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Rack 3 | # Acts as the glue between the HTTP/WebSocket world and the Firehose::Server class, 4 | # which talks directly to the Redis server. Also dispatches between HTTP and WebSocket 5 | # transport handlers depending on the clients' request. 6 | class App 7 | def initialize 8 | yield self if block_given? 9 | end 10 | 11 | def call(env) 12 | # Cache the parsed request so we don't need to re-parse it when we pass 13 | # control onto another app. 14 | req = env['parsed_request'] ||= ::Rack::Request.new(env) 15 | method = req.request_method 16 | 17 | case method 18 | when 'PUT' 19 | if Server.configuration.channel_deprecated?(req.path) 20 | Firehose.logger.warn "Publishing to DEPRECATED Channel: #{req.path}" 21 | end 22 | 23 | # Firehose::Client::Publisher PUT's payloads to the server. 24 | publisher.call(env) 25 | when 'HEAD' 26 | # HEAD requests are used to prevent sockets from timing out 27 | # from inactivity 28 | ping.call(env) 29 | else 30 | case req.path 31 | when "/metrics@firehose" 32 | metrics_api.call(env) 33 | else 34 | # TODO - 'harden' this up with a GET request and throw a "Bad Request" 35 | # HTTP error code. I'd do it now but I'm in a plane and can't think of it. 36 | consumer.call(env) 37 | end 38 | end 39 | end 40 | 41 | # The consumer pulls messages off of the backend and passes messages to the 42 | # connected HTTP or WebSocket client. This can be configured from the initialization 43 | # method of the rack app. 44 | def consumer 45 | @consumer ||= Consumer.new 46 | end 47 | 48 | private 49 | def publisher 50 | @publisher ||= Publisher.new 51 | end 52 | 53 | def ping 54 | @ping ||= Ping.new 55 | end 56 | 57 | def metrics_api 58 | @metrics_api ||= MetricsAPI.new 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/firehose/rack/consumer.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | module Firehose 4 | module Rack 5 | # Handles a subscription request over HTTP or WebSockets depeding on its abilities and 6 | # binds that to the Firehose::Server::Subscription class, which is bound to a channel that 7 | # gets published to. 8 | class Consumer 9 | class BasicHandler 10 | def last_message_sequence(subscription) 11 | # this is for backwards compatibility 12 | # prefer consistent :last_message_sequence from now on 13 | lms = subscription[:last_message_sequence] || subscription[:message_sequence] 14 | 15 | if subscription[:message_sequence] 16 | Firehose.logger.warn "Client used deprecated :message_sequence subscription format: #{subscription.inspect}" 17 | end 18 | 19 | lms.to_i 20 | end 21 | end 22 | 23 | # Rack consumer transports 24 | autoload :HttpLongPoll, 'firehose/rack/consumer/http_long_poll' 25 | autoload :WebSocket, 'firehose/rack/consumer/web_socket' 26 | 27 | MULTIPLEX_CHANNEL = "channels@firehose" 28 | 29 | def self.multiplexing_request?(env) 30 | env["PATH_INFO"].include? MULTIPLEX_CHANNEL 31 | end 32 | 33 | def self.multiplex_subscriptions(request) 34 | if request.get? 35 | query_string_subscriptions(request.env) 36 | elsif request.post? 37 | post_subscriptions(request) 38 | end 39 | end 40 | 41 | def self.query_string_subscriptions(env) 42 | query_params = ::Rack::Utils.parse_query(env["QUERY_STRING"]) 43 | 44 | query_params["subscribe"].to_s.split(",").map do |sub| 45 | chan, last_sequence = sub.split("!") 46 | last_sequence = last_sequence.to_i 47 | last_sequence = 0 if last_sequence < 0 48 | { 49 | channel: chan, 50 | last_message_sequence: last_sequence 51 | } 52 | end 53 | end 54 | 55 | def self.post_subscriptions(request) 56 | body = request.body.read 57 | subs = JSON.parse(body).map do |chan, val| 58 | # Hash is the newer format subscription message that supports 59 | # params 60 | if val.is_a? Hash 61 | { 62 | channel: chan, 63 | last_message_sequence: val["last_message_sequence"], 64 | params: val["params"] 65 | } 66 | # Otherwise the value of the JSON hash is implicitly the message 67 | # sequence 68 | else 69 | { 70 | channel: chan, 71 | last_message_sequence: val 72 | } 73 | end 74 | end 75 | 76 | if subs.empty? 77 | Firehose.logger.warn "Consumer.post_subscriptions: Empty / Invalid subscriptions POST body: #{body.inspect}" 78 | end 79 | 80 | subs 81 | rescue JSON::ParserError => e 82 | handle_parse_error(request, body, e) 83 | [] 84 | end 85 | 86 | def self.handle_parse_error(request, body, error) 87 | Firehose.logger.warn "JSON::ParserError for request body: #{body.inspect}" 88 | end 89 | 90 | # Let the client configure the consumer on initialization. 91 | def initialize 92 | yield self if block_given? 93 | end 94 | 95 | def call(env) 96 | websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env) 97 | end 98 | 99 | # Memoized instance of web socket that can be configured from the rack app. 100 | def websocket 101 | @web_socket ||= WebSocket.new 102 | end 103 | 104 | # Memoized instance of http long poll handler that can be configured from the rack app. 105 | def http_long_poll 106 | @http_long_poll ||= HttpLongPoll.new 107 | end 108 | 109 | private 110 | # Determine if the incoming request is a websocket request. 111 | def websocket_request?(env) 112 | Firehose::Rack::Consumer::WebSocket.request?(env) 113 | end 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /lib/firehose/rack/consumer/http_long_poll.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | module Firehose 4 | module Rack 5 | class Consumer 6 | class HttpLongPoll 7 | # How long should we wait before closing out the consuming clients web connection 8 | # for long polling? Most browsers timeout after a connection has been idle for 30s. 9 | TIMEOUT = 20 10 | 11 | attr_accessor :timeout 12 | 13 | def initialize(timeout=TIMEOUT) 14 | @timeout = timeout 15 | yield self if block_given? 16 | end 17 | 18 | def call(env) 19 | if Consumer.multiplexing_request?(env) 20 | MultiplexingHandler.new(@timeout).call(env) 21 | else 22 | DefaultHandler.new(@timeout).call(env) 23 | end 24 | end 25 | 26 | class Handler < Firehose::Rack::Consumer::BasicHandler 27 | include Firehose::Rack::Helpers 28 | 29 | def initialize(timeout=TIMEOUT) 30 | @timeout = timeout 31 | yield self if block_given? 32 | end 33 | 34 | def call(env) 35 | request = request(env) 36 | method = request.request_method 37 | 38 | case method 39 | # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response, 40 | # close down the requeust, and the client then reconnects. 41 | when "GET" 42 | handle_request(request, env) 43 | return ASYNC_RESPONSE 44 | # we use post messages for http long poll multiplexing 45 | when "POST" 46 | if Consumer.multiplexing_request?(env) 47 | handle_request(request, env) 48 | return ASYNC_RESPONSE 49 | end 50 | end 51 | 52 | Firehose.logger.debug "HTTP #{method} not supported" 53 | response(405, "#{method} not supported.", "Allow" => "GET") 54 | end 55 | 56 | private 57 | 58 | # If the request is a CORS request, return those headers, otherwise don't worry 'bout it 59 | def response_headers(env) 60 | cors_origin(env) ? cors_headers(env) : {} 61 | end 62 | 63 | def cors_origin(env) 64 | env['HTTP_ORIGIN'] 65 | end 66 | 67 | def cors_headers(env) 68 | {'Access-Control-Allow-Origin' => cors_origin(env)} 69 | end 70 | 71 | def request(env) 72 | env['parsed_request'] ||= ::Rack::Request.new(env) 73 | end 74 | 75 | def async_callback(env, code, message = "", headers = nil) 76 | resp_headers = response_headers(env) 77 | 78 | if headers 79 | resp_headers.merge!(headers) 80 | end 81 | 82 | if cb = env["async.callback"] 83 | cb.call response(code, message, resp_headers) 84 | else 85 | Firehose.logger.error "async.callback not set for response: #{message.inspect}" 86 | end 87 | end 88 | 89 | def respond_async(channel, last_sequence, params, env) 90 | EM.next_tick do 91 | if last_sequence < 0 92 | async_callback env, 400, "The last_message_sequence parameter may not be less than zero" 93 | else 94 | begin 95 | chan_sub = Server::ChannelSubscription.new( 96 | channel, 97 | env, 98 | params: params, 99 | sequence: last_sequence, 100 | timeout: @timeout 101 | ) 102 | 103 | chan_sub.next_messages.callback do |messages| 104 | # TODO: Can we send all of these messages down in one request? Sending one message per 105 | # request is slow and inefficient. If we change the protocol (3.0?) we could batch the 106 | # messages and send them all down the pipe, then close the conneciton. 107 | message = messages.first 108 | async_callback env, 200, wrap_frame(channel, message) 109 | end.errback do |e| 110 | if e == :timeout 111 | Firehose::Server.metrics.timeout!(:http_subscribe_multiplexed, channel) 112 | async_callback env, 204 113 | else 114 | Firehose::Server.metrics.error!(:http_subscribe_multiplexed, channel) 115 | Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{channel}: #{e.inspect}" 116 | async_callback env, 500, "Unexpected error" 117 | end 118 | end 119 | rescue Server::ChannelSubscription::Failed => e 120 | Firehose::Server.metrics.error!(:http_subscribe_multiplexed_failed, channel) 121 | Firehose.logger.info "Subscription failed: #{e.message}" 122 | async_callback env, 123 | 400, 124 | JSON.generate(error: "Subscription failed", 125 | reason: e.message) 126 | end 127 | end 128 | end 129 | end 130 | end 131 | 132 | class DefaultHandler < Handler 133 | def wrap_frame(channel, message) 134 | JSON.generate message: message.payload, 135 | last_sequence: message.sequence 136 | end 137 | 138 | def log_request(path, last_sequence, env) 139 | Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect}" 140 | end 141 | 142 | def handle_request(request, env) 143 | # Get the Last Message Sequence from the query string. 144 | # Ideally we'd use an HTTP header, but android devices don't let us 145 | # set any HTTP headers for CORS requests. 146 | params = request.params 147 | last_sequence = params.delete('last_message_sequence').to_i 148 | channel = request.path 149 | 150 | Firehose::Server.metrics.channel_subscribed!(channel) 151 | 152 | log_request channel, last_sequence, env 153 | respond_async channel, last_sequence, params, env 154 | end 155 | end 156 | 157 | class MultiplexingHandler < Handler 158 | def wrap_frame(channel, message) 159 | JSON.generate channel: channel, 160 | message: message.payload, 161 | last_sequence: message.sequence 162 | end 163 | 164 | def log_request(request, subscriptions, env) 165 | if request.post? 166 | Firehose.logger.debug "HTTP multiplexing POST, subscribing #{subscriptions.inspect}" 167 | else 168 | Firehose.logger.debug "HTTP multiplexing GET with query #{env["QUERY_STRING"].inspect}" 169 | end 170 | end 171 | 172 | def handle_request(request, env) 173 | subscriptions = Consumer.multiplex_subscriptions(request) 174 | if subscriptions.empty? 175 | Firehose::Server.metrics.error!(:http_subscribe_multiplexed_empty) 176 | Firehose.logger.warn "Client tried to subscribe multiplexed via HTTP without any channel subscriptions." 177 | async_callback env, 178 | 400, 179 | JSON.generate(error: "Subscription failed", 180 | reason: "No subscription data given - can't subscribe to nothing") 181 | else 182 | log_request request, subscriptions, env 183 | channels = subscriptions.map{|s| s[:channel]} 184 | Firehose::Server.metrics.channels_subscribed_multiplexed_long_polling!(channels) 185 | subscriptions.each do |sub| 186 | respond_async(sub[:channel], sub[:last_message_sequence] || sub[:message_sequence], sub[:params], env) 187 | end 188 | end 189 | end 190 | end 191 | end 192 | end 193 | end 194 | end 195 | -------------------------------------------------------------------------------- /lib/firehose/rack/consumer/web_socket.rb: -------------------------------------------------------------------------------- 1 | require 'faye/websocket' 2 | require 'json' 3 | require "rack/utils" 4 | 5 | module Firehose 6 | module Rack 7 | class Consumer 8 | class WebSocket 9 | include Firehose::Rack::Helpers 10 | 11 | # Setup a handler for the websocket connection. 12 | def call(env) 13 | begin 14 | ws = Faye::WebSocket.new(env) 15 | if Consumer.multiplexing_request?(env) 16 | MultiplexingHandler.new(ws) 17 | else 18 | DefaultHandler.new(ws) 19 | end 20 | ws.rack_response 21 | rescue StandardError => e 22 | Firehose.logger.error "WS connection error: #{e.inspect}" 23 | Firehose::Server.metrics.error!(:ws_invalid_request) 24 | response(400, "Invalid WebSocket request") 25 | end 26 | end 27 | 28 | # Determine if the rack request is a WebSocket request. 29 | def self.request?(env) 30 | Faye::WebSocket.websocket?(env) 31 | end 32 | 33 | class Handler < Firehose::Rack::Consumer::BasicHandler 34 | def initialize(ws) 35 | @ws = ws 36 | @req = ::Rack::Request.new ws.env 37 | # Setup the event handlers from this class. 38 | @ws.onopen = method :open 39 | @ws.onclose = method :close 40 | @ws.onerror = method :error 41 | @ws.onmessage = method :message 42 | end 43 | 44 | def parse_message(event) 45 | JSON.parse(event.data, :symbolize_names => true) rescue {} 46 | end 47 | 48 | # Send a JSON message to the client 49 | # Expects message to be a Hash 50 | def send_message(message) 51 | @ws.send JSON.generate(message) 52 | end 53 | 54 | # Log errors if a socket fails. `close` will fire after this to clean up any 55 | # remaining connectons. 56 | def error(event) 57 | Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`" 58 | end 59 | end 60 | 61 | # Manages connection state for the web socket that's connected 62 | # by the Consumer::WebSocket class. Deals with message sequence, 63 | # connection, failures, and subscription state. 64 | class DefaultHandler < Handler 65 | # Manages messages sent from the connect client to the server. This is mostly 66 | # used to handle heart-beats that are designed to prevent the WebSocket connection 67 | # from timing out from inactivity. 68 | def message(event) 69 | msg = parse_message(event) 70 | seq = last_message_sequence(msg) 71 | if msg[:ping] == 'PING' 72 | Firehose.logger.debug "WS ping received, sending pong" 73 | send_message pong: "PONG" 74 | elsif !@subscribed && seq.kind_of?(Integer) 75 | Firehose.logger.debug "Subscribing at message_sequence #{seq}" 76 | subscribe seq, @req.params 77 | end 78 | end 79 | 80 | # Log a message that the client has connected. 81 | def open(event) 82 | Firehose.logger.debug "WebSocket subscribed to `#{@req.path}`. Waiting for last_message_sequence..." 83 | Firehose::Server.metrics.new_connection! 84 | end 85 | 86 | # Log a message that the client has disconnected and reset the state for the class. Clean 87 | # up the subscribers to the channels. 88 | def close(event) 89 | disconnect 90 | Firehose.logger.debug "WS connection `#{@req.path}` closing. Code: #{event.code.inspect}; Reason #{event.reason.inspect}" 91 | Firehose::Server.metrics.connection_closed! 92 | end 93 | 94 | def disconnect 95 | if @deferrable 96 | @deferrable.fail :disconnect 97 | @chan_sub.unsubscribe if @chan_sub 98 | end 99 | end 100 | 101 | # Subscribe the client to the channel on the server. Asks for 102 | # the last sequence for clients that reconnect. 103 | def subscribe(last_sequence, params) 104 | begin 105 | @subscribed = true 106 | @chan_sub = Server::ChannelSubscription.new @req.path, 107 | @ws.env, 108 | params: params, 109 | sequence: last_sequence 110 | @deferrable = @chan_sub.next_messages 111 | @deferrable.callback do |messages| 112 | messages.each do |message| 113 | Firehose.logger.debug "WS sent `#{message.payload}` to `#{@req.path}` with sequence `#{message.sequence}`" 114 | send_message message: message.payload, last_sequence: message.sequence 115 | end 116 | subscribe messages.last.sequence, params 117 | end 118 | @deferrable.errback do |e| 119 | unless e == :disconnect 120 | Firehose.logger.error "WS Error: #{e}" 121 | EM.next_tick { raise e.inspect } 122 | end 123 | end 124 | rescue Server::ChannelSubscription::Failed => e 125 | Firehose.logger.info "Subscription failed: #{e.message}" 126 | send_message error: "Subscription failed", reason: e.message 127 | disconnect 128 | end 129 | end 130 | end 131 | 132 | class MultiplexingHandler < Handler 133 | class Subscription < Struct.new(:channel, :deferrable) 134 | def close 135 | deferrable.fail :disconnect 136 | channel.unsubscribe 137 | end 138 | end 139 | 140 | def initialize(ws) 141 | super(ws) 142 | @subscriptions = {} 143 | subscribe_multiplexed Consumer.multiplex_subscriptions(@req) 144 | end 145 | 146 | def message(event) 147 | msg = parse_message(event) 148 | 149 | if subscriptions = msg[:multiplex_subscribe] 150 | subscriptions = [subscriptions] unless subscriptions.is_a?(Array) 151 | Firehose::Server.metrics.channels_subscribed_multiplexed_ws_dynamic!(subscriptions) 152 | return subscribe_multiplexed(subscriptions) 153 | end 154 | 155 | if channel_names = msg[:multiplex_unsubscribe] 156 | return unsubscribe(channel_names) 157 | end 158 | 159 | if msg[:ping] == 'PING' 160 | Firehose.logger.debug "WS ping received, sending pong" 161 | return send_message pong: "PONG" 162 | end 163 | end 164 | 165 | def open(event) 166 | Firehose.logger.debug "Multiplexing Websocket connected: #{@req.path}" 167 | end 168 | 169 | def close(event) 170 | @subscriptions.each_value(&:close) 171 | @subscriptions.clear 172 | end 173 | 174 | def subscribe_multiplexed(subscriptions) 175 | channels = subscriptions.map{|s| s[:channel]} 176 | Firehose::Server.metrics.channels_subscribed_multiplexed_ws!(channels) 177 | 178 | subscriptions.each do |sub| 179 | Firehose.logger.debug "Subscribing multiplexed to: #{sub}" 180 | 181 | channel, sequence = sub[:channel] 182 | 183 | next if channel.nil? 184 | 185 | if @subscriptions.include?(channel) 186 | Firehose.logger.warn "Duplicate (aborted) multiplexing WS channel subscription: #{channel}" 187 | Firehose::Server.metrics.duplicate_multiplex_ws_subscription! 188 | # skip duplicate channel subscriptions 189 | next 190 | end 191 | 192 | subscribe(channel, last_message_sequence(sub), sub[:params]) 193 | end 194 | end 195 | 196 | # Subscribe the client to the channel on the server. Asks for 197 | # the last sequence for clients that reconnect. 198 | def subscribe(channel_name, last_sequence, params) 199 | chan_sub = Server::ChannelSubscription.new( 200 | channel_name, 201 | @ws.env, 202 | params: params, 203 | sequence: last_sequence 204 | ) 205 | 206 | deferrable = chan_sub.next_messages 207 | subscription = Subscription.new(chan_sub, deferrable) 208 | 209 | @subscriptions[channel_name] = subscription 210 | 211 | deferrable.callback do |messages| 212 | messages.each do |message| 213 | send_message( 214 | channel: channel_name, 215 | message: message.payload, 216 | last_sequence: message.sequence 217 | ) 218 | Firehose.logger.debug "WS sent `#{message.payload}` to `#{channel_name}` with sequence `#{message.sequence}`" 219 | end 220 | subscribe channel_name, messages.last.sequence, params 221 | end 222 | 223 | deferrable.errback do |e| 224 | unless e == :disconnect 225 | Firehose::Server.metrics.error!(:ws_subscribe_multiplexed, channel_name) 226 | EM.next_tick { raise e.inspect } 227 | end 228 | end 229 | end 230 | 231 | def unsubscribe(channel_names) 232 | Firehose.logger.debug "Unsubscribing from channels: #{channel_names}" 233 | Array(channel_names).each do |chan| 234 | if sub = @subscriptions[chan] 235 | sub.close 236 | @subscriptions.delete(chan) 237 | end 238 | end 239 | end 240 | end 241 | end 242 | end 243 | end 244 | end 245 | -------------------------------------------------------------------------------- /lib/firehose/rack/metrics_api.rb: -------------------------------------------------------------------------------- 1 | require "json" 2 | 3 | module Firehose 4 | module Rack 5 | class MetricsAPI 6 | include Firehose::Rack::Helpers 7 | 8 | def initialize 9 | end 10 | 11 | def call(env) 12 | Firehose.logger.info "MetricsAPI GET request" 13 | response(200, Firehose::Server.metrics.to_json) 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/firehose/rack/ping.rb: -------------------------------------------------------------------------------- 1 | require 'em-hiredis' 2 | 3 | module Firehose 4 | module Rack 5 | # Allows the Firehose client to periodically "ping" the server 6 | # so that the connection isn't timed out by browsers or proxies from 7 | # inactivity. 8 | class Ping 9 | attr_reader :redis 10 | 11 | def initialize(redis=nil) 12 | @redis = redis 13 | end 14 | 15 | def call(env) 16 | PingCheck.new(env, redis).call 17 | ASYNC_RESPONSE 18 | end 19 | 20 | # Encapsulate this in a class so we aren't passing a bunch of variables around 21 | class PingCheck 22 | include Firehose::Rack::Helpers 23 | 24 | attr_reader :req, :env, :key, :redis 25 | 26 | TEST_VALUE = 'Firehose Healthcheck Test Value' 27 | SECONDS_TO_EXPIRE = 60 28 | 29 | def self.redis 30 | @redis ||= Firehose::Server.redis.connection 31 | end 32 | 33 | def initialize(env, redis=nil) 34 | @redis = redis || self.class.redis 35 | @env = env 36 | @req = env['parsed_request'] ||= ::Rack::Request.new(env) 37 | @key = "/firehose/ping/#{Time.now.to_i}/#{rand}" 38 | end 39 | 40 | def call 41 | log req, 'started' 42 | test_redis 43 | end 44 | 45 | 46 | private 47 | 48 | def log(req, msg) 49 | Firehose.logger.debug "HTTP PING request for path '#{req.path}': #{msg}" 50 | end 51 | 52 | def test_redis 53 | redis.set(key, TEST_VALUE). 54 | callback { expire_key }. 55 | callback { read_and_respond }. 56 | errback do |e| 57 | log req, "failed with write value to redis: #{e.inspect}" 58 | env['async.callback'].call response(500) 59 | end 60 | end 61 | 62 | def expire_key 63 | redis.expire(key, SECONDS_TO_EXPIRE). 64 | errback do 65 | log req, "failed to expire key #{key.inspect}. If this key is not manually deleted, it may cause a memory leak." 66 | end 67 | end 68 | 69 | def read_and_respond 70 | redis.get(key). 71 | callback do |val| 72 | if val == TEST_VALUE 73 | log req, 'succeeded' 74 | env['async.callback'].call response(200) 75 | else 76 | log req, "failed with unexpected value retrieved from redis: #{val.inspect}" 77 | env['async.callback'].call response(500) 78 | end 79 | end. 80 | errback do |e| 81 | log req, "failed with read value from redis: #{e.inspect}" 82 | env['async.callback'].call response(500) 83 | end 84 | end 85 | end 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/firehose/rack/publisher.rb: -------------------------------------------------------------------------------- 1 | require "rack/utils" 2 | 3 | module Firehose 4 | module Rack 5 | class Publisher 6 | include Firehose::Rack::Helpers 7 | 8 | def call(env) 9 | req = env['parsed_request'] ||= ::Rack::Request.new(env) 10 | path = req.path 11 | method = req.request_method 12 | cache_control = {} 13 | 14 | # Parse out cache control directives from the Cache-Control header. 15 | if cache_control_header = env['HTTP_CACHE_CONTROL'] 16 | cache_control = cache_control_header.split(',').map(&:strip).inject({}) do |memo, directive| 17 | key, value = directive.split('=') 18 | memo[key.downcase] = value 19 | memo 20 | end 21 | end 22 | 23 | # Read the max-age directive from the cache so that we can set a TTL on the redis key. This will 24 | # prevent stale content from being served up to the client. 25 | ttl = cache_control['max-age'] 26 | 27 | if method == 'PUT' 28 | EM.next_tick do 29 | body = env['rack.input'].read 30 | Firehose.logger.debug "HTTP published #{body.inspect} to #{path.inspect} with ttl #{ttl.inspect}" 31 | opts = { :ttl => ttl }.merge(parse_options(env)) 32 | publisher.publish(path, body, opts).callback do 33 | env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain') 34 | end.errback do |e| 35 | Firehose.logger.debug "Error publishing: #{e.inspect}" 36 | env['async.callback'].call response(500, 'Error when trying to publish', 'Content-Type' => 'text/plain') 37 | end 38 | end 39 | 40 | # Tell the web server that this will be an async response. 41 | ASYNC_RESPONSE 42 | else 43 | Firehose.logger.debug "HTTP #{method} not supported" 44 | msg = "#{method} not supported." 45 | [501, {'Content-Type' => 'text/plain', 'Content-Length' => msg.bytesize.to_s}, [msg]] 46 | end 47 | end 48 | 49 | private 50 | 51 | def parse_options(env) 52 | opts = {} 53 | if buffer_size = env["HTTP_X_FIREHOSE_BUFFER_SIZE"] 54 | opts[:buffer_size] = buffer_size.to_i 55 | end 56 | if deprecated = env["HTTP_X_FIREHOSE_DEPRECATED"] 57 | opts[:deprecated] = deprecated == "true" 58 | end 59 | if persist = env["HTTP_X_FIREHOSE_PERSIST"] 60 | opts[:persist] = persist == "true" 61 | end 62 | opts 63 | end 64 | 65 | def publisher 66 | @publisher ||= Firehose::Server::Publisher.new 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/firehose/rails.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Rails 3 | class Engine < ::Rails::Engine 4 | # enable rails to pick up lib/assets/* via sprockets. 5 | end 6 | end 7 | end -------------------------------------------------------------------------------- /lib/firehose/server.rb: -------------------------------------------------------------------------------- 1 | require 'faye/websocket' 2 | require 'em-hiredis' 3 | 4 | # Set the EM::Hiredis logger to be the same as the Firehose logger. 5 | EM::Hiredis.logger = Firehose.logger 6 | 7 | module Firehose 8 | # Firehose components that sit between the Rack HTTP software and the Redis server. 9 | # This mostly handles message sequencing and different HTTP channel names. 10 | module Server 11 | autoload :Configuration, 'firehose/server/configuration' 12 | autoload :Message, 'firehose/server/message' 13 | autoload :MessageFilter, 'firehose/server/message_filter' 14 | autoload :MessageBuffer, 'firehose/server/message_buffer' 15 | autoload :Subscriber, 'firehose/server/subscriber' 16 | autoload :Publisher, 'firehose/server/publisher' 17 | autoload :ChannelSubscription, 'firehose/server/channel_subscription' 18 | autoload :App, 'firehose/server/app' 19 | autoload :Redis, 'firehose/server/redis' 20 | autoload :Metrics, 'firehose/server/metrics' 21 | autoload :MetricsCollector, 'firehose/server/metrics_collector' 22 | 23 | def self.configuration 24 | @configuration ||= Configuration.new 25 | yield(@configuration) if block_given? 26 | @configuration 27 | end 28 | 29 | def self.redis 30 | configuration.redis 31 | end 32 | 33 | def self.metrics 34 | interval = ENV["METRICS_INTERVAL"].to_i 35 | interval = MetricsCollector.metrics_interval 36 | @metrics ||= Firehose::Server::Metrics::TimeSeries.new(seconds: interval) 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/firehose/server/app.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | # Configure servers that are booted with-out going through Rack. This is mostly used by 4 | # the `firehose server` CLI command or for testing. Production configurations are likely 5 | # to boot with custom rack configurations. 6 | class App 7 | def initialize(opts={}) 8 | @port = opts[:port] || Firehose::URI.port 9 | @host = opts[:host] || Firehose::URI.host 10 | @server = opts[:server] || :rainbows 11 | 12 | Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}" 13 | Firehose::Server::MetricsCollector.new.start 14 | end 15 | 16 | def start 17 | self.send("start_#{@server}") 18 | end 19 | 20 | private 21 | # Boot the Firehose server with the Rainbows app server. 22 | def start_rainbows 23 | require 'rainbows' 24 | Faye::WebSocket.load_adapter('rainbows') 25 | 26 | rackup = Unicorn::Configurator::RACKUP 27 | rackup[:port] = @port if @port 28 | rackup[:host] = @host if @host 29 | rackup[:set_listener] = true 30 | opts = rackup[:options] 31 | opts[:config_file] = File.expand_path('../../../../config/rainbows.rb', __FILE__) 32 | 33 | server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts) 34 | server.start.join 35 | end 36 | 37 | # Boot the Firehose server with the Thin app server. 38 | def start_thin 39 | require 'thin' 40 | 41 | Faye::WebSocket.load_adapter('thin') 42 | 43 | # TODO: See if we can just set Thin to use Firehose.logger instead of 44 | # printing out messages by itself. 45 | Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR 46 | 47 | server = Thin::Server.new(@host, @port) do 48 | run Firehose::Rack::App.new 49 | end.start 50 | end 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/firehose/server/channel_subscription.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | # Connects to a specific channel on Redis and listens for messages to notify subscribers. 4 | class ChannelSubscription 5 | # Can be raised if a Subscription failed for any reason. 6 | class Failed < StandardError 7 | end 8 | 9 | attr_reader :channel_key, :params, :sequence, :timeout 10 | 11 | def self.redis 12 | @redis ||= Firehose::Server.redis.connection 13 | end 14 | 15 | def self.subscriber 16 | @subscriber ||= Server::Subscriber.new 17 | end 18 | 19 | class ClientInfo 20 | attr_reader :ip, :referer, :user_agent 21 | def initialize(env) 22 | @ip = env["REMOTE_ADDR"] 23 | if realIp = env["HTTP_X_FORWARDED_FOR"] 24 | @ip = realIp 25 | end 26 | @referer = env["HTTP_REFERER"] 27 | @user_agent = env["HTTP_USER_AGENT"] 28 | end 29 | 30 | def to_s 31 | "ip=#{@ip.inspect} referer=#{@referer.inspect} user_agent=#{@user_agent.inspect}" 32 | end 33 | end 34 | 35 | def initialize(channel_key, env, sequence: 0, params: {}, timeout: nil) 36 | @redis = self.class.redis 37 | @subscriber = self.class.subscriber 38 | @sequence = sequence 39 | @timeout = timeout 40 | @channel_key = channel_key 41 | @client_info = ClientInfo.new(env) 42 | @deferrable = EM::DefaultDeferrable.new 43 | @deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) } 44 | if Server.configuration.channel_deprecated?(channel_key) 45 | Firehose.logger.warn "Subscription to DEPRECATED Channel: #{channel_key} from client: #{@client_info}" 46 | end 47 | on_subscribe(params) 48 | end 49 | 50 | def on_subscribe(params) 51 | message_filter.on_subscribe(params) 52 | end 53 | 54 | def on_unsubscribe 55 | message_filter.on_unsubscribe 56 | end 57 | 58 | def on_message(message) 59 | message_filter.process(message) 60 | end 61 | 62 | def next_messages 63 | list_key = Server::Redis.key(channel_key, :list) 64 | sequence_key = Server::Redis.key(channel_key, :sequence) 65 | 66 | @redis.multi 67 | @redis.get(sequence_key). 68 | errback {|e| @deferrable.fail e } 69 | # Fetch entire list: http://stackoverflow.com/questions/10703019/redis-fetch-all-value-of-list-without-iteration-and-without-popping 70 | @redis.lrange(list_key, 0, -1). 71 | errback {|e| @deferrable.fail e } 72 | @redis.exec.callback do |(channel_sequence, message_list)| 73 | # Reverse the messages so they can be correctly procesed by the MessageBuffer class. There's 74 | # a patch in the message-buffer-redis branch that moves this concern into the Publisher LUA 75 | # script. We kept it out of this for now because it represents a deployment risk and `reverse!` 76 | # is a cheap operation in Ruby. 77 | message_list.reverse! 78 | channel_sequence = [channel_sequence.to_i, message_list.size].max 79 | buffer = MessageBuffer.new(message_list, channel_sequence, @sequence) 80 | if buffer.remaining_messages.empty? 81 | Firehose.logger.debug "No messages in buffer, subscribing. sequence: `#{channel_sequence}` consumer_sequence: #{@sequence}" 82 | # Either this resource has never been seen before or we are all caught up. 83 | # Subscribe and hope something gets published to this end-point. 84 | subscribe 85 | else # Either the client is under water or caught up to head. 86 | @deferrable.succeed process_messages(buffer.remaining_messages) 87 | @deferrable.callback { on_unsubscribe } 88 | end 89 | end.errback {|e| @deferrable.fail e } 90 | # TODO -ARRRG!!!! We can't listen to deferrables here. Need to expose those 91 | # on this class because this is a class var. In practice thats how we roll anyway. 92 | @deferrable 93 | end 94 | 95 | def unsubscribe 96 | @subscriber.unsubscribe self 97 | end 98 | 99 | def process_messages(messages) 100 | messages = messages.map do |m| 101 | m = m.dup 102 | on_message(m) 103 | m 104 | end 105 | @deferrable.succeed messages 106 | end 107 | 108 | private 109 | 110 | def message_filter 111 | @message_filter ||= begin 112 | if mf = Server.configuration.message_filter 113 | mf.new(self) 114 | else 115 | Firehose::Server::MessageFilter.new(self) 116 | end 117 | end 118 | end 119 | 120 | def subscribe 121 | @subscriber.subscribe self 122 | if @timeout 123 | timer = EventMachine::Timer.new(@timeout) do 124 | @deferrable.fail :timeout 125 | unsubscribe 126 | end 127 | # Cancel the timer if when the deferrable succeeds 128 | @deferrable.callback { timer.cancel } 129 | end 130 | end 131 | end 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /lib/firehose/server/configuration.rb: -------------------------------------------------------------------------------- 1 | require "set" 2 | 3 | module Firehose 4 | module Server 5 | # Configuration object for Firehose server. 6 | class Configuration 7 | attr_accessor :message_filter, :redis 8 | attr_reader :deprecated_channels 9 | 10 | def initialize 11 | @redis = Redis.new 12 | @message_filter = MessageFilter 13 | @deprecated_channels = Set.new 14 | @deprecated_channel_check = -> (_channel) { false } 15 | end 16 | 17 | def deprecate_channel(channel) 18 | unless channel_deprecated?(channel) 19 | Firehose.logger.info "Deprecated channel: #{channel}" 20 | @deprecated_channels << channel 21 | end 22 | end 23 | 24 | def undeprecate_channel(channel) 25 | if channel_deprecated?(channel) 26 | Firehose.logger.info "Undeprecated channel: #{channel}" 27 | @deprecated_channels.delete channel 28 | end 29 | end 30 | 31 | def deprecated_channels=(channels) 32 | @deprecated_channels = Set.new(channels) 33 | end 34 | 35 | def deprecated_channel(&block) 36 | @deprecated_channel_check = block 37 | end 38 | 39 | def channel_deprecated?(channel) 40 | @deprecated_channels.include?(channel) || 41 | !!@deprecated_channel_check.call(channel) 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/firehose/server/message.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | Message = Struct.new(:payload, :sequence) do 4 | def json_payload 5 | JSON.parse payload 6 | end 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/firehose/server/message_buffer.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | # Encapsulates a sequence of messages from the server along with their 4 | # consumer_sequences calculate by offset. 5 | class MessageBuffer 6 | # Number of messages that Redis buffers for the client if its 7 | # connection drops, then reconnects. 8 | DEFAULT_SIZE = ENV.fetch("DEFAULT_BUFFER_SIZE", 100) 9 | 10 | def initialize(message_list, channel_sequence, consumer_sequence = nil) 11 | @message_list = message_list 12 | @channel_sequence = channel_sequence 13 | @consumer_sequence = consumer_sequence.to_i 14 | 15 | # Channel sequence is 10 16 | # Buffer size of 5 17 | # Start of sequence in buffer ... which would be 6 18 | @starting_channel_sequence = @channel_sequence - @message_list.size + 1 19 | end 20 | 21 | def remaining_messages 22 | messages.last(remaining_message_count) 23 | end 24 | 25 | private 26 | 27 | def remaining_message_count 28 | # Special case to always get the latest message. 29 | return 1 unless @consumer_sequence > 0 30 | 31 | count = @channel_sequence - @consumer_sequence 32 | 33 | if count < 0 34 | # UNEXPECTED: Somehow the sequence is ahead of the channel. 35 | # It is likely a bug in the consumer, but we'll assume 36 | # the consumer has all the messages. 37 | 0 38 | elsif count > @message_list.size 39 | # Consumer is under water since the last request. Just send the most recent message. 40 | 1 41 | else 42 | count 43 | end 44 | end 45 | 46 | # Calculates the last_message_sequence per message. 47 | # [a b c e f] 48 | def messages 49 | @messages ||= @message_list.map.with_index do |payload, index| 50 | Message.new(payload, @starting_channel_sequence + index) 51 | end 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/firehose/server/message_filter.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | # A no-op message filter. This class is meant to be 4 | # extended by users for implementing channel middleware. 5 | class MessageFilter 6 | attr_reader :channel, :params 7 | 8 | def initialize(channel) 9 | @channel = channel 10 | @params = {} 11 | end 12 | 13 | def process(message) 14 | end 15 | 16 | def on_subscribe(params) 17 | @params = params 18 | end 19 | 20 | def on_unsubscribe 21 | @params = {} 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/firehose/server/metrics.rb: -------------------------------------------------------------------------------- 1 | require "set" 2 | 3 | module Firehose::Server 4 | module Metrics 5 | class TimeSeries 6 | attr_reader :series 7 | 8 | def initialize(seconds: 5, keep_buckets: 2, gc: true) 9 | @seconds = seconds.to_i 10 | if @seconds < 1 11 | raise ArgumentError, "TimeSeries interval must be >= 1" 12 | end 13 | @keep_buckets = keep_buckets 14 | @track_gc_metrics = gc 15 | clear! 16 | end 17 | 18 | def method_missing(method, *args) 19 | current.send(method, *args) 20 | end 21 | 22 | def clear! 23 | @series = Hash.new do |h, k| 24 | bucket = bucket(k) 25 | h[bucket] = Firehose::Server::Metrics::Buffer.new(bucket, gc: @track_gc_metrics) 26 | end 27 | end 28 | 29 | def clear_old! 30 | # keep latest @keep_buckets buckets 31 | buckets = [] 32 | @keep_buckets.times do |i| 33 | bucket = bucket(Time.now) - (i * @seconds) 34 | buckets << [bucket, @series[bucket]] 35 | end 36 | clear! 37 | # use reverse_each to keep insertion order correct 38 | # (Hash keys are insertion ordered) 39 | buckets.reverse_each do |(bucket, buffer)| 40 | @series[bucket] = buffer 41 | end 42 | @series 43 | end 44 | 45 | def to_json 46 | JSON.generate @series.values.map(&:to_hash) 47 | end 48 | 49 | def empty? 50 | @series.empty? 51 | end 52 | 53 | # private 54 | 55 | def current 56 | @series[bucket(Time.now)] 57 | end 58 | 59 | def bucket(time) 60 | secs = time.to_i 61 | secs - (secs % @seconds) 62 | end 63 | end 64 | 65 | class Buffer 66 | def initialize(time_bucket, gc: true) 67 | @time_bucket = time_bucket 68 | @active_channels = Set.new 69 | @global = Hash.new { 0 } 70 | @channel_metrics = Hash.new 71 | if gc 72 | @gc_metrics = GC.stat 73 | end 74 | end 75 | 76 | # metric handlers 77 | 78 | def message_published!(channel, message = nil) 79 | @active_channels << channel 80 | incr_global! :published 81 | incr_channel! channel, :published 82 | 83 | cm = channel_metrics(channel) 84 | if message 85 | incr_channel! channel, :total_size, message.size 86 | end 87 | end 88 | 89 | def channel_subscribed!(channel) 90 | @active_channels << channel 91 | incr_global! :subscribed 92 | incr_channel! channel, :subscribed 93 | end 94 | 95 | def channels_subscribed_multiplexed_ws!(channels) 96 | channels.each do |channel| 97 | @active_channels << channel 98 | incr_global! :subscribed_multiplexed_ws 99 | incr_channel! channel, :subscribed_multiplexed_ws 100 | end 101 | end 102 | 103 | def channels_subscribed_multiplexed_long_polling!(channels) 104 | channels.each do |channel| 105 | @active_channels << channel 106 | incr_global! :subscribed_multiplexed_long_polling 107 | incr_channel! channel, :subscribed_multiplexed_long_polling 108 | end 109 | end 110 | 111 | def channels_subscribed_multiplexed_ws_dynamic!(subscriptions) 112 | incr_global! :subscribed_multiplexed_ws_dynamic, subscriptions.size 113 | end 114 | 115 | def duplicate_multiplex_ws_subscription! 116 | incr_global! :duplicate_multiplex_subscription 117 | end 118 | 119 | def new_connection! 120 | incr_global! :connections 121 | incr_global! :connections_opened 122 | end 123 | 124 | def connection_closed! 125 | incr_global! :connections_closed 126 | decr_global! :connections 127 | end 128 | 129 | def error!(error_tag, channel = nil) 130 | incr_global! :errors 131 | if channel 132 | incr_channel! channel, "error_#{error_tag}" 133 | else 134 | incr_global! "error_#{error_tag}" 135 | end 136 | end 137 | 138 | def timeout!(tag, channel = nil) 139 | incr_global! :timeouts 140 | if channel 141 | incr_channel! channel, "timeout_#{tag}" 142 | else 143 | incr_global! "timeout_#{tag}" 144 | end 145 | end 146 | 147 | # serialization helpers (used to store metrics to redis) 148 | 149 | def to_hash 150 | h = { 151 | time: @time_bucket, 152 | global: @global.merge(active_channels: @active_channels.size), 153 | channels: @channel_metrics 154 | } 155 | h.merge!(gc: @gc_metrics) if @gc_metrics 156 | h 157 | end 158 | 159 | def to_json 160 | JSON.generate self.to_hash 161 | end 162 | 163 | def == other 164 | self.to_hash == other.to_hash 165 | end 166 | 167 | private 168 | 169 | def incr_global!(name, increment = 1) 170 | @global[name] += increment 171 | end 172 | 173 | def decr_global!(name, decrement = 1) 174 | @global[name] -= decrement 175 | end 176 | 177 | def incr_channel!(channel, counter, increment = 1) 178 | channel_metrics(channel)[counter] += increment 179 | end 180 | 181 | def decr_channel!(channel, counter, decrement = 1) 182 | channel_metrics(channel)[counter] -= decrement 183 | end 184 | 185 | def channel_metrics(channel) 186 | @channel_metrics[channel] ||= Hash.new { 0 } 187 | end 188 | end 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /lib/firehose/server/metrics_collector.rb: -------------------------------------------------------------------------------- 1 | module Firehose::Server 2 | class MetricsCollector 3 | DEFAULT_INTERVAL = 10 # in seconds 4 | attr_reader :logger, :redis 5 | 6 | def initialize(logger = Firehose.logger, redis = Firehose::Server.redis) 7 | @logger = logger 8 | @redis = redis 9 | end 10 | 11 | def start 12 | logger.info "Starting MetricsCollector with an interval of #{metrics_interval}s" 13 | EM.next_tick do 14 | # Disable for now and just keep metrics in memory 15 | # EM.add_periodic_timer(metrics_interval) do 16 | # save_metrics 17 | # end 18 | EM.add_periodic_timer(metrics_interval * 1.3) do 19 | clear_old_metrics 20 | end 21 | end 22 | end 23 | 24 | def clear_old_metrics 25 | logger.debug "Clearing old metrics" 26 | Firehose::Server.metrics.clear_old! 27 | end 28 | 29 | def save_metrics 30 | unless Firehose::Server.metrics.empty? 31 | bucket = metrics_bucket 32 | logger.info "Saving metrics to Redis to bucket #{bucket.inspect}" 33 | redis.connection.set(bucket, metrics_data) 34 | end 35 | end 36 | 37 | def metrics_data 38 | Firehose::Server.metrics.to_json 39 | end 40 | 41 | def metrics_interval 42 | self.class.metrics_interval 43 | end 44 | 45 | def self.metrics_interval 46 | @interval ||= begin 47 | if (i = ENV["METRICS_INTERVAL"].to_i) > 0 48 | i 49 | else 50 | DEFAULT_INTERVAL 51 | end 52 | end 53 | end 54 | 55 | def metrics_bucket 56 | now = Time.now.to_i 57 | bucket_id = now - (now % metrics_interval) 58 | Firehose::Server::Redis.key :metrics, bucket_id 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/firehose/server/publisher.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | class Publisher 4 | # Seconds that the message buffer should live before Redis expires it. 5 | TTL = 60*60*24 6 | 7 | # Delimited used to frame different parts of a message that's published 8 | # over Firehose. 9 | PAYLOAD_DELIMITER = "\n" 10 | 11 | # Publish a message to a Firehose channel via Redis. 12 | def publish(channel_key, message, opts={}) 13 | # How long should we hang on to the resource once is published? 14 | ttl = (opts[:ttl] || TTL).to_i 15 | buffer_size = (opts[:buffer_size] || MessageBuffer::DEFAULT_SIZE).to_i 16 | persist = !!opts[:persist] 17 | 18 | if opts.include?(:deprecated) 19 | if opts[:deprecated] 20 | Server.configuration.deprecate_channel channel_key 21 | else 22 | Server.configuration.undeprecate_channel channel_key 23 | end 24 | end 25 | 26 | # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong 27 | # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in 28 | # em-hiredis or us a diff lib? 29 | if (deferrable = opts[:deferrable]).nil? 30 | deferrable = EM::DefaultDeferrable.new 31 | deferrable.errback do |e| 32 | # Handle missing Lua publishing script in cache 33 | # (such as Redis restarting or someone executing SCRIPT FLUSH) 34 | if e.message =~ /NOSCRIPT/ 35 | deferrable.succeed 36 | EM.next_tick do 37 | @publish_script_digest = nil 38 | combined_opts = opts.merge :deferrable => deferrable 39 | self.publish channel_key, message, combined_opts 40 | end 41 | else 42 | EM.next_tick { raise e } 43 | end 44 | end 45 | end 46 | 47 | if @publish_script_digest.nil? 48 | register_publish_script.errback do |e| 49 | deferrable.fail e 50 | end.callback do |digest| 51 | @publish_script_digest = digest 52 | Firehose.logger.debug "Registered Lua publishing script with Redis => #{digest}" 53 | eval_publish_script channel_key, message, ttl, buffer_size, persist, deferrable 54 | end 55 | else 56 | eval_publish_script channel_key, message, ttl, buffer_size, persist, deferrable 57 | end 58 | 59 | Firehose::Server.metrics.message_published!(channel_key, message) 60 | 61 | deferrable 62 | end 63 | 64 | private 65 | def redis 66 | @redis ||= Firehose::Server.redis.connection 67 | end 68 | 69 | # Serialize components of a message into something that can be dropped into Redis. 70 | def self.to_payload(channel_key, sequence, message) 71 | [channel_key, sequence, message].join(PAYLOAD_DELIMITER) 72 | end 73 | 74 | # Deserialize components of a message back into Ruby. 75 | def self.from_payload(payload) 76 | @payload_size ||= method(:to_payload).arity 77 | payload.split(PAYLOAD_DELIMITER, @payload_size) 78 | end 79 | 80 | # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted 81 | # characters and then escape or remove everything else. 82 | # See: http://en.wikibooks.org/wiki/Lua_Programming/How_to_Lua/escape_sequence 83 | def lua_escape(str) 84 | str.gsub(/\\/,'\\\\\\').gsub(/"/,'\"').gsub(/\n/,'\n').gsub(/\r/,'\r') 85 | end 86 | 87 | def register_publish_script 88 | redis.script 'LOAD', REDIS_PUBLISH_SCRIPT 89 | end 90 | 91 | def eval_publish_script(channel_key, message, ttl, buffer_size, persist, deferrable) 92 | list_key = Server::Redis.key(channel_key, :list) 93 | script_args = [ 94 | Server::Redis.key(channel_key, :sequence), 95 | list_key, 96 | Server::Redis.key(:channel_updates), 97 | ttl, 98 | message, 99 | buffer_size, 100 | persist, 101 | PAYLOAD_DELIMITER, 102 | channel_key 103 | ] 104 | 105 | redis.evalsha( 106 | @publish_script_digest, script_args.length, *script_args 107 | ).errback do |e| 108 | deferrable.fail e 109 | end.callback do |sequence| 110 | Firehose.logger.debug "Redis stored/published `#{message}` to list `#{list_key}` with sequence `#{sequence}`" 111 | deferrable.succeed 112 | end 113 | end 114 | 115 | REDIS_PUBLISH_SCRIPT = <<-LUA 116 | local sequence_key = KEYS[1] 117 | local list_key = KEYS[2] 118 | local channel_key = KEYS[3] 119 | local ttl = KEYS[4] 120 | local message = KEYS[5] 121 | local buffer_size = KEYS[6] + 0 122 | local persist = KEYS[7] == "true" 123 | local payload_delimiter = KEYS[8] 124 | local firehose_resource = KEYS[9] 125 | 126 | local current_sequence = redis.call('get', sequence_key) 127 | if current_sequence == nil or current_sequence == false then 128 | current_sequence = 0 129 | end 130 | 131 | local sequence = current_sequence + 1 132 | local message_payload = firehose_resource .. payload_delimiter .. sequence .. payload_delimiter .. message 133 | 134 | redis.call('set', sequence_key, sequence) 135 | if buffer_size > 0 then 136 | redis.call('lpush', list_key, message) 137 | redis.call('ltrim', list_key, 0, buffer_size - 1) 138 | else 139 | redis.call('del', list_key) 140 | end 141 | 142 | redis.call('publish', channel_key, message_payload) 143 | 144 | if persist then 145 | redis.call('persist', sequence_key) 146 | redis.call('persist', list_key) 147 | else 148 | redis.call('expire', sequence_key, ttl) 149 | redis.call('expire', list_key, ttl) 150 | end 151 | 152 | return sequence 153 | LUA 154 | 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /lib/firehose/server/redis.rb: -------------------------------------------------------------------------------- 1 | require "uri" 2 | 3 | module Firehose 4 | module Server 5 | # Manages redis configuration and connections. 6 | class Redis 7 | DEFAULT_URL = "redis://127.0.0.1:6379/0".freeze 8 | KEY_DELIMITER = ":".freeze 9 | ROOT_KEY = "firehose".freeze 10 | 11 | attr_accessor :url 12 | 13 | def initialize(url = self.class.url) 14 | @url = URI(url) 15 | end 16 | 17 | def connection 18 | EM::Hiredis.connect(@url) 19 | end 20 | 21 | # Generates keys for all firehose interactions with Redis. Ensures a root 22 | # key of `firehose` 23 | def self.key(*segments) 24 | segments.flatten.unshift(ROOT_KEY).join(KEY_DELIMITER) 25 | end 26 | 27 | def self.url 28 | ENV.fetch("REDIS_URL", DEFAULT_URL) 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/firehose/server/subscriber.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | module Server 3 | # Setups a connetion to Redis to listen for new resources... 4 | class Subscriber 5 | attr_reader :pubsub 6 | 7 | def initialize(redis = Firehose::Server.redis.connection) 8 | @pubsub = redis.pubsub 9 | # TODO: Instead of just raising an exception, it would probably be better 10 | # for the errback to set some sort of 'disconnected' state. Then 11 | # whenever a deferrable was 'subscribed' we could instantly fail 12 | # the deferrable with whatever connection error we had. 13 | # An alternative which would have a similar result would be to 14 | # subscribe lazily (i.e. not until we have a deferrable to subscribe). 15 | # Then, if connecting failed, it'd be super easy to fail the deferrable 16 | # with the same error. 17 | # The final goal is to allow the failed deferrable bubble back up 18 | # so we can send back a nice, clean 500 error to the client. 19 | channel_updates_key = Server::Redis.key('channel_updates') 20 | pubsub.subscribe(channel_updates_key). 21 | errback{|e| EM.next_tick { raise e } }. 22 | callback { Firehose.logger.debug "Redis subscribed to `#{channel_updates_key}`" } 23 | pubsub.on(:message) do |_, payload| 24 | channel_key, channel_sequence, message = Server::Publisher.from_payload(payload) 25 | messages = [ Message.new(message, channel_sequence.to_i) ] 26 | if channels = subscriptions.delete(channel_key) 27 | Firehose.logger.debug "Redis notifying #{channels.count} channels(s) at `#{channel_key}` with channel_sequence `#{channel_sequence}` and message `#{message}`" 28 | channels.each do |channel| 29 | Firehose.logger.debug "Sending message #{message} and channel_sequence #{channel_sequence} to client from subscriber" 30 | channel.process_messages messages 31 | end 32 | end 33 | end 34 | end 35 | 36 | def subscribe(channel) 37 | subscriptions[channel.channel_key].push channel 38 | end 39 | 40 | def unsubscribe(channel) 41 | subscriptions[channel.channel_key].delete channel 42 | subscriptions.delete(channel.channel_key) if subscriptions[channel.channel_key].empty? 43 | end 44 | 45 | private 46 | def subscriptions 47 | @subscriptions ||= Hash.new{|h,k| h[k] = []} 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/firehose/version.rb: -------------------------------------------------------------------------------- 1 | module Firehose 2 | VERSION = "1.4.6" 3 | CODENAME = "Inspectable Pipes" 4 | end 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firehose", 3 | "version": "1.3.3", 4 | "main": "lib/assets/javascripts/firehose.js.coffee", 5 | "devDependencies": { 6 | "coffee-script": "*", 7 | "jasmine-jquery": "git://github.com/velesin/jasmine-jquery.git", 8 | "karma": "~0.12.16", 9 | "karma-chrome-launcher": "~0.1.4", 10 | "karma-coffee-preprocessor": "~0.2.1", 11 | "karma-jasmine": "~0.2.0", 12 | "karma-junit-reporter": "^0.2.2", 13 | "karma-phantomjs-launcher": "^0.1.4", 14 | "karma-safari-launcher": "*", 15 | "karma-sprockets-mincer": "0.1.2" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /spec/integrations/integration_test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | module IntegrationTestHelper 4 | def start_server 5 | @server_pid = fork do 6 | Firehose::Server::App.new(:server => server, :port => uri.port).start 7 | end 8 | 9 | # Need to give the server a chance to boot up. 10 | sleep 1 11 | end 12 | 13 | def stop_server 14 | Process.kill 'INT', @server_pid 15 | end 16 | 17 | 18 | # Let isn't allowed in before(:all) 19 | def uri 20 | Firehose::URI 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/integrations/rainbows_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'integrations/shared_examples' 3 | 4 | describe "rainbows" do 5 | def server; :rainbows; end # Let isn't allowed in before(:all) 6 | it_behaves_like 'Firehose::Rack::App' 7 | end 8 | -------------------------------------------------------------------------------- /spec/integrations/shared_examples.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'integrations/integration_test_helper' 3 | require 'json' 4 | 5 | shared_examples_for 'Firehose::Rack::App' do 6 | include EM::TestHelper 7 | include IntegrationTestHelper 8 | 9 | before(:all) do 10 | Firehose::Client::Producer::Http.adapter = :em_http 11 | start_server 12 | end 13 | 14 | after(:all) do 15 | Firehose::Client::Producer::Http.adapter = nil 16 | stop_server 17 | end 18 | 19 | before(:each) { WebMock.disable! } 20 | after(:each) { WebMock.enable! } 21 | 22 | let(:app) { Firehose::Rack::App.new } 23 | let(:messages) { (1..200).map{|n| "msg-#{n}" } } 24 | let(:channel) { "/firehose/integration/#{Time.now.to_i}" } 25 | let(:http_url) { "http://#{uri.host}:#{uri.port}#{channel}" } 26 | let(:http_multi_url) { "http://#{uri.host}:#{uri.port}/channels@firehose" } 27 | let(:ws_url) { "ws://#{uri.host}:#{uri.port}#{channel}" } 28 | let(:multiplex_channels) { ["/foo/bar", "/bar/baz", "/baz/quux"] } 29 | let(:subscription_query) { multiplex_channels.map{|c| "#{c}!0"}.join(",") } 30 | 31 | it "supports pub-sub http and websockets" do 32 | # Setup variables that we'll use after we turn off EM to validate our 33 | # test assertions. 34 | outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []} 35 | 36 | # Our WS and Http clients call this when they have received their messages to determine 37 | # when to turn off EM and make the test assertion at the very bottom. 38 | succeed = Proc.new do 39 | # TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after 40 | # the test finishes running. However, without it the test will randomly fail with a 41 | # "Redis disconnected" error. 42 | em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size } 43 | end 44 | 45 | # Setup a publisher 46 | publish = Proc.new do 47 | Firehose::Client::Producer::Http.new.publish(outgoing.shift).to(channel, buffer_size: rand(100)) do 48 | # The random timer ensures that sometimes the clients will be behind 49 | # and sometimes they will be caught up. 50 | EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty? 51 | end 52 | end 53 | 54 | # Lets have an HTTP Long poll client 55 | http_long_poll = Proc.new do |cid, last_sequence| 56 | http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => last_sequence}) 57 | http.errback { em.stop } 58 | http.callback do 59 | frame = JSON.parse(http.response, :symbolize_names => true) 60 | received[cid] << frame[:message] 61 | if received[cid].size < messages.size 62 | # Add some jitter so the clients aren't syncronized 63 | EM::add_timer(rand*0.001) { http_long_poll.call cid, frame[:last_sequence] } 64 | else 65 | succeed.call cid 66 | end 67 | end 68 | end 69 | 70 | # And test a web socket client too, at the same time. 71 | websocket = Proc.new do |cid| 72 | ws = Faye::WebSocket::Client.new(ws_url) 73 | 74 | ws.onopen = lambda do |event| 75 | ws.send('{"last_message_sequence":0}') 76 | end 77 | 78 | ws.onmessage = lambda do |event| 79 | frame = JSON.parse(event.data, :symbolize_names => true) 80 | received[cid] << frame[:message] 81 | succeed.call cid unless received[cid].size < messages.size 82 | end 83 | 84 | ws.onclose = lambda do |event| 85 | ws = nil 86 | end 87 | 88 | ws.onerror = lambda do |event| 89 | raise 'ws failed' + "\n" + event.inspect 90 | end 91 | end 92 | 93 | # Great, we have all the pieces in order, lets run this thing in the reactor. 94 | em 180 do 95 | # Start the clients. 96 | websocket.call("ws1") 97 | websocket.call("ws2") 98 | http_long_poll.call("http1") 99 | http_long_poll.call("http2") 100 | 101 | # Wait a sec to let our clients set up. 102 | em.add_timer(1){ publish.call } 103 | end 104 | 105 | # When EM stops, these assertions will be made. 106 | expect(received.size).to eql(4) 107 | 108 | (1..2).each do |id| 109 | ws_msgs = received["ws#{id}"].sort 110 | http_msgs = received["http#{id}"].sort 111 | expect(ws_msgs.size).to eql(messages.size) 112 | expect(http_msgs.size).to be >= messages.size * 0.75 113 | expect(http_msgs.size).to be <= messages.size 114 | expect(ws_msgs).to eql(messages.sort) 115 | end 116 | end 117 | 118 | it "supports channel multiplexing for http_long_poll and websockets" do 119 | # Setup variables that we'll use after we turn off EM to validate our 120 | # test assertions. 121 | outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []} 122 | 123 | # Our WS and Http clients call this when they have received their messages to determine 124 | # when to turn off EM and make the test assertion at the very bottom. 125 | succeed = Proc.new do 126 | # TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after 127 | # the test finishes running. However, without it the test will randomly fail with a 128 | # "Redis disconnected" error. 129 | em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size } 130 | end 131 | 132 | # Lets have an HTTP Long poll client using channel multiplexing 133 | multiplexed_http_long_poll = Proc.new do |cid, last_sequence| 134 | http = EM::HttpRequest.new(http_multi_url).get(:query => {'subscribe' => subscription_query}) 135 | 136 | http.errback { em.stop } 137 | http.callback do 138 | frame = JSON.parse(http.response, :symbolize_names => true) 139 | received[cid] << frame[:message] 140 | if received[cid].size < messages.size 141 | # Add some jitter so the clients aren't syncronized 142 | EM::add_timer(rand*0.001) { multiplexed_http_long_poll.call cid, frame[:last_sequence] } 143 | else 144 | succeed.call cid 145 | end 146 | end 147 | end 148 | 149 | # Test multiplexed web socket client 150 | outgoing = messages.dup 151 | publish_multi = Proc.new do 152 | msg = outgoing.shift 153 | chan = multiplex_channels[rand(multiplex_channels.size)] 154 | Firehose::Client::Producer::Http.new.publish(msg).to(chan) do 155 | EM::add_timer(rand*0.005) { publish_multi.call } unless outgoing.empty? 156 | end 157 | end 158 | 159 | multiplexed_websocket = Proc.new do |cid| 160 | ws = Faye::WebSocket::Client.new("ws://#{uri.host}:#{uri.port}/channels@firehose?subscribe=#{subscription_query}") 161 | 162 | ws.onmessage = lambda do |event| 163 | frame = JSON.parse(event.data, :symbolize_names => true) 164 | received[cid] << frame[:message] 165 | succeed.call cid unless received[cid].size < messages.size 166 | end 167 | 168 | ws.onclose = lambda do |event| 169 | ws = nil 170 | end 171 | 172 | ws.onerror = lambda do |event| 173 | raise 'ws failed' + "\n" + event.inspect 174 | end 175 | end 176 | 177 | em 180 do 178 | # Start the clients. 179 | multiplexed_websocket.call("ws3") 180 | multiplexed_websocket.call("ws4") 181 | multiplexed_http_long_poll.call("http3") 182 | multiplexed_http_long_poll.call("http4") 183 | 184 | # Wait a sec to let our clients set up. 185 | em.add_timer(1){ publish_multi.call } 186 | end 187 | 188 | # When EM stops, these assertions will be made. 189 | expect(received.size).to eql(4) 190 | (3..4).each do |id| 191 | ws_msgs = received["ws#{id}"].sort 192 | http_msgs = received["http#{id}"].sort 193 | expect(ws_msgs.size).to eql(messages.size) 194 | expect(http_msgs.size).to be >= messages.size * 0.75 195 | expect(http_msgs.size).to be <= messages.size 196 | expect(ws_msgs).to eql(messages.sort) 197 | end 198 | end 199 | 200 | 201 | it "returns 400 error for long-polling when using http long polling and sequence header is < 0" do 202 | em 5 do 203 | http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => -1}) 204 | http.errback { |e| raise e.inspect } 205 | http.callback do 206 | expect(http.response_header.status).to eql(400) 207 | em.stop 208 | end 209 | end 210 | end 211 | end 212 | -------------------------------------------------------------------------------- /spec/integrations/thin_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'integrations/shared_examples' 3 | 4 | describe "thin" do 5 | def server; :thin; end # Let isn't allowed in before(:all) 6 | it_behaves_like 'Firehose::Rack::App' 7 | end 8 | -------------------------------------------------------------------------------- /spec/lib/client/consumer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Client::Consumer::WebSocket do 4 | context "transport" do 5 | # Transport for Firehose::Client::App class is tested via the spec/integrations suite. 6 | end 7 | end 8 | 9 | describe Firehose::Client::Consumer::HttpLongPoll do 10 | context "transport" do 11 | # Transport for Firehose::Client::App class is tested via the spec/integrations suite. 12 | end 13 | end -------------------------------------------------------------------------------- /spec/lib/client/producer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Client::Producer::Http do 4 | let(:channel) { "/channel-awesome" } 5 | let(:url) { "#{Firehose::URI}#{channel}"} 6 | let(:publish_stub) { stub_request(:put, url) } 7 | let(:message) { "hey dude" } 8 | 9 | before(:all) do 10 | Firehose::Client::Producer::Http.adapter = :net_http 11 | end 12 | 13 | after(:all) do 14 | Firehose::Client::Producer::Http.adapter = nil 15 | end 16 | 17 | it "publishes message to channel" do 18 | publish_stub.to_return(:body => "", :status => 202) 19 | 20 | Firehose::Client::Producer::Http.new.publish(message).to(channel) 21 | expect(WebMock).to have_requested(:put, url).with { |req| req.body == message } 22 | end 23 | 24 | context 'prefix is specified in URI' do 25 | let(:firehose_uri) {"#{Firehose::URI}/prefix"} 26 | let(:url) { "#{firehose_uri}#{channel}"} 27 | 28 | it "publishes message to channel" do 29 | publish_stub.to_return(:body => "", :status => 202) 30 | 31 | Firehose::Client::Producer::Http.new(firehose_uri).publish(message).to(channel) 32 | expect(WebMock).to have_requested(:put, url).with { |req| req.body == message } 33 | end 34 | end 35 | 36 | it "publishes message to channel with expiry headers" do 37 | publish_stub.to_return(:body => "", :status => 202) 38 | ttl = 20 39 | 40 | Firehose::Client::Producer::Http.new.publish(message).to(channel, :ttl => ttl) 41 | expect(WebMock).to have_requested(:put, url).with { |req| req.body == message and req.headers['Cache-Control'] == "max-age=#{ttl}" } 42 | end 43 | 44 | describe "connection error handling" do 45 | it "raises PublishError if not 201" do 46 | publish_stub.to_return(:body => "", :status => 500) 47 | 48 | expect { 49 | Firehose::Client::Producer::Http.new.publish(message).to(channel) 50 | }.to raise_exception(Firehose::Client::Producer::Http::PublishError) 51 | end 52 | 53 | it "uses .error_handler if not 201" do 54 | publish_stub.to_return(:body => "", :status => 500) 55 | 56 | producer = Firehose::Client::Producer::Http.new 57 | producer.on_error do |e| 58 | expect(e.message).to match(/could not publish.+to/i) 59 | end 60 | producer.publish(message).to(channel) 61 | end 62 | 63 | it "raises TimeoutError if timed out" do 64 | publish_stub.to_timeout 65 | 66 | expect { 67 | Firehose::Client::Producer::Http.new.publish(message).to(channel) 68 | }.to raise_exception(Firehose::Client::Producer::Http::TimeoutError) 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /spec/lib/firehose_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose do 4 | it "has 7474 for default port" do 5 | expect(Firehose::URI.to_s).to eql('//0.0.0.0:7474') 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/lib/rack/consumer/http_long_poll_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'rack/test' 3 | require 'async_rack_test' 4 | 5 | describe Firehose::Rack::Consumer::HttpLongPoll do 6 | include AsyncRackTest::Methods 7 | include EM::TestHelper 8 | 9 | let(:app) { Firehose::Rack::Consumer::HttpLongPoll.new } 10 | 11 | context "transport" do 12 | # Transport for Firehose::Rack::App class is tested via the spec/integrations suite. 13 | end 14 | 15 | context "configuration" do 16 | it "has #timeout" do 17 | expect(Firehose::Rack::Consumer::HttpLongPoll.new(200).timeout).to eql(200) 18 | end 19 | end 20 | 21 | describe "Single subscription" do 22 | context "GET request" do 23 | it "receives subscription params" do 24 | em do 25 | expect_any_instance_of(Firehose::Server::ChannelSubscription).to receive(:on_subscribe).with({"to" => "nuts"}) 26 | get "/soup?to=nuts&last_message_sequence=1" 27 | EM.next_tick { em.stop } 28 | end 29 | end 30 | end 31 | end 32 | 33 | describe "Multiplexing subscription" do 34 | context "POST request" do 35 | it "parses implicit message sequence" do 36 | em do 37 | expect_any_instance_of(Firehose::Server::ChannelSubscription).to receive(:next_messages).and_return(EM::DefaultDeferrable.new) 38 | post "/channels@firehose", JSON.generate("/soup" => 1) 39 | EM.next_tick { em.stop } 40 | end 41 | end 42 | it "parses explicit message sequence and params" do 43 | em do 44 | expect_any_instance_of(Firehose::Server::ChannelSubscription).to receive(:next_messages).and_return(EM::DefaultDeferrable.new) 45 | expect_any_instance_of(Firehose::Server::ChannelSubscription).to receive(:on_subscribe).with({"soup" => "nuts"}) 46 | post "/channels@firehose", JSON.generate({ 47 | "/soup" => { 48 | "last_message_sequence" => 1, 49 | "params" => { 50 | "soup" => "nuts" 51 | } 52 | } 53 | }) 54 | EM.next_tick { em.stop } 55 | end 56 | end 57 | end 58 | end 59 | 60 | context "POST request" do 61 | before do 62 | post "/soup" 63 | end 64 | 65 | it "returns 405 for POST" do 66 | expect(last_response.status).to eql(405) 67 | end 68 | 69 | it "specifies GET in the Allow header" do 70 | expect(last_response.headers["Allow"]).to eql("GET") 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/lib/rack/consumer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'rack/test' 3 | 4 | describe Firehose::Rack::Consumer, :type => :request do 5 | include Rack::Test::Methods 6 | let(:app) { Firehose::Rack::Consumer.new } 7 | let(:path) { "/test/path/#{Time.now.to_i}" } 8 | 9 | it "has Content-Length on OPTIONS request" 10 | it "has Content-Length on GET request" 11 | 12 | context "configuration" do 13 | let(:app) { Firehose::Rack::Consumer } 14 | 15 | it "configures long polling timeout" do 16 | expect(app.new{ |a| a.http_long_poll.timeout = 300 }.http_long_poll.timeout).to eql(300) 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /spec/lib/rack/ping_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'rack/test' 3 | require 'async_rack_test' 4 | 5 | describe Firehose::Rack::Ping, :type => :request do 6 | include AsyncRackTest::Methods 7 | let(:app) { Firehose::Rack::Ping.new dummy_redis } 8 | let(:path) { "/test/path/#{Time.now.to_i}" } 9 | let(:deferrable) { EM::DefaultDeferrable.new } 10 | let(:dummy_redis) { double 'redis', :set => deferrable, :get => deferrable, :expire => deferrable } 11 | 12 | context 'redis is available' do 13 | before { deferrable.succeed Firehose::Rack::Ping::PingCheck::TEST_VALUE } 14 | 15 | it "returns 200" do 16 | ahead path 17 | expect(last_response.status).to eql(200) 18 | end 19 | end 20 | 21 | context 'redis is not available' do 22 | before { deferrable.fail 'some error' } 23 | it "returns 500" do 24 | ahead path 25 | expect(last_response.status).to eql(500) 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/lib/rack/publisher_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'rack/test' 3 | require 'async_rack_test' 4 | 5 | describe Firehose::Rack::Publisher, :type => :request do 6 | include AsyncRackTest::Methods 7 | let(:app) { Firehose::Rack::Publisher.new } 8 | let(:path) { "/test/path/#{Time.now.to_i}" } 9 | let(:deferrable) { EM::DefaultDeferrable.new } 10 | 11 | context 'publishing is successful' do 12 | before { deferrable.succeed } 13 | 14 | it "returns 202" do 15 | app.stub(:publisher => double('publisher', :publish => deferrable)) 16 | aput path, :body => "some nice little message" 17 | expect(last_response.status).to eql(202) 18 | end 19 | 20 | it "has Content-Length of zero" do 21 | app.stub(:publisher => double('publisher', :publish => deferrable)) 22 | aput path, :body => "some nice little message" 23 | expect(last_response.headers['Content-Length']).to eql('0') 24 | end 25 | 26 | it "parses Cache-Control max-age" do 27 | body = "howdy dude!" 28 | ttl = '92' 29 | 30 | Firehose::Server::Publisher.any_instance.stub(:publish).with(path, body, :ttl => ttl).and_return(deferrable) 31 | aput path, body, 'HTTP_CACHE_CONTROL' => 'max-age=92' 32 | end 33 | end 34 | end -------------------------------------------------------------------------------- /spec/lib/server/app_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Rack::App do 4 | context "transport" do 5 | # Transport for Firehose::Rack::App class is tested via the spec/integrations suite. 6 | end 7 | 8 | context "configuration" do 9 | let(:app) { Firehose::Rack::App } 10 | 11 | it "configures long polling timeout" do 12 | expect(app.new{ |a| a.consumer.http_long_poll.timeout = 300 }.consumer.http_long_poll.timeout).to eql(300) 13 | end 14 | end 15 | end -------------------------------------------------------------------------------- /spec/lib/server/channel_subscription_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Server::ChannelSubscription do 4 | include EM::TestHelper 5 | 6 | let(:channel_key) { '/bears/are/mean' } 7 | let(:env) { 8 | { 9 | "REMOTE_ADDR" => "192.168.0.1", 10 | "HTTP_REFERER"=> "http://localhost:80/test", 11 | "HTTP_USER_AGENT" => "test/runner" 12 | } 13 | } 14 | let(:channel) do 15 | Firehose::Server::ChannelSubscription.new(channel_key, env, 16 | sequence: sequence, 17 | timeout: timeout) 18 | end 19 | let(:sequence) { 0 } 20 | let(:timeout) { nil } 21 | let(:message) { 'Raaaarrrrrr!!!!' } 22 | let(:publisher) { Firehose::Server::Publisher.new } 23 | 24 | # If you use the memoized redis and subscriber connection objects between test 25 | # runs, EM won't clean up connections properly, lock forever, fail all of your tests 26 | # and remind you that you're wasting your life fighting event machine. Go have a beer. 27 | before(:each) do 28 | Firehose::Server::ChannelSubscription.stub(:redis) { Firehose::Server.redis.connection } 29 | Firehose::Server::ChannelSubscription.stub(:subscriber) { Firehose::Server::Subscriber.new(Firehose::Server.redis.connection) } 30 | end 31 | 32 | def push_message 33 | redis_exec 'lpush', "firehose:#{channel_key}:list", message 34 | redis_exec 'set', "firehose:#{channel_key}:sequence", '100' 35 | end 36 | 37 | context "callbacks" do 38 | it "calls #on_message" do 39 | push_message 40 | em do 41 | expect(channel).to receive(:on_message).with(Firehose::Server::Message.new(message, 100)) 42 | channel.next_messages.callback { em.stop } 43 | end 44 | end 45 | 46 | it "calls #on_subscribe" do 47 | expect_any_instance_of(Firehose::Server::ChannelSubscription).to receive(:on_subscribe).with({}) 48 | em do 49 | channel 50 | em.next_tick { em.stop } 51 | end 52 | end 53 | 54 | it "calls #on_unsubscribe" do 55 | push_message 56 | em do 57 | expect(channel).to receive(:on_unsubscribe).once 58 | channel.next_messages.callback { em.stop } 59 | end 60 | end 61 | end 62 | 63 | describe "#next_messages" do 64 | it "waits for message if message was not published before subscription" do 65 | em do 66 | channel.next_messages.callback do |messages| 67 | msg = messages.first 68 | expect(msg.payload).to eql(message) 69 | expect(msg.sequence).to eql(1) 70 | em.next_tick { em.stop } 71 | end 72 | 73 | publisher.publish(channel_key, message) 74 | end 75 | end 76 | 77 | it "returns the latest message and sequence if no sequence is given" do 78 | push_message 79 | 80 | em do 81 | channel.next_messages.callback do |messages| 82 | msg = messages.first 83 | expect(msg.payload).to eql(message) 84 | expect(msg.sequence).to eql(100) 85 | 86 | # This must happen _after_ the callback runs in order to pass consistently. 87 | em.next_tick { em.stop } 88 | end 89 | end 90 | end 91 | 92 | context "most recent sequence" do 93 | let(:sequence) { 100 } 94 | 95 | it "waits for message if most recent sequence is given" do 96 | push_message 97 | 98 | em 3 do 99 | channel.next_messages.callback do |messages| 100 | msg = messages.first.payload 101 | seq = messages.first.sequence 102 | expect(msg).to eql(message) 103 | expect(seq).to eql(101) 104 | em.next_tick { em.stop } 105 | end.errback 106 | 107 | publisher.publish(channel_key, message) 108 | end 109 | end 110 | end 111 | 112 | 113 | context "future sequence" do 114 | let(:sequence) { 101 } 115 | 116 | it "waits for message if a future sequence is given" do 117 | push_message 118 | 119 | em 3 do 120 | channel.next_messages.callback do |messages| 121 | msg = messages.first.payload 122 | seq = messages.first.sequence 123 | expect(msg).to eql(message) 124 | expect(seq).to eql(101) 125 | em.next_tick { em.stop } 126 | end.errback 127 | 128 | publisher.publish(channel_key, message) 129 | end 130 | end 131 | end 132 | 133 | 134 | context "outdated sequence" do 135 | let(:sequence) { 2 } 136 | 137 | it "immediatly gets a message if message sequence is behind and in list" do 138 | messages = %w[a b c d e] 139 | 140 | em 3 do 141 | publish_messages(messages) do 142 | channel.next_messages.callback do |messages| 143 | msg = messages.first.payload 144 | seq = messages.first.sequence 145 | expect(msg).to eql('c') 146 | expect(seq).to eql(3) 147 | 148 | # This must happen _after_ the callback runs in order to pass consistently. 149 | em.next_tick { em.stop } 150 | end 151 | end 152 | end 153 | end 154 | end 155 | 156 | it "gets current message if sequence is really far behind in list" do 157 | messages = ('aa'..'zz').to_a 158 | 159 | em 3 do 160 | publish_messages(messages) do 161 | channel.next_messages.callback do |msgs| 162 | msg = msgs.last.payload 163 | seq = msgs.last.sequence 164 | expect(msg).to eql(messages.last) 165 | expect(seq).to eql(messages.size) 166 | 167 | # This must happen _after_ the callback runs in order to pass consistently. 168 | em.next_tick { em.stop } 169 | end 170 | end 171 | end 172 | end 173 | 174 | context "a timeout is set" do 175 | let(:sequence) { 100 } 176 | let(:timeout) { 1 } 177 | 178 | it "times out if message isn't published in time" do 179 | push_message 180 | 181 | em 3 do 182 | channel.next_messages.callback do |messages| 183 | msg = messages.first.payload 184 | seq = messages.first.sequence 185 | raise 'test failed' 186 | end.errback do |e| 187 | expect(e).to eql(:timeout) 188 | em.next_tick { em.stop } 189 | end 190 | 191 | EM::add_timer(2) do 192 | publisher.publish(channel_key, message) 193 | end 194 | end 195 | end 196 | 197 | context "larger timeout" do 198 | let(:timeout) { 2 } 199 | 200 | it "does not timeout if message is published in time" do 201 | push_message 202 | 203 | em 3 do 204 | d = channel.next_messages.callback do |messages| 205 | msg = messages.first.payload 206 | seq = messages.first.sequence 207 | expect(msg).to eql(message) 208 | expect(seq).to eql(101) 209 | EM::add_timer(1) do 210 | em.stop 211 | end 212 | end.errback do |e| 213 | raise 'test failed' 214 | end 215 | 216 | d.should_not_receive(:fail) 217 | 218 | EM::add_timer(1) do 219 | publisher.publish(channel_key, message) 220 | end 221 | end 222 | end 223 | end 224 | end 225 | end 226 | 227 | 228 | # Publishes the given messages and executes the given block when finished. 229 | def publish_messages(all_messages) 230 | publish = Proc.new do |messages_to_publish| 231 | msg = messages_to_publish.shift 232 | publisher.publish(channel_key, msg).callback do 233 | if messages_to_publish.empty? 234 | # Publishing is done, proceed with the test 235 | yield 236 | else 237 | # Publish the next message 238 | publish.call messages_to_publish 239 | end 240 | end 241 | end 242 | 243 | publish.call all_messages.dup 244 | end 245 | end 246 | -------------------------------------------------------------------------------- /spec/lib/server/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | class CustomFilter < Firehose::Server::MessageFilter 4 | end 5 | 6 | describe Firehose::Server::Configuration do 7 | include EM::TestHelper 8 | 9 | DEFAULT_REDIS_URL = URI(ENV.fetch("REDIS_URL", Firehose::Server::Redis::DEFAULT_URL)) 10 | 11 | describe "configuring Firehose" do 12 | it "sets the default values" do 13 | config = Firehose::Server.configuration 14 | expect(config.message_filter).to eql(Firehose::Server::MessageFilter) 15 | expect(config.redis.url).to eql(DEFAULT_REDIS_URL) 16 | expect(config.deprecated_channels.to_a).to eql([]) 17 | end 18 | 19 | it "overrides values when given a configuration block" do 20 | redis_url = URI("redis://foo@bar") 21 | 22 | config = Firehose::Server.configuration do |conf| 23 | conf.message_filter = CustomFilter 24 | end 25 | 26 | expect(config.message_filter).to eql(CustomFilter) 27 | 28 | config = Firehose::Server.configuration do |conf| 29 | conf.redis.url = redis_url 30 | end 31 | 32 | expect(config.redis.url).to eql(redis_url) 33 | 34 | config = Firehose::Server.configuration do |conf| 35 | conf.redis.url = DEFAULT_REDIS_URL 36 | end 37 | 38 | expect(config.redis.url).to eql(DEFAULT_REDIS_URL) 39 | expect(config.deprecated_channels.to_a).to eql([]) 40 | 41 | config = Firehose::Server.configuration do |conf| 42 | conf.deprecated_channels = ["/foo", "/foo/bar"] 43 | conf.deprecated_channel do |channel| 44 | channel =~ /^\/foo\/(\d+)$/ 45 | end 46 | end 47 | 48 | expect(config.deprecated_channels.to_a).to eql(["/foo", "/foo/bar"]) 49 | expect(config.channel_deprecated?("/foo")).to be true 50 | expect(config.channel_deprecated?("/foo/bar")).to be true 51 | expect(config.channel_deprecated?("/foobar")).to be false 52 | expect(config.channel_deprecated?("/foo/123")).to be true 53 | expect(config.channel_deprecated?("/foo/123abc")).to be false 54 | expect(config.channel_deprecated?("/foo/123/456")).to be false 55 | expect(config.channel_deprecated?("/foo/123/bar")).to be false 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/lib/server/message_buffer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Server::MessageBuffer do 4 | it "has 100 DEFAULT_SIZE" do 5 | expect(Firehose::Server::MessageBuffer::DEFAULT_SIZE).to eql(100) 6 | end 7 | 8 | subject { Firehose::Server::MessageBuffer.new(messages, channel_sequence, consumer_sequence) } 9 | 10 | context "no messages" do 11 | let(:messages) { [] } 12 | let(:channel_sequence) { 0 } 13 | shared_examples "empty channel" do 14 | it "has empty remaining" do 15 | expect(subject.remaining_messages).to be_empty 16 | end 17 | end 18 | context "nil sequence" do 19 | let(:consumer_sequence) { nil } 20 | it_behaves_like "empty channel" 21 | end 22 | context "0 sequence" do 23 | let(:consumer_sequence) { 0 } 24 | it_behaves_like "empty channel" 25 | end 26 | context "negative sequence" do 27 | let(:consumer_sequence) { -1 } 28 | it_behaves_like "empty channel" 29 | end 30 | context "positive sequence" do 31 | let(:consumer_sequence) { 100 } 32 | it_behaves_like "empty channel" 33 | end 34 | end 35 | 36 | context "has messages (buffer size of 5, at channel sequence 5)" do 37 | let(:messages) { %w[a b c d e] } 38 | let(:channel_sequence) { 5 } 39 | 40 | shared_examples "latest message" do 41 | it "returns just the latest messages" do 42 | expect(subject.remaining_messages.map(&:payload)).to eql(["e"]) 43 | end 44 | it "has the correct sequence" do 45 | expect(subject.remaining_messages.map(&:sequence)).to eql([5]) 46 | end 47 | end 48 | 49 | context "nil sequence" do 50 | let(:consumer_sequence) { nil } 51 | it_behaves_like "latest message" 52 | end 53 | context "0 sequence" do 54 | let(:consumer_sequence) { 0 } 55 | it_behaves_like "latest message" 56 | end 57 | context "negative sequence" do 58 | let(:consumer_sequence) { -1 } 59 | it_behaves_like "latest message" 60 | end 61 | 62 | context "running behind" do 63 | let(:consumer_sequence) { 2 } 64 | it "has some remaining messages" do 65 | expect(subject.remaining_messages.map(&:payload)).to eql(%w[c d e]) 66 | end 67 | it "has the correct sequences" do 68 | expect(subject.remaining_messages.map(&:sequence)).to eql((3..5).to_a) 69 | end 70 | end 71 | 72 | context "caught up" do 73 | let(:consumer_sequence) { 5 } 74 | it "has no remaining messages" do 75 | expect(subject.remaining_messages).to be_empty 76 | end 77 | end 78 | context "ahead" do 79 | let(:consumer_sequence) { 10 } 80 | it "has no remaining messages" do 81 | expect(subject.remaining_messages).to be_empty 82 | end 83 | end 84 | end 85 | context "has messages (buffer size of 5, at channel sequence 10)" do 86 | let(:messages) { %w[f g h i j] } 87 | let(:channel_sequence) { 10 } 88 | 89 | shared_examples "latest message" do 90 | it "returns just the latest messages" do 91 | expect(subject.remaining_messages.map(&:payload)).to eql(["j"]) 92 | end 93 | it "has the correct sequence" do 94 | expect(subject.remaining_messages.map(&:sequence)).to eql([10]) 95 | end 96 | end 97 | 98 | context "nil sequence" do 99 | let(:consumer_sequence) { nil } 100 | it_behaves_like "latest message" 101 | end 102 | context "0 sequence" do 103 | let(:consumer_sequence) { 0 } 104 | it_behaves_like "latest message" 105 | end 106 | context "negative sequence" do 107 | let(:consumer_sequence) { -1 } 108 | it_behaves_like "latest message" 109 | end 110 | context "underwater" do 111 | let(:consumer_sequence) { 2 } 112 | it_behaves_like "latest message" 113 | end 114 | 115 | context "almost underwater" do 116 | let(:consumer_sequence) { 5 } 117 | it "has all remaining messages" do 118 | expect(subject.remaining_messages.map(&:payload)).to eql(%w[f g h i j]) 119 | end 120 | it "has the correct sequences" do 121 | expect(subject.remaining_messages.map(&:sequence)).to eql((6..10).to_a) 122 | end 123 | end 124 | 125 | context "running behind" do 126 | let(:consumer_sequence) { 7 } 127 | it "has some remaining messages" do 128 | expect(subject.remaining_messages.map(&:payload)).to eql(%w[h i j]) 129 | end 130 | it "has the correct sequences" do 131 | expect(subject.remaining_messages.map(&:sequence)).to eql((8..10).to_a) 132 | end 133 | end 134 | 135 | context "caught up" do 136 | let(:consumer_sequence) { 10 } 137 | it "has no remaining messages" do 138 | expect(subject.remaining_messages).to be_empty 139 | end 140 | end 141 | context "ahead" do 142 | let(:consumer_sequence) { 15 } 143 | it "has no remaining messages" do 144 | expect(subject.remaining_messages).to be_empty 145 | end 146 | end 147 | end 148 | end 149 | -------------------------------------------------------------------------------- /spec/lib/server/message_filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | class MinValFilter < Firehose::Server::MessageFilter 4 | JSONError = JSON.generate(error: "RandomFilter failed") 5 | 6 | def initialize(channel, min) 7 | super(channel) 8 | @min = min 9 | end 10 | 11 | def on_subscribe(params) 12 | unless params["valid"] 13 | raise Firehose::Server::ChannelSubscription::Failed, "Invalid params" 14 | end 15 | end 16 | 17 | def process(message) 18 | if message.json_payload["val"] < @min 19 | message.payload = JSONError 20 | end 21 | end 22 | end 23 | 24 | describe Firehose::Server::MessageFilter do 25 | include EM::TestHelper 26 | 27 | let(:mf) { MinValFilter.new("/foo", 0.5) } 28 | let(:message_ok) { Firehose::Server::Message.new(json_ok, 0) } 29 | let(:message_fail) { Firehose::Server::Message.new(json_fail, 0) } 30 | let(:json_ok) { JSON.generate(val: 0.6) } 31 | let(:json_fail) { JSON.generate(val: 0.4) } 32 | let(:json_error) { MinValFilter::JSONError } 33 | 34 | describe "#on_subscribe" do 35 | it "raises an exception if params invalid" do 36 | expect { 37 | mf.on_subscribe "valid" => false 38 | }.to raise_error(Firehose::Server::ChannelSubscription::Failed) 39 | end 40 | 41 | it "does not raise an exception if params valid" do 42 | expect { 43 | mf.on_subscribe "valid" => true 44 | }.to_not raise_error 45 | end 46 | end 47 | 48 | describe "#process" do 49 | it "doesn't change the message if the min val is reached" do 50 | msg = message_ok 51 | mf.process(msg) 52 | expect(msg.payload).to eql(json_ok) 53 | end 54 | 55 | it "returns an error message if the min val is not reached" do 56 | msg = message_fail 57 | mf.process(msg) 58 | expect(msg.payload).to_not eql(json_fail) 59 | expect(msg.payload).to eql(json_error) 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /spec/lib/server/metrics_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | START_TIME = Time.now.to_i 4 | 5 | def set_time!(time) 6 | Time.stub(:now) { time } 7 | end 8 | 9 | describe Firehose::Server::Metrics::TimeSeries do 10 | describe "#initialize" do 11 | context "invalid interval" do 12 | it "raises an ArgumentError if the given interval is invalid" do 13 | expect { 14 | Firehose::Server::Metrics::TimeSeries.new(seconds: nil) 15 | }.to raise_error(ArgumentError) 16 | 17 | expect { 18 | Firehose::Server::Metrics::TimeSeries.new(seconds: -1) 19 | }.to raise_error(ArgumentError) 20 | 21 | expect { 22 | Firehose::Server::Metrics::TimeSeries.new(seconds: 0) 23 | }.to raise_error(ArgumentError) 24 | 25 | expect { 26 | Firehose::Server::Metrics::TimeSeries.new(seconds: 0.5) 27 | }.to raise_error(ArgumentError) 28 | 29 | expect { 30 | Firehose::Server::Metrics::TimeSeries.new(seconds: 1) 31 | }.to_not raise_error 32 | 33 | expect { 34 | Firehose::Server::Metrics::TimeSeries.new(seconds: 1.5) 35 | }.to_not raise_error 36 | end 37 | end 38 | end 39 | 40 | describe "#current" do 41 | let(:metrics) { Firehose::Server::Metrics::TimeSeries.new } 42 | 43 | before :each do 44 | Time.stub(:now) { Time.at(START_TIME) } 45 | end 46 | 47 | it "always returns the same buffer for the same time" do 48 | expect(metrics.bucket(Time.now)).to eql(metrics.bucket(Time.now)) 49 | expect(metrics.current).to eql(metrics.current) 50 | end 51 | end 52 | 53 | describe "#clear!" do 54 | let(:metrics) { Firehose::Server::Metrics::TimeSeries.new(seconds: 1) } 55 | 56 | it "resets the TimeSeries to be empty again" do 57 | metrics.message_published!("chan", "hello") 58 | expect(metrics.series).to_not eql({}) 59 | metrics.clear! 60 | expect(metrics.series).to eql({}) 61 | end 62 | end 63 | 64 | describe "#clear_old!" do 65 | let(:metrics) do 66 | Firehose::Server::Metrics::TimeSeries.new( 67 | seconds: seconds, 68 | keep_buckets: keep_buckets 69 | ) 70 | end 71 | let(:seconds) { 5 } 72 | 73 | context "keep only last buckets" do 74 | let(:keep_buckets) { 1 } 75 | 76 | it "deletes all but the last bucket" do 77 | now = Time.now 78 | metrics.message_published!("chan1", "hello1") 79 | set_time!(now + 6) 80 | metrics.message_published!("chan2", "hello2") 81 | set_time!(now + 11) 82 | metrics.message_published!("chan3", "hello3") 83 | expect(metrics.series.size).to eql(3) 84 | set_time!(now + 12) 85 | metrics.clear_old! 86 | expect(metrics.series.size).to eql(1) 87 | end 88 | end 89 | 90 | context "keep last 2 buckets" do 91 | let(:keep_buckets) { 2 } 92 | 93 | it "deletes all but the last 2 buckets" do 94 | now = Time.now 95 | metrics.message_published!("chan1", "hello1") 96 | set_time!(now + 6) 97 | metrics.message_published!("chan2", "hello2") 98 | set_time!(now + 11) 99 | metrics.message_published!("chan3", "hello3") 100 | expect(metrics.series.size).to eql(3) 101 | set_time!(now + 12) 102 | metrics.clear_old! 103 | expect(metrics.series.size).to eql(2) 104 | end 105 | end 106 | 107 | context "keep last 3 buckets" do 108 | let(:keep_buckets) { 3 } 109 | let(:seconds) { 2 } 110 | 111 | it "deletes all but the last 3 buckets" do 112 | now = Time.now 113 | metrics.message_published!("chan1", "hello1") 114 | set_time!(now + 2) 115 | metrics.message_published!("chan2", "hello2") 116 | set_time!(now + 4) 117 | metrics.message_published!("chan3", "hello3") 118 | set_time!(now + 6) 119 | metrics.message_published!("chan4", "hello4") 120 | expect(metrics.series.size).to eql(4) 121 | set_time!(now + 8) 122 | metrics.clear_old! 123 | expect(metrics.series.size).to eql(3) 124 | end 125 | end 126 | end 127 | 128 | describe "#bucket" do 129 | let(:interval) { 2 } 130 | let(:metrics) do 131 | Firehose::Server::Metrics::TimeSeries.new(seconds: interval) 132 | end 133 | 134 | it "returns the same bucket for a given time within the configured seconds" do 135 | b0 = metrics.bucket(Time.at(0)) 136 | b1 = metrics.bucket(Time.at(1)) 137 | 138 | b2 = metrics.bucket(Time.at(2)) 139 | b3 = metrics.bucket(Time.at(3)) 140 | 141 | b4 = metrics.bucket(Time.at(4)) 142 | b5 = metrics.bucket(Time.at(5)) 143 | 144 | expect(b0).to eql(b1) 145 | expect(b1).to_not eql(b2) 146 | 147 | expect(b2).to eql(b3) 148 | expect(b3).to_not eql(b4) 149 | 150 | expect(b4).to eql(b5) 151 | end 152 | 153 | context "interval of 1 second" do 154 | let(:interval) { 1 } 155 | 156 | it "returns a new bucket for each second" do 157 | b0 = metrics.bucket(Time.at(0)) 158 | b1 = metrics.bucket(Time.at(1)) 159 | b2 = metrics.bucket(Time.at(2)) 160 | 161 | expect(b0).to_not eql(b1) 162 | expect(b1).to_not eql(b2) 163 | expect(b2).to_not eql(b0) 164 | end 165 | end 166 | end 167 | 168 | describe "#series" do 169 | let(:metrics) do 170 | Firehose::Server::Metrics::TimeSeries.new(seconds: 2, gc: false) 171 | end 172 | 173 | context "without any metrics" do 174 | it "should be empty" do 175 | expect(metrics.series).to eql({}) 176 | end 177 | end 178 | 179 | context "with one bucket having metrics" do 180 | let(:channel1) { "/test/channel/1" } 181 | let(:channel2) { "/test/channel/2" } 182 | 183 | before :each do 184 | Time.stub(:now) { Time.at(START_TIME) } 185 | end 186 | 187 | it "should return the metrics for one bucket" do 188 | metrics.message_published!(channel1) 189 | metrics.message_published!(channel2) 190 | 191 | buffer = Firehose::Server::Metrics::Buffer.new(metrics.bucket(Time.now), gc: false) 192 | buffer.message_published!(channel1) 193 | buffer.message_published!(channel2) 194 | 195 | expect(metrics.series).to be == { 196 | metrics.bucket(Time.now) => buffer 197 | } 198 | end 199 | end 200 | 201 | context "with multiple buckets having metrics" do 202 | let(:channel1) { "/test/channel/1" } 203 | let(:channel2) { "/test/channel/2" } 204 | 205 | it "shoult return the metrics for 2 buckets" do 206 | t1 = Time.now 207 | t2 = t1 + 2 208 | 209 | metrics.message_published!(channel1) 210 | metrics.message_published!(channel2) 211 | set_time!(t2) 212 | metrics.message_published!(channel2) 213 | 214 | bucket1 = metrics.bucket(t1) 215 | bucket2 = metrics.bucket(t2) 216 | 217 | buf1 = Firehose::Server::Metrics::Buffer.new(bucket1, gc: false) 218 | buf2 = Firehose::Server::Metrics::Buffer.new(bucket2, gc: false) 219 | 220 | buf1.message_published!(channel1) 221 | buf1.message_published!(channel2) 222 | buf2.message_published!(channel2) 223 | 224 | expect(metrics.series).to be == { 225 | bucket1 => buf1, 226 | bucket2 => buf2 227 | } 228 | end 229 | end 230 | end 231 | end 232 | 233 | describe Firehose::Server::Metrics::Buffer do 234 | context "new metrics instance" do 235 | let(:time) { 0 } 236 | let(:metrics) { Firehose::Server::Metrics::Buffer.new(time, gc: false) } 237 | 238 | describe "#==" do 239 | let(:buf1) { Firehose::Server::Metrics::Buffer.new(time, gc: false) } 240 | let(:buf2) { Firehose::Server::Metrics::Buffer.new(time, gc: false) } 241 | 242 | it "returns true for empty buffers" do 243 | expect(buf1).to be == buf2 244 | end 245 | 246 | it "returns true for two buffers with the same data" do 247 | buf1.message_published!("foo") 248 | expect(buf1).to_not be == buf2 249 | buf2.message_published!("foo") 250 | expect(buf1).to be == buf2 251 | end 252 | end 253 | 254 | describe "#to_hash" do 255 | it "returns 0 values for the metrics counters" do 256 | expect(metrics.to_hash).to eql({ 257 | time: time, 258 | global: { 259 | active_channels: 0 260 | }, 261 | channels: {} 262 | }) 263 | end 264 | 265 | it "returns updated counters" do 266 | channel = "/test/channel" 267 | channel2 = "/test/channel/2" 268 | 269 | 3.times { metrics.new_connection! } 270 | metrics.connection_closed! 271 | metrics.message_published!(channel) 272 | 2.times { metrics.channel_subscribed!(channel) } 273 | metrics.channels_subscribed_multiplexed_ws!([channel, channel2]) 274 | metrics.channels_subscribed_multiplexed_long_polling!([channel, channel2]) 275 | metrics.channels_subscribed_multiplexed_ws_dynamic!([{channel: "foo"}, {channel: "bar"}]) 276 | 277 | expect(metrics.to_hash).to eql({ 278 | time: time, 279 | global: { 280 | active_channels: 2, 281 | connections: 2, 282 | connections_opened: 3, 283 | connections_closed: 1, 284 | published: 1, 285 | subscribed: 2, 286 | subscribed_multiplexed_ws: 2, 287 | subscribed_multiplexed_ws_dynamic: 2, 288 | subscribed_multiplexed_long_polling: 2 289 | }, 290 | channels: { 291 | channel => { 292 | published: 1, 293 | subscribed: 2, 294 | subscribed_multiplexed_ws: 1, 295 | subscribed_multiplexed_long_polling: 1 296 | }, 297 | channel2 => { 298 | subscribed_multiplexed_ws: 1, 299 | subscribed_multiplexed_long_polling: 1 300 | } 301 | } 302 | }) 303 | end 304 | end 305 | end 306 | end 307 | -------------------------------------------------------------------------------- /spec/lib/server/publisher_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Server::Publisher do 4 | include EM::TestHelper 5 | 6 | let(:publisher) { Firehose::Server::Publisher.new } 7 | let(:channel_key) { "/firehose/publisher/test/#{Time.now.to_i}" } 8 | let(:message) { "howdy friends!" } 9 | 10 | it "has 1 day TTL" do 11 | expect(Firehose::Server::Publisher::TTL).to eql(86400) 12 | end 13 | 14 | describe "#publish" do 15 | it "publishes message change" do 16 | em do 17 | hiredis = Firehose::Server.redis.connection.pubsub 18 | hiredis.subscribe "firehose:channel_updates" 19 | hiredis.on(:message) {|_, msg| 20 | expect(msg).to eql("#{channel_key}\n1\n#{message}") 21 | em.next_tick { em.stop } 22 | } 23 | Firehose::Server::Publisher.new.publish channel_key, message 24 | end 25 | end 26 | 27 | "\"'\r\t\n!@\#$%^&*()[]\v\f\a\b\e{}/=?+\\|".each_char do |char| 28 | it "publishes messages with the '#{char.inspect}' character" do 29 | msg = [char, message, char].join 30 | em 1 do 31 | Firehose::Server::Publisher.new.publish(channel_key, msg).callback { em.stop } 32 | end 33 | expect(redis_exec('lpop', "firehose:#{channel_key}:list")).to eql(msg) 34 | end 35 | end 36 | 37 | it "adds message to list" do 38 | em do 39 | Firehose::Server::Publisher.new.publish(channel_key, message).callback { em.stop } 40 | end 41 | expect(redis_exec('lpop', "firehose:#{channel_key}:list")).to eql(message) 42 | end 43 | 44 | it "limits list to DEFAULT_SIZE messages" do 45 | em do 46 | Firehose::Server::MessageBuffer::DEFAULT_SIZE.times do |n| 47 | publisher.publish(channel_key, message) 48 | end 49 | publisher.publish(channel_key, message).callback { em.stop } 50 | end 51 | expect(redis_exec('llen', "firehose:#{channel_key}:list")).to eql(Firehose::Server::MessageBuffer::DEFAULT_SIZE) 52 | end 53 | 54 | it "limits message list to a custom buffer size" do 55 | buffer_size = rand(100) 56 | em do 57 | Firehose::Server::MessageBuffer::DEFAULT_SIZE.times do |n| 58 | publisher.publish(channel_key, message) 59 | end 60 | publisher.publish(channel_key, message, buffer_size: buffer_size).callback { em.stop } 61 | end 62 | redis_exec('llen', "firehose:#{channel_key}:list").should == buffer_size 63 | end 64 | 65 | it "stores no messages if buffer size is 0" do 66 | buffer_size = 0 67 | em do 68 | Firehose::Server::MessageBuffer::DEFAULT_SIZE.times do |n| 69 | publisher.publish(channel_key, message) 70 | end 71 | publisher.publish(channel_key, message, buffer_size: buffer_size).callback { em.stop } 72 | end 73 | redis_exec('llen', "firehose:#{channel_key}:list").should == buffer_size 74 | end 75 | 76 | it "increments sequence" do 77 | sequence_key = "firehose:#{channel_key}:sequence" 78 | 79 | @done_counter = 0 80 | expect(redis_exec('get', sequence_key)).to be_nil 81 | em do 82 | publisher.publish(channel_key, message).callback { @done_counter += 1; em.next_tick { em.stop } if @done_counter > 1 } 83 | publisher.publish(channel_key, message).callback { @done_counter += 1; em.next_tick { em.stop } if @done_counter > 1 } 84 | end 85 | expect(redis_exec('get', sequence_key).to_i).to eql(2) 86 | end 87 | 88 | it "sets expiry on sequence and list keys" do 89 | ttl = 78 # Why 78? Why not! 90 | 91 | em do 92 | publisher.publish(channel_key, message, :ttl => 78).callback do 93 | # Allow for 1 second of slippage/delay 94 | expect(redis_exec('TTL', "firehose:#{channel_key}:sequence")).to be > (ttl- 1) 95 | expect(redis_exec('TTL', "firehose:#{channel_key}:list")).to be > (ttl - 1) 96 | em.stop 97 | end 98 | end 99 | end 100 | 101 | it "marks the channel as deprecated" do 102 | em do 103 | expect(Firehose::Server.configuration.channel_deprecated?(channel_key)).to be(false) 104 | publisher.publish(channel_key, message, deprecated: true).callback do 105 | expect(Firehose::Server.configuration.channel_deprecated?(channel_key)).to be(true) 106 | em.stop 107 | end 108 | end 109 | end 110 | 111 | it "marks the channel as undeprecated" do 112 | em do 113 | Firehose::Server.configuration.deprecate_channel(channel_key) 114 | expect(Firehose::Server.configuration.channel_deprecated?(channel_key)).to be(true) 115 | publisher.publish(channel_key, message, deprecated: false).callback do 116 | expect(Firehose::Server.configuration.channel_deprecated?(channel_key)).to be(false) 117 | em.stop 118 | end 119 | end 120 | end 121 | 122 | it "sets no ttl on the channel if message is to be persisted" do 123 | em do 124 | publisher.publish(channel_key, message, persist: true).callback do 125 | expect(redis_exec('TTL', "firehose:#{channel_key}:sequence")).to be -1 126 | expect(redis_exec('TTL', "firehose:#{channel_key}:list")).to be -1 127 | em.stop 128 | end 129 | end 130 | end 131 | 132 | it "sets no ttl on the channel if message is to be persisted even if ttl given" do 133 | em do 134 | publisher.publish(channel_key, message, ttl: 10, persist: true).callback do 135 | expect(redis_exec('TTL', "firehose:#{channel_key}:sequence")).to be -1 136 | expect(redis_exec('TTL', "firehose:#{channel_key}:list")).to be -1 137 | em.stop 138 | end 139 | end 140 | end 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /spec/lib/server/redis_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Server::Redis do 4 | describe ".key" do 5 | subject { Firehose::Server::Redis.key("list", "channel", ["fizz", "buzz"]) } 6 | it "has root `firehose` key" do 7 | expect(subject).to start_with("firehose:") 8 | end 9 | it "flattens all arguments into key" do 10 | expect(subject).to eql("firehose:list:channel:fizz:buzz") 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/lib/server/subscriber_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Firehose::Server::Subscriber do 4 | include EM::TestHelper 5 | 6 | let(:channel_key) { '/bears/are/mean' } 7 | let(:env) { 8 | { 9 | "REMOTE_ADDR" => "192.168.0.1", 10 | "HTTP_REFERER"=> "http://localhost:80/test", 11 | "HTTP_USER_AGENT" => "test/runner" 12 | } 13 | } 14 | let(:chan_sub) { Firehose::Server::ChannelSubscription.new(channel_key, env) } 15 | let(:subscriber) { Firehose::Server::Subscriber.new(Firehose::Server.redis.connection) } 16 | let(:dummy_subscriber){ Firehose::Server::Subscriber.new(double('redis', :pubsub => double('pubsub', :subscribe => EM::DefaultDeferrable.new, :on => nil))) } 17 | let(:message) { 'Raaaarrrrrr!!!!' } 18 | let(:publisher) { Firehose::Server::Publisher.new } 19 | 20 | describe "#subscribe" do 21 | it "adds the deferrable to the subscriptions hash" do 22 | em do 23 | dummy_subscriber.subscribe(chan_sub) 24 | expect(dummy_subscriber.send(:subscriptions)[channel_key]).to eql([chan_sub]) 25 | em.next_tick { em.stop } 26 | end 27 | end 28 | 29 | it "calls process_messages on the deferrable when a message is published" do 30 | em do 31 | expect(chan_sub).to receive(:process_messages)#.with([Firehose::Server::Message.new(message, 1)]) 32 | subscriber.subscribe(chan_sub) 33 | publisher.publish(channel_key, message).callback do 34 | em.next_tick { em.stop } 35 | end 36 | end 37 | end 38 | 39 | it "doesn't call process_messages on the deferrable when a 2nd message is published" do 40 | em do 41 | expect(chan_sub).to receive(:process_messages).with([Firehose::Server::Message.new(message, 1)]) 42 | expect(chan_sub).to_not receive(:process_messages).with([Firehose::Server::Message.new('2nd message', 2)]) 43 | 44 | subscriber.subscribe(chan_sub) 45 | publisher.publish(channel_key, message).callback do 46 | publisher.publish(channel_key, '2nd message').callback do 47 | em.next_tick { em.stop } 48 | end 49 | end 50 | end 51 | end 52 | end 53 | 54 | describe "#unsubscribe" do 55 | it "removes the deferrable from the subscriptions hash" do 56 | dummy_subscriber.subscribe(chan_sub) 57 | dummy_subscriber.unsubscribe(chan_sub) 58 | expect(dummy_subscriber.send(:subscriptions).has_key?(channel_key)).to be_falsey 59 | end 60 | 61 | it "doesn't call succeed on the deferrable when a message is published" do 62 | deferrable = EM::DefaultDeferrable.new 63 | deferrable.should_not_receive(:succeed).with(message, 1) # The publisher is fresh, so the sequence ID will be 1. 64 | em do 65 | subscriber.subscribe(chan_sub) 66 | subscriber.unsubscribe(chan_sub) 67 | publisher.publish(channel_key, message).callback do 68 | em.next_tick { em.stop } 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RACK_ENV'] ||= 'test' 2 | 3 | require 'logger' 4 | require 'eventmachine' 5 | require 'em-http' 6 | require 'faye/websocket' 7 | require 'hiredis' 8 | require 'firehose' 9 | 10 | # Test coverage reports on CodeClimate. 11 | begin 12 | require 'simplecov' 13 | SimpleCov.start 14 | rescue LoadError 15 | end 16 | 17 | # We use both EM::Http and Net::Http in this test lib. When EM:Http is fired up 18 | # we're usually hitting Rainbows! for integrations, and when Net::Http we want 19 | # to mock that up. 20 | require 'webmock/rspec' 21 | WebMock.allow_net_connect! 22 | 23 | module EM::TestHelper 24 | # Run test inside of reactor. 25 | def em(ttl=30, &block) 26 | if block 27 | # Run the block inside of a reactor 28 | EM.run do 29 | EM.add_timer(ttl) do 30 | EM.stop 31 | raise "Test timed-out" 32 | end 33 | block.call(EM) 34 | end 35 | else # or just grab em and go nuts. 36 | EM 37 | end 38 | end 39 | end 40 | 41 | module Hiredis::TestHelper 42 | def redis 43 | @conn ||= begin 44 | redis = Firehose::Server.configuration.redis 45 | conn = Hiredis::Connection.new 46 | conn.connect(redis.url.host, redis.url.port) 47 | conn 48 | end 49 | end 50 | 51 | def reset_redis 52 | redis_exec 'flushdb' 53 | end 54 | 55 | def redis_exec(*tokens) 56 | redis.write tokens 57 | redis.read 58 | end 59 | end 60 | 61 | # Configure RSpec runner. See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 62 | RSpec.configure do |config| 63 | config.treat_symbols_as_metadata_keys_with_true_values = true 64 | config.run_all_when_everything_filtered = true 65 | config.filter_run :focus 66 | config.include Hiredis::TestHelper 67 | config.before(:each) do 68 | reset_redis 69 | end 70 | end 71 | 72 | # Allow async responses to get through rack/lint 73 | require 'rack/lint' 74 | class Rack::Lint 75 | def check_status_with_async(status) 76 | check_status_without_async(status) unless status == -1 77 | end 78 | alias_method :check_status_without_async, :check_status 79 | alias_method :check_status, :check_status_with_async 80 | 81 | def check_content_type_with_async(status, headers) 82 | check_content_type_without_async(status, headers) unless status == -1 83 | end 84 | alias_method :check_content_type_without_async, :check_content_type 85 | alias_method :check_content_type, :check_content_type_with_async 86 | end 87 | --------------------------------------------------------------------------------