├── .eslintrc.js ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .prettierrc ├── .rubocop.yml ├── CHANGELOG ├── DEV.md ├── Dockerfile ├── Gemfile ├── Guardfile ├── LICENSE ├── README.md ├── Rakefile ├── assets ├── jquery-1.8.2.js ├── message-bus-ajax.js └── message-bus.js ├── bench ├── codecs │ ├── all_codecs.rb │ ├── marshal.rb │ ├── packed_string.rb │ └── string_hack.rb ├── codecs_large_user_list.rb └── codecs_standard_message.rb ├── docker-compose.yml ├── examples ├── bench │ ├── bench.lua │ ├── config.ru │ ├── puma.rb │ ├── unicorn.conf.rb │ └── wrk.sample ├── chat │ ├── Gemfile │ ├── chat.rb │ ├── config.ru │ └── docker_container │ │ ├── chat.yml │ │ └── update_chat └── minimal │ ├── Gemfile │ └── config.ru ├── lib ├── message_bus.rb └── message_bus │ ├── backends.rb │ ├── backends │ ├── base.rb │ ├── memory.rb │ ├── postgres.rb │ └── redis.rb │ ├── client.rb │ ├── codec │ ├── base.rb │ ├── json.rb │ └── oj.rb │ ├── connection_manager.rb │ ├── distributed_cache.rb │ ├── http_client.rb │ ├── http_client │ ├── channel.rb │ └── version.rb │ ├── message.rb │ ├── rack │ ├── middleware.rb │ └── thin_ext.rb │ ├── rails │ └── railtie.rb │ ├── timer_thread.rb │ └── version.rb ├── message_bus.gemspec ├── package-lock.json ├── package.json ├── spec ├── assets │ ├── SpecHelper.js │ └── message-bus.spec.js ├── fixtures │ └── test │ │ ├── Gemfile │ │ └── config.ru ├── helpers.rb ├── integration │ └── http_client_spec.rb ├── lib │ ├── fake_async_middleware.rb │ ├── message_bus │ │ ├── assets │ │ │ └── asset_encoding_spec.rb │ │ ├── backend_spec.rb │ │ ├── client_spec.rb │ │ ├── connection_manager_spec.rb │ │ ├── distributed_cache_spec.rb │ │ ├── multi_process_spec.rb │ │ ├── rack │ │ │ └── middleware_spec.rb │ │ └── timer_thread_spec.rb │ └── message_bus_spec.rb ├── performance │ ├── backlog.rb │ └── publish.rb ├── spec_helper.rb └── support │ └── jasmine-browser.json └── vendor └── assets └── javascripts └── .gitignore /.eslintrc.js: -------------------------------------------------------------------------------- 1 | /*global module*/ 2 | module.exports = { 3 | env: { 4 | browser: true, 5 | es2021: false, 6 | }, 7 | extends: 'eslint:recommended', 8 | parserOptions: { 9 | ecmaVersion: 2015, 10 | sourceType: 'module', 11 | }, 12 | rules: {}, 13 | ignorePatterns: ['/vendor', '/doc', '/assets/jquery-1.8.2.js'], 14 | overrides: [ 15 | { 16 | // Enable async/await in tests only 17 | files: ["spec/**/*"], 18 | parserOptions: { 19 | ecmaVersion: 2022, 20 | }, 21 | }, 22 | ], 23 | }; 24 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 5 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | 17 | - uses: ruby/setup-ruby@v1 18 | with: 19 | ruby-version: "3.3" 20 | bundler-cache: true 21 | 22 | - name: Set up Node.js 23 | uses: actions/setup-node@v4 24 | with: 25 | node-version: 20 26 | cache: npm 27 | 28 | - name: Setup npm 29 | run: npm install 30 | 31 | - name: Rubocop 32 | run: bundle exec rubocop 33 | 34 | - name: ESLint 35 | run: npx eslint . 36 | 37 | test: 38 | runs-on: ubuntu-latest 39 | name: Ruby ${{ matrix.ruby }} (${{ matrix.redis }}) 40 | timeout-minutes: 10 41 | 42 | env: 43 | PGHOST: localhost 44 | PGPASSWORD: postgres 45 | PGUSER: postgres 46 | 47 | strategy: 48 | fail-fast: false 49 | matrix: 50 | ruby: ["3.2", "3.3", "3.4"] 51 | redis: ["redis:5", "redis:6", "valkey/valkey"] 52 | 53 | services: 54 | postgres: 55 | image: postgres:16 56 | env: 57 | POSTGRES_DB: message_bus_test 58 | POSTGRES_PASSWORD: postgres 59 | ports: 60 | - 5432:5432 61 | options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 62 | redis: 63 | image: ${{ matrix.redis }} 64 | ports: 65 | - 6379:6379 66 | options: >- 67 | --health-cmd "redis-cli ping" 68 | --health-interval 10s 69 | --health-timeout 5s 70 | --health-retries 5 71 | 72 | steps: 73 | - uses: actions/checkout@v4 74 | 75 | - uses: ruby/setup-ruby@v1 76 | with: 77 | ruby-version: ${{ matrix.ruby }} 78 | bundler-cache: true 79 | 80 | - name: Tests 81 | env: 82 | TESTOPTS: --verbose 83 | run: bundle exec rake 84 | 85 | publish: 86 | if: github.event_name == 'push' && github.ref == 'refs/heads/main' 87 | needs: [lint, test] 88 | runs-on: ubuntu-latest 89 | 90 | steps: 91 | - uses: actions/checkout@v4 92 | 93 | - name: Release gem 94 | uses: discourse/publish-rubygems-action@v3 95 | id: publish-gem 96 | env: 97 | RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }} 98 | GIT_EMAIL: team@discourse.org 99 | GIT_NAME: discoursebot 100 | 101 | - name: Update package version 102 | if: steps.publish-gem.outputs.new_version == 'true' 103 | run: | 104 | VERSION=$(ruby -r './lib/message_bus/version' -e 'puts MessageBus::VERSION') 105 | sed -i "s/0.0.0-version-placeholder/$VERSION/" package.json 106 | git config --global user.email "ci@ci.invalid" 107 | git config --global user.name "Discourse CI" 108 | git add package.json 109 | git commit -m 'bump' 110 | 111 | - name: Publish package 112 | uses: JS-DevTools/npm-publish@v3 113 | if: steps.publish-gem.outputs.new_version == 'true' 114 | with: 115 | token: ${{ secrets.NPM_TOKEN }} 116 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | *.swp 19 | .rubocop-https---raw-githubusercontent-com-discourse-discourse-master--rubocop-yml 20 | .byebug_history 21 | node_modules/ 22 | yarn.lock 23 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_gem: 2 | rubocop-discourse: stree-compat.yml 3 | inherit_mode: 4 | merge: 5 | - Exclude 6 | 7 | AllCops: 8 | Exclude: 9 | - "examples/**/*" 10 | 11 | Discourse/Plugins: 12 | Enabled: false 13 | 14 | RSpec: 15 | Enabled: false 16 | -------------------------------------------------------------------------------- /DEV.md: -------------------------------------------------------------------------------- 1 | ### How to Publish to NPM 2 | 3 | 1. First, edit `package.json` and bump the version. 4 | 5 | 2. Log in to npm `yarn login` 6 | 7 | 3. Publish: `yarn publish` 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.7 2 | 3 | RUN cd /tmp && \ 4 | wget --quiet https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \ 5 | tar -xf phantomjs-2.1.1-linux-x86_64.tar.bz2 && \ 6 | mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin && \ 7 | rm -rf phantomjs* 8 | 9 | WORKDIR /usr/src/app 10 | 11 | RUN mkdir -p ./lib/message_bus 12 | COPY lib/message_bus/version.rb ./lib/message_bus 13 | COPY Gemfile *.gemspec ./ 14 | RUN bundle install 15 | 16 | COPY . . 17 | 18 | CMD ["rake"] 19 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | source 'https://rubygems.org' 3 | 4 | # Specify your gem's dependencies in message_bus.gemspec 5 | gemspec 6 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | # A sample Guardfile 3 | # More info at https://github.com/guard/guard#readme 4 | 5 | ## Uncomment and set this to only include directories you want to watch 6 | directories %w(lib spec) 7 | 8 | ## Uncomment to clear the screen before every task 9 | # clearing :on 10 | 11 | ## Guard internally checks for changes in the Guardfile and exits. 12 | ## If you want Guard to automatically start up again, run guard in a 13 | ## shell loop, e.g.: 14 | ## 15 | ## $ while bundle exec guard; do echo "Restarting Guard..."; done 16 | ## 17 | ## Note: if you are using the `directories` clause above and you are not 18 | ## watching the project directory ('.'), then you will want to move 19 | ## the Guardfile to a watched dir and symlink it back, e.g. 20 | # 21 | # $ mkdir config 22 | # $ mv Guardfile config/ 23 | # $ ln -s config/Guardfile . 24 | # 25 | # and, you'll have to watch "config/Guardfile" instead of "Guardfile" 26 | 27 | # Note: The cmd option is now required due to the increasing number of ways 28 | # rspec may be run, below are examples of the most common uses. 29 | # * bundler: 'bundle exec rspec' 30 | # * bundler binstubs: 'bin/rspec' 31 | # * spring: 'bin/rspec' (This will use spring if running and you have 32 | # installed the spring binstubs per the docs) 33 | # * zeus: 'zeus rspec' (requires the server to be started separately) 34 | # * 'just' rspec: 'rspec' 35 | 36 | guard :rspec, cmd: "bundle exec rspec" do 37 | require "guard/rspec/dsl" 38 | dsl = Guard::RSpec::Dsl.new(self) 39 | 40 | # Feel free to open issues for suggestions and improvements 41 | 42 | # RSpec files 43 | rspec = dsl.rspec 44 | watch(rspec.spec_helper) { rspec.spec_dir } 45 | watch(rspec.spec_support) { rspec.spec_dir } 46 | watch(rspec.spec_files) 47 | 48 | # Ruby files 49 | ruby = dsl.ruby 50 | dsl.watch_spec_files_for(ruby.lib_files) 51 | end 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Sam Saffron 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'rubygems' 3 | require 'rake/testtask' 4 | require 'bundler' 5 | require 'bundler/gem_tasks' 6 | require 'bundler/setup' 7 | require 'yard' 8 | 9 | Bundler.require(:default, :test) 10 | 11 | YARD::Rake::YardocTask.new 12 | 13 | BACKENDS = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?.*).rb})[:backend] } - ["base"] 14 | SPEC_FILES = Dir['spec/**/*_spec.rb'] 15 | INTEGRATION_FILES = Dir['spec/integration/**/*_spec.rb'] 16 | 17 | module CustomBuild 18 | def build_gem 19 | `cp assets/message-bus* vendor/assets/javascripts` 20 | super 21 | end 22 | end 23 | 24 | module Bundler 25 | class GemHelper 26 | prepend CustomBuild 27 | end 28 | end 29 | 30 | desc "Generate documentation for Yard, and fail if there are any warnings" 31 | task :test_doc do 32 | sh "yard --fail-on-warning #{'--no-progress' if ENV['CI']}" 33 | end 34 | 35 | namespace :jasmine do 36 | desc "Run Jasmine tests in headless mode" 37 | task 'ci' do 38 | if !system("npx jasmine-browser-runner runSpecs") 39 | exit 1 40 | end 41 | end 42 | end 43 | 44 | namespace :spec do 45 | BACKENDS.each do |backend| 46 | desc "Run tests on the #{backend} backend" 47 | task backend do 48 | begin 49 | ENV['MESSAGE_BUS_BACKEND'] = backend 50 | Rake::TestTask.new(backend) do |t| 51 | t.test_files = SPEC_FILES - INTEGRATION_FILES 52 | end 53 | Rake::Task[backend].invoke 54 | ensure 55 | ENV.delete('MESSAGE_BUS_BACKEND') 56 | end 57 | end 58 | end 59 | 60 | desc "Run integration tests" 61 | task :integration do 62 | require "socket" 63 | 64 | def port_available?(port) 65 | server = TCPServer.open("0.0.0.0", port) 66 | server.close 67 | true 68 | rescue Errno::EADDRINUSE 69 | false 70 | end 71 | 72 | begin 73 | ENV['MESSAGE_BUS_BACKEND'] = 'memory' 74 | pid = spawn("bundle exec puma -p 9292 spec/fixtures/test/config.ru") 75 | sleep 1 while port_available?(9292) 76 | Rake::TestTask.new(:integration) do |t| 77 | t.test_files = INTEGRATION_FILES 78 | end 79 | Rake::Task[:integration].invoke 80 | ensure 81 | ENV.delete('MESSAGE_BUS_BACKEND') 82 | Process.kill('TERM', pid) if pid 83 | end 84 | end 85 | end 86 | 87 | desc "Run tests on all backends, plus client JS tests" 88 | task spec: BACKENDS.map { |backend| "spec:#{backend}" } + ["jasmine:ci", "spec:integration"] 89 | 90 | desc "Run performance benchmarks on all backends" 91 | task :performance do 92 | begin 93 | ENV['MESSAGE_BUS_BACKENDS'] = BACKENDS.join(",") 94 | sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/performance/*.rb'].to_a.join(' ')}" 95 | ensure 96 | ENV.delete('MESSAGE_BUS_BACKENDS') 97 | end 98 | end 99 | 100 | desc "Run all tests and confirm the documentation compiles without error" 101 | task default: [:spec, :test_doc] 102 | -------------------------------------------------------------------------------- /assets/message-bus-ajax.js: -------------------------------------------------------------------------------- 1 | // A bare-bones implementation of $.ajax that MessageBus will use 2 | // as a fallback if jQuery is not present 3 | // 4 | // Only implements methods & options used by MessageBus 5 | (function(global) { 6 | 'use strict'; 7 | if (!global.MessageBus){ 8 | throw new Error("MessageBus must be loaded before the ajax adapter"); 9 | } 10 | 11 | global.MessageBus.ajax = function(options){ 12 | var XHRImpl = (global.MessageBus && global.MessageBus.xhrImplementation) || global.XMLHttpRequest; 13 | var xhr = new XHRImpl(); 14 | xhr.dataType = options.dataType; 15 | xhr.open('POST', options.url); 16 | for (var name in options.headers){ 17 | xhr.setRequestHeader(name, options.headers[name]); 18 | } 19 | xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded'); 20 | if (options.messageBus.chunked){ 21 | options.messageBus.onProgressListener(xhr); 22 | } 23 | xhr.onreadystatechange = function(){ 24 | if (xhr.readyState === 4){ 25 | var status = xhr.status; 26 | if (status >= 200 && status < 300 || status === 304){ 27 | options.success(xhr.responseText); 28 | } else { 29 | options.error(xhr, xhr.statusText); 30 | } 31 | options.complete(); 32 | } 33 | } 34 | xhr.send(new URLSearchParams(options.data).toString()); 35 | return xhr; 36 | }; 37 | 38 | })(window); 39 | -------------------------------------------------------------------------------- /bench/codecs/all_codecs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative './packed_string' 4 | require_relative './string_hack' 5 | require_relative './marshal' 6 | 7 | def all_codecs 8 | { 9 | json: MessageBus::Codec::Json.new, 10 | oj: MessageBus::Codec::Oj.new, 11 | marshal: MarshalCodec.new, 12 | packed_string_4_bytes: PackedString.new("V"), 13 | packed_string_8_bytes: PackedString.new("Q"), 14 | string_hack: StringHack.new 15 | } 16 | end 17 | 18 | def bench_decode(hash, user_needle) 19 | encoded_data = all_codecs.map do |name, codec| 20 | [ 21 | name, codec, codec.encode(hash.dup) 22 | ] 23 | end 24 | 25 | Benchmark.ips do |x| 26 | 27 | encoded_data.each do |name, codec, encoded| 28 | x.report(name) do |n| 29 | while n > 0 30 | decoded = codec.decode(encoded) 31 | decoded["user_ids"].include?(user_needle) 32 | n -= 1 33 | end 34 | end 35 | end 36 | 37 | x.compare! 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /bench/codecs/marshal.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MarshalCodec 4 | def encode(hash) 5 | ::Marshal.dump(hash) 6 | end 7 | 8 | def decode(payload) 9 | ::Marshal.load(payload) # rubocop:disable Security/MarshalLoad 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /bench/codecs/packed_string.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class PackedString 4 | class FastIdList 5 | def self.from_array(array, pack_with) 6 | new(array.sort.pack("#{pack_with}*"), pack_with) 7 | end 8 | 9 | def self.from_string(string, pack_with) 10 | new(string, pack_with) 11 | end 12 | 13 | def initialize(packed, pack_with) 14 | raise "unknown pack format, expecting Q or V" if pack_with != "V" && pack_with != "Q" 15 | @packed = packed 16 | @pack_with = pack_with 17 | @slot_size = pack_with == "V" ? 4 : 8 18 | end 19 | 20 | def include?(id) 21 | found = (0...length).bsearch do |index| 22 | @packed.byteslice(index * @slot_size, @slot_size).unpack1(@pack_with) >= id 23 | end 24 | 25 | found && @packed.byteslice(found * @slot_size, @slot_size).unpack1(@pack_with) == id 26 | end 27 | 28 | def length 29 | @length ||= @packed.bytesize / @slot_size 30 | end 31 | 32 | def to_a 33 | @packed.unpack("#{@pack_with}*") 34 | end 35 | 36 | def to_s 37 | @packed 38 | end 39 | end 40 | 41 | def initialize(pack_with = "V") 42 | @pack_with = pack_with 43 | @oj_options = { mode: :compat } 44 | end 45 | 46 | def encode(hash) 47 | 48 | if user_ids = hash["user_ids"] 49 | hash["user_ids"] = FastIdList.from_array(hash["user_ids"], @pack_with).to_s 50 | end 51 | 52 | hash["data"] = ::Oj.dump(hash["data"], @oj_options) 53 | 54 | Marshal.dump(hash) 55 | end 56 | 57 | def decode(payload) 58 | result = Marshal.load(payload) # rubocop:disable Security/MarshalLoad 59 | result["data"] = ::Oj.load(result["data"], @oj_options) 60 | 61 | if str = result["user_ids"] 62 | result["user_ids"] = FastIdList.from_string(str, @pack_with) 63 | end 64 | 65 | result 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /bench/codecs/string_hack.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class StringHack 4 | class FastIdList 5 | def self.from_array(array) 6 | new(",#{array.join(",")},") 7 | end 8 | 9 | def self.from_string(string) 10 | new(string) 11 | end 12 | 13 | def initialize(packed) 14 | @packed = packed 15 | end 16 | 17 | def include?(id) 18 | @packed.include?(",#{id},") 19 | end 20 | 21 | def to_s 22 | @packed 23 | end 24 | end 25 | 26 | def initialize 27 | @oj_options = { mode: :compat } 28 | end 29 | 30 | def encode(hash) 31 | if user_ids = hash["user_ids"] 32 | hash["user_ids"] = FastIdList.from_array(user_ids).to_s 33 | end 34 | 35 | ::Oj.dump(hash, @oj_options) 36 | end 37 | 38 | def decode(payload) 39 | result = ::Oj.load(payload, @oj_options) 40 | 41 | if str = result["user_ids"] 42 | result["user_ids"] = FastIdList.from_string(str) 43 | end 44 | 45 | result 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /bench/codecs_large_user_list.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/inline' 4 | 5 | gemfile do 6 | source 'https://rubygems.org' 7 | gem 'message_bus', path: '../' 8 | gem 'benchmark-ips' 9 | gem 'oj' 10 | end 11 | 12 | require 'benchmark/ips' 13 | require 'message_bus' 14 | require_relative 'codecs/all_codecs' 15 | 16 | bench_decode({ 17 | "data" => "hello world", 18 | "user_ids" => (1..10000).to_a, 19 | "group_ids" => nil, 20 | "client_ids" => nil 21 | }, 5000 22 | ) 23 | 24 | # packed_string_4_bytes: 127176.1 i/s 25 | # packed_string_8_bytes: 94494.6 i/s - 1.35x (± 0.00) slower 26 | # string_hack: 26403.4 i/s - 4.82x (± 0.00) slower 27 | # marshal: 4985.5 i/s - 25.51x (± 0.00) slower 28 | # oj: 3072.9 i/s - 41.39x (± 0.00) slower 29 | # json: 2222.7 i/s - 57.22x (± 0.00) slower 30 | -------------------------------------------------------------------------------- /bench/codecs_standard_message.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/inline' 4 | 5 | gemfile do 6 | source 'https://rubygems.org' 7 | gem 'message_bus', path: '../' 8 | gem 'benchmark-ips' 9 | gem 'oj' 10 | end 11 | 12 | require 'benchmark/ips' 13 | require 'message_bus' 14 | require_relative 'codecs/all_codecs' 15 | 16 | bench_decode({ 17 | "data" => { amazing: "hello world this is an amazing message hello there!!!", another_key: [2, 3, 4] }, 18 | "user_ids" => [1, 2, 3], 19 | "group_ids" => [1], 20 | "client_ids" => nil 21 | }, 2 22 | ) 23 | 24 | # marshal: 504885.6 i/s 25 | # json: 401050.9 i/s - 1.26x (± 0.00) slower 26 | # oj: 340847.4 i/s - 1.48x (± 0.00) slower 27 | # string_hack: 296741.6 i/s - 1.70x (± 0.00) slower 28 | # packed_string_4_bytes: 207942.6 i/s - 2.43x (± 0.00) slower 29 | # packed_string_8_bytes: 206093.0 i/s - 2.45x (± 0.00) slower 30 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | volumes: 3 | bundle_config: 4 | bundle: 5 | redis_data: 6 | postgres_data: 7 | services: 8 | tests: 9 | build: 10 | context: . 11 | environment: 12 | BUNDLE_TO: /usr/local/bundle 13 | REDISURL: redis://redis:6379 14 | PGHOST: postgres 15 | PGUSER: postgres 16 | PGPASSWORD: "1234" 17 | volumes: 18 | - .:/usr/src/app 19 | - bundle_config:/home/src/app/.bundle 20 | - bundle:/usr/local/bundle 21 | depends_on: 22 | - postgres 23 | - redis 24 | docs: 25 | build: 26 | context: . 27 | command: yard server --bind 0.0.0.0 --reload 28 | environment: 29 | BUNDLE_TO: /usr/local/bundle 30 | volumes: 31 | - .:/usr/src/app 32 | - bundle_config:/home/src/app/.bundle 33 | - bundle:/usr/local/bundle 34 | ports: 35 | - 8808:8808 36 | example: 37 | build: 38 | context: . 39 | command: bash -c "cd examples/chat && bundle install && bundle exec rackup --server puma --host 0.0.0.0" 40 | environment: 41 | BUNDLE_TO: /usr/local/bundle 42 | REDISURL: redis://redis:6379 43 | volumes: 44 | - .:/usr/src/app 45 | - bundle_config:/home/src/app/.bundle 46 | - bundle:/usr/local/bundle 47 | ports: 48 | - 9292:9292 49 | redis: 50 | image: redis:5.0 51 | volumes: 52 | - redis_data:/data 53 | postgres: 54 | image: postgres:11.0 55 | volumes: 56 | - postgres_data:/var/lib/postgresql/data 57 | environment: 58 | - POSTGRES_PASSWORD=1234 59 | - POSTGRES_DB=message_bus_test 60 | -------------------------------------------------------------------------------- /examples/bench/bench.lua: -------------------------------------------------------------------------------- 1 | -- wrk returns lots of read errors, this is unavoidable cause 2 | -- 3 | -- 1. There is no internal implementation of chunked encoding in wrk (which would be ideal) 4 | -- 5 | -- 2. MessageBus gem does not provide http keepalive (by design), and can not provide content length 6 | -- if MessageBus provided keepalive it would have to be able to re-dispatch requests to rack, something 7 | -- that is not supported by the underlying rack hijack protocol, once a req is hijacked it can not be 8 | -- returned 9 | -- 10 | -- This leads to certain read errors while the bench runs cause wrk can not figure out cleanly that 11 | -- MessageBus is done with a request 12 | -- 13 | 14 | wrk.method = "POST" 15 | wrk.body = "" 16 | wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" 17 | 18 | -- chunking is not supported internally to wrk 19 | wrk.headers["Dont-Chunk"] = "true" 20 | wrk.headers["Connection"] = "Close" 21 | 22 | request = function() 23 | local hexdict = {48,49,50,51,52,53,54,55,56,57,97,98,99,100,101,102} 24 | local randstr = {} 25 | for i=1, 32 do 26 | randstr[i] = hexdict[math.random(1, 16)] 27 | end 28 | local path = wrk.path .. "message-bus/" .. string.char(unpack(randstr)) .. "/poll" 29 | return wrk.format(nil, path) 30 | end 31 | -------------------------------------------------------------------------------- /examples/bench/config.ru: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | $LOAD_PATH.unshift File.expand_path('../../../lib', __FILE__) 3 | 4 | require 'message_bus' 5 | require 'stackprof' 6 | 7 | if defined?(PhusionPassenger) 8 | PhusionPassenger.on_event(:starting_worker_process) do |forked| 9 | if forked 10 | # We're in smart spawning mode. 11 | MessageBus.after_fork 12 | else 13 | # We're in conservative spawning mode. We don't need to do anything. 14 | end 15 | end 16 | end 17 | 18 | # require 'rack-mini-profiler' 19 | 20 | # Rack::MiniProfiler.config.storage = Rack::MiniProfiler::MemoryStore 21 | 22 | # use Rack::MiniProfiler 23 | # StackProf.start(mode: :cpu) 24 | # Thread.new { 25 | # sleep 10 26 | # StackProf.stop 27 | # File.write("test.prof",Marshal.dump(StackProf.results)) 28 | # } 29 | 30 | MessageBus.long_polling_interval = 1000 * 2 31 | MessageBus.max_active_clients = 10000 32 | use MessageBus::Rack::Middleware 33 | run lambda { |_env| [200, { "Content-Type" => "text/html" }, ["Howdy"]] } 34 | -------------------------------------------------------------------------------- /examples/bench/puma.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'message_bus' 3 | on_worker_boot do 4 | MessageBus.after_fork 5 | end 6 | -------------------------------------------------------------------------------- /examples/bench/unicorn.conf.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'message_bus' 3 | after_fork do |_server, _worker| 4 | MessageBus.after_fork 5 | end 6 | -------------------------------------------------------------------------------- /examples/bench/wrk.sample: -------------------------------------------------------------------------------- 1 | wrk -c100 -d2m --timeout=30s --latency -s bench.lua http://127.0.0.1:3000 2 | -------------------------------------------------------------------------------- /examples/chat/Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | gem 'puma' 3 | gem 'redis' 4 | gem 'sinatra' 5 | -------------------------------------------------------------------------------- /examples/chat/chat.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH.unshift File.expand_path('../../../lib', __FILE__) 4 | require 'message_bus' 5 | require 'sinatra' 6 | require 'sinatra/base' 7 | require 'set' 8 | require 'json' 9 | 10 | $online = Hash.new 11 | 12 | MessageBus.subscribe "/presence" do |msg| 13 | if user = msg.data["enter"] 14 | $online[user] = Time.now 15 | end 16 | if user = msg.data["leave"] 17 | $online.delete user 18 | end 19 | end 20 | 21 | MessageBus.user_id_lookup do |env| 22 | MessageBus.logger = env['rack.logger'] 23 | name = env["HTTP_X_NAME"] 24 | if name 25 | unless $online[name] 26 | MessageBus.publish "/presence", enter: name 27 | end 28 | $online[name] = Time.now 29 | end 30 | name 31 | end 32 | 33 | def expire_old_sessions 34 | $online.each do |name, time| 35 | if (Time.now - (5 * 60)) > time 36 | puts "forcing leave for #{name} session timed out" 37 | MessageBus.publish "/presence", leave: name 38 | end 39 | end 40 | rescue => e 41 | # need to make $online thread safe 42 | p e 43 | end 44 | Thread.new do 45 | while true 46 | expire_old_sessions 47 | sleep 1 48 | end 49 | end 50 | 51 | class Chat < Sinatra::Base 52 | set :public_folder, File.expand_path('../../../assets', __FILE__) 53 | 54 | use MessageBus::Rack::Middleware 55 | 56 | post '/enter' do 57 | name = params["name"] 58 | i = 1 59 | while $online.include? name 60 | name = "#{params["name"]}#{i}" 61 | i += 1 62 | end 63 | MessageBus.publish '/presence', enter: name 64 | { users: $online.keys, name: name }.to_json 65 | end 66 | 67 | post '/leave' do 68 | # puts "Got leave for #{params["name"]}" 69 | MessageBus.publish '/presence', leave: params["name"] 70 | end 71 | 72 | post '/message' do 73 | msg = { data: params["data"][0..500], name: params["name"][0..100] } 74 | MessageBus.publish '/message', msg 75 | 76 | "OK" 77 | end 78 | 79 | get '/' do 80 | <<~HTML 81 | 82 | 83 | 84 | 85 | 86 | 103 | 104 | 105 |

This is a trivial chat demo... It is implemented as a Sinatra app. The message_bus can easily be added to any Rails/Rack app. This app can be deployed with Discourse Docker using this template.

106 | 107 | 111 | 117 |
Enter your name: 118 | 119 | 218 | 219 | 220 | 221 | HTML 222 | end 223 | 224 | run! if app_file == $0 225 | end 226 | -------------------------------------------------------------------------------- /examples/chat/config.ru: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require './chat' 4 | run Chat 5 | -------------------------------------------------------------------------------- /examples/chat/docker_container/chat.yml: -------------------------------------------------------------------------------- 1 | base_image: "discourse/base:2.0.20171204" 2 | 3 | update_pups: false 4 | 5 | 6 | params: 7 | home: /var/www/message_bus 8 | 9 | templates: 10 | - "templates/redis.template.yml" 11 | 12 | expose: 13 | - "80:80" 14 | - "443:443" 15 | 16 | volumes: 17 | - volume: 18 | host: /var/docker/shared/chat 19 | guest: /shared 20 | - volume: 21 | host: /etc/letsencrypt 22 | guest: /etc/letsencrypt 23 | 24 | hooks: 25 | after_redis: 26 | - exec: 27 | cmd: 28 | - useradd chat -s /bin/bash -m -U 29 | - exec: 30 | background: true 31 | cmd: "sudo -u redis /usr/bin/redis-server /etc/redis/redis.conf --dbfilename test.rdb" 32 | - exec: mkdir -p /var/www 33 | - exec: cd /var/www && git clone --depth 1 https://github.com/SamSaffron/message_bus.git 34 | - exec: 35 | cmd: 36 | - gem install puma 37 | - gem install redis 38 | - gem install sinatra 39 | - file: 40 | path: /etc/service/puma/run 41 | chmod: "+x" 42 | contents: | 43 | #!/bin/bash 44 | exec 2>&1 45 | # redis 46 | cd $home/examples/chat 47 | exec sudo -E -u chat LD_PRELOAD=/usr/lib/libjemalloc.so.1 puma -p 8080 -e production 48 | - exec: rm /etc/nginx/sites-enabled/default 49 | - replace: 50 | filename: /etc/nginx/nginx.conf 51 | from: pid /run/nginx.pid; 52 | to: daemon off; 53 | - exec: 54 | cmd: 55 | - "mkdir -p /shared/ssl" 56 | - "[ -e /shared/ssl/dhparams.pem ] || openssl dhparam -out /shared/ssl/dhparams.pem 2048" 57 | - file: 58 | path: /etc/nginx/conf.d/chat.conf 59 | contents: | 60 | upstream chat { 61 | server localhost:8080; 62 | } 63 | server { 64 | listen 80; 65 | rewrite ^ https://chat.samsaffron.com$request_uri? permanent; 66 | } 67 | server { 68 | listen 443 ssl http2; 69 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 70 | ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA; 71 | ssl_prefer_server_ciphers on; 72 | ssl_session_tickets off; 73 | ssl_session_timeout 1d; 74 | ssl_session_cache shared:SSL:1m; 75 | ssl_dhparam /shared/ssl/dhparams.pem; 76 | ssl_certificate /etc/letsencrypt/live/chat.samsaffron.com/fullchain.pem; 77 | ssl_certificate_key /etc/letsencrypt/live/chat.samsaffron.com/privkey.pem; 78 | gzip on; 79 | gzip_types application/json text/css application/x-javascript; 80 | gzip_min_length 1000; 81 | server_name chat.samsaffron.com; 82 | keepalive_timeout 65; 83 | root /shared/letsencrypt; 84 | 85 | location /message-bus/ { 86 | proxy_set_header Host $http_host; 87 | proxy_set_header X-Real-IP $remote_addr; 88 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 89 | proxy_set_header X-Forwarded-Proto http; 90 | proxy_http_version 1.1; 91 | proxy_buffering off; 92 | proxy_pass http://chat; 93 | break; 94 | } 95 | 96 | location / { 97 | try_files $uri @chat; 98 | break; 99 | } 100 | 101 | location @chat { 102 | proxy_set_header Host $http_host; 103 | proxy_set_header X-Real-IP $remote_addr; 104 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 105 | proxy_set_header X-Forwarded-Proto http; 106 | proxy_http_version 1.1; 107 | proxy_pass http://chat; 108 | break; 109 | } 110 | } 111 | - file: 112 | path: /etc/service/nginx/run 113 | chmod: "+x" 114 | contents: | 115 | #!/bin/sh 116 | exec 2>&1 117 | exec /usr/sbin/nginx 118 | 119 | -------------------------------------------------------------------------------- /examples/chat/docker_container/update_chat: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | result=`cd /root/message_bus && git pull | grep "Already up-to-date"` 4 | 5 | if [ -z "$result" ]; then 6 | echo "updating..." 7 | cd /var/docker && ./launcher rebuild chat --skip-prereqs 8 | fi 9 | 10 | -------------------------------------------------------------------------------- /examples/minimal/Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | source 'https://rubygems.org' 3 | 4 | gem 'message_bus' 5 | -------------------------------------------------------------------------------- /examples/minimal/config.ru: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'message_bus' 3 | 4 | # MessageBus.long_polling_interval = 1000 * 2 5 | 6 | use MessageBus::Rack::Middleware 7 | run lambda { |_env| [200, { "Content-Type" => "text/html" }, ["Howdy"]] } 8 | -------------------------------------------------------------------------------- /lib/message_bus/backends.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | # @see MessageBus::Backends::Base 5 | module Backends 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/message_bus/backends/base.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | module Backends 5 | # Backends provide a consistent API over a variety of options for persisting 6 | # published messages. The API they present is around the publication to and 7 | # reading of messages from those backlogs in a manner consistent with 8 | # message_bus' philosophy. 9 | # 10 | # The heart of the message bus, a backend acts as two things: 11 | # 12 | # 1. A channel multiplexer 13 | # 2. Backlog storage per-multiplexed channel. 14 | # 15 | # Backends manage and expose multiple backlogs: 16 | # 17 | # * A backlog for each channel, in which messages that were published to 18 | # that channel are stored. 19 | # * A global backlog, which conceptually stores all published messages, 20 | # regardless of the channel to which they were published. 21 | # 22 | # Backlog storage mechanisms and schemas are up to each individual backend 23 | # implementation, and some backends store messages very differently than 24 | # others. It is not necessary in order to be considered a valid backend, 25 | # to, for example, store each channel backlog as a separate collection. 26 | # As long as the API is presented per this documentation, the backend is 27 | # free to make its own storage and performance optimisations. 28 | # 29 | # The concept of a per-channel backlog permits for lookups of messages in 30 | # a manner that is optimised for the use case of a subscriber catching up 31 | # from a message pointer, while a global backlog allows for optimising the 32 | # case where another system subscribes to the firehose of messages, for 33 | # example a message_bus server receiving all publications for delivery 34 | # to subscribed clients. 35 | # 36 | # Backends are fully responsible for maintaining their storage, including 37 | # any pruning or expiration of that storage that is necessary. message_bus 38 | # allows for several options for limiting the required storage capacity 39 | # by either backlog size or the TTL of messages in a backlog. Backends take 40 | # these settings and effect them either forcibly or by delegating to their 41 | # storage mechanism. 42 | # 43 | # Message which are published to message_bus have two IDs; one which they 44 | # are known by in the channel-specific backlog that they are published to, 45 | # and another (the "global ID") which is unique across all channels and by 46 | # which the message can be found in the global backlog. IDs are all 47 | # sequential integers starting at 0. 48 | # 49 | # @abstract 50 | class Base 51 | 52 | # Raised to indicate that the concrete backend implementation does not implement part of the API 53 | ConcreteClassMustImplementError = Class.new(StandardError) 54 | 55 | # @return [String] a special message published to trigger termination of backend subscriptions 56 | UNSUB_MESSAGE = "$$UNSUBSCRIBE" 57 | 58 | # @return [Boolean] The subscription state of the backend 59 | attr_reader :subscribed 60 | # @return [Integer] the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped. 61 | attr_accessor :max_backlog_size 62 | # @return [Integer] the largest permitted size (number of messages) for the global backlog; beyond this capacity, old messages will be dropped. 63 | attr_accessor :max_global_backlog_size 64 | # @return [Integer] the longest amount of time a message may live in a backlog before being removed, in seconds. 65 | attr_accessor :max_backlog_age 66 | # Typically, backlogs are trimmed whenever we publish to them. This setting allows some tolerance in order to improve performance. 67 | # @return [Integer] the interval of publications between which the backlog will not be cleared. 68 | attr_accessor :clear_every 69 | # @return [Integer] the largest permitted size (number of messages) to be held in a memory buffer when publication fails, for later re-publication. 70 | attr_accessor :max_in_memory_publish_backlog 71 | 72 | # @param [Hash] config backend-specific configuration options; see the concrete class for details 73 | # @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped. 74 | def initialize(config = {}, max_backlog_size = 1000); end 75 | 76 | # Performs routines specific to the backend that are necessary after a process fork, typically triggered by a forking webserver. Typically this re-opens sockets to the backend. 77 | def after_fork 78 | raise ConcreteClassMustImplementError 79 | end 80 | 81 | # Deletes all message_bus data from the backend. Use with extreme caution. 82 | def reset! 83 | raise ConcreteClassMustImplementError 84 | end 85 | 86 | # Closes all open connections to the storage. 87 | def destroy 88 | raise ConcreteClassMustImplementError 89 | end 90 | 91 | # Deletes all backlogs and their data. Does not delete non-backlog data that message_bus may persist, depending on the concrete backend implementation. Use with extreme caution. 92 | # @abstract 93 | def expire_all_backlogs! 94 | raise ConcreteClassMustImplementError 95 | end 96 | 97 | # Publishes a message to a channel 98 | # 99 | # @param [String] channel the name of the channel to which the message should be published 100 | # @param [JSON] data some data to publish to the channel. Must be an object that can be encoded as JSON 101 | # @param [Hash] opts 102 | # @option opts [Boolean] :queue_in_memory (true) whether or not to hold the message in an in-memory buffer if publication fails, to be re-tried later 103 | # @option opts [Integer] :max_backlog_age (`self.max_backlog_age`) the longest amount of time a message may live in a backlog before being removed, in seconds 104 | # @option opts [Integer] :max_backlog_size (`self.max_backlog_size`) the largest permitted size (number of messages) for the channel backlog; beyond this capacity, old messages will be dropped 105 | # 106 | # @return [Integer] the channel-specific ID the message was given 107 | def publish(channel, data, opts = nil) 108 | raise ConcreteClassMustImplementError 109 | end 110 | 111 | # Get the ID of the last message published on a channel 112 | # 113 | # @param [String] channel the name of the channel in question 114 | # 115 | # @return [Integer] the channel-specific ID of the last message published to the given channel 116 | def last_id(channel) 117 | raise ConcreteClassMustImplementError 118 | end 119 | 120 | # Get the ID of the last message published on multiple channels 121 | # 122 | # @param [Array] channels - array of channels to fetch 123 | # 124 | # @return [Array] the channel-specific IDs of the last message published to each requested channel 125 | def last_ids(*channels) 126 | raise ConcreteClassMustImplementError 127 | end 128 | 129 | # Get messages from a channel backlog 130 | # 131 | # @param [String] channel the name of the channel in question 132 | # @param [#to_i] last_id the channel-specific ID of the last message that the caller received on the specified channel 133 | # 134 | # @return [Array] all messages published to the specified channel since the specified last ID 135 | def backlog(channel, last_id = 0) 136 | raise ConcreteClassMustImplementError 137 | end 138 | 139 | # Get messages from the global backlog 140 | # 141 | # @param [#to_i] last_id the global ID of the last message that the caller received 142 | # 143 | # @return [Array] all messages published on any channel since the specified last ID 144 | def global_backlog(last_id = 0) 145 | raise ConcreteClassMustImplementError 146 | end 147 | 148 | # Get a specific message from a channel 149 | # 150 | # @param [String] channel the name of the channel in question 151 | # @param [Integer] message_id the channel-specific ID of the message required 152 | # 153 | # @return [MessageBus::Message, nil] the requested message, or nil if it does not exist 154 | def get_message(channel, message_id) 155 | raise ConcreteClassMustImplementError 156 | end 157 | 158 | # Subscribe to messages on a particular channel. Each message since the 159 | # last ID specified will be delivered by yielding to the passed block as 160 | # soon as it is available. This will block until subscription is terminated. 161 | # 162 | # @param [String] channel the name of the channel to which we should subscribe 163 | # @param [#to_i] last_id the channel-specific ID of the last message that the caller received on the specified channel 164 | # 165 | # @yield [message] a message-handler block 166 | # @yieldparam [MessageBus::Message] message each message as it is delivered 167 | # 168 | # @return [nil] 169 | def subscribe(channel, last_id = nil) 170 | raise ConcreteClassMustImplementError 171 | end 172 | 173 | # Causes all subscribers to the bus to unsubscribe, and terminates the local connection. Typically used to reset tests. 174 | def global_unsubscribe 175 | raise ConcreteClassMustImplementError 176 | end 177 | 178 | # Subscribe to messages on all channels. Each message since the last ID 179 | # specified will be delivered by yielding to the passed block as soon as 180 | # it is available. This will block until subscription is terminated. 181 | # 182 | # @param [#to_i] last_id the global ID of the last message that the caller received 183 | # 184 | # @yield [message] a message-handler block 185 | # @yieldparam [MessageBus::Message] message each message as it is delivered 186 | # 187 | # @return [nil] 188 | def global_subscribe(last_id = nil) 189 | raise ConcreteClassMustImplementError 190 | end 191 | 192 | # rubocop:enable Lint/UnusedMethodArgument 193 | end 194 | end 195 | end 196 | -------------------------------------------------------------------------------- /lib/message_bus/backends/memory.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | module Backends 5 | # The memory backend stores published messages in a simple array per 6 | # channel, and does not store a separate global backlog. 7 | # 8 | # @note This backend diverges from the standard in Base in the following ways: 9 | # 10 | # * Does not support forking 11 | # * Does not support in-memory buffering of messages on publication (redundant) 12 | # 13 | # @see Base general information about message_bus backends 14 | class Memory < Base 15 | class Client 16 | attr_accessor :max_backlog_age 17 | 18 | class Listener 19 | attr_reader :do_sub, :do_unsub, :do_message 20 | 21 | def subscribe(&block) 22 | @do_sub = block 23 | end 24 | 25 | def unsubscribe(&block) 26 | @do_unsub = block 27 | end 28 | 29 | def message(&block) 30 | @do_message = block 31 | end 32 | end 33 | 34 | class Channel 35 | attr_accessor :backlog, :ttl 36 | 37 | def initialize(ttl:) 38 | @backlog = [] 39 | @ttl = ttl 40 | end 41 | 42 | def expired? 43 | last_publication_time = nil 44 | backlog.each do |_id, _value, published_at| 45 | if !last_publication_time || published_at > last_publication_time 46 | last_publication_time = published_at 47 | end 48 | end 49 | return true unless last_publication_time 50 | 51 | last_publication_time < Time.now - ttl 52 | end 53 | end 54 | 55 | def initialize(_config) 56 | @mutex = Mutex.new 57 | @listeners = [] 58 | @timer_thread = MessageBus::TimerThread.new 59 | @timer_thread.on_error do |e| 60 | logger.warn "Failed to process job: #{e} #{e.backtrace}" 61 | end 62 | @timer_thread.every(1) { expire } 63 | reset! 64 | end 65 | 66 | def add(channel, value, max_backlog_age:) 67 | listeners = nil 68 | id = nil 69 | sync do 70 | id = @global_id += 1 71 | channel_object = chan(channel) 72 | channel_object.backlog << [id, value, Time.now] 73 | if max_backlog_age 74 | channel_object.ttl = max_backlog_age 75 | end 76 | listeners = @listeners.dup 77 | end 78 | msg = MessageBus::Message.new id, id, channel, value 79 | payload = msg.encode 80 | listeners.each { |l| l.push(payload) } 81 | id 82 | end 83 | 84 | def expire 85 | sync do 86 | @channels.delete_if { |_name, channel| channel.expired? } 87 | end 88 | end 89 | 90 | def clear_global_backlog(backlog_id, num_to_keep) 91 | if backlog_id > num_to_keep 92 | oldest = backlog_id - num_to_keep 93 | sync do 94 | @channels.each_value do |channel| 95 | channel.backlog.delete_if { |id, _| id <= oldest } 96 | end 97 | end 98 | nil 99 | end 100 | end 101 | 102 | def clear_channel_backlog(channel, backlog_id, num_to_keep) 103 | oldest = backlog_id - num_to_keep 104 | sync { chan(channel).backlog.delete_if { |id, _| id <= oldest } } 105 | nil 106 | end 107 | 108 | def backlog(channel, backlog_id) 109 | sync { chan(channel).backlog.select { |id, _| id > backlog_id } } 110 | end 111 | 112 | def global_backlog(backlog_id) 113 | sync do 114 | @channels.dup.flat_map do |channel_name, channel| 115 | channel.backlog.select { |id, _| id > backlog_id }.map { |id, value| [id, channel_name, value] } 116 | end.sort 117 | end 118 | end 119 | 120 | def get_value(channel, id) 121 | sync { chan(channel).backlog.find { |i, _| i == id }[1] } 122 | end 123 | 124 | # Dangerous, drops the message_bus table containing the backlog if it exists. 125 | def reset! 126 | sync do 127 | @global_id = 0 128 | @channels = {} 129 | end 130 | end 131 | 132 | # use with extreme care, will nuke all of the data 133 | def expire_all_backlogs! 134 | sync do 135 | @channels = {} 136 | end 137 | end 138 | 139 | def max_id(channel = nil) 140 | if channel 141 | sync do 142 | if entry = chan(channel).backlog.last 143 | entry.first 144 | end 145 | end 146 | else 147 | sync { @global_id - 1 } 148 | end || 0 149 | end 150 | 151 | def subscribe 152 | listener = Listener.new 153 | yield listener 154 | 155 | q = Queue.new 156 | sync do 157 | @listeners << q 158 | end 159 | 160 | listener.do_sub.call 161 | while msg = q.pop 162 | listener.do_message.call(nil, msg) 163 | end 164 | listener.do_unsub.call 165 | sync do 166 | @listeners.delete(q) 167 | end 168 | 169 | nil 170 | end 171 | 172 | def unsubscribe 173 | sync { @listeners.each { |l| l.push(nil) } } 174 | end 175 | 176 | private 177 | 178 | def chan(channel) 179 | @channels[channel] ||= Channel.new(ttl: @max_backlog_age) 180 | end 181 | 182 | def sync 183 | @mutex.synchronize { yield } 184 | end 185 | end 186 | 187 | # @param [Hash] config 188 | # @option config [Logger] :logger a logger to which logs will be output 189 | # @option config [Integer] :clear_every the interval of publications between which the backlog will not be cleared 190 | # @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped. 191 | def initialize(config = {}, max_backlog_size = 1000) 192 | @config = config 193 | @max_backlog_size = max_backlog_size 194 | @max_global_backlog_size = 2000 195 | # after 7 days inactive backlogs will be removed 196 | self.max_backlog_age = 604800 197 | @clear_every = config[:clear_every] || 1 198 | end 199 | 200 | def max_backlog_age=(value) 201 | client.max_backlog_age = value 202 | end 203 | 204 | # No-op; this backend doesn't support forking. 205 | # @see Base#after_fork 206 | def after_fork 207 | nil 208 | end 209 | 210 | # (see Base#reset!) 211 | def reset! 212 | client.reset! 213 | end 214 | 215 | # No-op; this backend doesn't maintain any storage connections. 216 | # (see Base#destroy) 217 | def destroy 218 | nil 219 | end 220 | 221 | # (see Base#expire_all_backlogs!) 222 | def expire_all_backlogs! 223 | client.expire_all_backlogs! 224 | end 225 | 226 | # (see Base#publish) 227 | # @todo :queue_in_memory NOT SUPPORTED 228 | def publish(channel, data, opts = nil) 229 | c = client 230 | max_backlog_age = opts && opts[:max_backlog_age] 231 | backlog_id = c.add(channel, data, max_backlog_age: max_backlog_age) 232 | 233 | if backlog_id % clear_every == 0 234 | max_backlog_size = (opts && opts[:max_backlog_size]) || self.max_backlog_size 235 | c.clear_global_backlog(backlog_id, @max_global_backlog_size) 236 | c.clear_channel_backlog(channel, backlog_id, max_backlog_size) 237 | end 238 | 239 | backlog_id 240 | end 241 | 242 | # (see Base#last_id) 243 | def last_id(channel) 244 | client.max_id(channel) 245 | end 246 | 247 | # (see Base#last_ids) 248 | def last_ids(*channels) 249 | channels.map do |c| 250 | last_id(c) 251 | end 252 | end 253 | 254 | # (see Base#backlog) 255 | def backlog(channel, last_id = 0) 256 | items = client.backlog channel, last_id.to_i 257 | 258 | items.map! do |id, data| 259 | MessageBus::Message.new id, id, channel, data 260 | end 261 | end 262 | 263 | # (see Base#global_backlog) 264 | def global_backlog(last_id = 0) 265 | items = client.global_backlog last_id.to_i 266 | 267 | items.map! do |id, channel, data| 268 | MessageBus::Message.new id, id, channel, data 269 | end 270 | end 271 | 272 | # (see Base#get_message) 273 | def get_message(channel, message_id) 274 | if data = client.get_value(channel, message_id) 275 | MessageBus::Message.new message_id, message_id, channel, data 276 | else 277 | nil 278 | end 279 | end 280 | 281 | # (see Base#subscribe) 282 | def subscribe(channel, last_id = nil) 283 | # trivial implementation for now, 284 | # can cut down on connections if we only have one global subscriber 285 | raise ArgumentError unless block_given? 286 | 287 | global_subscribe(last_id) do |m| 288 | yield m if m.channel == channel 289 | end 290 | end 291 | 292 | # (see Base#global_unsubscribe) 293 | def global_unsubscribe 294 | client.unsubscribe 295 | @subscribed = false 296 | end 297 | 298 | # (see Base#global_subscribe) 299 | def global_subscribe(last_id = nil) 300 | raise ArgumentError unless block_given? 301 | 302 | highest_id = last_id 303 | 304 | begin 305 | client.subscribe do |on| 306 | h = {} 307 | 308 | on.subscribe do 309 | if highest_id 310 | process_global_backlog(highest_id) do |m| 311 | h[m.global_id] = true 312 | yield m 313 | end 314 | end 315 | @subscribed = true 316 | end 317 | 318 | on.unsubscribe do 319 | @subscribed = false 320 | end 321 | 322 | on.message do |_c, m| 323 | m = MessageBus::Message.decode m 324 | 325 | # we have 3 options 326 | # 327 | # 1. message came in the correct order GREAT, just deal with it 328 | # 2. message came in the incorrect order COMPLICATED, wait a tiny bit and clear backlog 329 | # 3. message came in the incorrect order and is lowest than current highest id, reset 330 | 331 | if h 332 | # If already yielded during the clear backlog when subscribing, 333 | # don't yield a duplicate copy. 334 | unless h.delete(m.global_id) 335 | h = nil if h.empty? 336 | yield m 337 | end 338 | else 339 | yield m 340 | end 341 | end 342 | end 343 | rescue => error 344 | @config[:logger].warn "#{error} subscribe failed, reconnecting in 1 second. Call stack\n#{error.backtrace.join("\n")}" 345 | sleep 1 346 | retry 347 | end 348 | end 349 | 350 | private 351 | 352 | def client 353 | @client ||= new_connection 354 | end 355 | 356 | def new_connection 357 | Client.new(@config) 358 | end 359 | 360 | def process_global_backlog(highest_id) 361 | if highest_id > client.max_id 362 | highest_id = 0 363 | end 364 | 365 | global_backlog(highest_id).each do |old| 366 | yield old 367 | highest_id = old.global_id 368 | end 369 | 370 | highest_id 371 | end 372 | 373 | MessageBus::BACKENDS[:memory] = self 374 | end 375 | end 376 | end 377 | -------------------------------------------------------------------------------- /lib/message_bus/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Represents a connected subscriber and delivers published messages over its 4 | # connected socket. 5 | class MessageBus::Client 6 | # @return [String] the unique ID provided by the client 7 | attr_accessor :client_id 8 | # @return [String,Integer] the user ID the client was authenticated for 9 | attr_accessor :user_id 10 | # @return [Array] the group IDs the authenticated client is a member of 11 | attr_accessor :group_ids 12 | # @return [Time] the time at which the client connected 13 | attr_accessor :connect_time 14 | # @return [String] the site ID the client was authenticated for; used for hosting multiple 15 | attr_accessor :site_id 16 | # @return [MessageBus::TimerThread::Cancelable] a timer job that is used to 17 | # auto-disconnect the client at the configured long-polling interval 18 | attr_accessor :cleanup_timer 19 | # @return [Thin::AsyncResponse, nil] 20 | attr_accessor :async_response 21 | # @return [IO] the HTTP socket the client is connected on 22 | attr_accessor :io 23 | # @return [Hash String>] custom headers to include in HTTP responses 24 | attr_accessor :headers 25 | # @return [Integer] the connection sequence number the client provided when connecting 26 | attr_accessor :seq 27 | # @return [Boolean] whether or not the client should use chunked encoding 28 | attr_accessor :use_chunked 29 | 30 | # @param [Hash] opts 31 | # @option opts [String] :client_id the unique ID provided by the client 32 | # @option opts [String,Integer] :user_id (`nil`) the user ID the client was authenticated for 33 | # @option opts [Array] :group_ids (`[]`) the group IDs the authenticated client is a member of 34 | # @option opts [String] :site_id (`nil`) the site ID the client was authenticated for; used for hosting multiple 35 | # applications or instances of an application against a single message_bus 36 | # @option opts [#to_i] :seq (`0`) the connection sequence number the client provided when connecting 37 | # @option opts [MessageBus::Instance] :message_bus (`MessageBus`) a specific instance of message_bus 38 | def initialize(opts) 39 | self.client_id = opts[:client_id] 40 | self.user_id = opts[:user_id] 41 | self.group_ids = opts[:group_ids] || [] 42 | self.site_id = opts[:site_id] 43 | self.seq = opts[:seq].to_i 44 | self.connect_time = Time.now 45 | @lock = Mutex.new 46 | @bus = opts[:message_bus] || MessageBus 47 | @subscriptions = {} 48 | @chunks_sent = 0 49 | @async_response = nil 50 | @io = nil 51 | @wrote_headers = false 52 | end 53 | 54 | # @yield executed with a lock on the Client instance 55 | # @return [void] 56 | def synchronize 57 | @lock.synchronize { yield } 58 | end 59 | 60 | # Closes the client connection 61 | def close 62 | if cleanup_timer 63 | # concurrency may nil cleanup timer 64 | cleanup_timer.cancel rescue nil 65 | self.cleanup_timer = nil 66 | end 67 | ensure_closed! 68 | end 69 | 70 | # Delivers a backlog of messages to the client, if there is anything in it. 71 | # If chunked encoding/streaming is in use, will keep the connection open; 72 | # if not, will close it. 73 | # 74 | # @param [Array] backlog the set of messages to deliver 75 | # @return [void] 76 | def deliver_backlog(backlog) 77 | if backlog.length > 0 78 | if use_chunked 79 | write_chunk(messages_to_json(backlog)) 80 | else 81 | write_and_close messages_to_json(backlog) 82 | end 83 | end 84 | end 85 | 86 | # If no data has yet been sent to the client, sends an empty chunk; prevents 87 | # clients from entering a timeout state if nothing is delivered initially. 88 | def ensure_first_chunk_sent 89 | if use_chunked && @chunks_sent == 0 90 | write_chunk("[]") 91 | end 92 | end 93 | 94 | # @return [Boolean] whether the connection is closed or not 95 | def closed? 96 | !@async_response && !@io 97 | end 98 | 99 | # Subscribes the client to messages on a channel, optionally from a 100 | # defined starting point. 101 | # 102 | # @param [String] channel the channel to subscribe to 103 | # @param [Integer, nil] last_seen_id the ID of the last message the client 104 | # received. If nil, will be subscribed from the head of the backlog. 105 | # @return [void] 106 | def subscribe(channel, last_seen_id) 107 | last_seen_id = nil if last_seen_id == "" 108 | last_seen_id ||= @bus.last_id(channel) 109 | @subscriptions[channel] = last_seen_id.to_i 110 | end 111 | 112 | # @return [Hash Integer>] the active subscriptions, mapping channel 113 | # names to last seen message IDs 114 | def subscriptions 115 | @subscriptions 116 | end 117 | 118 | # Delivers a message to the client, even if it's empty 119 | # @param [MessageBus::Message, nil] msg the message to deliver 120 | # @return [void] 121 | def <<(msg) 122 | json = messages_to_json([msg]) 123 | if use_chunked 124 | write_chunk json 125 | else 126 | write_and_close json 127 | end 128 | end 129 | 130 | # @param [MessageBus::Message] msg the message in question 131 | # @return [Boolean] whether or not the client has permission to receive the 132 | # passed message 133 | def allowed?(msg) 134 | client_allowed = !msg.client_ids || msg.client_ids.length == 0 || msg.client_ids.include?(self.client_id) 135 | 136 | user_allowed = false 137 | group_allowed = false 138 | 139 | has_users = msg.user_ids && msg.user_ids.length > 0 140 | has_groups = msg.group_ids && msg.group_ids.length > 0 141 | 142 | if has_users 143 | user_allowed = msg.user_ids.include?(self.user_id) 144 | end 145 | 146 | if has_groups 147 | group_allowed = ( 148 | msg.group_ids - (self.group_ids || []) 149 | ).length < msg.group_ids.length 150 | end 151 | 152 | has_permission = client_allowed && (user_allowed || group_allowed || (!has_users && !has_groups)) 153 | 154 | return has_permission if !has_permission 155 | 156 | filters_allowed = true 157 | 158 | len = @bus.client_message_filters.length 159 | while len > 0 160 | len -= 1 161 | channel_prefix, blk = @bus.client_message_filters[len] 162 | 163 | if msg.channel.start_with?(channel_prefix) 164 | filters_allowed = blk.call(msg) 165 | break if !filters_allowed 166 | end 167 | end 168 | 169 | filters_allowed 170 | end 171 | 172 | # @return [Array] the set of messages the client is due 173 | # to receive, based on its subscriptions and permissions. Includes status 174 | # message if any channels have no messages available and the client 175 | # requested a message newer than the newest on the channel, or when there 176 | # are messages available that the client doesn't have permission for. 177 | def backlog 178 | r = [] 179 | new_message_ids = nil 180 | 181 | last_bus_ids = @bus.last_ids(*@subscriptions.keys, site_id: site_id) 182 | 183 | @subscriptions.each do |k, v| 184 | last_client_id = v.to_i 185 | last_bus_id = last_bus_ids[k] 186 | 187 | if last_client_id < -1 # Client requesting backlog relative to bus position 188 | last_client_id = last_bus_id + last_client_id + 1 189 | last_client_id = 0 if last_client_id < 0 190 | elsif last_client_id == -1 # Client not requesting backlog 191 | next 192 | elsif last_client_id == last_bus_id # Client already up-to-date 193 | next 194 | elsif last_client_id > last_bus_id # Client ahead of the bus 195 | @subscriptions[k] = -1 196 | next 197 | end 198 | 199 | messages = @bus.backlog(k, last_client_id, site_id) 200 | 201 | messages.each do |msg| 202 | if allowed?(msg) 203 | r << msg 204 | else 205 | new_message_ids ||= {} 206 | new_message_ids[k] = msg.message_id 207 | end 208 | end 209 | end 210 | 211 | # stats message for all newly subscribed 212 | status_message = nil 213 | @subscriptions.each do |k, v| 214 | if v.to_i == -1 || (new_message_ids && new_message_ids[k]) 215 | status_message ||= {} 216 | @subscriptions[k] = status_message[k] = last_bus_ids[k] 217 | end 218 | end 219 | 220 | r << MessageBus::Message.new(-1, -1, '/__status', status_message) if status_message 221 | 222 | r || [] 223 | end 224 | 225 | private 226 | 227 | # heavily optimised to avoid all unneeded allocations 228 | NEWLINE = "\r\n".freeze 229 | COLON_SPACE = ": ".freeze 230 | HTTP_11 = "HTTP/1.1 200 OK\r\n".freeze 231 | CONTENT_LENGTH = "Content-Length: ".freeze 232 | CONNECTION_CLOSE = "Connection: close\r\n".freeze 233 | CHUNKED_ENCODING = "Transfer-Encoding: chunked\r\n".freeze 234 | NO_SNIFF = "X-Content-Type-Options: nosniff\r\n".freeze 235 | 236 | TYPE_TEXT = "Content-Type: text/plain; charset=utf-8\r\n".freeze 237 | TYPE_JSON = "Content-Type: application/json; charset=utf-8\r\n".freeze 238 | 239 | def write_headers 240 | @io.write(HTTP_11) 241 | @headers.each do |k, v| 242 | next if k == "Content-Type" 243 | 244 | @io.write(k) 245 | @io.write(COLON_SPACE) 246 | @io.write(v) 247 | @io.write(NEWLINE) 248 | end 249 | @io.write(CONNECTION_CLOSE) 250 | if use_chunked 251 | @io.write(TYPE_TEXT) 252 | else 253 | @io.write(TYPE_JSON) 254 | end 255 | end 256 | 257 | def write_chunk(data) 258 | @bus.logger.debug "Delivering messages #{data} to client #{client_id} for user #{user_id} (chunked)" 259 | if @io && !@wrote_headers 260 | write_headers 261 | @io.write(CHUNKED_ENCODING) 262 | # this is required otherwise chrome will delay onprogress calls 263 | @io.write(NO_SNIFF) 264 | @io.write(NEWLINE) 265 | @wrote_headers = true 266 | end 267 | 268 | # chunked encoding may be "re-chunked" by proxies, so add a separator 269 | postfix = NEWLINE + "|" + NEWLINE 270 | data = data.gsub(postfix, NEWLINE + "||" + NEWLINE) 271 | chunk_length = data.bytesize + postfix.bytesize 272 | 273 | @chunks_sent += 1 274 | 275 | if @async_response 276 | @async_response << chunk_length.to_s(16) 277 | @async_response << NEWLINE 278 | @async_response << data 279 | @async_response << postfix 280 | @async_response << NEWLINE 281 | elsif @io 282 | @io.write(chunk_length.to_s(16) << NEWLINE << data << postfix << NEWLINE) 283 | end 284 | end 285 | 286 | def write_and_close(data) 287 | @bus.logger.debug "Delivering messages #{data} to client #{client_id} for user #{user_id}" 288 | if @io 289 | write_headers 290 | @io.write(CONTENT_LENGTH) 291 | @io.write(data.bytes.to_a.length) 292 | @io.write(NEWLINE) 293 | @io.write(NEWLINE) 294 | @io.write(data) 295 | @io.close 296 | @io = nil 297 | else 298 | @async_response << data 299 | @async_response.done 300 | @async_response = nil 301 | end 302 | end 303 | 304 | def ensure_closed! 305 | return unless in_async? 306 | 307 | if use_chunked 308 | write_chunk("[]") 309 | if @io 310 | @io.write("0\r\n\r\n") 311 | @io.close 312 | @io = nil 313 | end 314 | if @async_response 315 | @async_response << ("0\r\n\r\n") 316 | @async_response.done 317 | @async_response = nil 318 | end 319 | else 320 | write_and_close "[]" 321 | end 322 | rescue 323 | # we may have a dead socket, just nil the @io 324 | @io = nil 325 | @async_response = nil 326 | end 327 | 328 | def messages_to_json(msgs) 329 | MessageBus::Rack::Middleware.backlog_to_json(msgs) 330 | end 331 | 332 | def in_async? 333 | @async_response || @io 334 | end 335 | end 336 | -------------------------------------------------------------------------------- /lib/message_bus/codec/base.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | module Codec 5 | class Base 6 | def encode(hash) 7 | raise ConcreteClassMustImplementError 8 | end 9 | 10 | def decode(payload) 11 | raise ConcreteClassMustImplementError 12 | end 13 | end 14 | 15 | autoload :Json, File.expand_path("json", __dir__) 16 | autoload :Oj, File.expand_path("oj", __dir__) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/message_bus/codec/json.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | module Codec 5 | class Json < Base 6 | def encode(hash) 7 | JSON.dump(hash) 8 | end 9 | 10 | def decode(payload) 11 | JSON.parse(payload) 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/message_bus/codec/oj.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'oj' unless defined? ::Oj 4 | 5 | module MessageBus 6 | module Codec 7 | class Oj < Base 8 | def initialize(options = { mode: :compat }) 9 | @options = options 10 | end 11 | 12 | def encode(hash) 13 | ::Oj.dump(hash, @options) 14 | end 15 | 16 | def decode(payload) 17 | ::Oj.load(payload, @options) 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/message_bus/connection_manager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'json' unless defined? ::JSON 4 | 5 | # Manages a set of subscribers with active connections to the server, such that 6 | # messages which are published during the connection may be dispatched. 7 | class MessageBus::ConnectionManager 8 | require 'monitor' 9 | include MonitorMixin 10 | 11 | # @param [MessageBus::Instance] bus the message bus for which to manage connections 12 | def initialize(bus = nil) 13 | @clients = {} 14 | @subscriptions = {} 15 | @bus = bus || MessageBus 16 | mon_initialize 17 | end 18 | 19 | # Dispatches a message to any connected clients which are permitted to receive it 20 | # @param [MessageBus::Message] msg the message to dispatch 21 | # @return [void] 22 | def notify_clients(msg) 23 | synchronize do 24 | begin 25 | site_subs = @subscriptions[msg.site_id] 26 | subscription = site_subs[msg.channel] if site_subs 27 | 28 | return unless subscription 29 | 30 | subscription.each do |client_id| 31 | client = @clients[client_id] 32 | if client && client.allowed?(msg) 33 | begin 34 | client.synchronize do 35 | client << msg 36 | end 37 | rescue 38 | # pipe may be broken, move on 39 | end 40 | # turns out you can delete from a set while iterating 41 | remove_client(client) if client.closed? 42 | end 43 | end 44 | rescue => e 45 | @bus.logger.error "notify clients crash #{e} : #{e.backtrace}" 46 | end 47 | end 48 | end 49 | 50 | # Keeps track of a client with an active connection 51 | # @param [MessageBus::Client] client the client to track 52 | # @return [void] 53 | def add_client(client) 54 | synchronize do 55 | existing = @clients[client.client_id] 56 | if existing && existing.seq > client.seq 57 | client.close 58 | else 59 | if existing 60 | remove_client(existing) 61 | existing.close 62 | end 63 | 64 | @clients[client.client_id] = client 65 | @subscriptions[client.site_id] ||= {} 66 | client.subscriptions.each do |k, _v| 67 | subscribe_client(client, k) 68 | end 69 | end 70 | end 71 | end 72 | 73 | # Removes a client 74 | # @param [MessageBus::Client] c the client to remove 75 | # @return [void] 76 | def remove_client(c) 77 | synchronize do 78 | @clients.delete c.client_id 79 | @subscriptions[c.site_id].each do |_k, set| 80 | set.delete c.client_id 81 | end 82 | if c.cleanup_timer 83 | # concurrency may cause this to fail 84 | c.cleanup_timer.cancel rescue nil 85 | end 86 | end 87 | end 88 | 89 | # Finds a client by ID 90 | # @param [String] client_id the client ID to search by 91 | # @return [MessageBus::Client] the client with the specified ID 92 | def lookup_client(client_id) 93 | synchronize do 94 | @clients[client_id] 95 | end 96 | end 97 | 98 | # @return [Integer] the number of tracked clients 99 | def client_count 100 | synchronize do 101 | @clients.length 102 | end 103 | end 104 | 105 | private 106 | 107 | def subscribe_client(client, channel) 108 | synchronize do 109 | set = @subscriptions[client.site_id][channel] 110 | unless set 111 | set = Set.new 112 | @subscriptions[client.site_id][channel] = set 113 | end 114 | set << client.client_id 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /lib/message_bus/distributed_cache.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'weakref' 4 | require 'base64' 5 | require 'securerandom' 6 | 7 | module MessageBus 8 | # Like a hash, just does its best to stay in sync across the farm. 9 | # On boot all instances are blank, but they populate as various processes 10 | # fill it up. 11 | class DistributedCache 12 | DEFAULT_SITE_ID = 'default' 13 | 14 | class Manager 15 | CHANNEL_NAME ||= '/distributed_hash'.freeze 16 | 17 | attr_accessor :app_version 18 | 19 | def initialize(message_bus = nil, publish_queue_in_memory: true) 20 | @subscribers = [] 21 | @subscribed = false 22 | @lock = Mutex.new 23 | @message_bus = message_bus || MessageBus 24 | @publish_queue_in_memory = publish_queue_in_memory 25 | @app_version = nil 26 | end 27 | 28 | def subscribers 29 | @subscribers 30 | end 31 | 32 | def process_message(message) 33 | i = @subscribers.length - 1 34 | 35 | payload = message.data 36 | 37 | while i >= 0 38 | begin 39 | current = @subscribers[i] 40 | 41 | next if payload["origin"] == current.identity 42 | next if current.key != payload["hash_key"] 43 | 44 | next if @app_version && payload["app_version"] != @app_version 45 | 46 | hash = current.hash(message.site_id || DEFAULT_SITE_ID) 47 | 48 | case payload["op"] 49 | # TODO: consider custom marshal support with a restricted set 50 | when "set" then hash[payload["key"]] = payload["marshalled"] ? Marshal.load(Base64.decode64(payload["value"])) : payload["value"] # rubocop:disable Security/MarshalLoad 51 | when "delete" then hash.delete(payload["key"]) 52 | when "clear" then hash.clear 53 | end 54 | rescue WeakRef::RefError 55 | @subscribers.delete_at(i) 56 | ensure 57 | i -= 1 58 | end 59 | end 60 | end 61 | 62 | def ensure_subscribe! 63 | return if @subscribed 64 | 65 | @lock.synchronize do 66 | return if @subscribed 67 | 68 | @message_bus.subscribe(CHANNEL_NAME) do |message| 69 | @lock.synchronize do 70 | process_message(message) 71 | end 72 | end 73 | @subscribed = true 74 | end 75 | end 76 | 77 | def publish(hash, message) 78 | message[:origin] = hash.identity 79 | message[:hash_key] = hash.key 80 | message[:app_version] = @app_version if @app_version 81 | 82 | @message_bus.publish(CHANNEL_NAME, message, 83 | user_ids: [-1], 84 | queue_in_memory: @publish_queue_in_memory 85 | ) 86 | end 87 | 88 | def set(hash, key, value) 89 | # special support for set 90 | marshal = (Set === value || Hash === value || Array === value) 91 | value = Base64.encode64(Marshal.dump(value)) if marshal 92 | publish(hash, op: :set, key: key, value: value, marshalled: marshal) 93 | end 94 | 95 | def delete(hash, key) 96 | publish(hash, op: :delete, key: key) 97 | end 98 | 99 | def clear(hash) 100 | publish(hash, op: :clear) 101 | end 102 | 103 | def register(hash) 104 | @lock.synchronize do 105 | @subscribers << WeakRef.new(hash) 106 | end 107 | end 108 | end 109 | 110 | @default_manager = Manager.new 111 | 112 | def self.default_manager 113 | @default_manager 114 | end 115 | 116 | attr_reader :key 117 | 118 | def initialize(key, manager: nil, namespace: true, app_version: nil) 119 | @key = key 120 | @data = {} 121 | @manager = manager || DistributedCache.default_manager 122 | @manager.app_version = app_version if app_version 123 | @namespace = namespace 124 | @app_version = app_version 125 | 126 | @manager.ensure_subscribe! 127 | @manager.register(self) 128 | end 129 | 130 | def identity 131 | # fork resilient / multi machine identity 132 | (@seed_id ||= SecureRandom.hex) + "#{Process.pid}" 133 | end 134 | 135 | def []=(k, v) 136 | k = k.to_s if Symbol === k 137 | @manager.set(self, k, v) 138 | hash[k] = v 139 | end 140 | 141 | def [](k) 142 | k = k.to_s if Symbol === k 143 | hash[k] 144 | end 145 | 146 | def delete(k, publish: true) 147 | k = k.to_s if Symbol === k 148 | @manager.delete(self, k) if publish 149 | hash.delete(k) 150 | end 151 | 152 | def clear 153 | @manager.clear(self) 154 | hash.clear 155 | end 156 | 157 | def hash(site_id_arg = nil) 158 | site_id = 159 | if @namespace 160 | site_id_arg || 161 | (MessageBus.site_id_lookup && MessageBus.site_id_lookup.call) || 162 | DEFAULT_SITE_ID 163 | else 164 | DEFAULT_SITE_ID 165 | end 166 | 167 | @data[site_id] ||= {} 168 | end 169 | end 170 | end 171 | -------------------------------------------------------------------------------- /lib/message_bus/http_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'securerandom' 3 | require 'net/http' 4 | require 'json' 5 | require 'uri' 6 | require 'message_bus/http_client/channel' 7 | 8 | module MessageBus 9 | # MessageBus client that enables subscription via long polling with support 10 | # for chunked encoding. Falls back to normal polling if long polling is not 11 | # available. 12 | # 13 | # @!attribute [r] channels 14 | # @return [Hash] a map of the channels that the client is subscribed to 15 | # @!attribute [r] stats 16 | # @return [Stats] a Struct containing the statistics of failed and successful 17 | # polling requests 18 | # 19 | # @!attribute enable_long_polling 20 | # @return [Boolean] whether long polling is enabled 21 | # @!attribute status 22 | # @return [HTTPClient::STOPPED, HTTPClient::STARTED] the status of the client. 23 | # @!attribute enable_chunked_encoding 24 | # @return [Boolean] whether chunked encoding is enabled 25 | # @!attribute min_poll_interval 26 | # @return [Float] the min poll interval for long polling in seconds 27 | # @!attribute max_poll_interval 28 | # @return [Float] the max poll interval for long polling in seconds 29 | # @!attribute background_callback_interval 30 | # @return [Float] the polling interval in seconds 31 | class HTTPClient 32 | class InvalidChannel < StandardError; end 33 | class MissingBlock < StandardError; end 34 | 35 | attr_reader :channels, 36 | :stats 37 | 38 | attr_accessor :enable_long_polling, 39 | :status, 40 | :enable_chunked_encoding, 41 | :min_poll_interval, 42 | :max_poll_interval, 43 | :background_callback_interval 44 | 45 | CHUNK_SEPARATOR = "\r\n|\r\n".freeze 46 | private_constant :CHUNK_SEPARATOR 47 | STATUS_CHANNEL = "/__status".freeze 48 | private_constant :STATUS_CHANNEL 49 | 50 | STOPPED = 0 51 | STARTED = 1 52 | 53 | Stats = Struct.new(:failed, :success) 54 | private_constant :Stats 55 | 56 | # @param base_url [String] Base URL of the message_bus server to connect to 57 | # @param enable_long_polling [Boolean] Enable long polling 58 | # @param enable_chunked_encoding [Boolean] Enable chunk encoding 59 | # @param min_poll_interval [Float, Integer] Min poll interval when long polling in seconds 60 | # @param max_poll_interval [Float, Integer] Max poll interval when long polling in seconds. 61 | # When requests fail, the client will backoff and this is the upper limit. 62 | # @param background_callback_interval [Float, Integer] Interval to poll when 63 | # when polling in seconds. 64 | # @param headers [Hash] extra HTTP headers to be set on the polling requests. 65 | # 66 | # @return [Object] Instance of MessageBus::HTTPClient 67 | def initialize(base_url, enable_long_polling: true, 68 | enable_chunked_encoding: true, 69 | min_poll_interval: 0.1, 70 | max_poll_interval: 180, 71 | background_callback_interval: 60, 72 | headers: {}) 73 | 74 | @uri = URI(base_url) 75 | @enable_long_polling = enable_long_polling 76 | @enable_chunked_encoding = enable_chunked_encoding 77 | @min_poll_interval = min_poll_interval 78 | @max_poll_interval = max_poll_interval 79 | @background_callback_interval = background_callback_interval 80 | @headers = headers 81 | @client_id = SecureRandom.hex 82 | @channels = {} 83 | @status = STOPPED 84 | @mutex = Mutex.new 85 | @stats = Stats.new(0, 0) 86 | end 87 | 88 | # Starts a background thread that polls the message bus endpoint 89 | # for the given base_url. 90 | # 91 | # Intervals for long polling can be configured via min_poll_interval and 92 | # max_poll_interval. 93 | # 94 | # Intervals for polling can be configured via background_callback_interval. 95 | # 96 | # @return [Object] Instance of MessageBus::HTTPClient 97 | def start 98 | @mutex.synchronize do 99 | return if started? 100 | 101 | @status = STARTED 102 | 103 | thread = Thread.new do 104 | begin 105 | while started? 106 | unless @channels.empty? 107 | poll 108 | @stats.success += 1 109 | @stats.failed = 0 110 | end 111 | 112 | sleep interval 113 | end 114 | rescue StandardError => e 115 | @stats.failed += 1 116 | warn("#{e.class} #{e.message}: #{e.backtrace.join("\n")}") 117 | sleep interval 118 | retry 119 | ensure 120 | stop 121 | end 122 | end 123 | 124 | thread.abort_on_exception = true 125 | end 126 | 127 | self 128 | end 129 | 130 | # Stops the client from polling the message bus endpoint. 131 | # 132 | # @return [Integer] the current status of the client 133 | def stop 134 | @status = STOPPED 135 | end 136 | 137 | # Subscribes to a channel which executes the given callback when a message 138 | # is published to the channel 139 | # 140 | # @example Subscribing to a channel for message 141 | # client = MessageBus::HTTPClient.new('http://some.test.com') 142 | # 143 | # client.subscribe("/test") do |payload, _message_id, _global_id| 144 | # puts payload 145 | # end 146 | # 147 | # A last_message_id may be provided. 148 | # * -1 will subscribe to all new messages 149 | # * -2 will receive last message + all new messages 150 | # * -3 will receive last 2 message + all new messages 151 | # 152 | # @example Subscribing to a channel with `last_message_id` 153 | # client.subscribe("/test", last_message_id: -2) do |payload| 154 | # puts payload 155 | # end 156 | # 157 | # @param channel [String] channel to listen for messages on 158 | # @param last_message_id [Integer] last message id to start polling on. 159 | # 160 | # @yield [data, message_id, global_id] 161 | # callback to be executed whenever a message is received 162 | # 163 | # @yieldparam data [Hash] data payload of the message received on the channel 164 | # @yieldparam message_id [Integer] id of the message in the channel 165 | # @yieldparam global_id [Integer] id of the message in the global backlog 166 | # @yieldreturn [void] 167 | # 168 | # @return [Integer] the current status of the client 169 | def subscribe(channel, last_message_id: nil, &callback) 170 | raise InvalidChannel unless channel.to_s.start_with?("/") 171 | raise MissingBlock unless block_given? 172 | 173 | last_message_id = -1 if last_message_id && !last_message_id.is_a?(Integer) 174 | 175 | @channels[channel] ||= Channel.new 176 | channel = @channels[channel] 177 | channel.last_message_id = last_message_id if last_message_id 178 | channel.callbacks.push(callback) 179 | start if stopped? 180 | end 181 | 182 | # unsubscribes from a channel 183 | # 184 | # @example Unsubscribing from a channel 185 | # client = MessageBus::HTTPClient.new('http://some.test.com') 186 | # callback = -> { |payload| puts payload } 187 | # client.subscribe("/test", &callback) 188 | # client.unsubscribe("/test") 189 | # 190 | # If a callback is given, only the specific callback will be unsubscribed. 191 | # 192 | # @example Unsubscribing a callback from a channel 193 | # client.unsubscribe("/test", &callback) 194 | # 195 | # When the client does not have any channels left, it will stop polling and 196 | # waits until a new subscription is started. 197 | # 198 | # @param channel [String] channel to unsubscribe 199 | # @yield [data, global_id, message_id] specific callback to unsubscribe 200 | # 201 | # @return [Integer] the current status of the client 202 | def unsubscribe(channel, &callback) 203 | if callback 204 | @channels[channel].callbacks.delete(callback) 205 | remove_channel(channel) if @channels[channel].callbacks.empty? 206 | else 207 | remove_channel(channel) 208 | end 209 | 210 | stop if @channels.empty? 211 | @status 212 | end 213 | 214 | private 215 | 216 | def stopped? 217 | @status == STOPPED 218 | end 219 | 220 | def started? 221 | @status == STARTED 222 | end 223 | 224 | def remove_channel(channel) 225 | @channels.delete(channel) 226 | end 227 | 228 | def interval 229 | if @enable_long_polling 230 | if (failed_count = @stats.failed) > 2 231 | (@min_poll_interval * 2**failed_count).clamp( 232 | @min_poll_interval, @max_poll_interval 233 | ) 234 | else 235 | @min_poll_interval 236 | end 237 | else 238 | @background_callback_interval 239 | end 240 | end 241 | 242 | def poll 243 | http = Net::HTTP.new(@uri.host, @uri.port) 244 | http.use_ssl = true if @uri.scheme == 'https' 245 | request = Net::HTTP::Post.new(request_path, headers) 246 | request.body = poll_payload 247 | 248 | if @enable_long_polling 249 | buffer = +"" 250 | 251 | http.request(request) do |response| 252 | response.read_body do |chunk| 253 | unless chunk.empty? 254 | buffer << chunk 255 | process_buffer(buffer) 256 | end 257 | end 258 | end 259 | else 260 | response = http.request(request) 261 | notify_channels(JSON.parse(response.body)) 262 | end 263 | end 264 | 265 | def is_chunked? 266 | !headers["Dont-Chunk"] 267 | end 268 | 269 | def process_buffer(buffer) 270 | index = buffer.index(CHUNK_SEPARATOR) 271 | 272 | if is_chunked? 273 | return unless index 274 | 275 | messages = buffer[0..(index - 1)] 276 | buffer.slice!("#{messages}#{CHUNK_SEPARATOR}") 277 | else 278 | messages = buffer[0..-1] 279 | buffer.slice!(messages) 280 | end 281 | 282 | notify_channels(JSON.parse(messages)) 283 | end 284 | 285 | def notify_channels(messages) 286 | messages.each do |message| 287 | current_channel = message['channel'] 288 | 289 | if current_channel == STATUS_CHANNEL 290 | message["data"].each do |channel_name, last_message_id| 291 | if (channel = @channels[channel_name]) 292 | channel.last_message_id = last_message_id 293 | end 294 | end 295 | else 296 | @channels.each do |channel_name, channel| 297 | next unless channel_name == current_channel 298 | 299 | channel.last_message_id = message['message_id'] 300 | 301 | channel.callbacks.each do |callback| 302 | callback.call( 303 | message['data'], 304 | channel.last_message_id, 305 | message['global_id'] 306 | ) 307 | end 308 | end 309 | end 310 | end 311 | end 312 | 313 | def poll_payload 314 | payload = {} 315 | 316 | @channels.each do |channel_name, channel| 317 | payload[channel_name] = channel.last_message_id 318 | end 319 | 320 | payload.to_json 321 | end 322 | 323 | def request_path 324 | "/message-bus/#{@client_id}/poll" 325 | end 326 | 327 | def headers 328 | headers = {} 329 | headers['Content-Type'] = 'application/json' 330 | headers['X-Silence-logger'] = 'true' 331 | 332 | if !@enable_long_polling || !@enable_chunked_encoding 333 | headers['Dont-Chunk'] = 'true' 334 | end 335 | 336 | headers.merge!(@headers) 337 | end 338 | end 339 | end 340 | -------------------------------------------------------------------------------- /lib/message_bus/http_client/channel.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | module MessageBus 3 | class HTTPClient 4 | # @private 5 | class Channel 6 | attr_accessor :last_message_id, :callbacks 7 | 8 | def initialize(last_message_id: -1, callbacks: []) 9 | @last_message_id = last_message_id 10 | @callbacks = callbacks 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/message_bus/http_client/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | class HTTPClient 5 | VERSION = '1.0.0.pre1' 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/message_bus/message.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Represents a published message and its encoding for persistence. 4 | class MessageBus::Message < Struct.new(:global_id, :message_id, :channel, :data) 5 | attr_accessor :site_id, :user_ids, :group_ids, :client_ids 6 | 7 | def self.decode(encoded) 8 | s1 = encoded.index("|") 9 | s2 = encoded.index("|", s1 + 1) 10 | s3 = encoded.index("|", s2 + 1) 11 | 12 | global_id = encoded[0, s1 + 1].to_i 13 | message_id = encoded[(s1 + 1), (s2 - s1 - 1)].to_i 14 | channel = encoded[(s2 + 1), (s3 - s2 - 1)] 15 | channel.gsub!("$$123$$", "|") 16 | data = encoded[(s3 + 1), encoded.size] 17 | 18 | MessageBus::Message.new(global_id, message_id, channel, data) 19 | end 20 | 21 | # only tricky thing to encode is pipes in a channel name ... do a straight replace 22 | def encode 23 | "#{global_id}|#{message_id}|#{channel.gsub("|", "$$123$$")}|#{data}" 24 | end 25 | 26 | def encode_without_ids 27 | "#{channel.gsub("|", "$$123$$")}|#{data}" 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/message_bus/rack/middleware.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'json' 4 | 5 | # our little message bus, accepts long polling and polling 6 | module MessageBus::Rack; end 7 | 8 | # Accepts requests from subscribers, validates and authenticates them, 9 | # delivers existing messages from the backlog and informs a 10 | # `MessageBus::ConnectionManager` of a connection which is remaining open. 11 | class MessageBus::Rack::Middleware 12 | # @param [Array] backlog a list of messages for delivery 13 | # @return [JSON] a JSON representation of the backlog, compliant with the 14 | # subscriber API specification 15 | def self.backlog_to_json(backlog) 16 | m = backlog.map do |msg| 17 | { 18 | global_id: msg.global_id, 19 | message_id: msg.message_id, 20 | channel: msg.channel, 21 | data: msg.data 22 | } 23 | end.to_a 24 | JSON.dump(m) 25 | end 26 | 27 | # @return [Boolean] whether the message listener (subscriber) is started or not) 28 | attr_reader :started_listener 29 | 30 | # Sets up the middleware to receive subscriber client requests and begins 31 | # listening for messages published on the bus for re-distribution (unless 32 | # the bus is disabled). 33 | # 34 | # @param [Proc] app the rack app 35 | # @param [Hash] config 36 | # @option config [MessageBus::Instance] :message_bus (`MessageBus`) a specific instance of message_bus 37 | def initialize(app, config = {}) 38 | @app = app 39 | @bus = config[:message_bus] || MessageBus 40 | @connection_manager = MessageBus::ConnectionManager.new(@bus) 41 | @started_listener = false 42 | @base_route = "#{@bus.base_route}message-bus/" 43 | @base_route_length = @base_route.length 44 | @broadcast_route = "#{@base_route}broadcast" 45 | start_listener unless @bus.off? 46 | end 47 | 48 | # Stops listening for messages on the bus 49 | # @return [void] 50 | def stop_listener 51 | if @subscription 52 | @bus.unsubscribe(&@subscription) 53 | @started_listener = false 54 | end 55 | end 56 | 57 | # Process an HTTP request from a subscriber client 58 | # @param [Rack::Request::Env] env the request environment 59 | def call(env) 60 | return @app.call(env) unless env['PATH_INFO'].start_with? @base_route 61 | 62 | handle_request(env) 63 | end 64 | 65 | private 66 | 67 | def handle_request(env) 68 | # Prevent simple polling from clobbering the session 69 | # See: https://github.com/discourse/message_bus/issues/257 70 | if (rack_session_options = env[Rack::RACK_SESSION_OPTIONS]) 71 | rack_session_options[:skip] = true 72 | end 73 | 74 | # special debug/test route 75 | if @bus.allow_broadcast? && env['PATH_INFO'] == @broadcast_route 76 | parsed = Rack::Request.new(env) 77 | @bus.publish parsed["channel"], parsed["data"] 78 | return [200, { "Content-Type" => "text/html" }, ["sent"]] 79 | end 80 | 81 | client_id = env['PATH_INFO'][@base_route_length..-1].split("/")[0] 82 | return [404, {}, ["not found"]] unless client_id 83 | 84 | headers = {} 85 | headers["Cache-Control"] = "must-revalidate, private, max-age=0" 86 | headers["Content-Type"] = "application/json; charset=utf-8" 87 | headers["Pragma"] = "no-cache" 88 | headers["Expires"] = "0" 89 | 90 | if @bus.extra_response_headers_lookup 91 | @bus.extra_response_headers_lookup.call(env).each do |k, v| 92 | headers[k] = v 93 | end 94 | end 95 | 96 | if env["REQUEST_METHOD"] == "OPTIONS" 97 | return [200, headers, ["OK"]] 98 | end 99 | 100 | user_id = @bus.user_id_lookup.call(env) if @bus.user_id_lookup 101 | group_ids = @bus.group_ids_lookup.call(env) if @bus.group_ids_lookup 102 | site_id = @bus.site_id_lookup.call(env) if @bus.site_id_lookup 103 | 104 | # close db connection as early as possible 105 | close_db_connection! 106 | 107 | client = MessageBus::Client.new(message_bus: @bus, client_id: client_id, 108 | user_id: user_id, site_id: site_id, group_ids: group_ids) 109 | 110 | if channels = env['message_bus.channels'] 111 | if seq = env['message_bus.seq'] 112 | client.seq = seq.to_i 113 | end 114 | channels.each do |k, v| 115 | client.subscribe(k, v) 116 | end 117 | else 118 | request = Rack::Request.new(env) 119 | is_json = request.content_type && request.content_type.include?('application/json') 120 | data = is_json ? JSON.parse(request.body.read) : request.POST 121 | data.each do |k, v| 122 | if k == "__seq" 123 | client.seq = v.to_i 124 | else 125 | client.subscribe(k, v) 126 | end 127 | end 128 | end 129 | 130 | long_polling = @bus.long_polling_enabled? && 131 | env['QUERY_STRING'] !~ /dlp=t/ && 132 | @connection_manager.client_count < @bus.max_active_clients 133 | 134 | allow_chunked = env['HTTP_VERSION'] == 'HTTP/1.1' 135 | allow_chunked &&= !env['HTTP_DONT_CHUNK'] 136 | allow_chunked &&= @bus.chunked_encoding_enabled? 137 | 138 | client.use_chunked = allow_chunked 139 | 140 | backlog = client.backlog 141 | 142 | if backlog.length > 0 && !allow_chunked 143 | client.close 144 | @bus.logger.debug "Delivering backlog #{backlog} to client #{client_id} for user #{user_id}" 145 | [200, headers, [self.class.backlog_to_json(backlog)]] 146 | elsif long_polling && env['rack.hijack'] && @bus.rack_hijack_enabled? 147 | io = env['rack.hijack'].call 148 | # TODO disable client till deliver backlog is called 149 | client.io = io 150 | client.headers = headers 151 | client.synchronize do 152 | client.deliver_backlog(backlog) 153 | add_client_with_timeout(client) 154 | client.ensure_first_chunk_sent 155 | end 156 | [418, {}, ["I'm a teapot, undefined in spec"]] 157 | elsif long_polling && env['async.callback'] 158 | response = nil 159 | # load extension if needed 160 | begin 161 | response = Thin::AsyncResponse.new(env) 162 | rescue NameError 163 | require 'message_bus/rack/thin_ext' 164 | response = Thin::AsyncResponse.new(env) 165 | end 166 | 167 | headers.each do |k, v| 168 | response.headers[k] = v 169 | end 170 | 171 | if allow_chunked 172 | response.headers["X-Content-Type-Options"] = "nosniff" 173 | response.headers["Transfer-Encoding"] = "chunked" 174 | response.headers["Content-Type"] = "text/plain; charset=utf-8" 175 | end 176 | 177 | response.status = 200 178 | client.async_response = response 179 | client.synchronize do 180 | add_client_with_timeout(client) 181 | client.deliver_backlog(backlog) 182 | client.ensure_first_chunk_sent 183 | end 184 | 185 | throw :async 186 | else 187 | [200, headers, [self.class.backlog_to_json(backlog)]] 188 | end 189 | rescue => e 190 | if @bus.on_middleware_error && result = @bus.on_middleware_error.call(env, e) 191 | result 192 | else 193 | raise 194 | end 195 | end 196 | 197 | def close_db_connection! 198 | # IMPORTANT 199 | # ConnectionManagement in Rails puts a BodyProxy around stuff 200 | # this means connections are not returned until rack.async is 201 | # closed 202 | if defined? ActiveRecord::Base.connection_handler 203 | if Gem::Version.new(Rails.version) >= "7.1" 204 | ActiveRecord::Base.connection_handler.clear_active_connections!(:all) 205 | else 206 | ActiveRecord::Base.connection_handler.clear_active_connections! 207 | end 208 | elsif defined? ActiveRecord::Base.clear_active_connections! 209 | ActiveRecord::Base.clear_active_connections! 210 | end 211 | end 212 | 213 | def add_client_with_timeout(client) 214 | @connection_manager.add_client(client) 215 | 216 | client.cleanup_timer = @bus.timer.queue(@bus.long_polling_interval.to_f / 1000) { 217 | begin 218 | client.close 219 | @connection_manager.remove_client(client) 220 | rescue 221 | @bus.logger.warn "Failed to clean up client properly: #{$!} #{$!.backtrace}" 222 | end 223 | } 224 | end 225 | 226 | def start_listener 227 | unless @started_listener 228 | 229 | thin = defined?(Thin::Server) && ObjectSpace.each_object(Thin::Server).to_a.first 230 | thin_running = thin && thin.running? 231 | 232 | @subscription = @bus.subscribe do |msg| 233 | run = proc do 234 | begin 235 | @connection_manager.notify_clients(msg) if @connection_manager 236 | rescue 237 | @bus.logger.warn "Failed to notify clients: #{$!} #{$!.backtrace}" 238 | end 239 | end 240 | 241 | if thin_running 242 | EM.next_tick(&run) 243 | else 244 | @bus.timer.queue(&run) 245 | end 246 | 247 | @started_listener = true 248 | end 249 | end 250 | end 251 | end 252 | -------------------------------------------------------------------------------- /lib/message_bus/rack/thin_ext.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # there is also another in cramp this is from https://github.com/macournoyer/thin_async/blob/master/lib/thin/async.rb 4 | module Thin 5 | unless defined?(DeferrableBody) 6 | # Based on version from James Tucker 7 | class DeferrableBody 8 | include ::EM::Deferrable 9 | 10 | def initialize 11 | @queue = [] 12 | @body_callback = nil 13 | end 14 | 15 | def call(body) 16 | @queue << body 17 | schedule_dequeue 18 | end 19 | 20 | def each(&blk) 21 | @body_callback = blk 22 | schedule_dequeue 23 | end 24 | 25 | private 26 | 27 | def schedule_dequeue 28 | return unless @body_callback 29 | 30 | ::EM.next_tick do 31 | next unless body = @queue.shift 32 | 33 | body.each do |chunk| 34 | @body_callback.call(chunk) 35 | end 36 | schedule_dequeue unless @queue.empty? 37 | end 38 | end 39 | end 40 | end 41 | 42 | # Response which body is sent asynchronously. 43 | class AsyncResponse 44 | include Rack::Response::Helpers 45 | 46 | attr_reader :headers, :callback, :closed 47 | attr_accessor :status 48 | 49 | def initialize(env, status = 200, headers = {}) 50 | @callback = env['async.callback'] 51 | @body = DeferrableBody.new 52 | @status = status 53 | @headers = headers 54 | @headers_sent = false 55 | end 56 | 57 | def send_headers 58 | return if @headers_sent 59 | 60 | @callback.call [@status, @headers, @body] 61 | @headers_sent = true 62 | end 63 | 64 | def write(body) 65 | send_headers 66 | @body.call(body.respond_to?(:each) ? body : [body]) 67 | end 68 | alias :<< :write 69 | 70 | # Tell Thin the response is complete and the connection can be closed. 71 | def done 72 | @closed = true 73 | send_headers 74 | ::EM.next_tick { @body.succeed } 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/message_bus/rails/railtie.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus; module Rails; end; end 4 | 5 | # rails engine for asset pipeline 6 | class MessageBus::Rails::Engine < ::Rails::Engine; end 7 | 8 | class MessageBus::Rails::Railtie < ::Rails::Railtie 9 | initializer "message_bus.configure_init" do |app| 10 | # We want MessageBus to show up after the session middleware, but depending on how 11 | # the Rails app is configured that might be ActionDispatch::Session::CookieStore, or potentially 12 | # ActionDispatch::Session::ActiveRecordStore. 13 | # 14 | # given https://github.com/rails/rails/commit/fedde239dcee256b417dc9bcfe5fef603bf0d952#diff-533a9a9cc17a8a899cb830626089e5f9 15 | # there is no way of walking the stack for operations 16 | if !skip_middleware?(app.config) 17 | if api_only?(app.config) 18 | app.middleware.use(MessageBus::Rack::Middleware) 19 | else 20 | app.middleware.insert_before(ActionDispatch::Flash, MessageBus::Rack::Middleware) 21 | end 22 | end 23 | 24 | MessageBus.logger = Rails.logger 25 | end 26 | 27 | def skip_middleware?(config) 28 | return false if !config.respond_to?(:skip_message_bus_middleware) 29 | 30 | config.skip_message_bus_middleware 31 | end 32 | 33 | def api_only?(config) 34 | return false if !config.respond_to?(:api_only) 35 | 36 | config.api_only 37 | end 38 | 39 | end 40 | -------------------------------------------------------------------------------- /lib/message_bus/timer_thread.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MessageBus::TimerThread 4 | attr_reader :jobs 5 | 6 | class Cancelable 7 | class NoOp 8 | def call 9 | end 10 | end 11 | 12 | # usually you could just use a blank lambda 13 | # but an object is ever so slightly faster 14 | NOOP = NoOp.new 15 | 16 | def initialize(job) 17 | @job = job 18 | end 19 | 20 | def cancel 21 | @job[1] = NOOP 22 | end 23 | end 24 | 25 | class CancelableEvery 26 | attr_accessor :cancelled, :current 27 | def cancel 28 | current.cancel if current 29 | @cancelled = true 30 | end 31 | end 32 | 33 | def initialize 34 | @stopped = false 35 | @jobs = [] 36 | @mutex = Mutex.new 37 | @next = nil 38 | @thread = Thread.new { do_work } 39 | @on_error = lambda { |e| STDERR.puts "Exception while processing Timer:\n #{e.backtrace.join("\n")}" } 40 | end 41 | 42 | def stop 43 | @stopped = true 44 | running = true 45 | while running 46 | @mutex.synchronize do 47 | running = @thread && @thread.alive? 48 | 49 | if running 50 | begin 51 | @thread.wakeup 52 | rescue ThreadError 53 | raise if @thread.alive? 54 | end 55 | end 56 | end 57 | sleep 0 58 | end 59 | end 60 | 61 | def every(delay, &block) 62 | result = CancelableEvery.new 63 | do_work = proc do 64 | begin 65 | block.call 66 | ensure 67 | result.current = queue(delay, &do_work) 68 | end 69 | end 70 | result.current = queue(delay, &do_work) 71 | result 72 | end 73 | 74 | # queue a block to run after a certain delay (in seconds) 75 | def queue(delay = 0, &block) 76 | queue_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + delay 77 | job = [queue_time, block] 78 | 79 | @mutex.synchronize do 80 | i = @jobs.length 81 | while i > 0 82 | i -= 1 83 | current, _ = @jobs[i] 84 | if current < queue_time 85 | i += 1 86 | break 87 | end 88 | end 89 | @jobs.insert(i, job) 90 | @next = queue_time if i == 0 91 | end 92 | 93 | unless @thread.alive? 94 | @mutex.synchronize do 95 | @thread = Thread.new { do_work } unless @thread.alive? 96 | end 97 | end 98 | 99 | if @thread.status == "sleep" 100 | @thread.wakeup 101 | end 102 | 103 | Cancelable.new(job) 104 | end 105 | 106 | def on_error(&block) 107 | @on_error = block 108 | end 109 | 110 | protected 111 | 112 | def do_work 113 | while !@stopped 114 | if @next && @next <= Process.clock_gettime(Process::CLOCK_MONOTONIC) 115 | _, blk = @mutex.synchronize { @jobs.shift } 116 | begin 117 | blk.call 118 | rescue => e 119 | @on_error.call(e) if @on_error 120 | end 121 | @mutex.synchronize do 122 | @next, _ = @jobs[0] 123 | end 124 | end 125 | unless @next && @next <= Process.clock_gettime(Process::CLOCK_MONOTONIC) 126 | sleep_time = 1000 127 | @mutex.synchronize do 128 | sleep_time = @next - Process.clock_gettime(Process::CLOCK_MONOTONIC) if @next 129 | end 130 | sleep [0, sleep_time].max 131 | end 132 | end 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /lib/message_bus/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module MessageBus 4 | VERSION = "4.4.1" 5 | end 6 | -------------------------------------------------------------------------------- /message_bus.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path('../lib/message_bus/version', __FILE__) 4 | 5 | Gem::Specification.new do |gem| 6 | gem.authors = ["Sam Saffron"] 7 | gem.email = ["sam.saffron@gmail.com"] 8 | gem.description = %q{A message bus for rack} 9 | gem.summary = %q{} 10 | gem.homepage = "https://github.com/discourse/message_bus" 11 | gem.license = "MIT" 12 | gem.files = `git ls-files`.split($\) + 13 | ["vendor/assets/javascripts/message-bus.js", "vendor/assets/javascripts/message-bus-ajax.js"] 14 | gem.name = "message_bus" 15 | gem.require_paths = ["lib"] 16 | gem.version = MessageBus::VERSION 17 | gem.required_ruby_version = ">= 2.6.0" 18 | 19 | gem.add_runtime_dependency 'rack', '>= 1.1.3' 20 | 21 | # Optional runtime dependencies 22 | gem.add_development_dependency 'redis' 23 | gem.add_development_dependency 'pg' 24 | gem.add_development_dependency 'concurrent-ruby' # for distributed-cache 25 | 26 | gem.add_development_dependency 'minitest' 27 | gem.add_development_dependency 'minitest-hooks' 28 | gem.add_development_dependency 'minitest-global_expectations' 29 | gem.add_development_dependency 'rake' 30 | gem.add_development_dependency 'http_parser.rb' 31 | gem.add_development_dependency 'thin' 32 | gem.add_development_dependency 'rack-test' 33 | gem.add_development_dependency 'puma' 34 | gem.add_development_dependency 'm' 35 | gem.add_development_dependency 'byebug' 36 | gem.add_development_dependency 'oj' 37 | gem.add_development_dependency 'yard' 38 | 39 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7.0') 40 | gem.add_development_dependency 'rubocop-discourse', '3.8.1' 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "message-bus-client", 3 | "version": "0.0.0-version-placeholder", 4 | "description": "A message bus client in Javascript", 5 | "main": "assets/message-bus.js", 6 | "keywords": [ 7 | "es6", 8 | "modules" 9 | ], 10 | "files": [ 11 | "assets/message-bus.js" 12 | ], 13 | "jsnext:main": "assets/message-bus.js", 14 | "module": "assets/message-bus.js", 15 | "repository": "https://github.com/discourse/message_bus", 16 | "author": "Sam Saffron, Robin Ward", 17 | "license": "MIT", 18 | "bugs": { 19 | "url": "https://github.com/discourse/message_bus/issues" 20 | }, 21 | "homepage": "https://github.com/discourse/message_bus#readme", 22 | "devDependencies": { 23 | "eslint": "^8.31.0", 24 | "jasmine-browser-runner": "^0.10.0", 25 | "jasmine-core": "^3.10.1" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /spec/assets/SpecHelper.js: -------------------------------------------------------------------------------- 1 | /* global beforeEach, afterEach, it, spyOn, MessageBus */ 2 | 3 | var message_id = 1; 4 | var SEPARATOR = "\r\n|\r\n"; 5 | 6 | var encodeChunks = function(xhr, chunks) { 7 | if (!chunks || !chunks.length){ 8 | return ''; 9 | } 10 | for (var i=0;i { 41 | if(this.statusText === "abort"){ 42 | return; 43 | } 44 | this.onprogress?.(); 45 | this.onreadystatechange(); 46 | } 47 | 48 | if(spec.delayResponsePromise){ 49 | spec.delayResponsePromise.then(() => complete()) 50 | spec.delayResponsePromise = null; 51 | }else{ 52 | complete(); 53 | } 54 | } 55 | 56 | MockedXMLHttpRequest.prototype.open = function(){ } 57 | 58 | MockedXMLHttpRequest.prototype.abort = function(){ 59 | this.readyState = 4 60 | this.responseText = ''; 61 | this.statusText = 'abort'; 62 | this.status = 0; 63 | this.onreadystatechange() 64 | } 65 | 66 | MockedXMLHttpRequest.prototype.setRequestHeader = function(k,v){ 67 | this.headers[k] = v; 68 | } 69 | 70 | MockedXMLHttpRequest.prototype.getResponseHeader = function(headerName){ 71 | return spec.responseHeaders[headerName]; 72 | } 73 | 74 | MessageBus.xhrImplementation = MockedXMLHttpRequest 75 | this.MockedXMLHttpRequest = MockedXMLHttpRequest 76 | 77 | this.responseChunks = [ 78 | {channel: '/test', data: {password: 'MessageBusRocks!'}} 79 | ]; 80 | 81 | this.responseStatus = 200; 82 | this.responseHeaders = { 83 | "Content-Type": 'text/plain; charset=utf-8', 84 | }; 85 | 86 | MessageBus.enableChunkedEncoding = true; 87 | MessageBus.firstChunkTimeout = 3000; 88 | MessageBus.retryChunkedAfterRequests = 1; 89 | 90 | MessageBus.start(); 91 | }); 92 | 93 | afterEach(function(){ 94 | MessageBus.stop() 95 | MessageBus.callbacks.splice(0, MessageBus.callbacks.length) 96 | MessageBus.shouldLongPollCallback = null; 97 | }); 98 | 99 | window.testMB = function(description, testFn, path, data){ 100 | this.responseChunks = [ 101 | {channel: path || '/test', data: data || {password: 'MessageBusRocks!'}} 102 | ]; 103 | it(description, function(done){ 104 | var spec = this; 105 | var promisy = { 106 | finally: function(fn){ 107 | this.resolve = fn; 108 | } 109 | } 110 | this.perform = function(specFn){ 111 | var xhrRequest = null; 112 | spyOn(this.MockedXMLHttpRequest.prototype, 'open').and.callFake(function(method, url){ 113 | xhrRequest = this; 114 | xhrRequest.url = url 115 | xhrRequest.method = method 116 | spec.MockedXMLHttpRequest.prototype.open.and.callThrough(this, method, url); 117 | }) 118 | MessageBus.subscribe(path || '/test', function(message){ 119 | try { 120 | specFn.call(spec, message, xhrRequest); 121 | } catch( error ){ 122 | promisy.resolve.call(spec); 123 | throw(error); 124 | } 125 | promisy.resolve.call(spec); 126 | done(); 127 | }); 128 | return promisy; 129 | }; 130 | testFn.call(this); 131 | }); 132 | 133 | } 134 | -------------------------------------------------------------------------------- /spec/assets/message-bus.spec.js: -------------------------------------------------------------------------------- 1 | /* eslint-env es2022 */ 2 | /* global describe, it, spyOn, MessageBus, expect, jasmine, testMB */ 3 | 4 | function approxEqual(valueOne, valueTwo) { 5 | return Math.abs(valueOne - valueTwo) < 500; 6 | } 7 | 8 | describe("Messagebus", function () { 9 | it("submits change requests", function(done){ 10 | spyOn(this.MockedXMLHttpRequest.prototype, 'send').and.callThrough(); 11 | var spec = this; 12 | MessageBus.subscribe('/test', function(){ 13 | expect(spec.MockedXMLHttpRequest.prototype.send) 14 | .toHaveBeenCalled() 15 | var data = spec.MockedXMLHttpRequest.prototype.send.calls.argsFor(0)[0]; 16 | var params = new URLSearchParams(data); 17 | expect(params.get("/test")).toEqual("-1"); 18 | expect(params.get("__seq")).toMatch(/\d+/); 19 | done(); 20 | }); 21 | }); 22 | 23 | it("calls callbacks", function(done){ 24 | MessageBus.subscribe('/test', function(message){ 25 | expect(message.password).toEqual('MessageBusRocks!'); 26 | done(); 27 | }); 28 | }); 29 | 30 | it('returns status', function(done){ 31 | MessageBus.pause(); 32 | expect(MessageBus.status()).toMatch("paused"); 33 | MessageBus.resume(); 34 | expect(MessageBus.status()).toMatch("started"); 35 | MessageBus.stop(); 36 | expect(MessageBus.status()).toMatch("stopped"); 37 | done(); 38 | }); 39 | 40 | it('stores messages when paused, then delivers them when resumed', function(done){ 41 | MessageBus.pause() 42 | spyOn(this.MockedXMLHttpRequest.prototype, 'send').and.callThrough(); 43 | var spec = this; 44 | var onMessageSpy = jasmine.createSpy('onMessageSpy'); 45 | MessageBus.subscribe('/test', onMessageSpy); 46 | setTimeout(function(){ 47 | expect(spec.MockedXMLHttpRequest.prototype.send).toHaveBeenCalled() 48 | expect(onMessageSpy).not.toHaveBeenCalled() 49 | MessageBus.resume() 50 | }, 1010) // greater than delayPollTimeout of 500 + 500 random 51 | setTimeout(function(){ 52 | expect(onMessageSpy).toHaveBeenCalled() 53 | done() 54 | }, 1050) // greater than first timeout above 55 | }); 56 | 57 | it('can unsubscribe from callbacks', function(done){ 58 | var onMessageSpy = jasmine.createSpy('onMessageSpy'); 59 | MessageBus.subscribe('/test', onMessageSpy); 60 | MessageBus.unsubscribe('/test', onMessageSpy); 61 | MessageBus.subscribe('/test', function(){ 62 | expect(onMessageSpy).not.toHaveBeenCalled() 63 | done() 64 | }); 65 | }); 66 | 67 | testMB('sets dlp parameter when longPolling is disabled', function(){ 68 | MessageBus.enableLongPolling = false 69 | this.perform(function(message, xhr){ 70 | expect(xhr.url).toMatch("dlp=t"); 71 | }).finally(function(){ 72 | MessageBus.enableLongPolling = true 73 | }) 74 | }); 75 | 76 | testMB('respects baseUrl setting', function(){ 77 | MessageBus.baseUrl = "/a/test/base/url/"; 78 | this.perform(function(message, xhr){ 79 | expect(xhr.url).toMatch("/a/test/base/url/"); 80 | }).finally(function(){ 81 | MessageBus.baseUrl = "/"; 82 | }) 83 | }); 84 | 85 | it('respects minPollInterval setting with defaults', function(){ 86 | expect(MessageBus.minPollInterval).toEqual(100); 87 | MessageBus.minPollInterval = 1000; 88 | expect(MessageBus.minPollInterval).toEqual(1000); 89 | }); 90 | 91 | testMB('sends using custom header', function(){ 92 | MessageBus.headers['X-MB-TEST-VALUE'] = '42'; 93 | this.perform(function(message, xhr){ 94 | expect(xhr.headers).toEqual({ 95 | 'X-SILENCE-LOGGER': 'true', 96 | 'X-MB-TEST-VALUE': '42', 97 | 'Content-Type': 'application/x-www-form-urlencoded' 98 | }); 99 | }).finally(function(){ 100 | MessageBus.headers = {}; 101 | }) 102 | }); 103 | 104 | it("respects Retry-After response header when larger than 15 seconds", async function () { 105 | spyOn(this.MockedXMLHttpRequest.prototype, "send").and.callThrough(); 106 | spyOn(window, "setTimeout").and.callThrough(); 107 | 108 | this.responseStatus = 429; 109 | this.responseHeaders["Retry-After"] = "23"; 110 | 111 | await new Promise((resolve) => MessageBus.subscribe("/test", resolve)); 112 | 113 | const nextPollScheduledIn = window.setTimeout.calls.mostRecent().args[1]; 114 | expect(nextPollScheduledIn).toEqual(23000); 115 | }); 116 | 117 | it("retries after 15s for lower retry-after values", async function () { 118 | spyOn(this.MockedXMLHttpRequest.prototype, "send").and.callThrough(); 119 | spyOn(window, "setTimeout").and.callThrough(); 120 | 121 | this.responseStatus = 429; 122 | this.responseHeaders["Retry-After"] = "13"; 123 | 124 | await new Promise((resolve) => MessageBus.subscribe("/test", resolve)); 125 | 126 | const nextPollScheduledIn = window.setTimeout.calls.mostRecent().args[1]; 127 | expect(nextPollScheduledIn).toEqual(15000); 128 | }); 129 | 130 | it("waits for callbackInterval after receiving data in chunked long-poll mode", async function () { 131 | // The callbackInterval is equal to the length of the server response in chunked long-poll mode, so 132 | // this ultimately ends up being a continuous stream of requests 133 | 134 | spyOn(this.MockedXMLHttpRequest.prototype, "send").and.callThrough(); 135 | spyOn(window, "setTimeout").and.callThrough(); 136 | 137 | await new Promise((resolve) => MessageBus.subscribe("/test", resolve)); 138 | 139 | const nextPollScheduledIn = window.setTimeout.calls.mostRecent().args[1]; 140 | expect( 141 | approxEqual(nextPollScheduledIn, MessageBus.callbackInterval) 142 | ).toEqual(true); 143 | }); 144 | 145 | it("waits for backgroundCallbackInterval after receiving data in non-long-poll mode", async function () { 146 | spyOn(this.MockedXMLHttpRequest.prototype, "send").and.callThrough(); 147 | spyOn(window, "setTimeout").and.callThrough(); 148 | MessageBus.shouldLongPollCallback = () => false; 149 | MessageBus.enableChunkedEncoding = false; 150 | 151 | await new Promise((resolve) => MessageBus.subscribe("/test", resolve)); 152 | 153 | const nextPollScheduledIn = window.setTimeout.calls.mostRecent().args[1]; 154 | expect( 155 | approxEqual(nextPollScheduledIn, MessageBus.backgroundCallbackInterval) 156 | ).toEqual(true); 157 | }); 158 | 159 | it("re-polls immediately after receiving data in non-chunked long-poll mode", async function () { 160 | spyOn(this.MockedXMLHttpRequest.prototype, "send").and.callThrough(); 161 | spyOn(window, "setTimeout").and.callThrough(); 162 | MessageBus.enableChunkedEncoding = false; 163 | 164 | await new Promise((resolve) => MessageBus.subscribe("/test", resolve)); 165 | 166 | const nextPollScheduledIn = window.setTimeout.calls.mostRecent().args[1]; 167 | expect(nextPollScheduledIn).toEqual(MessageBus.minPollInterval); 168 | }); 169 | 170 | it("enters don't-chunk-mode if first chunk times out", async function () { 171 | spyOn(this.MockedXMLHttpRequest.prototype, "send").and.callThrough(); 172 | spyOn( 173 | this.MockedXMLHttpRequest.prototype, 174 | "setRequestHeader" 175 | ).and.callThrough(); 176 | 177 | let resolveFirstResponse; 178 | this.delayResponsePromise = new Promise( 179 | (resolve) => (resolveFirstResponse = resolve) 180 | ); 181 | MessageBus.firstChunkTimeout = 50; 182 | 183 | await new Promise((resolve) => MessageBus.subscribe("/test", resolve)); 184 | resolveFirstResponse(); 185 | 186 | const calls = 187 | this.MockedXMLHttpRequest.prototype.setRequestHeader.calls.all(); 188 | 189 | const dontChunkCalls = calls.filter((c) => c.args[0] === "Dont-Chunk"); 190 | expect(dontChunkCalls.length).toEqual(1); 191 | }); 192 | 193 | it("doesn't enter don't-chunk-mode if aborted before first chunk", async function () { 194 | spyOn( 195 | this.MockedXMLHttpRequest.prototype, 196 | "setRequestHeader" 197 | ).and.callThrough(); 198 | 199 | this.delayResponsePromise = new Promise(() => {}); 200 | MessageBus.firstChunkTimeout = 300; 201 | 202 | const requestWasStarted = new Promise( 203 | (resolve) => (this.requestStarted = resolve) 204 | ); 205 | 206 | // Trigger request 207 | const subscribedPromise = new Promise((resolve) => 208 | MessageBus.subscribe("/test", resolve) 209 | ); 210 | 211 | await requestWasStarted; 212 | 213 | // Change subscription (triggers an abort and re-poll) 214 | MessageBus.subscribe("/test2", () => {}); 215 | 216 | // Wait for stuff to settle 217 | await subscribedPromise; 218 | 219 | // Wait 300ms to ensure dontChunk timeout has passed 220 | await new Promise((resolve) => setTimeout(resolve, 300)); 221 | 222 | const calls = 223 | this.MockedXMLHttpRequest.prototype.setRequestHeader.calls.all(); 224 | 225 | const dontChunkCalls = calls.filter((c) => c.args[0] === "Dont-Chunk"); 226 | expect(dontChunkCalls.length).toEqual(0); 227 | }); 228 | }); 229 | -------------------------------------------------------------------------------- /spec/fixtures/test/Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | source 'https://rubygems.org' 3 | 4 | gem 'message_bus' 5 | -------------------------------------------------------------------------------- /spec/fixtures/test/config.ru: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'message_bus' 3 | 4 | MessageBus.config[:backend] = :memory 5 | MessageBus.long_polling_interval = 1000 6 | use MessageBus::Rack::Middleware 7 | 8 | run ->(env) do 9 | if env["REQUEST_METHOD"] == "GET" && env["REQUEST_PATH"] == "/publish" 10 | payload = { hello: "world" } 11 | 12 | ["/test", "/test2"].each do |channel| 13 | MessageBus.publish(channel, payload) 14 | end 15 | end 16 | 17 | [200, { "Content-Type" => "text/html" }, ["Howdy"]] 18 | end 19 | -------------------------------------------------------------------------------- /spec/helpers.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'logger' 4 | require 'method_source' 5 | 6 | def wait_for(timeout_milliseconds = 2000, &blk) 7 | timeout = timeout_milliseconds / 1000.0 8 | finish = Time.now + timeout 9 | result = nil 10 | 11 | while Time.now < finish && !(result = blk.call) 12 | sleep(0.001) 13 | end 14 | 15 | flunk("wait_for timed out:\n#{blk.source}") if !result 16 | end 17 | 18 | def test_config_for_backend(backend) 19 | config = { 20 | backend: backend, 21 | logger: Logger.new(IO::NULL), 22 | } 23 | 24 | case backend 25 | when :redis 26 | config[:url] = ENV['REDISURL'] 27 | when :postgres 28 | config[:backend_options] = { 29 | host: ENV['PGHOST'], 30 | user: ENV['PGUSER'] || ENV['USER'], 31 | password: ENV['PGPASSWORD'], 32 | dbname: ENV['PGDATABASE'] || 'message_bus_test' 33 | } 34 | end 35 | config 36 | end 37 | -------------------------------------------------------------------------------- /spec/integration/http_client_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../spec_helper' 4 | require 'message_bus/http_client' 5 | 6 | describe MessageBus::HTTPClient do 7 | let(:base_url) { "http://0.0.0.0:9292" } 8 | let(:client) { MessageBus::HTTPClient.new(base_url) } 9 | let(:headers) { client.send(:headers) } 10 | let(:channel) { "/test" } 11 | let(:channel2) { "/test2" } 12 | let(:stats) { client.stats } 13 | 14 | def publish_message 15 | response = Net::HTTP.get_response(URI("#{base_url}/publish")) 16 | assert_equal("200", response.code) 17 | end 18 | 19 | after do 20 | client.stop 21 | end 22 | 23 | describe '#start and #stop' do 24 | it 'should be able to start and stop polling correctly' do 25 | threads = Thread.list 26 | 27 | assert_equal(MessageBus::HTTPClient::STOPPED, client.status) 28 | 29 | client.start 30 | new_threads = Thread.list - threads 31 | 32 | assert_equal(1, new_threads.size) 33 | assert_equal(MessageBus::HTTPClient::STARTED, client.status) 34 | 35 | client.start 36 | 37 | assert_equal(new_threads, Thread.list - threads) 38 | end 39 | 40 | describe 'when an error is encountered while trying to poll' do 41 | let(:base_url) { "http://0.0.0.0:12312123" } 42 | 43 | let(:client) do 44 | MessageBus::HTTPClient.new(base_url, min_poll_interval: 1) 45 | end 46 | 47 | it 'should handle errors correctly' do 48 | begin 49 | original_stderr = $stderr 50 | $stderr = fake = StringIO.new 51 | 52 | client.channels[channel] = MessageBus::HTTPClient::Channel.new( 53 | callbacks: [-> {}] 54 | ) 55 | 56 | client.start 57 | 58 | assert_equal(MessageBus::HTTPClient::STARTED, client.status) 59 | 60 | while stats.failed < 1 do 61 | sleep 0.05 62 | end 63 | 64 | # Sleep for more than the default min_poll_interval to ensure 65 | # that we sleep for the right interval after failure 66 | sleep 0.5 67 | 68 | assert_match(/Errno::ECONNREFUSED|SocketError/, fake.string) 69 | ensure 70 | $stderr = original_stderr 71 | end 72 | end 73 | end 74 | end 75 | 76 | describe '#subscribe' do 77 | it 'should be able to subscribe to channels for messages' do 78 | called = 0 79 | called2 = 0 80 | 81 | client.subscribe(channel, last_message_id: -1) do |data| 82 | called += 1 83 | assert_equal("world", data["hello"]) 84 | end 85 | 86 | client.subscribe(channel2) do |data| 87 | called2 += 1 88 | assert_equal("world", data["hello"]) 89 | end 90 | 91 | while called < 2 && called2 < 2 92 | publish_message 93 | sleep 0.05 94 | end 95 | 96 | while stats.success < 1 97 | sleep 0.05 98 | end 99 | 100 | assert_equal(0, stats.failed) 101 | end 102 | 103 | describe 'supports including extra headers' do 104 | let(:client) do 105 | MessageBus::HTTPClient.new(base_url, headers: { 106 | 'Dont-Chunk' => "true" 107 | }) 108 | end 109 | 110 | it 'should include the header in the request' do 111 | called = 0 112 | 113 | client.subscribe(channel) do |data| 114 | called += 1 115 | assert_equal("world", data["hello"]) 116 | end 117 | 118 | while called < 2 119 | publish_message 120 | sleep 0.05 121 | end 122 | end 123 | end 124 | 125 | describe 'when chunked encoding is disabled' do 126 | let(:client) do 127 | MessageBus::HTTPClient.new(base_url, enable_chunked_encoding: false) 128 | end 129 | 130 | it 'should still be able to subscribe to channels for messages' do 131 | called = 0 132 | 133 | client.subscribe(channel) do |data| 134 | called += 1 135 | assert_equal("world", data["hello"]) 136 | end 137 | 138 | while called < 2 139 | publish_message 140 | sleep 0.05 141 | end 142 | end 143 | end 144 | 145 | describe 'when enable_long_polling is disabled' do 146 | let(:client) do 147 | MessageBus::HTTPClient.new(base_url, 148 | enable_long_polling: false, 149 | background_callback_interval: 0.01) 150 | end 151 | 152 | it 'should still be able to subscribe to channels for messages' do 153 | called = 0 154 | 155 | client.subscribe(channel) do |data| 156 | called += 1 157 | assert_equal("world", data["hello"]) 158 | end 159 | 160 | while called < 2 161 | publish_message 162 | sleep 0.05 163 | end 164 | end 165 | end 166 | 167 | describe 'when channel name is invalid' do 168 | it 'should raise the right error' do 169 | ["test", 1, :test].each do |invalid_channel| 170 | assert_raises MessageBus::HTTPClient::InvalidChannel do 171 | client.subscribe(invalid_channel) 172 | end 173 | end 174 | end 175 | end 176 | 177 | describe 'when a block is not given' do 178 | it 'should raise the right error' do 179 | assert_raises MessageBus::HTTPClient::MissingBlock do 180 | client.subscribe(channel) 181 | end 182 | end 183 | end 184 | 185 | describe 'with last_message_id' do 186 | describe 'when invalid' do 187 | it 'should subscribe from the latest message' do 188 | client.subscribe(channel, last_message_id: 'haha') {} 189 | assert_equal(-1, client.channels[channel].last_message_id) 190 | end 191 | end 192 | 193 | describe 'when valid' do 194 | it 'should subscribe from the right message' do 195 | client.subscribe(channel, last_message_id: -2) {} 196 | assert_equal(-2, client.channels[channel].last_message_id) 197 | end 198 | end 199 | end 200 | end 201 | 202 | describe '#unsubscribe' do 203 | it 'should be able to unsubscribe a channel' do 204 | client.subscribe(channel) { raise "Not called" } 205 | assert(client.channels[channel]) 206 | 207 | client.unsubscribe(channel) 208 | assert_nil(client.channels[channel]) 209 | end 210 | 211 | describe 'with callback' do 212 | it 'should be able to unsubscribe a callback for a particular channel' do 213 | callback = -> { raise "Not called" } 214 | callback2 = -> { raise "Not called2" } 215 | 216 | client.subscribe(channel, &callback) 217 | client.subscribe(channel, &callback2) 218 | assert_equal([callback, callback2], client.channels[channel].callbacks) 219 | 220 | client.unsubscribe(channel, &callback) 221 | assert_equal([callback2], client.channels[channel].callbacks) 222 | 223 | client.unsubscribe(channel, &callback2) 224 | assert_nil(client.channels[channel]) 225 | end 226 | end 227 | end 228 | end 229 | -------------------------------------------------------------------------------- /spec/lib/fake_async_middleware.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'http/parser' 3 | class FakeAsyncMiddleware 4 | def initialize(app, config = {}) 5 | @app = app 6 | @bus = config[:message_bus] || MessageBus 7 | @simulate_thin_async = false 8 | @simulate_hijack = false 9 | @in_async = false 10 | @allow_chunked = false 11 | end 12 | 13 | def app 14 | @app 15 | end 16 | 17 | def simulate_thin_async 18 | @simulate_thin_async = true 19 | @simulate_hijack = false 20 | end 21 | 22 | def simulate_hijack 23 | @simulate_thin_async = false 24 | @simulate_hijack = true 25 | end 26 | 27 | def allow_chunked 28 | @allow_chunked = true 29 | end 30 | 31 | def in_async? 32 | @in_async 33 | end 34 | 35 | def simulate_thin_async? 36 | @simulate_thin_async && @bus.long_polling_enabled? 37 | end 38 | 39 | def simulate_hijack? 40 | @simulate_hijack && @bus.long_polling_enabled? 41 | end 42 | 43 | def call(env) 44 | unless @allow_chunked 45 | env['HTTP_DONT_CHUNK'] = 'True' 46 | end 47 | if simulate_thin_async? 48 | call_thin_async(env) 49 | elsif simulate_hijack? 50 | call_rack_hijack(env) 51 | else 52 | @app.call(env) 53 | end 54 | end 55 | 56 | def translate_io_result(io) 57 | data = io.string 58 | body = +"" 59 | 60 | parser = Http::Parser.new 61 | parser.on_body = proc { |chunk| body << chunk } 62 | parser << data 63 | 64 | [parser.status_code, parser.headers, [body]] 65 | end 66 | 67 | def call_rack_hijack(env) 68 | # this is not to spec, the spec actually return, but here we will simply simulate and block 69 | result = nil 70 | hijacked = false 71 | io = nil 72 | 73 | EM.run { 74 | env['rack.hijack'] = lambda { 75 | hijacked = true 76 | io = StringIO.new 77 | } 78 | 79 | env['rack.hijack_io'] = io 80 | 81 | result = @app.call(env) 82 | 83 | EM::Timer.new(1) { EM.stop } 84 | 85 | defer = lambda { 86 | if !io || !io.closed? 87 | @in_async = true 88 | EM.next_tick do 89 | defer.call 90 | end 91 | else 92 | if io.closed? 93 | result = translate_io_result(io) 94 | end 95 | EM.next_tick { EM.stop } 96 | end 97 | } 98 | 99 | if !hijacked 100 | EM.next_tick { EM.stop } 101 | else 102 | defer.call 103 | end 104 | } 105 | 106 | @in_async = false 107 | result || [500, {}, ['timeout']] 108 | end 109 | 110 | def call_thin_async(env) 111 | result = nil 112 | EM.run { 113 | env['async.callback'] = lambda { |r| 114 | # more judo with deferrable body, at this point we just have headers 115 | r[2].callback do 116 | # even more judo cause rack test does not call each like the spec says 117 | body = +"" 118 | r[2].each do |m| 119 | body << m 120 | end 121 | r[2] = [body] 122 | result = r 123 | end 124 | } 125 | catch(:async) { 126 | result = @app.call(env) 127 | } 128 | 129 | EM::Timer.new(1) { EM.stop } 130 | 131 | defer = lambda { 132 | if !result 133 | @in_async = true 134 | EM.next_tick do 135 | defer.call 136 | end 137 | else 138 | EM.next_tick { EM.stop } 139 | end 140 | } 141 | defer.call 142 | } 143 | 144 | @in_async = false 145 | result || [500, {}, ['timeout']] 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /spec/lib/message_bus/assets/asset_encoding_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative '../../../spec_helper' 3 | asset_directory = File.expand_path('../../../../../assets', __FILE__) 4 | asset_file_paths = Dir.glob(File.join(asset_directory, 'message-bus.js')) 5 | asset_file_names = asset_file_paths.map { |e| File.basename(e) } 6 | 7 | describe asset_file_names do 8 | it 'should contain .js files' do 9 | asset_file_names.must_include('message-bus.js') 10 | end 11 | end 12 | 13 | asset_file_paths.each do |path| 14 | describe "Asset file #{File.basename(path).inspect}" do 15 | it 'should be encodable as UTF8' do 16 | binary_data = File.open(path, 'rb') { |f| f.read } 17 | binary_data.encode(Encoding::UTF_8) 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/lib/message_bus/backend_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../spec_helper' 4 | require 'message_bus' 5 | 6 | describe BACKEND_CLASS do 7 | before do 8 | @bus = BACKEND_CLASS.new(test_config_for_backend(CURRENT_BACKEND)) 9 | end 10 | 11 | after do 12 | @bus.reset! 13 | @bus.destroy 14 | end 15 | 16 | describe "API parity" do 17 | it "has the same public methods as the base class" do 18 | @bus.public_methods.sort.must_equal MessageBus::Backends::Base.new(test_config_for_backend(CURRENT_BACKEND)).public_methods.sort 19 | end 20 | end 21 | 22 | it "should be able to access the backlog" do 23 | @bus.publish "/foo", "bar" 24 | @bus.publish "/foo", "baz" 25 | 26 | @bus.backlog("/foo", 0).to_a.must_equal [ 27 | MessageBus::Message.new(1, 1, '/foo', 'bar'), 28 | MessageBus::Message.new(2, 2, '/foo', 'baz') 29 | ] 30 | end 31 | 32 | it "should initialize with max_backlog_size" do 33 | BACKEND_CLASS.new({}, 2000).max_backlog_size.must_equal 2000 34 | end 35 | 36 | it "should truncate channels correctly" do 37 | @bus.max_backlog_size = 2 38 | [ 39 | "one", 40 | "two", 41 | "three", 42 | "four", 43 | ].each do |t| 44 | @bus.publish "/foo", t 45 | end 46 | 47 | @bus.backlog("/foo").to_a.must_equal [ 48 | MessageBus::Message.new(3, 3, '/foo', 'three'), 49 | MessageBus::Message.new(4, 4, '/foo', 'four'), 50 | ] 51 | end 52 | 53 | it "should truncate global backlog correctly" do 54 | @bus.max_global_backlog_size = 2 55 | @bus.publish "/foo", "one" 56 | @bus.publish "/bar", "two" 57 | @bus.publish "/baz", "three" 58 | 59 | @bus.global_backlog.length.must_equal 2 60 | end 61 | 62 | it "should be able to grab a message by id" do 63 | id1 = @bus.publish "/foo", "bar" 64 | id2 = @bus.publish "/foo", "baz" 65 | @bus.get_message("/foo", id2).must_equal MessageBus::Message.new(2, 2, "/foo", "baz") 66 | @bus.get_message("/foo", id1).must_equal MessageBus::Message.new(1, 1, "/foo", "bar") 67 | end 68 | 69 | it "should have the correct number of messages for multi threaded access" do 70 | threads = [] 71 | 4.times do 72 | threads << Thread.new do 73 | 25.times { 74 | @bus.publish "/foo", "foo" 75 | } 76 | end 77 | end 78 | 79 | threads.each(&:join) 80 | @bus.backlog("/foo").length.must_equal 100 81 | end 82 | 83 | it "should be able to encode and decode messages properly" do 84 | m = MessageBus::Message.new 1, 2, '||', '||' 85 | MessageBus::Message.decode(m.encode).must_equal m 86 | end 87 | 88 | it "should allow us to get last id on a channel" do 89 | @bus.last_id("/foo").must_equal 0 90 | @bus.publish("/foo", "one") 91 | @bus.last_id("/foo").must_equal 1 92 | end 93 | 94 | it "should allow us to get multiple last_ids" do 95 | @bus.last_ids("/foo", "/bar", "/foobar").must_equal [0, 0, 0] 96 | 97 | @bus.publish("/foo", "one") 98 | @bus.publish("/foo", "two") 99 | @bus.publish("/foobar", "three") 100 | 101 | @bus.last_ids("/foo", "/bar", "/foobar").must_equal( 102 | [ 103 | @bus.last_id("/foo"), 104 | @bus.last_id("/bar"), 105 | @bus.last_id("/foobar") 106 | ] 107 | ) 108 | end 109 | 110 | it "can set backlog age" do 111 | @bus.max_backlog_age = 1 112 | 113 | expected_backlog_size = 0 114 | 115 | # Start at time = 0s 116 | @bus.publish "/foo", "bar" 117 | expected_backlog_size += 1 118 | 119 | @bus.global_backlog.length.must_equal expected_backlog_size 120 | @bus.backlog("/foo", 0).length.must_equal expected_backlog_size 121 | 122 | sleep 1.25 # Should now be at time =~ 1.25s. Our backlog should have expired by now. 123 | expected_backlog_size = 0 124 | 125 | case CURRENT_BACKEND 126 | when :postgres 127 | # Force triggering backlog expiry: postgres backend doesn't expire backlogs on a timer, but at publication time. 128 | @bus.global_backlog.length.wont_equal expected_backlog_size 129 | @bus.backlog("/foo", 0).length.wont_equal expected_backlog_size 130 | @bus.publish "/foo", "baz" 131 | expected_backlog_size += 1 132 | end 133 | 134 | # Assert that the backlog did expire, and now has only the new publication in it. 135 | @bus.global_backlog.length.must_equal expected_backlog_size 136 | @bus.backlog("/foo", 0).length.must_equal expected_backlog_size 137 | 138 | sleep 0.75 # Should now be at time =~ 2s 139 | 140 | @bus.publish "/foo", "baz" # Publish something else before another expiry 141 | expected_backlog_size += 1 142 | 143 | sleep 0.75 # Should now be at time =~ 2.75s 144 | # Our oldest message is now 1.5s old, but we didn't cease publishing for a period of 1s at a time, so we should not have expired the backlog. 145 | 146 | @bus.publish "/foo", "baz" # Publish something else to ward off another expiry 147 | expected_backlog_size += 1 148 | 149 | case CURRENT_BACKEND 150 | when :postgres 151 | # Postgres expires individual messages that have lived longer than the TTL, not whole backlogs 152 | expected_backlog_size -= 1 153 | else 154 | # Assert that the backlog did not expire, and has all of our publications since the last expiry. 155 | end 156 | @bus.global_backlog.length.must_equal expected_backlog_size 157 | @bus.backlog("/foo", 0).length.must_equal expected_backlog_size 158 | end 159 | 160 | it "can set backlog age on publish" do 161 | @bus.max_backlog_age = 100 162 | 163 | expected_backlog_size = 0 164 | 165 | initial_id = @bus.last_id("/foo") 166 | 167 | # Start at time = 0s 168 | @bus.publish "/foo", "bar", max_backlog_age: 1 169 | expected_backlog_size += 1 170 | 171 | @bus.global_backlog.length.must_equal expected_backlog_size 172 | @bus.backlog("/foo", 0).length.must_equal expected_backlog_size 173 | 174 | sleep 1.25 # Should now be at time =~ 1.25s. Our backlog should have expired by now. 175 | expected_backlog_size = 0 176 | 177 | case CURRENT_BACKEND 178 | when :postgres 179 | # Force triggering backlog expiry: postgres backend doesn't expire backlogs on a timer, but at publication time. 180 | @bus.global_backlog.length.wont_equal expected_backlog_size 181 | @bus.backlog("/foo", 0).length.wont_equal expected_backlog_size 182 | @bus.publish "/foo", "baz", max_backlog_age: 1 183 | expected_backlog_size += 1 184 | end 185 | 186 | # Assert that the backlog did expire, and now has only the new publication in it. 187 | @bus.global_backlog.length.must_equal expected_backlog_size 188 | @bus.backlog("/foo", 0).length.must_equal expected_backlog_size 189 | 190 | # for the time being we can give pg a pass here 191 | # TODO: make the implementation here consistent 192 | if CURRENT_BACKEND != :postgres 193 | # ids are not opaque we expect them to be reset on our channel if it 194 | # got cleared due to an expire, the reason for this is cause we will leak entries due to tracking 195 | # this in turn can bloat storage for the backend 196 | @bus.last_id("/foo").must_equal initial_id 197 | end 198 | 199 | sleep 0.75 # Should now be at time =~ 2s 200 | 201 | @bus.publish "/foo", "baz", max_backlog_age: 1 # Publish something else before another expiry 202 | expected_backlog_size += 1 203 | 204 | sleep 0.75 # Should now be at time =~ 2.75s 205 | # Our oldest message is now 1.5s old, but we didn't cease publishing for a period of 1s at a time, so we should not have expired the backlog. 206 | 207 | @bus.publish "/foo", "baz", max_backlog_age: 1 # Publish something else to ward off another expiry 208 | expected_backlog_size += 1 209 | 210 | case CURRENT_BACKEND 211 | when :postgres 212 | # Postgres expires individual messages that have lived longer than the TTL, not whole backlogs 213 | expected_backlog_size -= 1 214 | else 215 | # Assert that the backlog did not expire, and has all of our publications since the last expiry. 216 | end 217 | @bus.global_backlog.length.must_equal expected_backlog_size 218 | @bus.backlog("/foo", 0).length.must_equal expected_backlog_size 219 | end 220 | 221 | it "can set backlog size on publish" do 222 | @bus.max_backlog_size = 100 223 | 224 | @bus.publish "/foo", "bar", max_backlog_size: 2 225 | @bus.publish "/foo", "bar", max_backlog_size: 2 226 | @bus.publish "/foo", "bar", max_backlog_size: 2 227 | 228 | @bus.backlog("/foo").length.must_equal 2 229 | end 230 | 231 | it "should be able to access the global backlog" do 232 | @bus.publish "/foo", "bar" 233 | @bus.publish "/hello", "world" 234 | @bus.publish "/foo", "baz" 235 | @bus.publish "/hello", "planet" 236 | 237 | expected_messages = case CURRENT_BACKEND 238 | when :redis 239 | # Redis has channel-specific message IDs 240 | [ 241 | MessageBus::Message.new(1, 1, "/foo", "bar"), 242 | MessageBus::Message.new(2, 1, "/hello", "world"), 243 | MessageBus::Message.new(3, 2, "/foo", "baz"), 244 | MessageBus::Message.new(4, 2, "/hello", "planet") 245 | ] 246 | else 247 | [ 248 | MessageBus::Message.new(1, 1, "/foo", "bar"), 249 | MessageBus::Message.new(2, 2, "/hello", "world"), 250 | MessageBus::Message.new(3, 3, "/foo", "baz"), 251 | MessageBus::Message.new(4, 4, "/hello", "planet") 252 | ] 253 | end 254 | 255 | @bus.global_backlog.to_a.must_equal expected_messages 256 | end 257 | 258 | it "should correctly omit dropped messages from the global backlog" do 259 | @bus.max_backlog_size = 1 260 | @bus.publish "/foo", "a1" 261 | @bus.publish "/foo", "b1" 262 | @bus.publish "/bar", "a1" 263 | @bus.publish "/bar", "b1" 264 | 265 | expected_messages = case CURRENT_BACKEND 266 | when :redis 267 | # Redis has channel-specific message IDs 268 | [ 269 | MessageBus::Message.new(2, 2, "/foo", "b1"), 270 | MessageBus::Message.new(4, 2, "/bar", "b1") 271 | ] 272 | else 273 | [ 274 | MessageBus::Message.new(2, 2, "/foo", "b1"), 275 | MessageBus::Message.new(4, 4, "/bar", "b1") 276 | ] 277 | end 278 | 279 | @bus.global_backlog.to_a.must_equal expected_messages 280 | end 281 | 282 | it "should cope with a storage reset cleanly" do 283 | @bus.publish("/foo", "one") 284 | got = [] 285 | 286 | t = Thread.new do 287 | @bus.subscribe("/foo") do |msg| 288 | got << msg 289 | end 290 | end 291 | 292 | # sleep 50ms to allow the bus to correctly subscribe, 293 | # I thought about adding a subscribed callback, but outside of testing it matters less 294 | sleep 0.05 295 | 296 | @bus.publish("/foo", "two") 297 | 298 | @bus.reset! 299 | 300 | @bus.publish("/foo", "three") 301 | 302 | wait_for(100) do 303 | got.length == 2 304 | end 305 | 306 | t.kill 307 | 308 | got.map { |m| m.data }.must_equal ["two", "three"] 309 | got[1].global_id.must_equal 1 310 | end 311 | 312 | it "should support clear_every setting" do 313 | @bus.clear_every = 5 314 | @bus.max_global_backlog_size = 2 315 | @bus.publish "/foo", "11" 316 | @bus.publish "/bar", "21" 317 | @bus.publish "/baz", "31" 318 | @bus.publish "/bar", "41" 319 | @bus.global_backlog.length.must_equal 4 320 | 321 | @bus.publish "/baz", "51" 322 | @bus.global_backlog.length.must_equal 2 323 | end 324 | 325 | it "should be able to subscribe globally with recovery" do 326 | @bus.publish("/foo", "11") 327 | @bus.publish("/bar", "12") 328 | got = [] 329 | 330 | t = Thread.new do 331 | @bus.global_subscribe(0) do |msg| 332 | got << msg 333 | end 334 | end 335 | 336 | @bus.publish("/bar", "13") 337 | 338 | wait_for(100) do 339 | got.length == 3 340 | end 341 | 342 | t.kill 343 | 344 | got.length.must_equal 3 345 | got.map { |m| m.data }.must_equal ["11", "12", "13"] 346 | end 347 | 348 | it "should handle subscribe on single channel, with recovery" do 349 | @bus.publish("/foo", "11") 350 | @bus.publish("/bar", "12") 351 | got = [] 352 | 353 | t = Thread.new do 354 | @bus.subscribe("/foo", 0) do |msg| 355 | got << msg 356 | end 357 | end 358 | 359 | @bus.publish("/foo", "13") 360 | 361 | wait_for(100) do 362 | got.length == 2 363 | end 364 | 365 | t.kill 366 | 367 | got.map { |m| m.data }.must_equal ["11", "13"] 368 | end 369 | 370 | it "should not get backlog if subscribe is called without params" do 371 | @bus.publish("/foo", "11") 372 | got = [] 373 | 374 | t = Thread.new do 375 | @bus.subscribe("/foo") do |msg| 376 | got << msg 377 | end 378 | end 379 | 380 | # sleep 50ms to allow the bus to correctly subscribe, 381 | # I thought about adding a subscribed callback, but outside of testing it matters less 382 | sleep 0.05 383 | 384 | @bus.publish("/foo", "12") 385 | 386 | wait_for(100) do 387 | got.length == 1 388 | end 389 | 390 | t.kill 391 | 392 | got.map { |m| m.data }.must_equal ["12"] 393 | end 394 | 395 | end 396 | -------------------------------------------------------------------------------- /spec/lib/message_bus/client_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../spec_helper' 4 | require 'message_bus' 5 | 6 | describe MessageBus::Client do 7 | describe "subscriptions" do 8 | def setup_client(client_id) 9 | MessageBus::Client.new client_id: client_id, message_bus: @bus 10 | end 11 | 12 | before do 13 | @bus = MessageBus::Instance.new 14 | @bus.configure(test_config_for_backend(CURRENT_BACKEND)) 15 | @client = setup_client('abc') 16 | end 17 | 18 | after do 19 | @client.close 20 | @bus.reset! 21 | @bus.destroy 22 | end 23 | 24 | def http_parse(message) 25 | lines = message.split("\r\n") 26 | 27 | status = lines.shift.split(" ")[1] 28 | headers = {} 29 | chunks = [] 30 | 31 | while line = lines.shift 32 | break if line == "" 33 | 34 | name, val = line.split(": ") 35 | headers[name] = val 36 | end 37 | 38 | while line = lines.shift 39 | length = line.to_i(16) 40 | break if length == 0 41 | 42 | rest = lines.join("\r\n") 43 | chunks << rest[0...length] 44 | lines = (rest[length + 2..-1] || "").split("\r\n") 45 | end 46 | 47 | # split/join gets tricky 48 | chunks[-1] << "\r\n" 49 | 50 | [status, headers, chunks] 51 | end 52 | 53 | def parse_chunk(data) 54 | payload, _ = data.split(/\r\n\|\r\n/m) 55 | JSON.parse(payload) 56 | end 57 | 58 | it "can chunk replies" do 59 | @client.use_chunked = true 60 | r, w = IO.pipe 61 | @client.io = w 62 | @client.headers = { "Content-Type" => "application/json; charset=utf-8" } 63 | @client << MessageBus::Message.new(1, 1, '/test', 'test') 64 | @client << MessageBus::Message.new(2, 2, '/test', "a|\r\n|\r\n|b") 65 | 66 | lines = r.read_nonblock(8000) 67 | 68 | status, headers, chunks = http_parse(lines) 69 | 70 | headers["Content-Type"].must_equal "text/plain; charset=utf-8" 71 | status.must_equal "200" 72 | chunks.length.must_equal 2 73 | 74 | chunk1 = parse_chunk(chunks[0]) 75 | chunk1.length.must_equal 1 76 | chunk1.first["data"].must_equal 'test' 77 | 78 | chunk2 = parse_chunk(chunks[1]) 79 | chunk2.length.must_equal 1 80 | chunk2.first["data"].must_equal "a|\r\n|\r\n|b" 81 | 82 | @client << MessageBus::Message.new(3, 3, '/test', 'test3') 83 | @client.close 84 | 85 | data = r.read 86 | 87 | data[-5..-1].must_equal "0\r\n\r\n" 88 | 89 | _, _, chunks = http_parse(+"HTTP/1.1 200 OK\r\n\r\n" << data) 90 | 91 | chunks.length.must_equal 2 92 | 93 | chunk1 = parse_chunk(chunks[0]) 94 | chunk1.length.must_equal 1 95 | chunk1.first["data"].must_equal 'test3' 96 | 97 | # end with [] 98 | chunk2 = parse_chunk(chunks[1]) 99 | chunk2.length.must_equal 0 100 | end 101 | 102 | it "does not raise an error when trying to write a message to a closed client using chunked encoding" do 103 | @client.use_chunked = true 104 | assert(@client.closed?) 105 | @client << MessageBus::Message.new(1, 1, "/test", "test") 106 | end 107 | 108 | it "does not bleed data across sites" do 109 | @client.site_id = "test" 110 | 111 | @client.subscribe('/hello', nil) 112 | @bus.publish '/hello', 'world' 113 | log = @client.backlog 114 | log.length.must_equal 0 115 | end 116 | 117 | it "does not bleed status across sites" do 118 | @client.site_id = "test" 119 | 120 | @client.subscribe('/hello', -1) 121 | @bus.publish '/hello', 'world' 122 | log = @client.backlog 123 | log[0].data.must_equal("/hello" => 0) 124 | end 125 | 126 | it "allows negative subscribes to look behind" do 127 | @bus.publish '/hello', 'world' 128 | @bus.publish '/hello', 'sam' 129 | 130 | @client.subscribe('/hello', -2) 131 | 132 | log = @client.backlog 133 | log.length.must_equal(1) 134 | log[0].data.must_equal("sam") 135 | end 136 | 137 | it "provides status" do 138 | @client.subscribe('/hello', -1) 139 | log = @client.backlog 140 | log.length.must_equal 1 141 | log[0].data.must_equal("/hello" => 0) 142 | end 143 | 144 | it 'provides status updates to clients that are not allowed to a message' do 145 | another_client = setup_client('def') 146 | clients = [@client, another_client] 147 | 148 | channel = SecureRandom.hex 149 | 150 | clients.each { |client| client.subscribe(channel, nil) } 151 | 152 | @bus.publish(channel, "world", client_ids: ['abc']) 153 | 154 | log = @client.backlog 155 | log.length.must_equal 1 156 | log[0].channel.must_equal channel 157 | log[0].data.must_equal 'world' 158 | 159 | log = another_client.backlog 160 | log.length.must_equal 1 161 | log[0].channel.must_equal '/__status' 162 | log[0].data.must_equal(channel => 1) 163 | end 164 | 165 | it "should provide a list of subscriptions" do 166 | @client.subscribe('/hello', nil) 167 | @client.subscriptions['/hello'].wont_equal nil 168 | end 169 | 170 | it "should provide backlog for subscribed channel" do 171 | @client.subscribe('/hello', nil) 172 | @bus.publish '/hello', 'world' 173 | log = @client.backlog 174 | log.length.must_equal 1 175 | log[0].channel.must_equal '/hello' 176 | log[0].data.must_equal 'world' 177 | end 178 | 179 | describe '#allowed?' do 180 | it "allows only client_id in list if message contains client_ids" do 181 | @message = MessageBus::Message.new(1, 2, '/test', 'hello') 182 | @message.client_ids = ["1", "2"] 183 | @client.client_id = "2" 184 | @client.allowed?(@message).must_equal true 185 | 186 | @client.client_id = "3" 187 | @client.allowed?(@message).must_equal false 188 | 189 | @message.client_ids = [] 190 | 191 | @client.client_id = "3" 192 | @client.allowed?(@message).must_equal true 193 | 194 | @message.client_ids = nil 195 | 196 | @client.client_id = "3" 197 | @client.allowed?(@message).must_equal true 198 | end 199 | 200 | describe 'targeted at user' do 201 | before do 202 | @message = MessageBus::Message.new(1, 2, '/test', 'hello') 203 | @message.user_ids = [1, 2, 3] 204 | end 205 | 206 | it "allows client with user_id that is included in message's user_ids" do 207 | @client.user_id = 1 208 | @client.allowed?(@message).must_equal(true) 209 | end 210 | 211 | it "denies client with user_id that is not included in message's user_ids" do 212 | @client.user_id = 4 213 | @client.allowed?(@message).must_equal(false) 214 | end 215 | 216 | it "denies client with nil user_id" do 217 | @client.user_id = nil 218 | 219 | @client.allowed?(@message).must_equal(false) 220 | end 221 | 222 | it "allows client if message's user_ids is not set" do 223 | @message.user_ids = nil 224 | @client.user_id = 4 225 | @client.allowed?(@message).must_equal(true) 226 | end 227 | 228 | it "allows client if message's user_ids is empty" do 229 | @message.user_ids = [] 230 | @client.user_id = 4 231 | @client.allowed?(@message).must_equal(true) 232 | end 233 | 234 | it "allows client with client_id that is included in message's client_ids" do 235 | @message.client_ids = ["1", "2"] 236 | @client.client_id = "1" 237 | @client.user_id = 1 238 | 239 | @client.allowed?(@message).must_equal(true) 240 | end 241 | 242 | it "denies client with client_id that is not included in message's client_ids" do 243 | @message.client_ids = ["1", "2"] 244 | @client.client_id = "3" 245 | @client.user_id = 1 246 | 247 | @client.allowed?(@message).must_equal(false) 248 | end 249 | end 250 | 251 | describe "targeted at group" do 252 | before do 253 | @message = MessageBus::Message.new(1, 2, '/test', 'hello') 254 | @message.group_ids = [1, 2, 3] 255 | end 256 | 257 | it "denies client that are not members of group" do 258 | @client.group_ids = [77, 0, 10] 259 | @client.allowed?(@message).must_equal false 260 | end 261 | 262 | it 'denies client with nil group_ids' do 263 | @client.group_ids = nil 264 | @client.allowed?(@message).must_equal false 265 | end 266 | 267 | it "allows client that are members of group" do 268 | @client.group_ids = [1, 2, 3] 269 | @client.allowed?(@message).must_equal true 270 | end 271 | 272 | it "allows any client if message's group_ids is not set" do 273 | @message.group_ids = nil 274 | @client.group_ids = [77, 0, 10] 275 | @client.allowed?(@message).must_equal true 276 | end 277 | 278 | it "allows any client if message's group_ids is empty" do 279 | @message.group_ids = [] 280 | @client.group_ids = [77, 0, 10] 281 | @client.allowed?(@message).must_equal true 282 | end 283 | 284 | it "allows client with client_id that is included in message's client_ids" do 285 | @message.client_ids = ["1", "2"] 286 | @client.client_id = "1" 287 | @client.group_ids = [1] 288 | 289 | @client.allowed?(@message).must_equal(true) 290 | end 291 | 292 | it "denies client with client_id that is not included in message's client_ids" do 293 | @message.client_ids = ["1", "2"] 294 | @client.client_id = "3" 295 | @client.group_ids = [1] 296 | 297 | @client.allowed?(@message).must_equal(false) 298 | end 299 | end 300 | 301 | describe 'targeted at group and user' do 302 | before do 303 | @message = MessageBus::Message.new(1, 2, '/test', 'hello') 304 | @message.group_ids = [1, 2, 3] 305 | @message.user_ids = [4, 5, 6] 306 | end 307 | 308 | it "allows client with user_id that is included in message's user_ids" do 309 | @client.user_id = 4 310 | @client.allowed?(@message).must_equal(true) 311 | end 312 | 313 | it "denies client with user_id that is not included in message's user_ids" do 314 | @client.user_id = 1 315 | @client.allowed?(@message).must_equal(false) 316 | end 317 | 318 | it "allows client with group_ids that is included in message's group_ids" do 319 | @client.group_ids = [1, 0, 3] 320 | @client.allowed?(@message).must_equal(true) 321 | end 322 | 323 | it "denies client with group_ids that is not included in message's group_ids" do 324 | @client.group_ids = [8, 9, 10] 325 | @client.allowed?(@message).must_equal(false) 326 | end 327 | 328 | it "allows client with allowed client_id and user_id" do 329 | @message.client_ids = ["1", "2"] 330 | @client.user_id = 4 331 | @client.client_id = "2" 332 | 333 | @client.allowed?(@message).must_equal(true) 334 | end 335 | 336 | it "denies client with allowed client_id but disallowed user_id" do 337 | @message.client_ids = ["1", "2"] 338 | @client.user_id = 99 339 | @client.client_id = "2" 340 | 341 | @client.allowed?(@message).must_equal(false) 342 | end 343 | 344 | it "allows client with allowed client_id and group_id" do 345 | @message.client_ids = ["1", "2"] 346 | @client.group_ids = [1] 347 | @client.client_id = "2" 348 | 349 | @client.allowed?(@message).must_equal(true) 350 | end 351 | 352 | it "denies client with allowed client_id but disallowed group_id" do 353 | @message.client_ids = ["1", "2"] 354 | @client.group_ids = [99] 355 | @client.client_id = "2" 356 | 357 | @client.allowed?(@message).must_equal(false) 358 | end 359 | end 360 | 361 | describe 'when MessageBus#client_message_filters has been configured' do 362 | 363 | it 'filters messages correctly' do 364 | message = MessageBus::Message.new(1, 2, '/test/5', 'hello') 365 | @client.allowed?(message).must_equal(true) 366 | 367 | @bus.register_client_message_filter('/test') do |m| 368 | m.data != 'hello' 369 | end 370 | 371 | @client.allowed?(message).must_equal(false) 372 | end 373 | 374 | it 'filters messages correctly when multiple filters have been configured' do 375 | 376 | bob_message = MessageBus::Message.new(1, 2, '/test/5', 'bob') 377 | fred_message = MessageBus::Message.new(1, 2, '/test/5', 'fred') 378 | random_message = MessageBus::Message.new(1, 2, '/test/77', 'random') 379 | 380 | @bus.register_client_message_filter('/test') do |message| 381 | message.data == 'bob' || message.data == 'fred' 382 | end 383 | 384 | @bus.register_client_message_filter('/test') do |message| 385 | message.data == 'fred' 386 | end 387 | 388 | @client.allowed?(fred_message).must_equal(true) 389 | @client.allowed?(bob_message).must_equal(false) 390 | @client.allowed?(random_message).must_equal(false) 391 | end 392 | end 393 | end 394 | end 395 | end 396 | -------------------------------------------------------------------------------- /spec/lib/message_bus/connection_manager_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../spec_helper' 4 | require 'message_bus' 5 | 6 | class FakeAsync 7 | attr_accessor :cleanup_timer 8 | 9 | def initialize 10 | @sent = nil 11 | end 12 | 13 | def <<(val) 14 | sleep 0.01 # simulate IO 15 | @sent ||= +"" 16 | @sent << val 17 | end 18 | 19 | def sent 20 | @sent 21 | end 22 | 23 | def done 24 | @done = true 25 | end 26 | 27 | def done? 28 | @done 29 | end 30 | end 31 | 32 | class FakeTimer 33 | attr_accessor :cancelled 34 | def cancel 35 | @cancelled = true 36 | end 37 | end 38 | 39 | describe MessageBus::ConnectionManager do 40 | before do 41 | @bus = MessageBus 42 | @manager = MessageBus::ConnectionManager.new(@bus) 43 | @client = MessageBus::Client.new(client_id: "xyz", user_id: 1, site_id: 10) 44 | @resp = FakeAsync.new 45 | @client.async_response = @resp 46 | @client.subscribe('test', -1) 47 | @manager.add_client(@client) 48 | @client.cleanup_timer = FakeTimer.new 49 | end 50 | 51 | it "should cancel the timer after its responds" do 52 | m = MessageBus::Message.new(1, 1, "test", "data") 53 | m.site_id = 10 54 | @manager.notify_clients(m) 55 | @client.cleanup_timer.cancelled.must_equal true 56 | end 57 | 58 | it "should be able to lookup an identical client" do 59 | @manager.lookup_client(@client.client_id).must_equal @client 60 | end 61 | 62 | it "should not notify clients on incorrect site" do 63 | m = MessageBus::Message.new(1, 1, "test", "data") 64 | m.site_id = 9 65 | @manager.notify_clients(m) 66 | assert_nil @resp.sent 67 | end 68 | 69 | it "should notify clients on the correct site" do 70 | m = MessageBus::Message.new(1, 1, "test", "data") 71 | m.site_id = 10 72 | @manager.notify_clients(m) 73 | @resp.sent.wont_equal nil 74 | end 75 | 76 | it "should strip site id and user id from the payload delivered" do 77 | m = MessageBus::Message.new(1, 1, "test", "data") 78 | m.user_ids = [1] 79 | m.site_id = 10 80 | @manager.notify_clients(m) 81 | parsed = JSON.parse(@resp.sent) 82 | assert_nil parsed[0]["site_id"] 83 | assert_nil parsed[0]["user_id"] 84 | end 85 | 86 | it "should not deliver unselected" do 87 | m = MessageBus::Message.new(1, 1, "test", "data") 88 | m.user_ids = [5] 89 | m.site_id = 10 90 | @manager.notify_clients(m) 91 | assert_nil @resp.sent 92 | end 93 | end 94 | 95 | describe MessageBus::ConnectionManager, "notifying and subscribing concurrently" do 96 | it "does not subscribe incorrect clients" do 97 | manager = MessageBus::ConnectionManager.new 98 | 99 | client1 = MessageBus::Client.new(client_id: "a", seq: 1) 100 | client2 = MessageBus::Client.new(client_id: "a", seq: 2) 101 | 102 | manager.add_client(client2) 103 | manager.add_client(client1) 104 | 105 | manager.lookup_client("a").must_equal client2 106 | end 107 | 108 | it "is thread-safe" do 109 | @bus = MessageBus 110 | @manager = MessageBus::ConnectionManager.new(@bus) 111 | 112 | client_threads = 10.times.map do |id| 113 | Thread.new do 114 | @client = MessageBus::Client.new(client_id: "xyz_#{id}", site_id: 10) 115 | @resp = FakeAsync.new 116 | @client.async_response = @resp 117 | @client.subscribe("test", -1) 118 | @manager.add_client(@client) 119 | @client.cleanup_timer = FakeTimer.new 120 | 1 121 | end 122 | end 123 | 124 | subscriber_threads = 10.times.map do |id| 125 | Thread.new do 126 | m = MessageBus::Message.new(1, id, "test", "data_#{id}") 127 | m.site_id = 10 128 | @manager.notify_clients(m) 129 | 1 130 | end 131 | end 132 | 133 | client_threads.each(&:join).map(&:value).must_equal([1] * 10) 134 | subscriber_threads.each(&:join).map(&:value).must_equal([1] * 10) 135 | end 136 | end 137 | -------------------------------------------------------------------------------- /spec/lib/message_bus/distributed_cache_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../spec_helper' 4 | require 'minitest/hooks/default' 5 | require 'message_bus' 6 | require 'message_bus/distributed_cache' 7 | 8 | describe MessageBus::DistributedCache do 9 | before do 10 | @bus = MessageBus::Instance.new 11 | @bus.configure(backend: :memory) 12 | @manager = MessageBus::DistributedCache::Manager.new(@bus) 13 | @cache1 = cache(cache_name) 14 | @cache2 = cache(cache_name) 15 | end 16 | 17 | after do 18 | @bus.reset! 19 | @bus.destroy 20 | end 21 | 22 | def cache(name) 23 | MessageBus::DistributedCache.new(name, manager: @manager) 24 | end 25 | 26 | let :cache_name do 27 | SecureRandom.hex 28 | end 29 | 30 | it 'supports arrays with hashes' do 31 | c1 = cache("test1") 32 | c2 = cache("test1") 33 | 34 | c1["test"] = [{ test: :test }] 35 | 36 | wait_for do 37 | c2["test"] == [{ test: :test }] 38 | end 39 | 40 | expect(c2[:test]).must_equal([{ test: :test }]) 41 | end 42 | 43 | it 'allows us to store Set' do 44 | c1 = cache("test1") 45 | c2 = cache("test1") 46 | 47 | set = Set.new 48 | set << 1 49 | set << "b" 50 | set << 92803984 51 | set << 93739739873973 52 | 53 | c1["cats"] = set 54 | 55 | wait_for do 56 | c2["cats"] == set 57 | end 58 | 59 | expect(c2["cats"]).must_equal(set) 60 | 61 | set << 5 62 | 63 | c2["cats"] = set 64 | 65 | wait_for do 66 | c1["cats"] == set 67 | end 68 | 69 | expect(c1["cats"]).must_equal(set) 70 | end 71 | 72 | it 'does not leak state across caches' do 73 | c2 = cache("test1") 74 | c3 = cache("test1") 75 | c2["hi"] = "hi" 76 | wait_for do 77 | c3["hi"] == "hi" 78 | end 79 | 80 | Thread.pass 81 | assert_nil(@cache1["hi"]) 82 | end 83 | 84 | it 'allows coerces symbol keys to strings' do 85 | @cache1[:key] = "test" 86 | expect(@cache1["key"]).must_equal("test") 87 | 88 | wait_for do 89 | @cache2[:key] == "test" 90 | end 91 | expect(@cache2["key"]).must_equal("test") 92 | end 93 | 94 | it 'sets other caches' do 95 | @cache1["test"] = "world" 96 | wait_for do 97 | @cache2["test"] == "world" 98 | end 99 | end 100 | 101 | it 'deletes from other caches' do 102 | @cache1["foo"] = "bar" 103 | 104 | wait_for do 105 | @cache2["foo"] == "bar" 106 | end 107 | 108 | @cache1.delete("foo") 109 | assert_nil(@cache1["foo"]) 110 | 111 | wait_for do 112 | @cache2["foo"] == nil 113 | end 114 | end 115 | 116 | it 'clears cache on request' do 117 | @cache1["foo"] = "bar" 118 | 119 | wait_for do 120 | @cache2["foo"] == "bar" 121 | end 122 | 123 | @cache1.clear 124 | assert_nil(@cache1["foo"]) 125 | wait_for do 126 | @cache2["boom"] == nil 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /spec/lib/message_bus/multi_process_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../spec_helper' 4 | require 'message_bus' 5 | 6 | describe BACKEND_CLASS do 7 | def self.error! 8 | @error = true 9 | end 10 | 11 | def self.error? 12 | defined?(@error) 13 | end 14 | 15 | def new_bus 16 | BACKEND_CLASS.new(test_config_for_backend(CURRENT_BACKEND).merge(db: 10)) 17 | end 18 | 19 | def work_it 20 | bus = new_bus 21 | bus.subscribe("/echo", 0) do |msg| 22 | if msg.data == "done" 23 | bus.global_unsubscribe 24 | else 25 | bus.publish("/response", "#{msg.data}-#{Process.pid}") 26 | end 27 | end 28 | ensure 29 | bus.destroy 30 | exit!(0) 31 | end 32 | 33 | def spawn_child 34 | r = fork 35 | if r.nil? 36 | work_it 37 | else 38 | r 39 | end 40 | end 41 | 42 | n = ENV['MULTI_PROCESS_TIMES'].to_i 43 | n = 1 if n < 1 44 | n.times do 45 | it 'gets every response from child processes' do 46 | test_never :memory 47 | skip("previous error") if self.class.error? 48 | GC.start 49 | bus = new_bus 50 | bus.reset! 51 | 52 | begin 53 | pids = (1..10).map { spawn_child } 54 | expected_responses = pids.map { |x| (0...10).map { |i| "0#{i}-#{x}" } }.flatten 55 | unexpected_responses = [] 56 | 57 | t = Thread.new do 58 | bus.subscribe("/response", 0) do |msg| 59 | if expected_responses.include?(msg.data) 60 | expected_responses.delete(msg.data) 61 | else 62 | unexpected_responses << msg.data 63 | end 64 | end 65 | end 66 | 67 | 10.times { |i| bus.publish("/echo", "0#{i}") } 68 | 69 | wait_for(2000) do 70 | expected_responses.empty? 71 | end 72 | 73 | bus.publish("/echo", "done") 74 | bus.global_unsubscribe 75 | t.join 76 | 77 | expected_responses.must_be :empty? 78 | unexpected_responses.must_be :empty? 79 | rescue Exception 80 | self.class.error! 81 | raise 82 | ensure 83 | if pids 84 | pids.each do |pid| 85 | begin 86 | Process.kill("KILL", pid) 87 | rescue SystemCallError 88 | end 89 | Process.wait(pid) 90 | end 91 | end 92 | 93 | bus.global_unsubscribe 94 | bus.reset! 95 | bus.destroy 96 | end 97 | end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /spec/lib/message_bus/rack/middleware_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | # coding: utf-8 3 | 4 | require_relative '../../../spec_helper' 5 | require 'message_bus' 6 | require 'rack/test' 7 | 8 | describe MessageBus::Rack::Middleware do 9 | include Rack::Test::Methods 10 | let(:extra_middleware) { nil } 11 | let(:base_route) { nil } 12 | 13 | before do 14 | bus = @bus = MessageBus::Instance.new 15 | @bus.configure(test_config_for_backend(CURRENT_BACKEND)) 16 | @bus.long_polling_enabled = false 17 | @bus.base_route = base_route if base_route 18 | 19 | e_m = extra_middleware 20 | builder = Rack::Builder.new { 21 | use FakeAsyncMiddleware, message_bus: bus 22 | use e_m if e_m 23 | use MessageBus::Rack::Middleware, message_bus: bus 24 | run lambda { |_env| [500, { 'Content-Type' => 'text/html' }, 'should not be called'] } 25 | } 26 | 27 | @async_middleware = builder.to_app 28 | @message_bus_middleware = @async_middleware.app 29 | end 30 | 31 | after do 32 | @message_bus_middleware.stop_listener 33 | @bus.reset! 34 | @bus.destroy 35 | end 36 | 37 | def app 38 | @async_middleware 39 | end 40 | 41 | module LongPolling 42 | extend Minitest::Spec::DSL 43 | 44 | before do 45 | @bus.long_polling_enabled = true 46 | end 47 | 48 | describe "with altered base_route" do 49 | let(:base_route) { "/base/route/" } 50 | 51 | it "should respond as normal" do 52 | post "/base/route/message-bus/ABC?dlp=t", '/foo1' => 0 53 | @async_middleware.in_async?.must_equal false 54 | last_response.ok?.must_equal true 55 | end 56 | end 57 | 58 | it "should respond right away if dlp=t" do 59 | post "/message-bus/ABC?dlp=t", '/foo1' => 0 60 | @async_middleware.in_async?.must_equal false 61 | last_response.ok?.must_equal true 62 | end 63 | 64 | it "should respond with a 404 if the client_id is missing" do 65 | post "/message-bus/?dlp=t", '/foo1' => 0 66 | last_response.not_found?.must_equal true 67 | end 68 | 69 | it "should respond right away to long polls that are polling on -1 with the last_id" do 70 | post "/message-bus/ABC", '/foo' => -1 71 | last_response.ok?.must_equal true 72 | parsed = JSON.parse(last_response.body) 73 | parsed.length.must_equal 1 74 | parsed[0]["channel"].must_equal "/__status" 75 | parsed[0]["data"]["/foo"].must_equal @bus.last_id("/foo") 76 | end 77 | 78 | it "should respond to long polls when data is available" do 79 | middleware = @async_middleware 80 | bus = @bus 81 | 82 | @bus.extra_response_headers_lookup do |_env| 83 | { "FOO" => "BAR" } 84 | end 85 | 86 | t = Thread.new do 87 | wait_for(2000) { middleware.in_async? } 88 | bus.publish "/foo", "םוֹלשָׁ" 89 | end 90 | 91 | post "/message-bus/ABC", '/foo' => nil 92 | 93 | last_response.ok?.must_equal true 94 | parsed = JSON.parse(last_response.body) 95 | parsed.length.must_equal 1 96 | parsed[0]["data"].must_equal "םוֹלשָׁ" 97 | 98 | last_response.headers["FOO"].must_equal "BAR" 99 | t.join 100 | end 101 | 102 | it "should timeout within its alloted slot" do 103 | begin 104 | @bus.long_polling_interval = 10 105 | s = Time.now.to_f * 1000 106 | post "/message-bus/ABC", '/foo' => nil 107 | # allow for some jitter 108 | (Time.now.to_f * 1000 - s).must_be :<, 100 109 | ensure 110 | @bus.long_polling_interval = 5000 111 | end 112 | end 113 | end 114 | 115 | describe "thin async" do 116 | before do 117 | @async_middleware.simulate_thin_async 118 | end 119 | 120 | include LongPolling 121 | end 122 | 123 | describe "hijack" do 124 | before do 125 | @async_middleware.simulate_hijack 126 | @bus.rack_hijack_enabled = true 127 | end 128 | 129 | include LongPolling 130 | end 131 | 132 | describe "start listener" do 133 | let(:app) { ->(_) { [200, {}, []] } } 134 | 135 | it "never subscribes" do 136 | bus = MessageBus::Instance.new 137 | bus.off 138 | 139 | middleware = MessageBus::Rack::Middleware.new(app, message_bus: bus) 140 | 141 | middleware.started_listener.must_equal false 142 | end 143 | end 144 | 145 | describe "polling" do 146 | before do 147 | @bus.long_polling_enabled = false 148 | end 149 | 150 | it "should include access control headers" do 151 | @bus.extra_response_headers_lookup do |_env| 152 | { "FOO" => "BAR" } 153 | end 154 | 155 | client_id = "ABCD" 156 | 157 | # client always keeps a list of channels with last message id they got on each 158 | post "/message-bus/#{client_id}", 159 | '/foo' => nil, 160 | '/bar' => nil 161 | 162 | last_response.headers["FOO"].must_equal "BAR" 163 | end 164 | 165 | it "should respond with a 200 to a subscribe" do 166 | client_id = "ABCD" 167 | 168 | # client always keeps a list of channels with last message id they got on each 169 | post "/message-bus/#{client_id}", 170 | '/foo' => nil, 171 | '/bar' => nil 172 | 173 | last_response.ok?.must_equal true 174 | end 175 | 176 | # this means we recover from redis reset 177 | it "should understand that larger than position is the same as -1" do 178 | @bus.publish('/foo', 'bar') 179 | @bus.publish('/baz', 'test') 180 | @bus.publish('/boom', 'bang') 181 | 182 | post "/message-bus/ABCD", 183 | '/foo' => 1_000_000, 184 | '/baz' => @bus.last_id('/baz') + 1, 185 | '/boom' => 1_000_000 186 | 187 | last_response.ok?.must_equal true 188 | parsed = JSON.parse(last_response.body) 189 | 190 | parsed.length.must_equal 1 191 | parsed[0]["channel"].must_equal "/__status" 192 | parsed[0]["data"]["/foo"].must_equal @bus.last_id("/foo") 193 | parsed[0]["data"]["/boom"].must_equal @bus.last_id("/boom") 194 | end 195 | 196 | it "should correctly understand that -1 means stuff from now onwards" do 197 | # even if allow chunked 198 | @bus.chunked_encoding_enabled = true 199 | 200 | @bus.publish('/foo', 'bar') 201 | @bus.publish('/baz', 'test') 202 | @bus.publish('/boom', 'bang') 203 | 204 | post "/message-bus/ABCD", 205 | '/foo' => -1, 206 | '/baz' => @bus.last_id('/baz') + 1, 207 | '/boom' => -1 208 | 209 | last_response.ok?.must_equal true 210 | parsed = JSON.parse(last_response.body) 211 | 212 | parsed.length.must_equal 1 213 | parsed[0]["channel"].must_equal "/__status" 214 | parsed[0]["data"]["/foo"].must_equal @bus.last_id("/foo") 215 | parsed[0]["data"]["/boom"].must_equal @bus.last_id("/boom") 216 | end 217 | 218 | it "should respond with the data if messages exist in the backlog" do 219 | id = @bus.last_id('/foo') 220 | 221 | @bus.publish("/foo", "barbs") 222 | @bus.publish("/foo", "borbs") 223 | 224 | client_id = "ABCD" 225 | post "/message-bus/#{client_id}", 226 | '/foo' => id, 227 | '/bar' => nil 228 | 229 | parsed = JSON.parse(last_response.body) 230 | parsed.length.must_equal 2 231 | parsed[0]["data"].must_equal "barbs" 232 | parsed[1]["data"].must_equal "borbs" 233 | end 234 | 235 | it "should use the correct client ID" do 236 | id = @bus.last_id('/foo') 237 | 238 | client_id = "aBc123" 239 | @bus.publish("/foo", "msg1", client_ids: [client_id]) 240 | @bus.publish("/foo", "msg2", client_ids: ["not_me#{client_id}"]) 241 | 242 | post "/message-bus/#{client_id}", 243 | '/foo' => id 244 | 245 | parsed = JSON.parse(last_response.body) 246 | parsed.length.must_equal 2 247 | parsed[0]["data"].must_equal("msg1") 248 | parsed[1]["data"].wont_equal("msg2") 249 | end 250 | 251 | it "should use the correct client ID with additional path" do 252 | id = @bus.last_id('/foo') 253 | 254 | client_id = "aBc123" 255 | @bus.publish("/foo", "msg1", client_ids: [client_id]) 256 | @bus.publish("/foo", "msg2", client_ids: ["not_me#{client_id}"]) 257 | 258 | post "/message-bus/#{client_id}/path/not/needed", 259 | '/foo' => id 260 | 261 | parsed = JSON.parse(last_response.body) 262 | parsed.length.must_equal 2 263 | parsed[0]["data"].must_equal("msg1") 264 | parsed[1]["data"].wont_equal("msg2") 265 | end 266 | 267 | it "should have no cross talk" do 268 | seq = 0 269 | @bus.site_id_lookup do 270 | (seq += 1).to_s 271 | end 272 | 273 | # published on channel 1 274 | msg = @bus.publish("/foo", "test") 275 | 276 | # subscribed on channel 2 277 | post "/message-bus/ABCD", 278 | '/foo' => (msg - 1) 279 | 280 | parsed = JSON.parse(last_response.body) 281 | parsed.length.must_equal 0 282 | end 283 | 284 | it "should have global cross talk" do 285 | seq = 0 286 | @bus.site_id_lookup do 287 | (seq += 1).to_s 288 | end 289 | 290 | msg = @bus.publish("/global/foo", "test") 291 | 292 | post "/message-bus/ABCD", 293 | '/global/foo' => (msg - 1) 294 | 295 | parsed = JSON.parse(last_response.body) 296 | parsed.length.must_equal 1 297 | end 298 | 299 | it "should not get consumed messages" do 300 | @bus.publish("/foo", "barbs") 301 | id = @bus.last_id('/foo') 302 | 303 | client_id = "ABCD" 304 | post "/message-bus/#{client_id}", 305 | '/foo' => id 306 | 307 | parsed = JSON.parse(last_response.body) 308 | parsed.length.must_equal 0 309 | end 310 | 311 | it "should filter by user correctly" do 312 | id = @bus.publish("/foo", "test", user_ids: [1]) 313 | @bus.user_id_lookup do |_env| 314 | 0 315 | end 316 | 317 | client_id = "ABCD" 318 | post "/message-bus/#{client_id}", 319 | '/foo' => id - 1 320 | 321 | parsed = JSON.parse(last_response.body) 322 | parsed.length.must_equal 1 323 | 324 | message = parsed.first 325 | 326 | message["channel"].must_equal "/__status" 327 | message["data"].must_equal("/foo" => 1) 328 | 329 | @bus.user_id_lookup do |_env| 330 | 1 331 | end 332 | 333 | post "/message-bus/#{client_id}", 334 | '/foo' => id - 1 335 | 336 | parsed = JSON.parse(last_response.body) 337 | parsed.length.must_equal 1 338 | end 339 | 340 | it "should filter by group correctly" do 341 | id = @bus.publish("/foo", "test", group_ids: [3, 4, 5]) 342 | @bus.group_ids_lookup do |_env| 343 | [0, 1, 2] 344 | end 345 | 346 | client_id = "ABCD" 347 | post "/message-bus/#{client_id}", 348 | '/foo' => id - 1 349 | 350 | parsed = JSON.parse(last_response.body) 351 | message = parsed.first 352 | 353 | message["channel"].must_equal "/__status" 354 | message["data"].must_equal("/foo" => 1) 355 | 356 | @bus.group_ids_lookup do |_env| 357 | [1, 7, 4, 100] 358 | end 359 | 360 | post "/message-bus/#{client_id}", 361 | '/foo' => id - 1 362 | 363 | parsed = JSON.parse(last_response.body) 364 | parsed.length.must_equal 1 365 | end 366 | 367 | it "can decode a JSON encoded request" do 368 | id = @bus.last_id('/foo') 369 | @bus.publish("/foo", json: true) 370 | post("/message-bus/1234", 371 | JSON.generate('/foo' => id), 372 | "CONTENT_TYPE" => "application/json") 373 | JSON.parse(last_response.body).first["data"].must_equal('json' => true) 374 | end 375 | 376 | it "should tell Rack to skip committing the session" do 377 | post "/message-bus/1234", {}, { "rack.session.options" => {} } 378 | last_request.env["rack.session.options"][:skip].must_equal true 379 | end 380 | 381 | describe "on_middleware_error handling" do 382 | it "allows error handling of middleware failures" do 383 | @bus.on_middleware_error do |_env, err| 384 | if ArgumentError === err 385 | [407, {}, []] 386 | end 387 | end 388 | 389 | @bus.group_ids_lookup do |_env| 390 | raise ArgumentError 391 | end 392 | 393 | post("/message-bus/1234", 394 | JSON.generate('/foo' => 1), 395 | "CONTENT_TYPE" => "application/json") 396 | 397 | last_response.status.must_equal 407 398 | end 399 | 400 | it "does not handle exceptions from downstream middleware" do 401 | @bus.on_middleware_error do |_env, err| 402 | [404, {}, []] 403 | end 404 | 405 | get("/") 406 | 407 | last_response.status.must_equal 500 408 | last_response.body.must_equal 'should not be called' 409 | end 410 | end 411 | 412 | describe "messagebus.channels env support" do 413 | let(:extra_middleware) do 414 | Class.new do 415 | attr_reader :app 416 | 417 | def initialize(app) 418 | @app = app 419 | end 420 | 421 | def call(env) 422 | @app.call(env.merge('message_bus.channels' => { '/foo' => 0 })) 423 | end 424 | end 425 | end 426 | 427 | it "should respect messagebus.channels in the environment to force channels" do 428 | @message_bus_middleware = @async_middleware.app.app 429 | foo_id = @bus.publish("/foo", "testfoo") 430 | bar_id = @bus.publish("/bar", "testbar") 431 | 432 | post "/message-bus/ABCD", 433 | '/foo' => foo_id - 1 434 | 435 | parsed = JSON.parse(last_response.body) 436 | parsed.first['data'].must_equal 'testfoo' 437 | 438 | post "/message-bus/ABCD", 439 | '/bar' => bar_id - 1 440 | 441 | parsed = JSON.parse(last_response.body) 442 | parsed.first['data'].must_equal 'testfoo' 443 | end 444 | end 445 | end 446 | end 447 | -------------------------------------------------------------------------------- /spec/lib/message_bus/timer_thread_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../../spec_helper' 4 | require 'message_bus/timer_thread' 5 | 6 | describe MessageBus::TimerThread do 7 | before do 8 | @timer = MessageBus::TimerThread.new 9 | end 10 | 11 | after do 12 | @timer.stop 13 | end 14 | 15 | it "allows you to queue every jobs" do 16 | i = 0 17 | m = Mutex.new 18 | every = @timer.every(0.001) { m.synchronize { i += 1 if i < 3 } } 19 | # allow lots of time, cause in test mode stuff can be slow 20 | wait_for(1000) do 21 | m.synchronize do 22 | every.cancel if i == 3 23 | i == 3 24 | end 25 | end 26 | sleep 0.002 27 | i.must_equal 3 28 | end 29 | 30 | it "allows you to cancel timers" do 31 | success = true 32 | @timer.queue(0.005) { success = false }.cancel 33 | sleep(0.006) 34 | success.must_equal true 35 | end 36 | 37 | it "queues jobs in the correct order" do 38 | results = [] 39 | (0..3).to_a.reverse.each do |i| 40 | @timer.queue(0.005 * i) do 41 | results << i 42 | end 43 | end 44 | 45 | wait_for(3000) { 46 | 4 == results.length 47 | } 48 | 49 | results.must_equal [0, 1, 2, 3] 50 | end 51 | 52 | it "should call the error callback if something goes wrong" do 53 | error = nil 54 | 55 | @timer.on_error do |e| 56 | error = e 57 | end 58 | 59 | @timer.queue do 60 | boom 61 | end 62 | 63 | wait_for(100) do 64 | error 65 | end 66 | 67 | error.class.must_equal NameError 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /spec/lib/message_bus_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../spec_helper' 4 | require 'message_bus' 5 | 6 | describe MessageBus do 7 | before do 8 | @bus = MessageBus::Instance.new 9 | @bus.site_id_lookup do 10 | "magic" 11 | end 12 | @bus.configure(test_config_for_backend(CURRENT_BACKEND)) 13 | end 14 | 15 | after do 16 | @bus.reset! 17 | @bus.destroy 18 | end 19 | 20 | it "destroying immediately after `after_fork` does not lock" do 21 | 10.times do 22 | @bus.on 23 | @bus.after_fork 24 | @bus.destroy 25 | end 26 | end 27 | 28 | describe "#base_route=" do 29 | it "adds leading and trailing slashes" do 30 | @bus.base_route = "my/base/route" 31 | @bus.base_route.must_equal '/my/base/route/' 32 | end 33 | 34 | it "leaves existing leading and trailing slashes" do 35 | @bus.base_route = "/my/base/route/" 36 | @bus.base_route.must_equal '/my/base/route/' 37 | end 38 | 39 | it "removes duplicate slashes" do 40 | @bus.base_route = "//my///base/route" 41 | @bus.base_route.must_equal '/my/base/route/' 42 | end 43 | end 44 | 45 | it "can subscribe from a point in time" do 46 | @bus.publish("/minion", "banana") 47 | 48 | data1 = [] 49 | data2 = [] 50 | data3 = [] 51 | 52 | @bus.subscribe("/minion") do |msg| 53 | data1 << msg.data 54 | end 55 | 56 | @bus.subscribe("/minion", 0) do |msg| 57 | data2 << msg.data 58 | end 59 | 60 | @bus.subscribe("/minion", 1) do |msg| 61 | data3 << msg.data 62 | end 63 | 64 | @bus.publish("/minion", "bananana") 65 | @bus.publish("/minion", "it's so fluffy") 66 | 67 | wait_for(2000) do 68 | data3.length == 3 && data2.length == 3 && data1.length == 2 69 | end 70 | 71 | data1.must_equal ['bananana', "it's so fluffy"] 72 | data2.must_equal ['banana', 'bananana', "it's so fluffy"] 73 | data3.must_equal ['banana', 'bananana', "it's so fluffy"] 74 | end 75 | 76 | it "can transmit client_ids" do 77 | client_ids = nil 78 | 79 | @bus.subscribe("/chuck") do |msg| 80 | client_ids = msg.client_ids 81 | end 82 | 83 | @bus.publish("/chuck", { yeager: true }, client_ids: ['a', 'b']) 84 | wait_for(2000) { client_ids } 85 | 86 | client_ids.must_equal ['a', 'b'] 87 | end 88 | 89 | it "should recover from a reset" do 90 | data = nil 91 | @bus.subscribe("/chuck") do |msg| 92 | data = msg.data 93 | end 94 | @bus.publish("/chuck", norris: true) 95 | @bus.publish("/chuck", norris: true) 96 | @bus.publish("/chuck", norris: true) 97 | 98 | @bus.backend_instance.reset! 99 | 100 | @bus.publish("/chuck", yeager: true) 101 | 102 | wait_for(2000) { data && data["yeager"] } 103 | 104 | data["yeager"].must_equal true 105 | end 106 | 107 | it "should recover from a backlog expiring" do 108 | data = nil 109 | @bus.subscribe("/chuck") do |msg| 110 | data = msg.data 111 | end 112 | @bus.publish("/chuck", norris: true) 113 | @bus.publish("/chuck", norris: true) 114 | @bus.publish("/chuck", norris: true) 115 | 116 | @bus.backend_instance.expire_all_backlogs! 117 | 118 | @bus.publish("/chuck", yeager: true) 119 | 120 | wait_for(2000) { data && data["yeager"] } 121 | 122 | data["yeager"].must_equal true 123 | end 124 | 125 | it "should automatically decode hashed messages" do 126 | data = nil 127 | @bus.subscribe("/chuck") do |msg| 128 | data = msg.data 129 | end 130 | @bus.publish("/chuck", norris: true) 131 | wait_for(2000) { data } 132 | 133 | data["norris"].must_equal true 134 | end 135 | 136 | it "should get a message if it subscribes to it" do 137 | user_ids, data, site_id, channel = nil 138 | 139 | @bus.subscribe("/chuck") do |msg| 140 | data = msg.data 141 | site_id = msg.site_id 142 | channel = msg.channel 143 | user_ids = msg.user_ids 144 | end 145 | 146 | @bus.publish("/chuck", "norris", user_ids: [1, 2, 3]) 147 | 148 | wait_for(2000) { data } 149 | 150 | data.must_equal 'norris' 151 | site_id.must_equal 'magic' 152 | channel.must_equal '/chuck' 153 | user_ids.to_a.must_equal [1, 2, 3] 154 | end 155 | 156 | it "should get global messages if it subscribes to them" do 157 | data, site_id, channel = nil 158 | 159 | @bus.subscribe do |msg| 160 | data = msg.data 161 | site_id = msg.site_id 162 | channel = msg.channel 163 | end 164 | 165 | @bus.publish("/chuck", "norris") 166 | 167 | wait_for(2000) { data } 168 | 169 | data.must_equal 'norris' 170 | site_id.must_equal 'magic' 171 | channel.must_equal '/chuck' 172 | end 173 | 174 | it "should have the ability to grab the backlog messages in the correct order" do 175 | id = @bus.publish("/chuck", "norris") 176 | @bus.publish("/chuck", "foo") 177 | @bus.publish("/chuck", "bar") 178 | 179 | r = @bus.backlog("/chuck", id) 180 | 181 | r.map { |i| i.data }.to_a.must_equal ['foo', 'bar'] 182 | end 183 | 184 | it "should correctly get full backlog of a channel" do 185 | @bus.publish("/chuck", "norris") 186 | @bus.publish("/chuck", "foo") 187 | @bus.publish("/chuckles", "bar") 188 | 189 | @bus.backlog("/chuck").map { |i| i.data }.to_a.must_equal ['norris', 'foo'] 190 | end 191 | 192 | it "should correctly restrict the backlog size of a channel" do 193 | @bus.publish("/chuck", "norris") 194 | @bus.publish("/chuck", "foo", max_backlog_size: 1) 195 | 196 | @bus.backlog("/chuck").map { |i| i.data }.to_a.must_equal ['foo'] 197 | end 198 | 199 | it "can be turned off" do 200 | @bus.off 201 | 202 | @bus.off?.must_equal true 203 | 204 | @bus.publish("/chuck", "norris") 205 | 206 | @bus.backlog("/chuck").to_a.must_equal [] 207 | end 208 | 209 | it "can be turned off only for subscriptions" do 210 | @bus.off(disable_publish: false) 211 | 212 | @bus.off?.must_equal true 213 | 214 | data = [] 215 | 216 | @bus.subscribe("/chuck") do |msg| 217 | data << msg.data 218 | end 219 | 220 | @bus.publish("/chuck", "norris") 221 | 222 | @bus.on 223 | 224 | @bus.subscribe("/chuck") do |msg| 225 | data << msg.data 226 | end 227 | 228 | @bus.publish("/chuck", "berry") 229 | 230 | wait_for(2000) { data.length > 0 } 231 | 232 | data.must_equal ["berry"] 233 | 234 | @bus.backlog("/chuck").map(&:data).to_a.must_equal ["norris", "berry"] 235 | end 236 | 237 | it "can call destroy multiple times" do 238 | @bus.destroy 239 | @bus.destroy 240 | @bus.destroy 241 | end 242 | 243 | it "can be turned on after destroy" do 244 | @bus.destroy 245 | 246 | @bus.on 247 | 248 | @bus.after_fork 249 | end 250 | 251 | it "allows you to look up last_message" do 252 | @bus.publish("/bob", "dylan") 253 | @bus.publish("/bob", "marley") 254 | @bus.last_message("/bob").data.must_equal "marley" 255 | assert_nil @bus.last_message("/nothing") 256 | end 257 | 258 | describe "#publish" do 259 | it "allows publishing to a explicit site" do 260 | data, site_id, channel = nil 261 | 262 | @bus.subscribe do |msg| 263 | data = msg.data 264 | site_id = msg.site_id 265 | channel = msg.channel 266 | end 267 | 268 | @bus.publish("/chuck", "norris", site_id: "law-and-order") 269 | 270 | wait_for(2000) { data } 271 | 272 | data.must_equal 'norris' 273 | site_id.must_equal 'law-and-order' 274 | channel.must_equal '/chuck' 275 | end 276 | end 277 | 278 | describe "global subscriptions" do 279 | before do 280 | seq = 0 281 | @bus.site_id_lookup do 282 | (seq += 1).to_s 283 | end 284 | end 285 | 286 | it "can get last_message" do 287 | @bus.publish("/global/test", "test") 288 | @bus.last_message("/global/test").data.must_equal "test" 289 | end 290 | 291 | it "can subscribe globally" do 292 | data = nil 293 | @bus.subscribe do |message| 294 | data = message.data 295 | end 296 | 297 | @bus.publish("/global/test", "test") 298 | wait_for(1000) { data } 299 | 300 | data.must_equal "test" 301 | end 302 | 303 | it "can subscribe to channel" do 304 | data = nil 305 | @bus.subscribe("/global/test") do |message| 306 | data = message.data 307 | end 308 | 309 | @bus.publish("/global/test", "test") 310 | wait_for(1000) { data } 311 | 312 | data.must_equal "test" 313 | end 314 | 315 | it "should exception if publishing restricted messages to user" do 316 | assert_raises(MessageBus::InvalidMessage) do 317 | @bus.publish("/global/test", "test", user_ids: [1]) 318 | end 319 | end 320 | 321 | it "should exception if publishing restricted messages to group" do 322 | assert_raises(MessageBus::InvalidMessage) do 323 | @bus.publish("/global/test", "test", user_ids: [1]) 324 | end 325 | end 326 | 327 | it "should raise if we publish to an empty group or user list" do 328 | assert_raises(MessageBus::InvalidMessageTarget) do 329 | @bus.publish "/foo", "bar", user_ids: [] 330 | end 331 | 332 | assert_raises(MessageBus::InvalidMessageTarget) do 333 | @bus.publish "/foo", "bar", group_ids: [] 334 | end 335 | 336 | assert_raises(MessageBus::InvalidMessageTarget) do 337 | @bus.publish "/foo", "bar", client_ids: [] 338 | end 339 | 340 | assert_raises(MessageBus::InvalidMessageTarget) do 341 | @bus.publish "/foo", "bar", group_ids: [], user_ids: [1] 342 | end 343 | 344 | assert_raises(MessageBus::InvalidMessageTarget) do 345 | @bus.publish "/foo", "bar", group_ids: [1], user_ids: [] 346 | end 347 | end 348 | end 349 | 350 | it "should support forking properly" do 351 | test_never :memory 352 | 353 | data = [] 354 | @bus.subscribe do |msg| 355 | data << msg.data 356 | end 357 | 358 | @bus.publish("/hello", "pre-fork") 359 | wait_for(2000) { data.length == 1 } 360 | 361 | if child = Process.fork 362 | # The child was forked and we received its PID 363 | 364 | # Wait for fork to finish so we're asserting that we can still publish after it has 365 | Process.wait(child) 366 | 367 | @bus.publish("/hello", "continuation") 368 | else 369 | begin 370 | @bus.after_fork 371 | GC.start 372 | @bus.publish("/hello", "from-fork") 373 | ensure 374 | exit!(0) 375 | end 376 | end 377 | 378 | wait_for(2000) { data.length == 3 } 379 | 380 | @bus.publish("/hello", "after-fork") 381 | 382 | wait_for(2000) { data.length == 4 } 383 | 384 | data.must_equal(["pre-fork", "from-fork", "continuation", "after-fork"]) 385 | end 386 | 387 | describe '#register_client_message_filter' do 388 | it 'should register the message filter correctly' do 389 | @bus.register_client_message_filter('/test') 390 | 391 | @bus.client_message_filters.must_equal([]) 392 | 393 | @bus.register_client_message_filter('/test') { puts "hello world" } 394 | 395 | channel, blk = @bus.client_message_filters[0] 396 | 397 | blk.must_respond_to(:call) 398 | channel.must_equal('/test') 399 | end 400 | end 401 | end 402 | -------------------------------------------------------------------------------- /spec/performance/backlog.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH << File.join(File.dirname(__FILE__), '..', '..', 'lib') 4 | require 'logger' 5 | require 'benchmark' 6 | require 'message_bus' 7 | 8 | require_relative "../helpers" 9 | 10 | backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym) 11 | channel = "/foo" 12 | iterations = 10_000 13 | results = [] 14 | 15 | puts "Running backlog benchmark with #{iterations} iterations on backends: #{backends.inspect}" 16 | 17 | run_benchmark = lambda do |bm, backend| 18 | bus = MessageBus::Instance.new 19 | bus.configure(test_config_for_backend(backend)) 20 | 21 | bus.backend_instance.max_backlog_size = 100 22 | bus.backend_instance.max_global_backlog_size = 1000 23 | 24 | channel_names = 10.times.map { |i| "channel#{i}" } 25 | 26 | 100.times do |i| 27 | channel_names.each do |ch| 28 | bus.publish(ch, { message_number_is: i }) 29 | end 30 | end 31 | 32 | last_ids = channel_names.map { |ch| [ch, bus.last_id(ch)] }.to_h 33 | 34 | 1000.times do 35 | # Warmup 36 | client = MessageBus::Client.new(message_bus: bus) 37 | channel_names.each { |ch| client.subscribe(ch, -1) } 38 | client.backlog 39 | end 40 | 41 | bm.report("#{backend} - #backlog with no backlogs requested") do 42 | iterations.times do 43 | client = MessageBus::Client.new(message_bus: bus) 44 | channel_names.each { |ch| client.subscribe(ch, -1) } 45 | client.backlog 46 | end 47 | end 48 | 49 | (0..5).each do |ch_i| 50 | channels_with_messages = (ch_i) * 2 51 | 52 | bm.report("#{backend} - #backlog when #{channels_with_messages}/10 channels have new messages") do 53 | iterations.times do 54 | client = MessageBus::Client.new(message_bus: bus) 55 | channel_names.each_with_index do |ch, i| 56 | client.subscribe(ch, last_ids[ch] + ((i < channels_with_messages) ? -1 : 0)) 57 | end 58 | result = client.backlog 59 | if result.length != channels_with_messages 60 | raise "Result has #{result.length} messages. Expected #{channels_with_messages}" 61 | end 62 | end 63 | end 64 | end 65 | 66 | bus.reset! 67 | bus.destroy 68 | end 69 | 70 | puts 71 | 72 | Benchmark.benchmark(" duration\n", 60, "%10.2rs\n", "") do |bm| 73 | backends.each do |backend| 74 | run_benchmark.call(bm, backend) 75 | end 76 | end 77 | puts 78 | results.each do |result| 79 | puts result 80 | end 81 | -------------------------------------------------------------------------------- /spec/performance/publish.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH << File.join(File.dirname(__FILE__), '..', '..', 'lib') 4 | require 'logger' 5 | require 'benchmark' 6 | require 'message_bus' 7 | 8 | require_relative "../helpers" 9 | 10 | backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym) 11 | channel = "/foo" 12 | iterations = 100_000 13 | results = [] 14 | 15 | puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}" 16 | 17 | benchmark_publication_only = lambda do |bm, backend| 18 | bus = MessageBus::Instance.new 19 | bus.configure(test_config_for_backend(backend)) 20 | 21 | bm.report("#{backend} - publication only") do 22 | iterations.times { bus.publish(channel, "Hello world") } 23 | end 24 | 25 | bus.reset! 26 | bus.destroy 27 | end 28 | 29 | benchmark_subscription_no_trimming = lambda do |bm, backend| 30 | test_title = "#{backend} - subscription no trimming" 31 | 32 | bus = MessageBus::Instance.new 33 | bus.configure(test_config_for_backend(backend)) 34 | 35 | bus.backend_instance.max_backlog_size = iterations 36 | bus.backend_instance.max_global_backlog_size = iterations 37 | 38 | messages_received = 0 39 | bus.after_fork 40 | bus.subscribe(channel) do |_message| 41 | messages_received += 1 42 | end 43 | 44 | bm.report(test_title) do 45 | iterations.times { bus.publish(channel, "Hello world") } 46 | wait_for(60000) { messages_received == iterations } 47 | end 48 | 49 | results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" 50 | 51 | bus.reset! 52 | bus.destroy 53 | end 54 | 55 | benchmark_subscription_with_trimming = lambda do |bm, backend| 56 | test_title = "#{backend} - subscription with trimming" 57 | 58 | bus = MessageBus::Instance.new 59 | bus.configure(test_config_for_backend(backend)) 60 | 61 | bus.backend_instance.max_backlog_size = (iterations / 10) 62 | bus.backend_instance.max_global_backlog_size = (iterations / 10) 63 | 64 | messages_received = 0 65 | bus.after_fork 66 | bus.subscribe(channel) do |_message| 67 | messages_received += 1 68 | end 69 | 70 | bm.report(test_title) do 71 | iterations.times { bus.publish(channel, "Hello world") } 72 | wait_for(60000) { messages_received == iterations } 73 | end 74 | 75 | results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" 76 | 77 | bus.reset! 78 | bus.destroy 79 | end 80 | 81 | benchmark_subscription_with_trimming_and_clear_every = lambda do |bm, backend| 82 | test_title = "#{backend} - subscription with trimming and clear_every=50" 83 | 84 | bus = MessageBus::Instance.new 85 | bus.configure(test_config_for_backend(backend)) 86 | 87 | bus.backend_instance.max_backlog_size = (iterations / 10) 88 | bus.backend_instance.max_global_backlog_size = (iterations / 10) 89 | bus.backend_instance.clear_every = 50 90 | 91 | messages_received = 0 92 | bus.after_fork 93 | bus.subscribe(channel) do |_message| 94 | messages_received += 1 95 | end 96 | 97 | bm.report(test_title) do 98 | iterations.times { bus.publish(channel, "Hello world") } 99 | wait_for(60000) { messages_received == iterations } 100 | end 101 | 102 | results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" 103 | 104 | bus.reset! 105 | bus.destroy 106 | end 107 | 108 | puts 109 | Benchmark.bm(60) do |bm| 110 | backends.each do |backend| 111 | benchmark_publication_only.call(bm, backend) 112 | end 113 | 114 | puts 115 | 116 | backends.each do |backend| 117 | benchmark_subscription_no_trimming.call(bm, backend) 118 | end 119 | 120 | results << nil 121 | puts 122 | 123 | backends.each do |backend| 124 | benchmark_subscription_with_trimming.call(bm, backend) 125 | end 126 | 127 | backends.each do |backend| 128 | benchmark_subscription_with_trimming_and_clear_every.call(bm, backend) 129 | end 130 | end 131 | puts 132 | 133 | results.each do |result| 134 | puts result 135 | end 136 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $: << File.dirname(__FILE__) 4 | $: << File.join(File.dirname(__FILE__), '..', 'lib') 5 | require 'thin' 6 | require 'lib/fake_async_middleware' 7 | require 'message_bus' 8 | 9 | require 'minitest/autorun' 10 | require 'minitest/global_expectations' 11 | 12 | require_relative "helpers" 13 | 14 | CURRENT_BACKEND = (ENV['MESSAGE_BUS_BACKEND'] || :redis).to_sym 15 | 16 | require "message_bus/backends/#{CURRENT_BACKEND}" 17 | BACKEND_CLASS = MessageBus::BACKENDS.fetch(CURRENT_BACKEND) 18 | 19 | puts "Running with backend: #{CURRENT_BACKEND}" 20 | 21 | def test_only(*backends) 22 | skip "Test doesn't apply to #{CURRENT_BACKEND}" if backends.exclude?(CURRENT_BACKEND) 23 | end 24 | 25 | def test_never(*backends) 26 | skip "Test doesn't apply to #{CURRENT_BACKEND}" if backends.include?(CURRENT_BACKEND) 27 | end 28 | -------------------------------------------------------------------------------- /spec/support/jasmine-browser.json: -------------------------------------------------------------------------------- 1 | { 2 | "srcDir": "assets", 3 | "srcFiles": [ 4 | "message-bus.js", 5 | "message-bus-ajax.js" 6 | ], 7 | "specDir": "spec/assets", 8 | "specFiles": [ 9 | "message-bus.spec.js" 10 | ], 11 | "helpers": [ 12 | "SpecHelper.js" 13 | ], 14 | "random": true, 15 | "browser": "headlessChrome" 16 | } 17 | -------------------------------------------------------------------------------- /vendor/assets/javascripts/.gitignore: -------------------------------------------------------------------------------- 1 | message-bus.js 2 | message-bus-ajax.js 3 | --------------------------------------------------------------------------------