├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── Guardfile ├── LICENSE.txt ├── README.md ├── examples ├── basic.rb ├── binary.rb ├── get_vs_new.rb └── long_running.rb ├── lib ├── resilient.rb └── resilient │ ├── circuit_breaker.rb │ ├── circuit_breaker │ ├── metrics.rb │ ├── metrics │ │ ├── bucket.rb │ │ ├── bucket_range.rb │ │ ├── bucket_size.rb │ │ ├── storage │ │ │ └── memory.rb │ │ └── window_size.rb │ ├── properties.rb │ └── registry.rb │ ├── instrumenters │ ├── memory.rb │ └── noop.rb │ ├── key.rb │ ├── test │ ├── circuit_breaker_interface.rb │ ├── circuit_breaker_registry_interface.rb │ ├── metrics_interface.rb │ ├── metrics_storage_interface.rb │ └── properties_interface.rb │ └── version.rb ├── resilient.gemspec ├── script ├── bootstrap ├── console ├── release ├── test └── watch └── test ├── resilient ├── circuit_breaker │ ├── metrics │ │ └── storage │ │ │ └── memory_test.rb │ ├── metrics_test.rb │ ├── properties_test.rb │ └── registry_test.rb ├── circuit_breaker_instrumentation_test.rb ├── circuit_breaker_integration_test.rb ├── circuit_breaker_test.rb ├── instrumenters │ ├── memory_test.rb │ └── noop_test.rb └── key_test.rb ├── resilient_test.rb └── test_helper.rb /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | ruby: ['2.5', '2.6', '2.7'] 9 | steps: 10 | - name: Check out repository code 11 | uses: actions/checkout@v2 12 | - name: Do some action caching 13 | uses: actions/cache@v1 14 | with: 15 | path: vendor/bundle 16 | key: ${{ runner.os }}-gem-${{ hashFiles('**/Gemfile.lock') }} 17 | restore-keys: | 18 | ${{ runner.os }}-gem- 19 | - name: Set up Ruby 20 | uses: actions/setup-ruby@v1 21 | with: 22 | ruby-version: ${{ matrix.ruby }} 23 | - name: Install bundler 24 | run: gem install bundler 25 | - name: Run bundler 26 | run: bundle install --jobs 4 --retry 3 27 | - name: Run Tests 28 | run: script/test 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.gem 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.5.1 2 | 3 | * [Added two new events open_circuit and close_circuit](https://github.com/jnunemaker/resilient/pull/16) and [renamed them here](https://github.com/jnunemaker/resilient/commit/407e673eb5268912814398bebdeb173793d0af05). 4 | 5 | ## 0.5.0 6 | 7 | * [tweaked interface for metrics](https://github.com/jnunemaker/resilient/pull/15) 8 | * [stop rounding error percentage](https://github.com/jnunemaker/resilient/commit/6cdade5c3fc5f75a6ffde87082771608d582026c) 9 | 10 | ## prior to 0.5.0 11 | 12 | * No changelog kept so just checkout the commits. There aren't too terrible many. 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | gemspec 3 | 4 | gem "timecop", "~> 0.8" 5 | gem "minitest", "~> 5.8" 6 | 7 | group :development do 8 | gem "guard", "~> 2.13.0" 9 | gem "guard-minitest", "~> 2.4.4" 10 | end 11 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | guard :minitest do 2 | watch(%r{^test/(.*)\/?(.*)_test\.rb$}) { "test" } 3 | watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { "test" } 4 | watch(%r{^test/test_helper\.rb$}) { "test" } 5 | end 6 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 John Nunemaker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Resilient 2 | 3 | Some tools to aid in resiliency in Ruby. For now, just a circuit breaker (~~stolen from~~ based on [hystrix](https://github.com/netflix/hystrix)). Soon much more... 4 | 5 | Nothing asynchronous or thread safe yet either, but open to it and would like to see more around it in the future. See more here: [jnunemaker/resilient#18](https://github.com/jnunemaker/resilient/issues/18). 6 | 7 | ## Installation 8 | 9 | Add this line to your application's Gemfile: 10 | 11 | ```ruby 12 | gem "resilient" 13 | ``` 14 | 15 | And then execute: 16 | 17 | $ bundle 18 | 19 | Or install it yourself as: 20 | 21 | $ gem install resilient 22 | 23 | ## Usage 24 | 25 | ```ruby 26 | require "resilient/circuit_breaker" 27 | 28 | # default properties for circuit, CircuitBreaker.get is used instead of 29 | # CircuitBreaker.new as `get` keeps a registry of circuits by key to prevent 30 | # creating multiple instances of the same circuit breaker for a key; not using 31 | # `get` means you would have multiple instances of the circuit breaker and thus 32 | # separate state and metrics; you can read more in examples/get_vs_new.rb 33 | circuit_breaker = Resilient::CircuitBreaker.get("example") 34 | if circuit_breaker.allow_request? 35 | begin 36 | # do something expensive 37 | circuit_breaker.success 38 | rescue => boom 39 | circuit_breaker.failure 40 | # do fallback 41 | end 42 | else 43 | # do fallback 44 | end 45 | ``` 46 | 47 | customize properties of circuit: 48 | 49 | ```ruby 50 | circuit_breaker = Resilient::CircuitBreaker.get("example", { 51 | # at what percentage of errors should we open the circuit 52 | error_threshold_percentage: 50, 53 | # do not try request again for 5 seconds 54 | sleep_window_seconds: 5, 55 | # do not open circuit until at least 5 requests have happened 56 | request_volume_threshold: 5, 57 | }) 58 | # etc etc etc 59 | ``` 60 | 61 | force the circuit to be always open: 62 | 63 | ```ruby 64 | circuit_breaker = Resilient::CircuitBreaker.get("example", force_open: true) 65 | # etc etc etc 66 | ``` 67 | 68 | force the circuit to be always closed (great way to test in production with no impact, all instrumentation still runs which means you can measure in production with config and gain confidence while never actually opening a circuit incorrectly): 69 | 70 | ```ruby 71 | circuit_breaker = Resilient::CircuitBreaker.get("example", force_closed: true) 72 | # etc etc etc 73 | ``` 74 | 75 | customize rolling window to be 10 buckets of 1 second each (10 seconds in all): 76 | 77 | ```ruby 78 | circuit_breaker = Resilient::CircuitBreaker.get("example", { 79 | window_size_in_seconds: 10, 80 | bucket_size_in_seconds: 1, 81 | }) 82 | # etc etc etc 83 | ``` 84 | 85 | ## Default Properties 86 | 87 | Property | Default | Notes 88 | --------------------------------|------------------------|-------- 89 | **:force_open** | `false` | allows forcing the circuit open (stopping all requests) 90 | **:force_closed** | `false` | allows ignoring errors and therefore never trip "open" (e.g. allow all traffic through); normal instrumentation will still happen, thus allowing you to "test" configuration live without impact 91 | **:instrumenter** | `Instrumenters::Noop` | what to use to instrument all events that happen (e.g. `ActiveSupport::Notifications`) 92 | **:sleep_window_seconds** | `5` | seconds after tripping circuit before allowing retry 93 | **:request_volume_threshold** | `20` | number of requests that must be made within a statistical window before open/close decisions are made using stats 94 | **:error_threshold_percentage** | `50` | % of "marks" that must be failed to trip the circuit 95 | **:window_size_in_seconds** | `60` | number of seconds in the statistical window 96 | **:bucket_size_in_seconds** | `10` | size of buckets in statistical window 97 | **:metrics** | `Resilient::Metrics.new` | metrics instance used to keep track of success and failure 98 | 99 | ## Tests 100 | 101 | To ensure that you have clean circuit breakers for each test case, be sure to run the following in the setup for your tests (which resets the default registry and thus clears all the registered circuits) either before every test case or at a minimum each test case that uses circuit breakers. 102 | 103 | ```ruby 104 | Resilient::CircuitBreaker::Registry.reset 105 | ``` 106 | 107 | **Note**: If you use a non-default registry, you'll need to reset that on your own. If you don't know what I'm talking about, you are fine. 108 | 109 | ## Development 110 | 111 | ```bash 112 | # install dependencies 113 | script/bootstrap 114 | 115 | # run tests 116 | script/test 117 | 118 | # ...or to auto run tests with guard 119 | script/watch 120 | 121 | # to get a shell to play in 122 | script/console 123 | ``` 124 | 125 | ## Contributing 126 | 127 | Bug reports and pull requests are welcome on GitHub at https://github.com/jnunemaker/resilient. 128 | 129 | ## License 130 | 131 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 132 | 133 | ## Release (for maintainers) 134 | 135 | * increment version based on semver 136 | * git commit version change 137 | * script/release (releases to rubygems and git tags) 138 | -------------------------------------------------------------------------------- /examples/basic.rb: -------------------------------------------------------------------------------- 1 | # setting up load path 2 | require "pathname" 3 | root_path = Pathname(__FILE__).dirname.join("..").expand_path 4 | lib_path = root_path.join("lib") 5 | $:.unshift(lib_path) 6 | 7 | # requiring stuff for this example 8 | require "pp" 9 | require "resilient/circuit_breaker" 10 | 11 | circuit_breaker = Resilient::CircuitBreaker.get("example", { 12 | sleep_window_seconds: 1, 13 | request_volume_threshold: 10, 14 | error_threshold_percentage: 25, 15 | }) 16 | 17 | # success 18 | if circuit_breaker.allow_request? 19 | begin 20 | puts "do expensive thing" 21 | circuit_breaker.success 22 | rescue => boom 23 | # won't get here in this example 24 | circuit_breaker.failure 25 | end 26 | else 27 | raise "will not get here" 28 | end 29 | 30 | # failure 31 | if circuit_breaker.allow_request? 32 | begin 33 | raise 34 | rescue => boom 35 | circuit_breaker.failure 36 | puts "failed slow, do fallback" 37 | end 38 | else 39 | raise "will not get here" 40 | end 41 | 42 | # trip circuit, imagine this being same as above but in real life... 43 | # also, we have to fail at least the request volume threshold number of times 44 | circuit_breaker.properties.request_volume_threshold.times do 45 | circuit_breaker.failure 46 | end 47 | 48 | # fail fast 49 | if circuit_breaker.allow_request? 50 | raise "will not get here" 51 | else 52 | puts "failed fast, do fallback" 53 | end 54 | 55 | now = Time.now 56 | 57 | while Time.now - now < 3 58 | if circuit_breaker.allow_request? 59 | puts "doing a single attempt as we've failed fast for sleep_window_seconds" 60 | break 61 | else 62 | puts "failed fast, do fallback" 63 | end 64 | sleep rand(0.1) 65 | end 66 | 67 | if circuit_breaker.allow_request? 68 | raise "will not get here" 69 | else 70 | puts "request denied because single request has not been marked success yet" 71 | end 72 | 73 | puts "marking single request as success" 74 | circuit_breaker.success 75 | 76 | if circuit_breaker.allow_request? 77 | puts "circuit reset and back closed now, allowing requests" 78 | else 79 | raise "will not get here" 80 | end 81 | -------------------------------------------------------------------------------- /examples/binary.rb: -------------------------------------------------------------------------------- 1 | # setting up load path 2 | require "pathname" 3 | root_path = Pathname(__FILE__).dirname.join("..").expand_path 4 | lib_path = root_path.join("lib") 5 | $:.unshift(lib_path) 6 | 7 | # requiring stuff for this example 8 | require "pp" 9 | require "minitest/autorun" 10 | require "resilient/circuit_breaker" 11 | require "resilient/test/metrics_interface" 12 | 13 | # Metrics class that closes circuit on every success call and opens circuit for 14 | # sleep_window_seconds on every failure. 15 | class BinaryMetrics 16 | def initialize(options = {}) 17 | reset 18 | end 19 | 20 | def success 21 | reset 22 | nil 23 | end 24 | 25 | def failure 26 | @closed = false 27 | nil 28 | end 29 | 30 | def reset 31 | @closed = true 32 | nil 33 | end 34 | 35 | def under_request_volume_threshold?(request_volume_threshold) 36 | false 37 | end 38 | 39 | def under_error_threshold_percentage?(error_threshold_percentage) 40 | @closed 41 | end 42 | end 43 | 44 | class BinaryMetricsTest < Minitest::Test 45 | def setup 46 | @object = BinaryMetrics.new 47 | end 48 | 49 | include Resilient::Test::MetricsInterface 50 | end 51 | 52 | circuit_breaker = Resilient::CircuitBreaker.get("example", { 53 | sleep_window_seconds: 1, 54 | metrics: BinaryMetrics.new, 55 | }) 56 | 57 | # success 58 | if circuit_breaker.allow_request? 59 | begin 60 | puts "do expensive thing" 61 | circuit_breaker.success 62 | rescue => boom 63 | # won't get here in this example 64 | circuit_breaker.failure 65 | end 66 | else 67 | raise "will not get here" 68 | end 69 | 70 | # failure 71 | if circuit_breaker.allow_request? 72 | begin 73 | raise 74 | rescue => boom 75 | circuit_breaker.failure 76 | puts "failed slow, do fallback" 77 | end 78 | else 79 | raise "will not get here" 80 | end 81 | 82 | # fail fast 83 | if circuit_breaker.allow_request? 84 | raise "will not get here" 85 | else 86 | puts "failed fast, do fallback" 87 | end 88 | 89 | start = Time.now 90 | 91 | while (Time.now - start) < 3 92 | if circuit_breaker.allow_request? 93 | puts "doing a single attempt as we've failed fast for sleep_window_seconds" 94 | break 95 | else 96 | puts "failed fast, do fallback" 97 | end 98 | sleep rand(0.1) 99 | end 100 | 101 | if circuit_breaker.allow_request? 102 | raise "will not get here" 103 | else 104 | puts "request denied because single request has not been marked success yet" 105 | end 106 | 107 | puts "marking single request as success" 108 | circuit_breaker.success 109 | 110 | if circuit_breaker.allow_request? 111 | puts "circuit reset and back closed now, allowing requests" 112 | else 113 | raise "will not get here" 114 | end 115 | -------------------------------------------------------------------------------- /examples/get_vs_new.rb: -------------------------------------------------------------------------------- 1 | # setting up load path 2 | require "pathname" 3 | root_path = Pathname(__FILE__).dirname.join("..").expand_path 4 | lib_path = root_path.join("lib") 5 | $:.unshift(lib_path) 6 | 7 | # by default new is private so people don't use it, this makes it possible to 8 | # use it as resilient checks for this env var prior to privatizing new 9 | ENV["RESILIENT_PUBLICIZE_NEW"] = "1" 10 | 11 | # requiring stuff for this example 12 | require "pp" 13 | require "resilient/circuit_breaker" 14 | 15 | instance = Resilient::CircuitBreaker.get("example") 16 | instance_using_get = Resilient::CircuitBreaker.get("example") 17 | instance_using_new = Resilient::CircuitBreaker.new("example") 18 | 19 | puts "instance equals instance_using_get: #{instance.equal?(instance_using_get)}" 20 | puts "instance equals instance_using_new: #{instance.equal?(instance_using_new)}" 21 | 22 | instance.properties.request_volume_threshold.times do 23 | instance.failure 24 | end 25 | 26 | puts "instance allow_request?: #{instance.allow_request?}" 27 | puts "instance_using_get allow_request?: #{instance_using_get.allow_request?}" 28 | 29 | # this instance allows the request because it isn't sharing internal state and 30 | # metrics due to being a new allocated instance; the for instance does not 31 | # suffer this because it looks up instances in a registry rather than always 32 | # generating a new instance even if you use the exact same key as it bypasses 33 | # the registry 34 | puts "instance_using_new allow_request?: #{instance_using_new.allow_request?}" 35 | 36 | # instance equals instance_using_get: true 37 | # instance equals instance_using_new: false 38 | # instance allow_request?: false 39 | # instance_using_get allow_request?: false 40 | # instance_using_new allow_request?: true 41 | -------------------------------------------------------------------------------- /examples/long_running.rb: -------------------------------------------------------------------------------- 1 | # setting up load path 2 | require "pathname" 3 | root_path = Pathname(__FILE__).dirname.join("..").expand_path 4 | lib_path = root_path.join("lib") 5 | $:.unshift(lib_path) 6 | 7 | # requiring stuff for this example 8 | require "pp" 9 | require "resilient/circuit_breaker" 10 | 11 | circuit_breaker = Resilient::CircuitBreaker.get("example", { 12 | sleep_window_seconds: 5, 13 | request_volume_threshold: 20, 14 | error_threshold_percentage: 10, 15 | window_size_in_seconds: 60, 16 | bucket_size_in_seconds: 1, 17 | }) 18 | 19 | iterations = 0 20 | loop do 21 | if circuit_breaker.allow_request? 22 | begin 23 | puts "request allowed" 24 | raise if rand(100) < 10 25 | puts "request succeeded" 26 | circuit_breaker.success 27 | rescue => boom 28 | puts "request failed" 29 | circuit_breaker.failure 30 | end 31 | else 32 | puts "request denied" 33 | end 34 | puts "\n" 35 | sleep 0.1 36 | iterations += 1 37 | 38 | if iterations % 10 == 0 39 | p successes: circuit_breaker.metrics.successes, failures: circuit_breaker.metrics.failures, error_percentage: circuit_breaker.metrics.error_percentage, buckets: circuit_breaker.metrics.buckets.length 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/resilient.rb: -------------------------------------------------------------------------------- 1 | require "resilient/version" 2 | 3 | module Resilient 4 | end 5 | 6 | require "resilient/circuit_breaker" 7 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker.rb: -------------------------------------------------------------------------------- 1 | require "resilient/key" 2 | require "resilient/circuit_breaker/metrics" 3 | require "resilient/circuit_breaker/properties" 4 | require "resilient/circuit_breaker/registry" 5 | require "forwardable" 6 | 7 | module Resilient 8 | class CircuitBreaker 9 | extend Forwardable 10 | 11 | # Public: Returns an instance of circuit breaker based on key and registry. 12 | # Default registry is used if none is provided. If key does not exist, it is 13 | # registered. If key does exist, it returns registered instance instead of 14 | # allocating a new instance in order to ensure that state/metrics are the 15 | # same per key. 16 | # 17 | # See #initialize for docs on key and properties. 18 | def self.get(key, properties = nil, registry = nil) 19 | key = Key.wrap(key) 20 | (registry || Registry.default).fetch(key) { 21 | new(key, properties) 22 | } 23 | end 24 | 25 | unless ENV.key?("RESILIENT_PUBLICIZE_NEW") 26 | class << self 27 | private :new 28 | private :allocate 29 | end 30 | end 31 | 32 | def_delegator :@properties, :metrics 33 | 34 | attr_reader :key 35 | attr_reader :open 36 | attr_reader :opened_or_last_checked_at_epoch 37 | attr_reader :properties 38 | 39 | # Private: Builds new instance of a CircuitBreaker. 40 | # 41 | # key - The String or Resilient::Key that determines uniqueness of the 42 | # circuit breaker in the registry and for instrumentation. 43 | # 44 | # properties - The Hash or Resilient::CircuitBreaker::Properties that determine how the 45 | # circuit breaker should behave. Optional. Defaults to new 46 | # Resilient::CircuitBreaker::Properties instance. 47 | # 48 | # Returns CircuitBreaker instance. 49 | def initialize(key, properties = nil) 50 | raise ArgumentError, "key argument is required" if key.nil? 51 | 52 | @key = Key.wrap(key) 53 | @properties = Properties.wrap(properties) 54 | @open = false 55 | @opened_or_last_checked_at_epoch = 0 56 | end 57 | 58 | def allow_request? 59 | instrument("resilient.circuit_breaker.allow_request", key: @key) { |payload| 60 | payload[:result] = if payload[:force_open] = @properties.force_open 61 | false 62 | else 63 | # we still want to simulate normal behavior/metrics like open, allow 64 | # single request, etc. so it is possible to test properties in 65 | # production without impact using force_closed so we run these here 66 | # instead of in the else below 67 | allow_request = !open? || allow_single_request? 68 | 69 | if payload[:force_closed] = @properties.force_closed 70 | true 71 | else 72 | allow_request 73 | end 74 | end 75 | } 76 | end 77 | 78 | def success 79 | instrument("resilient.circuit_breaker.success", key: @key) { |payload| 80 | if @open 81 | payload[:closed_the_circuit] = true 82 | close_circuit 83 | else 84 | metrics.success 85 | end 86 | nil 87 | } 88 | end 89 | 90 | def failure 91 | instrument("resilient.circuit_breaker.failure", key: @key) { |payload| 92 | metrics.failure 93 | nil 94 | } 95 | end 96 | 97 | def reset 98 | instrument("resilient.circuit_breaker.reset", key: @key) { |payload| 99 | @open = false 100 | @opened_or_last_checked_at_epoch = 0 101 | metrics.reset 102 | nil 103 | } 104 | end 105 | 106 | private 107 | 108 | def open_circuit 109 | instrument("resilient.circuit_breaker.open_circuit", key: @key) { |payload| 110 | @opened_or_last_checked_at_epoch = Time.now.to_i 111 | @open = true 112 | payload[:open] = @open 113 | } 114 | end 115 | 116 | def close_circuit 117 | instrument("resilient.circuit_breaker.close_circuit", key: @key) { |payload| 118 | @open = false 119 | payload[:open] = @open 120 | payload[:interval] = Time.now.to_i - @opened_or_last_checked_at_epoch 121 | @opened_or_last_checked_at_epoch = 0 122 | } 123 | metrics.reset 124 | end 125 | 126 | def under_request_volume_threshold? 127 | metrics.under_request_volume_threshold?(@properties.request_volume_threshold) 128 | end 129 | 130 | def under_error_threshold_percentage? 131 | metrics.under_error_threshold_percentage?(@properties.error_threshold_percentage) 132 | end 133 | 134 | def open? 135 | instrument("resilient.circuit_breaker.open", key: @key) { |payload| 136 | payload[:result] = if @open 137 | true 138 | else 139 | if under_request_volume_threshold? 140 | false 141 | else 142 | if under_error_threshold_percentage? 143 | false 144 | else 145 | open_circuit 146 | true 147 | end 148 | end 149 | end 150 | } 151 | end 152 | 153 | def allow_single_request? 154 | instrument("resilient.circuit_breaker.allow_single_request", key: @key) { |payload| 155 | now = Time.now.to_i 156 | 157 | payload[:result] = if @open && now > (@opened_or_last_checked_at_epoch + @properties.sleep_window_seconds) 158 | @opened_or_last_checked_at_epoch = now 159 | true 160 | else 161 | false 162 | end 163 | } 164 | end 165 | 166 | def instrument(name, payload = {}, &block) 167 | properties.instrumenter.instrument(name, payload, &block) 168 | end 169 | end 170 | end 171 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/metrics.rb: -------------------------------------------------------------------------------- 1 | require "resilient/circuit_breaker/metrics/storage/memory" 2 | require "resilient/circuit_breaker/metrics/bucket_range" 3 | require "resilient/circuit_breaker/metrics/bucket_size" 4 | require "resilient/circuit_breaker/metrics/window_size" 5 | 6 | module Resilient 7 | class CircuitBreaker 8 | class Metrics 9 | attr_reader :window_size_in_seconds 10 | attr_reader :bucket_size_in_seconds 11 | attr_reader :buckets 12 | attr_reader :storage 13 | 14 | StorageSuccessKeys = [ 15 | :successes, 16 | ].freeze 17 | 18 | StorageFailureKeys = [ 19 | :failures, 20 | ].freeze 21 | 22 | StorageKeys = (StorageSuccessKeys + StorageFailureKeys).freeze 23 | 24 | def initialize(options = {}) 25 | @window_size_in_seconds = options.fetch(:window_size_in_seconds, 60) 26 | @bucket_size_in_seconds = options.fetch(:bucket_size_in_seconds, 10) 27 | @window_size = WindowSize.new(@window_size_in_seconds) 28 | @bucket_size = BucketSize.new(@bucket_size_in_seconds) 29 | @storage = options.fetch(:storage) { Storage::Memory.new } 30 | @buckets = [] 31 | end 32 | 33 | def under_request_volume_threshold?(request_volume_threshold) 34 | requests < request_volume_threshold 35 | end 36 | 37 | def under_error_threshold_percentage?(error_threshold_percentage) 38 | error_percentage < error_threshold_percentage 39 | end 40 | 41 | def success 42 | @storage.increment(current_bucket, StorageSuccessKeys) 43 | prune_buckets 44 | nil 45 | end 46 | 47 | def failure 48 | @storage.increment(current_bucket, StorageFailureKeys) 49 | prune_buckets 50 | nil 51 | end 52 | 53 | def reset 54 | @storage.prune(@buckets, StorageKeys) 55 | nil 56 | end 57 | 58 | private 59 | 60 | def requests 61 | prune_buckets 62 | requests = 0 63 | @storage.sum(@buckets, StorageKeys).each do |key, value| 64 | requests += value 65 | end 66 | requests 67 | end 68 | 69 | def error_percentage 70 | prune_buckets 71 | 72 | result = @storage.sum(@buckets, StorageKeys) 73 | successes = result[:successes] 74 | failures = result[:failures] 75 | 76 | requests = successes + failures 77 | return 0 if failures == 0 || requests == 0 78 | 79 | (failures / requests.to_f) * 100 80 | end 81 | 82 | def current_bucket(timestamp = Time.now.to_i) 83 | bucket = @buckets.detect { |bucket| bucket.include?(timestamp) } 84 | return bucket if bucket 85 | 86 | bucket = @bucket_size.bucket(timestamp) 87 | @buckets.push bucket 88 | 89 | bucket 90 | end 91 | 92 | def prune_buckets(timestamp = Time.now.to_i) 93 | pruned_buckets = [] 94 | bucket_range = BucketRange.generate(timestamp, @window_size, @bucket_size) 95 | 96 | @buckets.delete_if { |bucket| 97 | if bucket_range.prune?(bucket) 98 | pruned_buckets << bucket 99 | true 100 | end 101 | } 102 | 103 | if pruned_buckets.any? 104 | @storage.prune(pruned_buckets, StorageKeys) 105 | end 106 | end 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/metrics/bucket.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class CircuitBreaker 3 | class Metrics 4 | class Bucket 5 | attr_reader :timestamp_start 6 | attr_reader :timestamp_end 7 | 8 | def initialize(timestamp_start, timestamp_end) 9 | @timestamp_start = timestamp_start 10 | @timestamp_end = timestamp_end 11 | end 12 | 13 | def prune_before(window_size) 14 | @timestamp_end - window_size.seconds 15 | end 16 | 17 | def include?(timestamp) 18 | timestamp >= @timestamp_start && timestamp <= @timestamp_end 19 | end 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/metrics/bucket_range.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class CircuitBreaker 3 | class Metrics 4 | class BucketRange 5 | def self.generate(timestamp, window_size, bucket_size) 6 | end_bucket = bucket_size.bucket(timestamp) 7 | start_bucket = bucket_size.bucket(end_bucket.prune_before(window_size)) 8 | bucket_range = new(start_bucket, end_bucket) 9 | end 10 | 11 | attr_reader :start_bucket 12 | attr_reader :end_bucket 13 | 14 | def initialize(start_bucket, end_bucket) 15 | @start_bucket = start_bucket 16 | @end_bucket = end_bucket 17 | end 18 | 19 | def prune?(bucket) 20 | bucket.timestamp_end <= @start_bucket.timestamp_end 21 | end 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/metrics/bucket_size.rb: -------------------------------------------------------------------------------- 1 | require "resilient/circuit_breaker/metrics/bucket" 2 | 3 | module Resilient 4 | class CircuitBreaker 5 | class Metrics 6 | class BucketSize 7 | attr_reader :seconds 8 | 9 | def initialize(seconds) 10 | @seconds = seconds 11 | end 12 | 13 | def aligned_start(timestamp = Time.now.to_i) 14 | timestamp / @seconds * @seconds 15 | end 16 | 17 | def aligned_end(timestamp = Time.now.to_i) 18 | aligned_start(timestamp) + @seconds - 1 19 | end 20 | 21 | def bucket(timestamp = Time.now.to_i) 22 | Bucket.new aligned_start(timestamp), aligned_end(timestamp) 23 | end 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/metrics/storage/memory.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class CircuitBreaker 3 | class Metrics 4 | module Storage 5 | class Memory 6 | attr_reader :source 7 | 8 | def initialize 9 | @source = Hash.new { |h, k| h[k] = Hash.new(0) } 10 | end 11 | 12 | def increment(buckets, keys) 13 | Array(buckets).each do |bucket| 14 | Array(keys).each do |key| 15 | @source[bucket][key] += 1 16 | end 17 | end 18 | end 19 | 20 | def sum(buckets, keys) 21 | response = Hash.new(0) 22 | Array(buckets).each do |bucket| 23 | Array(keys).each do |key| 24 | response[key] += @source[bucket][key] 25 | end 26 | end 27 | response 28 | end 29 | 30 | def prune(buckets, keys) 31 | Array(buckets).each do |bucket| 32 | @source.delete(bucket) 33 | end 34 | end 35 | end 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/metrics/window_size.rb: -------------------------------------------------------------------------------- 1 | require "resilient/circuit_breaker/metrics/bucket" 2 | 3 | module Resilient 4 | class CircuitBreaker 5 | class Metrics 6 | class WindowSize 7 | attr_reader :seconds 8 | 9 | def initialize(seconds) 10 | @seconds = seconds 11 | end 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/properties.rb: -------------------------------------------------------------------------------- 1 | require "resilient/instrumenters/noop" 2 | 3 | module Resilient 4 | class CircuitBreaker 5 | class Properties 6 | 7 | # Internal: Takes a string name or instance of a Key and always returns a 8 | # Key instance. 9 | def self.wrap(hash_or_instance) 10 | case hash_or_instance 11 | when self 12 | hash_or_instance 13 | when Hash 14 | new(hash_or_instance) 15 | when NilClass 16 | new 17 | else 18 | raise TypeError, "properties must be Hash or Resilient::Properties instance" 19 | end 20 | end 21 | 22 | # allows forcing the circuit open (stopping all requests) 23 | attr_reader :force_open 24 | 25 | # allows ignoring errors and therefore never trip "open" 26 | # (ie. allow all traffic through); normal instrumentation will still 27 | # happen, thus allowing you to "test" configuration live without impact 28 | attr_reader :force_closed 29 | 30 | # what to use to instrument all events that happen 31 | # (ie: ActiveSupport::Notifications) 32 | attr_reader :instrumenter 33 | 34 | # seconds after tripping circuit before allowing retry 35 | attr_reader :sleep_window_seconds 36 | 37 | # number of requests that must be made within a statistical window before 38 | # open/close decisions are made using stats 39 | attr_reader :request_volume_threshold 40 | 41 | # % of "marks" that must be failed to trip the circuit 42 | attr_reader :error_threshold_percentage 43 | 44 | # number of seconds in the statistical window 45 | attr_reader :window_size_in_seconds 46 | 47 | # size of buckets in statistical window 48 | attr_reader :bucket_size_in_seconds 49 | 50 | # metrics instance used to keep track of success and failure 51 | attr_reader :metrics 52 | 53 | def initialize(options = {}) 54 | @force_open = options.fetch(:force_open, false) 55 | @force_closed = options.fetch(:force_closed, false) 56 | @instrumenter = options.fetch(:instrumenter, Instrumenters::Noop) 57 | @sleep_window_seconds = options.fetch(:sleep_window_seconds, 5) 58 | @request_volume_threshold = options.fetch(:request_volume_threshold, 20) 59 | @error_threshold_percentage = options.fetch(:error_threshold_percentage, 50) 60 | @window_size_in_seconds = options.fetch(:window_size_in_seconds, 60) 61 | @bucket_size_in_seconds = options.fetch(:bucket_size_in_seconds, 10) 62 | 63 | if @bucket_size_in_seconds >= @window_size_in_seconds 64 | raise ArgumentError, "bucket_size_in_seconds must be smaller than window_size_in_seconds" 65 | end 66 | 67 | if @window_size_in_seconds % @bucket_size_in_seconds != 0 68 | raise ArgumentError, 69 | "window_size_in_seconds must be perfectly divisible by" + 70 | " bucket_size_in_seconds in order to evenly partition the buckets" 71 | end 72 | 73 | @metrics = options.fetch(:metrics) { 74 | Metrics.new({ 75 | window_size_in_seconds: @window_size_in_seconds, 76 | bucket_size_in_seconds: @bucket_size_in_seconds, 77 | }) 78 | } 79 | end 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /lib/resilient/circuit_breaker/registry.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class CircuitBreaker 3 | class Registry 4 | # Internal: Default registry to use for circuit breakers. 5 | def self.default 6 | @default 7 | end 8 | 9 | # Internal: Allows overriding default registry for circuit breakers. 10 | def self.default=(value) 11 | @default = value 12 | end 13 | 14 | # Public: Reset the default registry. This completely wipes all instances 15 | # by swapping out the default registry for a new one and letting the old 16 | # one get GC'd. Useful in tests to get a completely clean slate. 17 | def self.reset 18 | default.reset 19 | end 20 | 21 | def initialize(source = nil) 22 | @source = source || {} 23 | end 24 | 25 | # Setup new instance as default. Needs to be after initialize so hash gets 26 | # initialize correctly. 27 | @default = new 28 | 29 | # Internal: To be used by CircuitBreaker to either get an instance for a 30 | # key or set a new instance for a key. 31 | # 32 | # Raises KeyError if key not found and no block provided. 33 | def fetch(key, &block) 34 | if value = @source[key] 35 | value 36 | else 37 | if block_given? 38 | @source[key] = yield 39 | else 40 | @source.fetch(key) 41 | end 42 | end 43 | end 44 | 45 | # Internal: To be used by CircuitBreaker to reset the stored circuit 46 | # breakers, which should only really be used for cleaning up in 47 | # test environment. 48 | def reset 49 | @source = {} 50 | nil 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/resilient/instrumenters/memory.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | module Instrumenters 3 | # Instrumentor that is useful for tests as it stores each of the events that 4 | # are instrumented. 5 | class Memory 6 | Event = Struct.new(:name, :payload, :result) 7 | 8 | attr_reader :events 9 | 10 | def initialize 11 | reset 12 | end 13 | 14 | def instrument(name, payload = {}) 15 | # Copy the payload to guard against later modifications to it, and to 16 | # ensure that all instrumentation code uses the payload passed to the 17 | # block rather than the one passed to #instrument. 18 | payload = payload.dup 19 | 20 | result = if block_given? 21 | yield payload 22 | else 23 | nil 24 | end 25 | @events << Event.new(name, payload, result) 26 | result 27 | end 28 | 29 | def reset 30 | @events = [] 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/resilient/instrumenters/noop.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | module Instrumenters 3 | class Noop 4 | def self.instrument(name, payload = {}) 5 | yield payload if block_given? 6 | end 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/resilient/key.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class Key 3 | 4 | # Internal: Takes a string name or instance of a Key and always returns a 5 | # Key instance. 6 | def self.wrap(string_or_instance) 7 | case string_or_instance 8 | when self, NilClass 9 | string_or_instance 10 | else 11 | new(string_or_instance) 12 | end 13 | end 14 | 15 | attr_reader :name 16 | 17 | def initialize(name) 18 | raise TypeError, "name must be a String" unless name.is_a?(String) 19 | @name = name 20 | end 21 | 22 | def ==(other) 23 | self.class == other.class && name == other.name 24 | end 25 | alias_method :eql?, :== 26 | 27 | def hash 28 | @name.hash 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/resilient/test/circuit_breaker_interface.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class Test 3 | module CircuitBreakerInterface 4 | def test_responds_to_key 5 | assert_respond_to @object, :key 6 | end 7 | 8 | def test_responds_to_allow_request 9 | assert_respond_to @object, :allow_request? 10 | end 11 | 12 | def test_responds_to_success 13 | assert_respond_to @object, :success 14 | end 15 | 16 | def test_success_returns_nothing 17 | assert_nil @object.success 18 | end 19 | 20 | def test_responds_to_failure 21 | assert_respond_to @object, :failure 22 | end 23 | 24 | def test_failure_returns_nothing 25 | assert_nil @object.failure 26 | end 27 | 28 | def test_responds_to_reset 29 | assert_respond_to @object, :reset 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/resilient/test/circuit_breaker_registry_interface.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class Test 3 | module CircuitBreakerRegistryInterface 4 | def test_responds_to_fetch 5 | assert_respond_to @object, :fetch 6 | end 7 | 8 | def test_responds_to_reset 9 | assert_respond_to @object, :reset 10 | end 11 | 12 | def test_fetch 13 | key = "foo" 14 | value = "bar".freeze 15 | 16 | assert_raises(KeyError) { @object.fetch(key) } 17 | assert_equal value, @object.fetch(key) { value } 18 | assert_equal value, @object.fetch(key) 19 | assert @object.fetch(key).equal?(value) 20 | end 21 | 22 | def test_reset 23 | original_foo = @object.fetch("foo") { Object.new } 24 | original_bar = @object.fetch("bar") { Object.new } 25 | 26 | assert_nil @object.reset 27 | 28 | foo = @object.fetch("foo") { Object.new } 29 | bar = @object.fetch("bar") { Object.new } 30 | 31 | # assert that the objects before and after reset are not the same object 32 | refute original_foo.equal?(foo) 33 | refute original_bar.equal?(bar) 34 | end 35 | 36 | def test_reset_empty_registry 37 | assert_nil @object.reset 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/resilient/test/metrics_interface.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class Test 3 | module MetricsInterface 4 | def test_responds_to_under_request_volume_threshold_predicate 5 | assert_respond_to @object, :under_request_volume_threshold? 6 | assert_equal 1, @object.method(:under_request_volume_threshold?).arity 7 | end 8 | 9 | def test_responds_to_under_error_threshold_percentage_predicate 10 | assert_respond_to @object, :under_error_threshold_percentage? 11 | assert_equal 1, @object.method(:under_error_threshold_percentage?).arity 12 | end 13 | 14 | def test_responds_to_success 15 | assert_respond_to @object, :success 16 | end 17 | 18 | def test_success_returns_nothing 19 | assert_nil @object.success 20 | end 21 | 22 | def test_responds_to_failure 23 | assert_respond_to @object, :failure 24 | end 25 | 26 | def test_failure_returns_nothing 27 | assert_nil @object.failure 28 | end 29 | 30 | def test_responds_to_reset 31 | assert_respond_to @object, :reset 32 | end 33 | 34 | def test_reset_returns_nothing 35 | assert_nil @object.reset 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/resilient/test/metrics_storage_interface.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class Test 3 | module MetricsStorageInterface 4 | def test_responds_to_increment 5 | assert_respond_to @object, :increment 6 | end 7 | 8 | def test_responds_to_sum 9 | assert_respond_to @object, :sum 10 | end 11 | 12 | def test_responds_to_prune 13 | assert_respond_to @object, :prune 14 | end 15 | 16 | def test_increment 17 | buckets = [ 18 | Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5), 19 | Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10), 20 | ] 21 | keys = [ 22 | :successes, 23 | :failures, 24 | ] 25 | @object.increment(buckets, keys) 26 | assert_equal 1, @object.source[buckets[0]][:successes] 27 | assert_equal 1, @object.source[buckets[0]][:failures] 28 | assert_equal 1, @object.source[buckets[1]][:successes] 29 | assert_equal 1, @object.source[buckets[1]][:failures] 30 | end 31 | 32 | def test_sum_defaults 33 | buckets = [ 34 | Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5), 35 | Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10), 36 | ] 37 | keys = [ 38 | :successes, 39 | :failures, 40 | ] 41 | result = @object.sum(buckets, keys) 42 | assert_equal 0, result[:successes] 43 | assert_equal 0, result[:failures] 44 | end 45 | 46 | def test_sum_with_values 47 | buckets = [ 48 | Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5), 49 | Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10), 50 | ] 51 | keys = [ 52 | :successes, 53 | :failures, 54 | ] 55 | @object.increment(buckets, keys) 56 | @object.increment(buckets, keys) 57 | @object.increment(buckets[0], keys) 58 | 59 | assert_equal 5, @object.sum(buckets, [:successes])[:successes] 60 | assert_equal 5, @object.sum(buckets, [:failures])[:failures] 61 | assert_equal 10, @object.sum(buckets, keys).values.inject(0) { |sum, value| sum += value } 62 | 63 | assert_equal 3, @object.sum(buckets[0], [:successes])[:successes] 64 | assert_equal 3, @object.sum(buckets[0], [:failures])[:failures] 65 | assert_equal 6, @object.sum(buckets[0], keys).values.inject(0) { |sum, value| sum += value } 66 | 67 | assert_equal 2, @object.sum(buckets[1], [:successes])[:successes] 68 | assert_equal 2, @object.sum(buckets[1], [:failures])[:failures] 69 | assert_equal 4, @object.sum(buckets[1], keys).values.inject(0) { |sum, value| sum += value } 70 | end 71 | 72 | def test_prune 73 | buckets = [ 74 | Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5), 75 | Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10), 76 | ] 77 | keys = [ 78 | :successes, 79 | :failures, 80 | ] 81 | @object.increment(buckets, keys) 82 | @object.increment(buckets, keys) 83 | @object.increment(buckets[0], keys) 84 | @object.prune(buckets, keys) 85 | result = @object.sum(buckets, keys) 86 | assert_equal 0, result[:successes] 87 | assert_equal 0, result[:failures] 88 | end 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/resilient/test/properties_interface.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | class Test 3 | module PropertiesInterface 4 | def test_responds_to_force_open 5 | assert_respond_to @object, :force_open 6 | end 7 | 8 | def test_responds_to_force_closed 9 | assert_respond_to @object, :force_closed 10 | end 11 | 12 | def test_responds_to_instrumenter 13 | assert_respond_to @object, :instrumenter 14 | end 15 | 16 | def test_responds_to_sleep_window_seconds 17 | assert_respond_to @object, :sleep_window_seconds 18 | end 19 | 20 | def test_responds_to_request_volume_threshold 21 | assert_respond_to @object, :request_volume_threshold 22 | end 23 | 24 | def test_responds_to_error_threshold_percentage 25 | assert_respond_to @object, :error_threshold_percentage 26 | end 27 | 28 | def test_responds_to_bucket_size_in_seconds 29 | assert_respond_to @object, :bucket_size_in_seconds 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/resilient/version.rb: -------------------------------------------------------------------------------- 1 | module Resilient 2 | VERSION = "0.5.1" 3 | end 4 | -------------------------------------------------------------------------------- /resilient.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "resilient/version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "resilient" 8 | spec.version = Resilient::VERSION 9 | spec.authors = ["John Nunemaker"] 10 | spec.email = ["nunemaker@gmail.com"] 11 | 12 | spec.summary = %q{toolkit for resilient ruby apps} 13 | spec.homepage = "https://github.com/jnunemaker/resilient" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 17 | spec.bindir = "exe" 18 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_development_dependency "bundler", "< 3.0" 22 | spec.add_development_dependency "minitest", "~> 5.8" 23 | spec.add_development_dependency "timecop", "~> 0.8.0" 24 | end 25 | -------------------------------------------------------------------------------- /script/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #/ Usage: bootstrap 3 | #/ 4 | #/ Get the dependencies needed to work on the gem. 5 | #/ 6 | 7 | set -e 8 | cd $(dirname "$0")/.. 9 | 10 | [ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && { 11 | grep '^#/' <"$0"| cut -c4- 12 | exit 0 13 | } 14 | 15 | bundle --quiet 16 | -------------------------------------------------------------------------------- /script/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "resilient" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /script/release: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #/ Usage: release 3 | #/ 4 | #/ Tag the version in the repo and push the gem. 5 | #/ 6 | 7 | set -e 8 | cd $(dirname "$0")/.. 9 | 10 | [ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && { 11 | grep '^#/' <"$0"| cut -c4- 12 | exit 0 13 | } 14 | 15 | gem_name=resilient 16 | 17 | # Build a new gem archive. 18 | rm -rf $gem_name-*.gem 19 | gem build -q $gem_name.gemspec 20 | 21 | # Make sure we're on the master branch. 22 | (git branch | grep -q '* master') || { 23 | echo "Only release from the master branch." 24 | exit 1 25 | } 26 | 27 | # Figure out what version we're releasing. 28 | tag=v`ls $gem_name-*.gem | sed "s/^$gem_name-\(.*\)\.gem$/\1/"` 29 | 30 | echo "Releasing $tag" 31 | 32 | # Make sure we haven't released this version before. 33 | git fetch -t origin 34 | 35 | (git tag -l | grep -q "$tag") && { 36 | echo "Whoops, there's already a '${tag}' tag." 37 | exit 1 38 | } 39 | 40 | # Tag it and bag it. 41 | gem push $gem_name-*.gem && git tag "$tag" && 42 | git push origin master && git push origin "$tag" 43 | -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #/ Usage: test [individual test file] 3 | #/ 4 | #/ Bootstrap and run all tests or an individual test. 5 | #/ 6 | #/ Examples: 7 | #/ 8 | #/ # run all tests 9 | #/ test 10 | #/ 11 | #/ # run individual test 12 | #/ test test/controller_instrumentation_test.rb 13 | #/ 14 | 15 | set -e 16 | cd $(dirname "$0")/.. 17 | 18 | [ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && { 19 | grep '^#/' <"$0"| cut -c4- 20 | exit 0 21 | } 22 | 23 | ruby -I lib -I test -r rubygems \ 24 | -e 'require "bundler/setup"' \ 25 | -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@" 26 | -------------------------------------------------------------------------------- /script/watch: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #/ Usage: watch 3 | #/ 4 | #/ Startup guard. 5 | #/ 6 | 7 | set -e 8 | cd $(dirname "$0")/.. 9 | 10 | [ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && { 11 | grep '^#/' <"$0"| cut -c4- 12 | exit 0 13 | } 14 | 15 | bundle exec guard 16 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker/metrics/storage/memory_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker/metrics/storage/memory" 3 | require "resilient/test/metrics_storage_interface" 4 | 5 | module Resilient 6 | class CircuitBreaker 7 | class Metrics 8 | module Storage 9 | class MemoryTest < Test 10 | def setup 11 | super 12 | @object = Memory.new 13 | end 14 | 15 | include Test::MetricsStorageInterface 16 | end 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker/metrics_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker/metrics" 3 | require "resilient/test/metrics_interface" 4 | 5 | module Resilient 6 | class CircuitBreaker 7 | class MetricsTest < Test 8 | def setup 9 | super 10 | @object = Metrics.new(window_size_in_seconds: 5, bucket_size_in_seconds: 1) 11 | end 12 | 13 | include Test::MetricsInterface 14 | 15 | def test_success 16 | @object.success 17 | assert_successes @object, 1 18 | end 19 | 20 | def test_success_prunes 21 | now = Time.now 22 | 23 | Timecop.freeze(now) do 24 | @object.success 25 | assert_equal 1, @object.buckets.length, debug_metrics(@object) 26 | end 27 | 28 | Timecop.freeze(now + 1) do 29 | @object.success 30 | assert_equal 2, @object.buckets.length, debug_metrics(@object) 31 | end 32 | 33 | Timecop.freeze(now + 4) do 34 | @object.success 35 | assert_equal 3, @object.buckets.length, debug_metrics(@object) 36 | end 37 | 38 | Timecop.freeze(now + 10) do 39 | @object.success 40 | assert_equal 1, @object.buckets.length, debug_metrics(@object) 41 | end 42 | end 43 | 44 | def test_success_prunes_with_greater_than_one_second_bucket_size 45 | now = Time.now 46 | metrics = Metrics.new(window_size_in_seconds: 60, bucket_size_in_seconds: 10) 47 | 48 | Timecop.freeze(now) do 49 | metrics.success 50 | assert_equal 1, metrics.buckets.length, debug_metrics(metrics) 51 | end 52 | 53 | Timecop.freeze(now + 10) do 54 | metrics.success 55 | assert_equal 2, metrics.buckets.length, debug_metrics(metrics) 56 | end 57 | 58 | Timecop.freeze(now + 40) do 59 | metrics.success 60 | assert_equal 3, metrics.buckets.length, debug_metrics(metrics) 61 | end 62 | 63 | Timecop.freeze(now + 100) do 64 | metrics.success 65 | assert_equal 1, metrics.buckets.length, debug_metrics(metrics) 66 | end 67 | end 68 | 69 | def test_failure 70 | @object.failure 71 | assert_failures @object, 1 72 | end 73 | 74 | def test_failure_prunes 75 | now = Time.now 76 | 77 | Timecop.freeze(now) do 78 | @object.failure 79 | assert_equal 1, @object.buckets.length, debug_metrics(@object) 80 | end 81 | 82 | Timecop.freeze(now + 1) do 83 | @object.failure 84 | assert_equal 2, @object.buckets.length, debug_metrics(@object) 85 | end 86 | 87 | Timecop.freeze(now + 4) do 88 | @object.failure 89 | assert_equal 3, @object.buckets.length, debug_metrics(@object) 90 | end 91 | 92 | Timecop.freeze(now + 9) do 93 | @object.failure 94 | assert_equal 1, @object.buckets.length, debug_metrics(@object) 95 | end 96 | end 97 | 98 | def test_failure_prunes_with_greater_than_one_second_bucket_size 99 | now = Time.now 100 | metrics = Metrics.new(window_size_in_seconds: 60, bucket_size_in_seconds: 10) 101 | 102 | Timecop.freeze(now) do 103 | metrics.failure 104 | assert_equal 1, metrics.buckets.length, debug_metrics(metrics) 105 | end 106 | 107 | Timecop.freeze(now + 10) do 108 | metrics.failure 109 | assert_equal 2, metrics.buckets.length, debug_metrics(metrics) 110 | end 111 | 112 | Timecop.freeze(now + 40) do 113 | metrics.failure 114 | assert_equal 3, metrics.buckets.length, debug_metrics(metrics) 115 | end 116 | 117 | Timecop.freeze(now + 100) do 118 | metrics.failure 119 | assert_equal 1, metrics.buckets.length, debug_metrics(metrics) 120 | end 121 | end 122 | 123 | def test_under_request_volume_threshold 124 | assert @object.under_request_volume_threshold?(1) 125 | refute @object.under_request_volume_threshold?(0) 126 | 10.times { @object.success } 127 | assert @object.under_request_volume_threshold?(11) 128 | refute @object.under_request_volume_threshold?(10) 129 | refute @object.under_request_volume_threshold?(9) 130 | end 131 | 132 | def test_under_request_volume_threshold_is_pruned 133 | now = Time.now 134 | Timecop.freeze(now) do 135 | @object.success 136 | @object.success 137 | @object.success 138 | assert_successes @object, 3 139 | end 140 | 141 | Timecop.freeze(now + @object.window_size_in_seconds) do 142 | @object.under_request_volume_threshold?(1) 143 | assert_successes @object, 0 144 | end 145 | end 146 | 147 | def test_under_error_threshold_percentage 148 | @object.success 149 | @object.failure 150 | @object.failure 151 | assert @object.under_error_threshold_percentage?(68) 152 | assert @object.under_error_threshold_percentage?(67) 153 | refute @object.under_error_threshold_percentage?(66) 154 | end 155 | 156 | def test_under_error_threshold_percentage_with_zero_requests 157 | assert @object.under_error_threshold_percentage?(10) 158 | end 159 | 160 | def test_under_error_threshold_percentage_with_zero_failures 161 | @object.success 162 | assert @object.under_error_threshold_percentage?(10) 163 | end 164 | 165 | def test_under_error_threshold_percentage_is_pruned 166 | now = Time.now 167 | Timecop.freeze(now) do 168 | @object.success 169 | @object.success 170 | @object.success 171 | assert_successes @object, 3 172 | end 173 | 174 | Timecop.freeze(now + @object.window_size_in_seconds) do 175 | @object.under_error_threshold_percentage?(1) 176 | assert_successes @object, 0 177 | end 178 | end 179 | 180 | def test_reset 181 | @object.success 182 | @object.failure 183 | assert_successes @object, 1 184 | assert_failures @object, 1 185 | 186 | @object.reset 187 | 188 | assert_successes @object, 0 189 | assert_failures @object, 0 190 | end 191 | 192 | private 193 | 194 | def assert_successes(metrics, expected_successes) 195 | actual_successes = metrics.storage.sum(metrics.buckets, Metrics::StorageSuccessKeys)[Metrics::StorageSuccessKeys.first] 196 | assert_equal expected_successes, actual_successes 197 | end 198 | 199 | def assert_failures(metrics, expected_failures) 200 | actual_failures = metrics.storage.sum(metrics.buckets, Metrics::StorageFailureKeys)[Metrics::StorageFailureKeys.first] 201 | assert_equal expected_failures, actual_failures 202 | end 203 | end 204 | end 205 | end 206 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker/properties_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker/properties" 3 | require "resilient/instrumenters/noop" 4 | require "resilient/instrumenters/memory" 5 | require "resilient/test/properties_interface" 6 | 7 | module Resilient 8 | class CircuitBreaker 9 | class PropertiesTest < Test 10 | def setup 11 | super 12 | @object = Properties.new 13 | end 14 | 15 | include Test::PropertiesInterface 16 | 17 | def test_wrap_with_hash 18 | properties = Properties.wrap(force_open: true) 19 | assert_instance_of Properties, properties 20 | assert properties.force_open 21 | end 22 | 23 | def test_wrap_with_instance 24 | original_properties = Properties.new(force_open: true) 25 | properties = Properties.wrap(original_properties) 26 | assert_instance_of Properties, properties 27 | assert_equal original_properties.force_open, properties.force_open 28 | end 29 | 30 | def test_wrap_with_nil 31 | properties = Properties.wrap(nil) 32 | assert_instance_of Properties, properties 33 | end 34 | 35 | def test_wrap_with_unsupported_type 36 | assert_raises TypeError do 37 | Properties.wrap(Object.new) 38 | end 39 | end 40 | 41 | def test_defaults_force_open 42 | assert_equal false, @object.force_open 43 | end 44 | 45 | def test_allows_overriding_force_open 46 | assert_equal true, Properties.new(force_open: true).force_open 47 | end 48 | 49 | def test_defaults_force_closed 50 | assert_equal false, @object.force_closed 51 | end 52 | 53 | def test_allows_overriding_force_closed 54 | assert_equal true, Properties.new(force_closed: true).force_closed 55 | end 56 | 57 | def test_defaults_instrumenter 58 | assert_equal Instrumenters::Noop, @object.instrumenter 59 | end 60 | 61 | def test_allows_overriding_instrumenter 62 | instrumenter = Instrumenters::Memory.new 63 | assert_equal instrumenter, Properties.new(instrumenter: instrumenter).instrumenter 64 | end 65 | 66 | def test_defaults_sleep_window_seconds 67 | assert_equal 5, @object.sleep_window_seconds 68 | end 69 | 70 | def test_allows_overriding_sleep_window_seconds 71 | assert_equal 2, Properties.new(sleep_window_seconds: 2).sleep_window_seconds 72 | end 73 | 74 | def test_defaults_request_volume_threshold 75 | assert_equal 20, @object.request_volume_threshold 76 | end 77 | 78 | def test_allows_overriding_request_volume_threshold 79 | assert_equal 1, Properties.new(request_volume_threshold: 1).request_volume_threshold 80 | end 81 | 82 | def test_defaults_error_threshold_percentage 83 | assert_equal 50, @object.error_threshold_percentage 84 | end 85 | 86 | def test_allows_overriding_error_threshold_percentage 87 | assert_equal 12, Properties.new(error_threshold_percentage: 12).error_threshold_percentage 88 | end 89 | 90 | def test_defaults_window_size_in_seconds 91 | assert_equal 60, @object.window_size_in_seconds 92 | end 93 | 94 | def test_allows_overriding_window_size_in_seconds 95 | assert_equal 120, Properties.new(window_size_in_seconds: 120).window_size_in_seconds 96 | end 97 | 98 | def test_defaults_bucket_size_in_seconds 99 | assert_equal 10, @object.bucket_size_in_seconds 100 | end 101 | 102 | def test_allows_overriding_bucket_size_in_seconds 103 | assert_equal 2, Properties.new(bucket_size_in_seconds: 2).bucket_size_in_seconds 104 | end 105 | 106 | def test_raises_bucket_size_greater_than_window_size 107 | assert_raises ArgumentError do 108 | Properties.new({ 109 | window_size_in_seconds: 8, 110 | bucket_size_in_seconds: 10, 111 | }) 112 | end 113 | end 114 | 115 | def test_raises_bucket_size_equal_to_window_size 116 | assert_raises ArgumentError do 117 | Properties.new({ 118 | window_size_in_seconds: 8, 119 | bucket_size_in_seconds: 8, 120 | }) 121 | end 122 | end 123 | 124 | def test_raises_if_window_size_not_perfectly_divisible_by_bucket_size 125 | assert_raises ArgumentError do 126 | Properties.new({ 127 | window_size_in_seconds: 21, 128 | bucket_size_in_seconds: 4, 129 | }) 130 | end 131 | end 132 | 133 | def test_defaults_metrics_based_on_window_and_bucket_size 134 | properties = Properties.new({ 135 | window_size_in_seconds: 60, 136 | bucket_size_in_seconds: 10, 137 | }) 138 | assert_equal 60, properties.metrics.window_size_in_seconds 139 | assert_equal 10, properties.metrics.bucket_size_in_seconds 140 | end 141 | 142 | def test_allows_overriding_metrics 143 | metrics = Metrics.new({ 144 | window_size_in_seconds: 60, 145 | bucket_size_in_seconds: 10, 146 | }) 147 | properties = Properties.new({ 148 | window_size_in_seconds: 50, 149 | bucket_size_in_seconds: 25, 150 | metrics: metrics, 151 | }) 152 | assert_equal 60, properties.metrics.window_size_in_seconds 153 | assert_equal 10, properties.metrics.bucket_size_in_seconds 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker/registry_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker/registry" 3 | require "resilient/test/circuit_breaker_registry_interface" 4 | 5 | module Resilient 6 | class CircuitBreaker 7 | class RegistryTest < Test 8 | def setup 9 | super 10 | @object = Registry.new 11 | end 12 | 13 | include Test::CircuitBreakerRegistryInterface 14 | 15 | def test_default_class_accessors 16 | original_default = Registry.default 17 | assert_instance_of Registry, Registry.default 18 | Registry.default = @object 19 | assert_equal @object, Registry.default 20 | ensure 21 | Registry.default = original_default 22 | end 23 | 24 | def test_class_reset 25 | Registry.default.fetch("foo") { "bar" } 26 | Registry.reset 27 | assert_equal "reset!", Registry.default.fetch("foo") { "reset!" } 28 | end 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker_instrumentation_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker" 3 | require "resilient/instrumenters/memory" 4 | require "resilient/circuit_breaker/properties" 5 | require "resilient/circuit_breaker/metrics/storage/memory" 6 | 7 | module Resilient 8 | class CircuitBreakerInstrumentationTest < Test 9 | def test_instruments_allow_request 10 | instrumenter = Instrumenters::Memory.new 11 | circuit_breaker = CircuitBreaker.get("test", instrumenter: instrumenter) 12 | assert circuit_breaker.allow_request? 13 | event = instrumenter.events.detect { |event| event.name =~ /allow_request/ } 14 | refute_nil event 15 | assert_equal "resilient.circuit_breaker.allow_request", event.name 16 | assert_equal "test", event.payload.fetch(:key).name 17 | assert_equal false, event.payload[:force_open] 18 | assert_equal false, event.payload[:force_closed] 19 | assert_equal true, event.payload[:result] 20 | 21 | event = instrumenter.events.reverse.detect { |event| event.name =~ /open/ } 22 | assert event 23 | assert_equal "test", event.payload.fetch(:key).name 24 | assert_equal false, event[:result] 25 | 26 | refute instrumenter.events.detect { |event| event.name =~ /allow_single_request/ } 27 | end 28 | 29 | def test_instruments_allow_request_force_open 30 | instrumenter = Instrumenters::Memory.new 31 | circuit_breaker = CircuitBreaker.get("test", { 32 | instrumenter: instrumenter, 33 | force_open: true, 34 | }) 35 | refute circuit_breaker.allow_request? 36 | event = instrumenter.events.first 37 | refute_nil event 38 | assert_equal "resilient.circuit_breaker.allow_request", event.name 39 | assert_equal "test", event.payload.fetch(:key).name 40 | assert_equal true, event.payload[:force_open] 41 | assert_equal false, event.payload[:result] 42 | 43 | refute instrumenter.events.detect { |event| event.name =~ /open/ } 44 | refute instrumenter.events.detect { |event| event.name =~ /allow_single_request/ } 45 | end 46 | 47 | def test_instruments_allow_request_force_closed 48 | instrumenter = Instrumenters::Memory.new 49 | circuit_breaker = CircuitBreaker.get("test", { 50 | instrumenter: instrumenter, 51 | force_closed: true, 52 | }) 53 | assert circuit_breaker.allow_request? 54 | event = instrumenter.events.detect { |event| event.name =~ /allow_request/ } 55 | refute_nil event 56 | assert_equal "resilient.circuit_breaker.allow_request", event.name 57 | assert_equal "test", event.payload.fetch(:key).name 58 | assert_equal true, event.payload[:force_closed] 59 | assert_equal true, event.payload[:result] 60 | 61 | event = instrumenter.events.reverse.detect { |event| event.name =~ /open/ } 62 | assert event 63 | assert_equal "test", event.payload.fetch(:key).name 64 | assert_equal false, event[:result] 65 | 66 | refute instrumenter.events.detect { |event| event.name =~ /allow_single_request/ } 67 | end 68 | 69 | def test_instruments_allow_request_force_closed_when_normal_behavior_would_be_open 70 | instrumenter = Instrumenters::Memory.new 71 | circuit_breaker = CircuitBreaker.get("test", { 72 | instrumenter: instrumenter, 73 | force_closed: true, 74 | error_threshold_percentage: 50, 75 | request_volume_threshold: 0, 76 | }) 77 | circuit_breaker.failure 78 | assert circuit_breaker.allow_request? 79 | event = instrumenter.events.detect { |event| 80 | event.name == "resilient.circuit_breaker.allow_request" 81 | } 82 | refute_nil event 83 | assert_equal "resilient.circuit_breaker.allow_request", event.name 84 | assert_equal "test", event.payload.fetch(:key).name 85 | assert_equal false, event.payload[:force_open] 86 | assert_equal true, event.payload[:force_closed] 87 | assert_equal true, event.payload[:result] 88 | 89 | event = instrumenter.events.reverse.detect { |event| event.name =~ /open/ } 90 | assert event 91 | assert_equal "test", event.payload.fetch(:key).name 92 | assert_equal true, event[:result] 93 | 94 | event = instrumenter.events.detect { |event| event.name =~ /allow_single_request/ } 95 | assert event 96 | assert_equal "test", event.payload.fetch(:key).name 97 | assert_equal false, event[:result] 98 | end 99 | 100 | def test_instrument_allow_request_force_closed_when_normal_behavior_would_be_allow_single_request 101 | instrumenter = Instrumenters::Memory.new 102 | circuit_breaker = CircuitBreaker.get("test", { 103 | instrumenter: instrumenter, 104 | force_closed: true, 105 | error_threshold_percentage: 50, 106 | request_volume_threshold: 0, 107 | sleep_window_seconds: 1, 108 | }) 109 | circuit_breaker.failure 110 | assert circuit_breaker.allow_request? 111 | 112 | # force code path through allow single request by moving past sleep threshold 113 | Timecop.freeze(Time.now + circuit_breaker.properties.sleep_window_seconds + 1) { 114 | assert circuit_breaker.allow_request? 115 | } 116 | 117 | event = instrumenter.events.reverse.detect { |event| 118 | event.name == "resilient.circuit_breaker.allow_request" 119 | } 120 | refute_nil event 121 | assert_equal "resilient.circuit_breaker.allow_request", event.name 122 | assert_equal "test", event.payload.fetch(:key).name 123 | assert_equal false, event.payload[:force_open] 124 | assert_equal true, event.payload[:force_closed] 125 | assert_equal true, event.payload[:result] 126 | 127 | event = instrumenter.events.reverse.detect { |event| event.name =~ /open/ } 128 | assert event 129 | assert_equal "test", event.payload.fetch(:key).name 130 | assert_equal true, event[:result] 131 | 132 | event = instrumenter.events.reverse.detect { |event| event.name =~ /allow_single_request/ } 133 | assert event 134 | assert_equal "test", event.payload.fetch(:key).name 135 | assert_equal true, event[:result] 136 | end 137 | 138 | def test_instruments_allow_request_open_true_allow_single_request_false 139 | instrumenter = Instrumenters::Memory.new 140 | circuit_breaker = CircuitBreaker.get("test", { 141 | instrumenter: instrumenter, 142 | error_threshold_percentage: 50, 143 | request_volume_threshold: 0, 144 | }) 145 | circuit_breaker.properties.instrumenter.reset 146 | circuit_breaker.failure 147 | refute circuit_breaker.allow_request? 148 | event = instrumenter.events.detect { |event| 149 | event.name == "resilient.circuit_breaker.allow_request" 150 | } 151 | refute_nil event 152 | assert_equal "resilient.circuit_breaker.allow_request", event.name 153 | assert_equal "test", event.payload.fetch(:key).name 154 | assert_equal false, event.payload[:force_open] 155 | assert_equal false, event.payload[:force_closed] 156 | assert_equal false, event.payload[:result] 157 | 158 | event = instrumenter.events.reverse.detect { |event| event.name =~ /open/ } 159 | assert event 160 | assert_equal "test", event.payload.fetch(:key).name 161 | assert_equal true, event[:result] 162 | 163 | event = instrumenter.events.reverse.detect { |event| event.name =~ /allow_single_request/ } 164 | assert event 165 | assert_equal "test", event.payload.fetch(:key).name 166 | assert_equal false, event[:result] 167 | end 168 | 169 | def test_instruments_allow_request_open_true_allow_single_request_true 170 | instrumenter = Instrumenters::Memory.new 171 | circuit_breaker = CircuitBreaker.get("test", { 172 | instrumenter: instrumenter, 173 | error_threshold_percentage: 50, 174 | request_volume_threshold: 0, 175 | sleep_window_seconds: 1, 176 | }) 177 | circuit_breaker.failure 178 | refute circuit_breaker.allow_request? 179 | 180 | # force code path through allow single request by moving past sleep threshold 181 | Timecop.freeze(Time.now + circuit_breaker.properties.sleep_window_seconds + 1) { 182 | assert circuit_breaker.allow_request? 183 | } 184 | 185 | event = instrumenter.events.reverse.detect { |event| 186 | event.name == "resilient.circuit_breaker.allow_request" 187 | } 188 | refute_nil event 189 | assert_equal "resilient.circuit_breaker.allow_request", event.name 190 | assert_equal "test", event.payload.fetch(:key).name 191 | assert_equal false, event.payload[:force_open] 192 | assert_equal false, event.payload[:force_closed] 193 | assert_equal true, event.payload[:result] 194 | 195 | event = instrumenter.events.reverse.detect { |event| event.name =~ /open/ } 196 | assert event 197 | assert_equal "test", event.payload.fetch(:key).name 198 | assert_equal true, event[:result] 199 | 200 | event = instrumenter.events.reverse.detect { |event| event.name =~ /allow_single_request/ } 201 | assert event 202 | assert_equal "test", event.payload.fetch(:key).name 203 | assert_equal true, event[:result] 204 | end 205 | 206 | def test_instruments_success_when_circuit_closed 207 | instrumenter = Instrumenters::Memory.new 208 | circuit_breaker = CircuitBreaker.get("test", instrumenter: instrumenter) 209 | circuit_breaker.success 210 | event = instrumenter.events.first 211 | refute_nil event 212 | assert_equal "resilient.circuit_breaker.success", event.name 213 | assert_equal "test", event.payload.fetch(:key).name 214 | assert_nil event.payload[:closed_the_circuit] 215 | end 216 | 217 | def test_instruments_success_when_circuit_open 218 | instrumenter = Instrumenters::Memory.new 219 | circuit_breaker = CircuitBreaker.get("test", instrumenter: instrumenter) 220 | circuit_breaker.instance_variable_set("@open", true) 221 | circuit_breaker.success 222 | event = instrumenter.events.first 223 | refute_nil event 224 | assert_equal "resilient.circuit_breaker.close_circuit", event.name 225 | assert_equal "test", event.payload.fetch(:key).name 226 | assert_equal false, event.payload[:open] 227 | refute_nil event.payload[:interval] 228 | event = instrumenter.events[1] 229 | refute_nil event 230 | assert_equal "resilient.circuit_breaker.success", event.name 231 | assert_equal "test", event.payload.fetch(:key).name 232 | assert_equal true, event.payload[:closed_the_circuit] 233 | end 234 | 235 | def test_instruments_failure 236 | instrumenter = Instrumenters::Memory.new 237 | circuit_breaker = CircuitBreaker.get("test", instrumenter: instrumenter) 238 | circuit_breaker.failure 239 | event = instrumenter.events.first 240 | refute_nil event 241 | assert_equal "resilient.circuit_breaker.failure", event.name 242 | assert_equal "test", event.payload.fetch(:key).name 243 | end 244 | 245 | def test_instruments_reset 246 | instrumenter = Instrumenters::Memory.new 247 | circuit_breaker = CircuitBreaker.get("test", instrumenter: instrumenter) 248 | circuit_breaker.reset 249 | event = instrumenter.events.first 250 | refute_nil event 251 | assert_equal "resilient.circuit_breaker.reset", event.name 252 | assert_equal "test", event.payload.fetch(:key).name 253 | end 254 | end 255 | end 256 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker_integration_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker" 3 | 4 | module Resilient 5 | class CircuitBreakerIntegrationTest < Test 6 | def test_enough_failures_in_time_window_open_circuit 7 | circuit_breaker = CircuitBreaker.get("test", { 8 | error_threshold_percentage: 25, 9 | request_volume_threshold: 0, 10 | window_size_in_seconds: 60, 11 | bucket_size_in_seconds: 10, 12 | }) 13 | 70.times { circuit_breaker.success } 14 | assert circuit_breaker.allow_request?, 15 | debug_circuit_breaker(circuit_breaker) 16 | 17 | 20.times { circuit_breaker.failure } 18 | assert circuit_breaker.allow_request?, 19 | debug_circuit_breaker(circuit_breaker) 20 | 21 | 5.times { circuit_breaker.success } 22 | assert circuit_breaker.allow_request?, 23 | debug_circuit_breaker(circuit_breaker) 24 | 25 | 5.times { circuit_breaker.failure } 26 | refute circuit_breaker.allow_request?, 27 | debug_circuit_breaker(circuit_breaker) 28 | end 29 | 30 | def test_enough_failures_in_time_window_but_under_request_threshold_does_not_open_circuit 31 | circuit_breaker = CircuitBreaker.get("test", { 32 | error_threshold_percentage: 25, 33 | request_volume_threshold: 20, 34 | window_size_in_seconds: 60, 35 | bucket_size_in_seconds: 10, 36 | }) 37 | 18.times { circuit_breaker.failure } 38 | assert circuit_breaker.allow_request?, 39 | debug_circuit_breaker(circuit_breaker) 40 | 41 | 2.times { circuit_breaker.failure } 42 | refute circuit_breaker.allow_request?, 43 | debug_circuit_breaker(circuit_breaker) 44 | end 45 | 46 | def test_forced_open_does_not_allow_request_even_if_all_successes 47 | circuit_breaker = CircuitBreaker.get("test", { 48 | error_threshold_percentage: 25, 49 | request_volume_threshold: 0, 50 | force_open: true, 51 | }) 52 | refute circuit_breaker.allow_request?, 53 | debug_circuit_breaker(circuit_breaker) 54 | 55 | 20.times { circuit_breaker.success } 56 | refute circuit_breaker.allow_request?, 57 | debug_circuit_breaker(circuit_breaker) 58 | end 59 | 60 | def test_forced_close_allows_requests_even_if_all_failures 61 | circuit_breaker = CircuitBreaker.get("test", { 62 | error_threshold_percentage: 25, 63 | request_volume_threshold: 0, 64 | force_closed: true, 65 | }) 66 | assert circuit_breaker.allow_request?, 67 | debug_circuit_breaker(circuit_breaker) 68 | 69 | 20.times { circuit_breaker.failure } 70 | assert circuit_breaker.allow_request?, 71 | debug_circuit_breaker(circuit_breaker) 72 | end 73 | 74 | def test_force_open_takes_precedence_over_force_closed 75 | circuit_breaker = CircuitBreaker.get("test", { 76 | request_volume_threshold: 0, 77 | force_closed: true, 78 | force_open: true, 79 | }) 80 | refute circuit_breaker.allow_request?, 81 | debug_circuit_breaker(circuit_breaker) 82 | end 83 | 84 | def test_allow_request_denies_for_sleep_seconds_then_allows_single_request_which_if_successful_closes_circuit 85 | circuit_breaker = CircuitBreaker.get("test", { 86 | error_threshold_percentage: 25, 87 | request_volume_threshold: 0, 88 | window_size_in_seconds: 60, 89 | bucket_size_in_seconds: 10, 90 | }) 91 | now = Time.now 92 | bucket1 = now 93 | bucket2 = now + 10 94 | bucket3 = now + 20 95 | bucket4 = now + 30 96 | bucket5 = now + 40 97 | bucket6 = now + 50 98 | 99 | Timecop.freeze(bucket1) do 100 | 12.times { circuit_breaker.success } 101 | 2.times { circuit_breaker.failure } 102 | assert circuit_breaker.allow_request?, 103 | debug_circuit_breaker(circuit_breaker) 104 | end 105 | 106 | Timecop.freeze(bucket2) do 107 | 13.times { circuit_breaker.success } 108 | 3.times { circuit_breaker.failure } 109 | assert circuit_breaker.allow_request?, 110 | debug_circuit_breaker(circuit_breaker) 111 | end 112 | 113 | Timecop.freeze(bucket3) do 114 | 22.times { circuit_breaker.success } 115 | 10.times { circuit_breaker.failure } 116 | assert circuit_breaker.allow_request?, 117 | debug_circuit_breaker(circuit_breaker) 118 | end 119 | 120 | Timecop.freeze(bucket4) do 121 | 14.times { circuit_breaker.success } 122 | 3.times { circuit_breaker.failure } 123 | assert circuit_breaker.allow_request?, 124 | debug_circuit_breaker(circuit_breaker) 125 | end 126 | 127 | Timecop.freeze(bucket5) do 128 | 9.times { circuit_breaker.success } 129 | 4.times { circuit_breaker.failure } 130 | assert circuit_breaker.allow_request?, 131 | debug_circuit_breaker(circuit_breaker) 132 | end 133 | 134 | Timecop.freeze(bucket6) do 135 | 33.times { circuit_breaker.success } 136 | 13.times { circuit_breaker.failure } 137 | refute circuit_breaker.allow_request?, 138 | debug_circuit_breaker(circuit_breaker) 139 | end 140 | 141 | # single request is allowed now 142 | Timecop.freeze(bucket6 + circuit_breaker.properties.sleep_window_seconds + 1) do 143 | # allow single request through 144 | assert circuit_breaker.allow_request?, 145 | debug_circuit_breaker(circuit_breaker) 146 | 147 | # haven't marked success or failure yet so fail subsequent checks 148 | refute circuit_breaker.allow_request?, 149 | debug_circuit_breaker(circuit_breaker) 150 | 151 | circuit_breaker.success 152 | 153 | # success happened so we allow requests once again 154 | assert circuit_breaker.allow_request?, 155 | debug_circuit_breaker(circuit_breaker) 156 | end 157 | end 158 | end 159 | end 160 | -------------------------------------------------------------------------------- /test/resilient/circuit_breaker_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker" 3 | require "resilient/test/circuit_breaker_interface" 4 | 5 | module Resilient 6 | class CircuitBreakerTest < Test 7 | def setup 8 | super 9 | @object = CircuitBreaker.get("object") 10 | end 11 | 12 | include Test::CircuitBreakerInterface 13 | 14 | def test_get 15 | first_initialization = CircuitBreaker.get(Resilient::Key.new("longmire")) 16 | assert_instance_of CircuitBreaker, first_initialization 17 | 18 | second_initialization = CircuitBreaker.get(Resilient::Key.new("longmire")) 19 | assert_instance_of CircuitBreaker, second_initialization 20 | assert first_initialization.equal?(second_initialization), 21 | "#{first_initialization.inspect} is not the exact same object as #{second_initialization.inspect}" 22 | 23 | string_initialization = CircuitBreaker.get("longmire") 24 | assert_instance_of CircuitBreaker, string_initialization 25 | assert first_initialization.equal?(string_initialization), 26 | "#{first_initialization.inspect} is not the exact same object as #{string_initialization.inspect}" 27 | end 28 | 29 | def test_get_with_nil_key 30 | assert_raises ArgumentError do 31 | CircuitBreaker.get(nil) 32 | end 33 | end 34 | 35 | def test_get_with_properties_hash 36 | circuit_breaker = CircuitBreaker.get("test", error_threshold_percentage: 51) 37 | assert_instance_of CircuitBreaker::Properties, circuit_breaker.properties 38 | assert_equal 51, circuit_breaker.properties.error_threshold_percentage 39 | end 40 | 41 | def test_get_with_properties_instance 42 | circuit_breaker = CircuitBreaker.get("test", error_threshold_percentage: 51) 43 | assert_instance_of CircuitBreaker::Properties, circuit_breaker.properties 44 | assert_equal 51, circuit_breaker.properties.error_threshold_percentage 45 | end 46 | 47 | def test_get_with_different_properties_than_initially_provided 48 | key = Resilient::Key.new("longmire") 49 | circuit_breaker = CircuitBreaker.get(key, error_threshold_percentage: 10) 50 | different_properties_circuit_breaker = CircuitBreaker.get(key, error_threshold_percentage: 15) 51 | 52 | assert_equal 10, 53 | different_properties_circuit_breaker.properties.error_threshold_percentage 54 | end 55 | 56 | def test_new 57 | assert_raises NoMethodError do 58 | CircuitBreaker.new 59 | end 60 | end 61 | 62 | def test_allocate 63 | assert_raises NoMethodError do 64 | CircuitBreaker.allocate 65 | end 66 | end 67 | 68 | def test_key 69 | assert_equal "object", @object.key.name 70 | end 71 | 72 | def test_allow_request_when_under_error_threshold_percentage 73 | properties = default_test_properties_options({ 74 | error_threshold_percentage: 51, 75 | }) 76 | circuit_breaker = CircuitBreaker.get("test", properties) 77 | circuit_breaker.success 78 | circuit_breaker.failure 79 | 80 | assert circuit_breaker.allow_request?, 81 | debug_circuit_breaker(circuit_breaker) 82 | end 83 | 84 | def test_allow_request_when_over_error_threshold_percentage 85 | properties = default_test_properties_options({ 86 | error_threshold_percentage: 49, 87 | }) 88 | circuit_breaker = CircuitBreaker.get("test", properties) 89 | circuit_breaker.success 90 | circuit_breaker.failure 91 | 92 | refute circuit_breaker.allow_request?, 93 | debug_circuit_breaker(circuit_breaker) 94 | end 95 | 96 | def test_allow_request_when_at_error_threshold_percentage 97 | properties = default_test_properties_options({ 98 | error_threshold_percentage: 50, 99 | }) 100 | circuit_breaker = CircuitBreaker.get("test", properties) 101 | circuit_breaker.success 102 | circuit_breaker.failure 103 | 104 | refute circuit_breaker.allow_request?, 105 | debug_circuit_breaker(circuit_breaker) 106 | end 107 | 108 | def test_allow_request_when_under_request_volume_threshold 109 | properties = default_test_properties_options({ 110 | request_volume_threshold: 5, 111 | }) 112 | circuit_breaker = CircuitBreaker.get("test", properties) 113 | 4.times { circuit_breaker.metrics.failure } 114 | 115 | assert circuit_breaker.allow_request?, 116 | debug_circuit_breaker(circuit_breaker) 117 | end 118 | 119 | def test_allow_request_with_circuit_open_but_after_sleep_window_seconds 120 | now = Time.now 121 | circuit_breaker = CircuitBreaker.get("test", default_test_properties_options({ 122 | error_threshold_percentage: 49, 123 | sleep_window_seconds: 5, 124 | })) 125 | circuit_breaker.success 126 | circuit_breaker.failure 127 | 128 | assert_equal 0, circuit_breaker.opened_or_last_checked_at_epoch 129 | 130 | Timecop.freeze(now) do 131 | refute circuit_breaker.allow_request?, 132 | debug_circuit_breaker(circuit_breaker) 133 | assert_equal now.to_i, circuit_breaker.opened_or_last_checked_at_epoch 134 | end 135 | 136 | Timecop.freeze(now + circuit_breaker.properties.sleep_window_seconds - 1) do 137 | refute circuit_breaker.allow_request?, 138 | debug_circuit_breaker(circuit_breaker) 139 | assert_equal now.to_i, circuit_breaker.opened_or_last_checked_at_epoch 140 | end 141 | 142 | Timecop.freeze(now + circuit_breaker.properties.sleep_window_seconds) do 143 | refute circuit_breaker.allow_request?, 144 | debug_circuit_breaker(circuit_breaker) 145 | assert_equal now.to_i, circuit_breaker.opened_or_last_checked_at_epoch 146 | end 147 | 148 | Timecop.freeze(now + circuit_breaker.properties.sleep_window_seconds + 1) do 149 | assert circuit_breaker.allow_request?, 150 | debug_circuit_breaker(circuit_breaker) 151 | 152 | assert_equal (now + circuit_breaker.properties.sleep_window_seconds + 1).to_i, 153 | circuit_breaker.opened_or_last_checked_at_epoch 154 | end 155 | end 156 | 157 | def test_allow_request_when_forced_open_but_under_threshold 158 | properties = default_test_properties_options({ 159 | error_threshold_percentage: 51, 160 | force_open: true, 161 | }) 162 | circuit_breaker = CircuitBreaker.get("test", properties) 163 | circuit_breaker.success 164 | circuit_breaker.failure 165 | 166 | refute circuit_breaker.allow_request?, 167 | debug_circuit_breaker(circuit_breaker) 168 | end 169 | 170 | def test_allow_request_when_forced_closed_but_over_threshold 171 | properties = default_test_properties_options({ 172 | error_threshold_percentage: 49, 173 | request_volume_threshold: 0, 174 | force_closed: true, 175 | }) 176 | circuit_breaker = CircuitBreaker.get("test", properties) 177 | circuit_breaker.success 178 | circuit_breaker.failure 179 | 180 | assert circuit_breaker.allow_request?, 181 | debug_circuit_breaker(circuit_breaker) 182 | end 183 | 184 | def test_success_when_open_does_reset_metrics 185 | metrics = Minitest::Mock.new 186 | circuit_breaker = CircuitBreaker.get("test", metrics: metrics) 187 | circuit_breaker.instance_variable_set("@open", true) 188 | 189 | metrics.expect :reset, nil 190 | circuit_breaker.success 191 | metrics.verify 192 | end 193 | 194 | def test_success_when_not_open_calls_success_on_metrics 195 | metrics = Minitest::Mock.new 196 | circuit_breaker = CircuitBreaker.get("test", metrics: metrics) 197 | 198 | metrics.expect :success, nil 199 | circuit_breaker.success 200 | metrics.verify 201 | end 202 | 203 | def test_failure_calls_failure_on_metrics 204 | metrics = Minitest::Mock.new 205 | circuit_breaker = CircuitBreaker.get("test", metrics: metrics) 206 | 207 | metrics.expect :failure, nil 208 | circuit_breaker.failure 209 | metrics.verify 210 | end 211 | 212 | def test_reset_calls_reset_on_metrics 213 | metrics = Minitest::Mock.new 214 | circuit_breaker = CircuitBreaker.get("test", metrics: metrics) 215 | 216 | metrics.expect :reset, nil 217 | circuit_breaker.reset 218 | metrics.verify 219 | end 220 | 221 | def test_reset_sets_open_to_false 222 | circuit_breaker = CircuitBreaker.get("test") 223 | circuit_breaker.reset 224 | 225 | assert_equal false, circuit_breaker.open 226 | end 227 | 228 | def test_reset_sets_opened_or_last_checked_at_epoch_to_zero 229 | circuit_breaker = CircuitBreaker.get("test") 230 | circuit_breaker.reset 231 | 232 | assert_equal 0, circuit_breaker.opened_or_last_checked_at_epoch 233 | end 234 | 235 | private 236 | 237 | # Returns a Hash of default properties options set in a way that all the short 238 | # circuit properties options are turned off. 239 | def default_test_properties_options(options = {}) 240 | { 241 | request_volume_threshold: 0, 242 | force_closed: false, 243 | force_open: false, 244 | }.merge(options) 245 | end 246 | end 247 | end 248 | -------------------------------------------------------------------------------- /test/resilient/instrumenters/memory_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/instrumenters/memory" 3 | 4 | module Resilient 5 | module Instrumenters 6 | class MemoryTest < Test 7 | def test_initialize 8 | instrumenter = Memory.new 9 | assert_equal [], instrumenter.events 10 | end 11 | 12 | def test_keeps_track_of_events 13 | name = :foo 14 | payload = {pay: "load"} 15 | block_result = :result 16 | instrumenter = Memory.new 17 | instrumenter.instrument(name, payload) { block_result } 18 | event = instrumenter.events.first 19 | refute_nil event 20 | assert_equal name, event.name 21 | assert_equal payload, event.payload 22 | assert_equal block_result, event.result 23 | end 24 | 25 | def test_yields_payload_to_block 26 | yielded = nil 27 | payload = {pay: "load"} 28 | instrumenter = Memory.new 29 | instrumenter.instrument(:foo, payload) { |yielded_payload| 30 | yielded = yielded_payload 31 | } 32 | assert_equal yielded, payload 33 | end 34 | 35 | def test_dups_payload_to_avoid_mutation 36 | instrumenter = Memory.new 37 | payload = {} 38 | result = instrumenter.instrument(:foo, payload) 39 | refute instrumenter.events.first.payload.equal?(payload) 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /test/resilient/instrumenters/noop_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/instrumenters/noop" 3 | 4 | module Resilient 5 | module Instrumenters 6 | class NoopTest < Test 7 | def test_instrument_with_name 8 | yielded = false 9 | Noop.instrument(:foo) { yielded = true } 10 | assert yielded 11 | end 12 | 13 | def test_instrument_with_name_and_payload 14 | yielded = false 15 | Noop.instrument(:foo, {:pay => :load}) { yielded = true } 16 | assert yielded 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/resilient/key_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient/circuit_breaker" 3 | require "resilient/test/circuit_breaker_interface" 4 | 5 | module Resilient 6 | class KeyTest < Test 7 | def test_wrap_with_string 8 | key = Key.wrap("test") 9 | assert_instance_of Key, key 10 | assert_equal "test", key.name 11 | end 12 | 13 | def test_wrap_with_instance 14 | original_key = Key.new("test") 15 | key = Key.wrap(original_key) 16 | assert_instance_of Key, key 17 | assert original_key.equal?(key) 18 | end 19 | 20 | def test_wrap_with_nil 21 | assert_nil Key.wrap(nil) 22 | end 23 | 24 | def test_wrap_with_unsupported_type 25 | assert_raises TypeError do 26 | Key.wrap(Object.new) 27 | end 28 | end 29 | 30 | def test_initialize_with_string 31 | key = Key.new("test") 32 | assert_equal "test", key.name 33 | end 34 | 35 | def test_initialize_with_symbol 36 | assert_raises TypeError do 37 | Key.new(:test) 38 | end 39 | end 40 | 41 | def test_hash 42 | name = "test" 43 | key = Key.new(name) 44 | assert_equal name.hash, key.hash 45 | end 46 | 47 | def test_equality 48 | key_a = Key.new("a") 49 | key_b = Key.new("b") 50 | other_key_a = Key.new("a") 51 | not_key = Object.new 52 | 53 | assert key_a.eql?(other_key_a) 54 | refute key_a.eql?(not_key) 55 | refute key_a.eql?(key_b) 56 | 57 | assert key_a == other_key_a 58 | refute key_a == not_key 59 | refute key_a == key_b 60 | end 61 | 62 | def test_as_hash_key 63 | key_a = Key.new("a") 64 | key_b = Key.new("b") 65 | other_key_a = Key.new("a") 66 | hash = {} 67 | hash[key_a] = "a" 68 | hash[other_key_a] = "other_a" 69 | hash[key_b] = "b" 70 | 71 | assert_equal "other_a", hash[key_a] 72 | assert_equal "other_a", hash[other_key_a] 73 | assert_equal "b", hash[key_b] 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /test/resilient_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "resilient" 3 | 4 | module Resilient 5 | class ResilientTest < Test 6 | def test_that_it_has_a_version_number 7 | refute_nil Resilient::VERSION 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require "minitest/autorun" 2 | require "timecop" 3 | require "pathname" 4 | 5 | module Resilient 6 | class Test < Minitest::Test 7 | def setup 8 | CircuitBreaker::Registry.reset 9 | end 10 | 11 | def debug_metrics(metrics, indent: "") 12 | keys = [:success, :failure] 13 | result = Hash.new { |h, k| h[k] = Hash.new(0) } 14 | metrics.buckets.each do |bucket| 15 | keys.each do |key| 16 | result[bucket][key] = metrics.storage.sum(bucket, key)[key] 17 | end 18 | end 19 | 20 | max_successes = result.values.map { |value| value[:success] }.max || 0 21 | max_failures = result.values.map { |value| value[:failure] }.max || 0 22 | max_requests = result.values.map { |value| value[:success] + value[:failure] }.max || 0 23 | 24 | requests_pad = max_requests.to_s.length 25 | successes_pad = max_successes.to_s.length 26 | failures_pad = max_failures.to_s.length 27 | 28 | buckets_debug = metrics.buckets.map { |bucket| 29 | "%s%s - %s (%s): %s + %s = %s" % [ 30 | indent, 31 | bucket.timestamp_start, 32 | bucket.timestamp_end, 33 | bucket.timestamp_end - bucket.timestamp_start + 1, 34 | result[bucket][:success].to_s.rjust(successes_pad), 35 | result[bucket][:failure].to_s.rjust(failures_pad), 36 | (result[bucket][:success] + result[bucket][:failure]).to_s.rjust(requests_pad), 37 | ] 38 | }.join("\n") 39 | end 40 | 41 | def debug_properties(properties, indent: "") 42 | properties.instance_variables.map { |ivar| 43 | "%s%s: %s" % [ 44 | indent, 45 | ivar.to_s.sub("@", ""), 46 | properties.instance_variable_get(ivar), 47 | ] 48 | }.join("\n") 49 | end 50 | 51 | def debug_circuit_breaker(circuit_breaker) 52 | <<-EOS 53 | now: #{Time.now.utc} 54 | open: #{circuit_breaker.open} 55 | opened_or_last_checked_at_epoch: #{circuit_breaker.opened_or_last_checked_at_epoch} 56 | metrics: #{circuit_breaker.metrics.inspect} 57 | buckets: 58 | #{debug_metrics(circuit_breaker.metrics, indent: " ")} 59 | properties: 60 | #{debug_properties(circuit_breaker.properties, indent: " ")} 61 | EOS 62 | end 63 | end 64 | end 65 | --------------------------------------------------------------------------------