├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .markdownlint.yaml ├── .rspec ├── .rubocop.yml ├── .yamllint.yml ├── .yardopts ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── bin ├── benchmark ├── check-version ├── console ├── rspec ├── rubocop ├── yard ├── yardoc └── yri ├── faulty.gemspec ├── lib ├── faulty.rb └── faulty │ ├── cache.rb │ ├── cache │ ├── auto_wire.rb │ ├── circuit_proxy.rb │ ├── default.rb │ ├── fault_tolerant_proxy.rb │ ├── interface.rb │ ├── mock.rb │ ├── null.rb │ └── rails.rb │ ├── circuit.rb │ ├── circuit_registry.rb │ ├── deprecation.rb │ ├── error.rb │ ├── events.rb │ ├── events │ ├── callback_listener.rb │ ├── filter_notifier.rb │ ├── honeybadger_listener.rb │ ├── listener_interface.rb │ ├── log_listener.rb │ └── notifier.rb │ ├── immutable_options.rb │ ├── patch.rb │ ├── patch │ ├── base.rb │ ├── elasticsearch.rb │ ├── mysql2.rb │ ├── redis.rb │ └── redis │ │ ├── middleware.rb │ │ └── patch.rb │ ├── result.rb │ ├── status.rb │ ├── storage.rb │ ├── storage │ ├── auto_wire.rb │ ├── circuit_proxy.rb │ ├── fallback_chain.rb │ ├── fault_tolerant_proxy.rb │ ├── interface.rb │ ├── memory.rb │ ├── null.rb │ └── redis.rb │ └── version.rb └── spec ├── cache ├── auto_wire_spec.rb ├── circuit_proxy_spec.rb ├── default_spec.rb ├── fault_tolerant_proxy_spec.rb ├── mock_spec.rb ├── null_spec.rb └── rails_spec.rb ├── circuit_spec.rb ├── deprecation_spec.rb ├── events ├── callback_listener_spec.rb ├── filter_notifier_spec.rb ├── honeybadger_listener_spec.rb ├── log_listener_spec.rb └── notifier_spec.rb ├── faulty_spec.rb ├── immutable_options_spec.rb ├── patch ├── base_spec.rb ├── elasticsearch_spec.rb ├── mysql2_spec.rb └── redis_spec.rb ├── patch_spec.rb ├── result_spec.rb ├── spec_helper.rb ├── status_spec.rb ├── storage ├── auto_wire_spec.rb ├── circuit_proxy_spec.rb ├── fallback_chain_spec.rb ├── fault_tolerant_proxy_spec.rb ├── memory_spec.rb └── redis_spec.rb └── support └── concurrency.rb /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: CI 3 | on: 4 | push: 5 | tags: ['v*'] 6 | branches: [master] 7 | pull_request: 8 | branches: ['**'] 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | ruby: ['2.4', '2.5', '2.6', '2.7', '3.0', jruby-head, truffleruby-head] 16 | redis: ['4'] 17 | search: [['opensearch-ruby:2.1.0', 'opensearchproject/opensearch:2.2.1']] 18 | include: 19 | # Redis 3 20 | - ruby: '2.7' 21 | redis: '3' 22 | search: ['opensearch-ruby:2.1.0', 'opensearchproject/opensearch:2.2.1'] 23 | # Opensearch 1.0 24 | - ruby: '2.7' 25 | redis: '4' 26 | search: ['opensearch-ruby:1.0.1', 'opensearchproject/opensearch:1.0.1'] 27 | # Elasticsearch 7.13 28 | - ruby: '2.7' 29 | redis: '4' 30 | search: ['elasticsearch:7.13.3', 'elasticsearch:7.13.4'] 31 | # Redis 5 32 | - ruby: '2.7' 33 | redis: '5' 34 | search: ['opensearch-ruby:2.1.0', 'opensearchproject/opensearch:2.2.1'] 35 | # Ruby 2.3 & Elasticsearch 7.5 36 | - ruby: '2.3' 37 | redis: '4' 38 | search: ['elasticsearch:7.5.0', 'elasticsearch:7.13.4'] 39 | services: 40 | redis: 41 | image: redis 42 | ports: 43 | - 6379:6379 44 | search: 45 | image: ${{ matrix.search[1] }} 46 | ports: 47 | - 9200:9200 48 | env: 49 | discovery.type: single-node 50 | plugins.security.disabled: ${{ contains(matrix.search[1], 'opensearch') && 'true' || '' }} 51 | options: >- 52 | --health-cmd="curl http://localhost:9200/_cluster/health" 53 | --health-interval=3s 54 | --health-timeout=5s 55 | --health-retries=20 56 | 57 | env: 58 | REDIS_VERSION: ${{ matrix.redis }} 59 | SEARCH_GEM: ${{ matrix.search[0] }} 60 | 61 | steps: 62 | - uses: actions/checkout@v3 63 | - uses: ruby/setup-ruby@v1 64 | with: 65 | ruby-version: ${{ matrix.ruby }} 66 | bundler-cache: true 67 | - name: start MySQL 68 | run: sudo /etc/init.d/mysql start 69 | - run: bundle exec rspec --format doc 70 | env: 71 | MYSQL_USER: root 72 | MYSQL_PASSWORD: root 73 | - uses: codecov/codecov-action@v3 74 | if: matrix.ruby == '2.7' 75 | with: 76 | files: coverage/coverage.xml 77 | 78 | rubocop: 79 | runs-on: ubuntu-latest 80 | steps: 81 | - uses: actions/checkout@v3 82 | - uses: ruby/setup-ruby@v1 83 | with: 84 | ruby-version: '2.7' 85 | bundler-cache: true 86 | - run: bundle exec rubocop 87 | 88 | yard: 89 | runs-on: ubuntu-latest 90 | steps: 91 | - uses: actions/checkout@v3 92 | - uses: ruby/setup-ruby@v1 93 | with: 94 | ruby-version: '2.7' 95 | bundler-cache: true 96 | - run: bin/yardoc --fail-on-warning 97 | 98 | check_version: 99 | runs-on: ubuntu-latest 100 | if: startsWith(github.ref, 'refs/tags/v') 101 | steps: 102 | - uses: actions/checkout@v3 103 | - uses: ruby/setup-ruby@v1 104 | with: 105 | ruby-version: '2.7' 106 | bundler-cache: true 107 | - run: bin/check-version 108 | 109 | release: 110 | needs: [test, rubocop, yard, check_version] 111 | if: startsWith(github.ref, 'refs/tags/v') 112 | runs-on: ubuntu-latest 113 | steps: 114 | - uses: actions/checkout@v2 115 | - uses: dawidd6/action-publish-gem@v1 116 | with: 117 | api_key: ${{secrets.RUBYGEMS_API_KEY}} 118 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /Gemfile.lock 3 | /vendor/ 4 | /.ruby-version 5 | /*.gem 6 | 7 | /coverage/ 8 | /doc/ 9 | /.yardoc/ 10 | 11 | .byebug_history 12 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | default: true 2 | 3 | # MD024/no-duplicate-heading/no-duplicate-header 4 | MD024: false 5 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | --- 2 | require: 3 | - rubocop-rspec 4 | 5 | AllCops: 6 | TargetRubyVersion: 2.3 7 | 8 | Gemspec/DeprecatedAttributeAssignment: { Enabled: true } 9 | Gemspec/RequireMFA: { Enabled: true } 10 | 11 | Layout/ArgumentAlignment: { EnforcedStyle: with_fixed_indentation } 12 | Layout/CaseIndentation: { EnforcedStyle: end } 13 | Layout/EndAlignment: { EnforcedStyleAlignWith: start_of_line } 14 | Layout/FirstArgumentIndentation: { EnforcedStyle: consistent } 15 | Layout/FirstArrayElementIndentation: { EnforcedStyle: consistent } 16 | Layout/FirstHashElementIndentation: { EnforcedStyle: consistent } 17 | Layout/LineContinuationLeadingSpace: { Enabled: true } 18 | Layout/LineContinuationSpacing: { Enabled: true } 19 | Layout/LineEndStringConcatenationIndentation: { Enabled: true } 20 | Layout/LineLength: { Max: 120 } 21 | Layout/MultilineMethodCallIndentation: { EnforcedStyle: indented } 22 | Layout/ParameterAlignment: { EnforcedStyle: with_fixed_indentation } 23 | Layout/RescueEnsureAlignment: { Enabled: false } 24 | Layout/SpaceBeforeBrackets: { Enabled: true } 25 | 26 | Lint/AmbiguousAssignment: { Enabled: true } 27 | Lint/AmbiguousOperatorPrecedence: { Enabled: true } 28 | Lint/AmbiguousRange: { Enabled: true } 29 | Lint/ConstantOverwrittenInRescue: { Enabled: true } 30 | Lint/DeprecatedConstants: { Enabled: true } 31 | Lint/DuplicateBranch: { Enabled: true } 32 | Lint/DuplicateRegexpCharacterClassElement: { Enabled: true } 33 | Lint/EmptyBlock: { Enabled: true } 34 | Lint/EmptyClass: { Enabled: true } 35 | Lint/EmptyInPattern: { Enabled: true } 36 | Lint/IncompatibleIoSelectWithFiberScheduler: { Enabled: true } 37 | Lint/LambdaWithoutLiteralBlock: { Enabled: true } 38 | Lint/NoReturnInBeginEndBlocks: { Enabled: true } 39 | Lint/NonAtomicFileOperation: { Enabled: true } 40 | Lint/NumberedParameterAssignment: { Enabled: true } 41 | Lint/OrAssignmentToConstant: { Enabled: true } 42 | Lint/RaiseException: { Enabled: true } 43 | Lint/RedundantDirGlobSort: { Enabled: true } 44 | Lint/RefinementImportMethods: { Enabled: true } 45 | Lint/RequireRangeParentheses: { Enabled: true } 46 | Lint/RequireRelativeSelfPath: { Enabled: true } 47 | Lint/StructNewOverride: { Enabled: true } 48 | Lint/SymbolConversion: { Enabled: true } 49 | Lint/ToEnumArguments: { Enabled: true } 50 | Lint/TripleQuotes: { Enabled: true } 51 | Lint/UnexpectedBlockArity: { Enabled: true } 52 | Lint/UnmodifiedReduceAccumulator: { Enabled: true } 53 | Lint/UselessRuby2Keywords: { Enabled: true } 54 | 55 | RSpec/BeEq: { Enabled: true } 56 | RSpec/BeNil: { Enabled: true } 57 | RSpec/Capybara/SpecificMatcher: { Enabled: true } 58 | RSpec/ChangeByZero: { Enabled: true } 59 | RSpec/ExampleLength: { Enabled: false } 60 | RSpec/ExcessiveDocstringSpacing: { Enabled: true } 61 | RSpec/FactoryBot/SyntaxMethods: { Enabled: true } 62 | RSpec/FilePath: { Enabled: false } 63 | RSpec/IdenticalEqualityAssertion: { Enabled: true } 64 | RSpec/MessageSpies: { Enabled: false } 65 | RSpec/MultipleExpectations: { Enabled: false } 66 | RSpec/MultipleMemoizedHelpers: { Enabled: false } 67 | RSpec/NamedSubject: { Enabled: false } 68 | RSpec/Rails/AvoidSetupHook: { Enabled: true } 69 | RSpec/Rails/HaveHttpStatus: { Enabled: true } 70 | RSpec/SubjectDeclaration: { Enabled: true } 71 | RSpec/SubjectStub: { Enabled: false } 72 | RSpec/VerifiedDoubleReference: { Enabled: true } 73 | 74 | Metrics/AbcSize: { Max: 40 } 75 | Metrics/BlockLength: { Enabled: false } 76 | Metrics/ClassLength: { Enabled: false } 77 | Metrics/CyclomaticComplexity: { Enabled: false } 78 | Metrics/MethodLength: { Max: 30 } 79 | Metrics/PerceivedComplexity: { Enabled: false } 80 | 81 | Naming/BlockForwarding: { Enabled: true } 82 | Naming/MethodParameterName: { MinNameLength: 1 } 83 | 84 | Security/CompoundHash: { Enabled: true } 85 | Security/IoMethods: { Enabled: true } 86 | 87 | Style/ArgumentsForwarding: { Enabled: true } 88 | Style/ClassEqualityComparison: { Enabled: false } 89 | Style/CollectionCompact: { Enabled: true } 90 | Style/DocumentDynamicEvalDefinition: { Enabled: true } 91 | Style/Documentation: { Enabled: false } 92 | Style/EmptyHeredoc: { Enabled: true } 93 | Style/EmptyMethod: { EnforcedStyle: expanded } 94 | Style/EndlessMethod: { Enabled: true } 95 | Style/EnvHome: { Enabled: true } 96 | Style/FetchEnvVar: { Enabled: true } 97 | Style/FileRead: { Enabled: true } 98 | Style/FileWrite: { Enabled: true } 99 | Style/FrozenStringLiteralComment: { Enabled: true, EnforcedStyle: always } 100 | Style/GuardClause: { Enabled: false } 101 | Style/HashConversion: { Enabled: true } 102 | Style/HashEachMethods: { Enabled: true } 103 | Style/HashExcept: { Enabled: true } 104 | Style/HashTransformKeys: { Enabled: false } 105 | Style/HashTransformValues: { Enabled: false } 106 | Style/IfUnlessModifier: { Enabled: false } 107 | Style/IfWithBooleanLiteralBranches: { Enabled: true } 108 | Style/InPatternThen: { Enabled: true } 109 | Style/MapCompactWithConditionalBlock: { Enabled: true } 110 | Style/MapToHash: { Enabled: true } 111 | Style/MultilineInPatternThen: { Enabled: true } 112 | Style/NegatedIfElseCondition: { Enabled: true } 113 | Style/NestedFileDirname: { Enabled: true } 114 | Style/NilLambda: { Enabled: true } 115 | Style/NumberedParameters: { Enabled: true } 116 | Style/NumberedParametersLimit: { Enabled: true } 117 | Style/ObjectThen: { Enabled: true } 118 | Style/OpenStructUse: { Enabled: true } 119 | Style/QuotedSymbols: { Enabled: true } 120 | Style/RedundantArgument: { Enabled: true } 121 | Style/RedundantInitialize: { Enabled: true } 122 | Style/RedundantSelfAssignmentBranch: { Enabled: true } 123 | Style/SelectByRegexp: { Enabled: true } 124 | Style/StringChars: { Enabled: true } 125 | Style/SwapValues: { Enabled: true } 126 | -------------------------------------------------------------------------------- /.yamllint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | rules: 3 | braces: 4 | min-spaces-inside: 1 5 | max-spaces-inside: 1 6 | 7 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --markup markdown 2 | --markup-provider redcarpet 3 | - LICENSE.txt CHANGELOG.md 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | gemspec 6 | 7 | # We add non-essential gems like debugging tools and CI dependencies 8 | # here. This also allows us to use conditional dependencies that depend on the 9 | # platform 10 | 11 | not_jruby = %i[ruby mingw x64_mingw].freeze 12 | 13 | gem 'activesupport', '>= 4.2' 14 | gem 'byebug', platforms: not_jruby 15 | gem 'irb', '~> 1.0' 16 | # Minimum of 0.5.0 for specific error classes 17 | gem 'mysql2', '>= 0.5.0', platforms: not_jruby 18 | gem 'redcarpet', '~> 3.5', platforms: not_jruby 19 | gem 'rspec_junit_formatter', '~> 0.4' 20 | gem 'yard', '~> 0.9.25', platforms: not_jruby 21 | 22 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.4') 23 | gem 'honeybadger', '>= 2.0' 24 | end 25 | 26 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') 27 | gem 'rubocop', '~> 1.32.0' 28 | gem 'rubocop-rspec', '~> 2.12' 29 | gem 'simplecov', '>= 0.17.1' 30 | gem 'simplecov-cobertura', '~> 2.1' 31 | end 32 | 33 | if ENV['REDIS_VERSION'] 34 | gem 'redis', "~> #{ENV['REDIS_VERSION']}" 35 | end 36 | 37 | if ENV['SEARCH_GEM'] 38 | name, version = ENV['SEARCH_GEM'].split(':') 39 | name = 'opensearch-ruby' if name == 'opensearch' 40 | gem name, "~> #{version}" 41 | else 42 | gem 'opensearch-ruby', '~> 2.1' 43 | end 44 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2020 ParentSquare 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the “Software”), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /bin/benchmark: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'benchmark' 6 | require 'faulty' 7 | require 'redis' 8 | require 'json' 9 | 10 | n = 100_000 11 | width = 25 12 | puts "In memory circuits x#{n}" 13 | Benchmark.bm(width) do |b| 14 | in_memory = Faulty.new(listeners: []) 15 | b.report('memory storage') do 16 | n.times { in_memory.circuit(:memory).run { true } } 17 | end 18 | 19 | b.report('memory storage failures') do 20 | n.times do 21 | begin 22 | in_memory.circuit(:memory_fail, sample_threshold: n + 1).run { raise 'fail' } 23 | rescue StandardError 24 | # Expected to raise here 25 | end 26 | end 27 | end 28 | 29 | in_memory_large = Faulty.new(listeners: [], storage: Faulty::Storage::Memory.new(max_sample_size: 1000)) 30 | b.report('large memory storage') do 31 | n.times { in_memory_large.circuit(:memory_large).run { true } } 32 | end 33 | end 34 | 35 | n = 1000 36 | puts "\n\Redis circuits x#{n}" 37 | Benchmark.bm(width) do |b| 38 | redis = Faulty.new(listeners: [], storage: Faulty::Storage::Redis.new) 39 | b.report('redis storage') do 40 | n.times { redis.circuit(:memory).run { true } } 41 | end 42 | 43 | b.report('redis storage failures') do 44 | n.times do 45 | begin 46 | redis.circuit(:memory_fail, sample_threshold: n + 1).run { raise 'fail' } 47 | rescue StandardError 48 | # Expected to raise here 49 | end 50 | end 51 | end 52 | 53 | redis_large = Faulty.new(listeners: [], storage: Faulty::Storage::Redis.new(max_sample_size: 1000)) 54 | b.report('large redis storage') do 55 | n.times { redis_large.circuit(:memory).run { true } } 56 | end 57 | end 58 | 59 | n = 1_000_000 60 | puts "\n\nExtra x#{n}" 61 | Benchmark.bm(width) do |b| 62 | in_memory = Faulty.new(listeners: []) 63 | 64 | log_listener = Faulty::Events::LogListener.new(Logger.new(File::NULL)) 65 | log_circuit = in_memory.circuit(:log_listener) 66 | log_status = log_circuit.status 67 | b.report('log listener success') do 68 | n.times { log_listener.handle(:circuit_success, circuit: log_circuit, status: log_status) } 69 | end 70 | 71 | log_error = StandardError.new('test error') 72 | b.report('log listener failure') do 73 | n.times { log_listener.handle(:circuit_failure, error: log_error, circuit: log_circuit, status: log_status) } 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /bin/check-version: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | tag="$(git describe --abbrev=0 2>/dev/null || echo)" 6 | echo "Tag: ${tag}" 7 | tag="${tag#v}" 8 | echo "Git Version: ${tag}" 9 | [ "$tag" = '' ] && exit 0 10 | gem_version="$(ruby -r ./lib/faulty/version -e "puts Faulty.version" | tail -n1)" 11 | echo "Gem Version: ${gem_version}" 12 | 13 | tag_gt_version="$(ruby -r ./lib/faulty/version -e "puts Faulty.version >= Gem::Version.new('${tag}')" | tail -n1)" 14 | test "$tag_gt_version" = true 15 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'faulty' 6 | require 'byebug' if Gem.loaded_specs['byebug'] 7 | require 'irb' 8 | 9 | # For default cache support 10 | require 'active_support' 11 | 12 | IRB.start 13 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rspec' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require 'pathname' 12 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path('bundle', __dir__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require 'rubygems' 27 | require 'bundler/setup' 28 | 29 | load Gem.bin_path('rspec-core', 'rspec') 30 | -------------------------------------------------------------------------------- /bin/rubocop: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rubocop' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require 'pathname' 12 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path('bundle', __dir__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require 'rubygems' 27 | require 'bundler/setup' 28 | 29 | load Gem.bin_path('rubocop', 'rubocop') 30 | -------------------------------------------------------------------------------- /bin/yard: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'yard' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require 'pathname' 12 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path('bundle', __dir__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require 'rubygems' 27 | require 'bundler/setup' 28 | 29 | load Gem.bin_path('yard', 'yard') 30 | -------------------------------------------------------------------------------- /bin/yardoc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'yardoc' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require 'pathname' 12 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path('bundle', __dir__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require 'rubygems' 27 | require 'bundler/setup' 28 | 29 | load Gem.bin_path('yard', 'yardoc') 30 | -------------------------------------------------------------------------------- /bin/yri: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'yri' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require 'pathname' 12 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path('bundle', __dir__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require 'rubygems' 27 | require 'bundler/setup' 28 | 29 | load Gem.bin_path('yard', 'yri') 30 | -------------------------------------------------------------------------------- /faulty.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path('lib', __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require 'faulty/version' 6 | 7 | Gem::Specification.new do |spec| 8 | spec.name = 'faulty' 9 | spec.version = Faulty.version 10 | spec.authors = ['Justin Howard'] 11 | spec.email = ['jmhoward0@gmail.com'] 12 | spec.licenses = ['MIT'] 13 | spec.summary = 'Fault-tolerance tools for ruby based on circuit-breakers' 14 | spec.homepage = 'https://github.com/ParentSquare/faulty' 15 | 16 | rubydoc = 'https://www.rubydoc.info/gems' 17 | spec.metadata['rubygems_mfa_required'] = 'true' 18 | spec.metadata['changelog_uri'] = "#{spec.homepage}/blob/master/CHANGELOG.md" 19 | spec.metadata['documentation_uri'] = "#{rubydoc}/#{spec.name}/#{spec.version}" 20 | 21 | spec.files = Dir['lib/**/*.rb', '*.md', '*.txt', '.yardopts'] 22 | spec.require_paths = ['lib'] 23 | 24 | spec.required_ruby_version = '>= 2.3' 25 | 26 | spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0' 27 | 28 | # Only essential development tools and dependencies go here. 29 | # Other non-essential development dependencies go in the Gemfile. 30 | spec.add_development_dependency 'connection_pool', '~> 2.0' 31 | spec.add_development_dependency 'json' 32 | spec.add_development_dependency 'redis', '>= 3.0' 33 | spec.add_development_dependency 'rspec', '~> 3.8' 34 | spec.add_development_dependency 'timecop', '>= 0.9' 35 | end 36 | -------------------------------------------------------------------------------- /lib/faulty.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'securerandom' 4 | require 'forwardable' 5 | require 'concurrent' 6 | 7 | require 'faulty/deprecation' 8 | require 'faulty/immutable_options' 9 | require 'faulty/cache' 10 | require 'faulty/circuit' 11 | require 'faulty/error' 12 | require 'faulty/events' 13 | require 'faulty/patch' 14 | require 'faulty/circuit_registry' 15 | require 'faulty/result' 16 | require 'faulty/status' 17 | require 'faulty/storage' 18 | 19 | # The {Faulty} class has class-level methods for global state or can be 20 | # instantiated to create an independent configuration. 21 | # 22 | # If you are using global state, call {Faulty#init} during your application's 23 | # initialization. This is the simplest way to use {Faulty}. If you prefer, you 24 | # can also call {Faulty.new} to create independent {Faulty} instances. 25 | class Faulty 26 | class << self 27 | # Start the Faulty environment 28 | # 29 | # This creates a global shared Faulty state for configuration and for 30 | # re-using State objects. 31 | # 32 | # Not thread safe, should be executed before any worker threads 33 | # are spawned. 34 | # 35 | # If you prefer dependency-injection instead of global state, you can skip 36 | # `init` and use {Faulty.new} to pass an instance directoy to your 37 | # dependencies. 38 | # 39 | # @param default_name [Symbol] The name of the default instance. Can be set 40 | # to `nil` to skip creating a default instance. 41 | # @param config [Hash] Attributes for {Faulty::Options} 42 | # @yield [Faulty::Options] For setting options in a block 43 | # @return [self] 44 | def init(default_name = :default, **config, &block) 45 | raise AlreadyInitializedError if @instances 46 | 47 | @default_instance = default_name 48 | @instances = Concurrent::Map.new 49 | register(default_name, new(**config, &block)) unless default_name.nil? 50 | self 51 | rescue StandardError 52 | @instances = nil 53 | raise 54 | end 55 | 56 | # Get the default instance given during {.init} 57 | # 58 | # @return [Faulty, nil] The default instance if it is registered 59 | def default 60 | raise UninitializedError unless @instances 61 | raise MissingDefaultInstanceError unless @default_instance 62 | 63 | self[@default_instance] 64 | end 65 | 66 | # Get an instance by name 67 | # 68 | # @return [Faulty, nil] The named instance if it is registered 69 | def [](name) 70 | raise UninitializedError unless @instances 71 | 72 | @instances[name.to_s] 73 | end 74 | 75 | # Register an instance to the global Faulty state 76 | # 77 | # Will not replace an existing instance with the same name. Check the 78 | # return value if you need to know whether the instance already existed. 79 | # 80 | # @param name [Symbol] The name of the instance to register 81 | # @param instance [Faulty] The instance to register. If nil, a new instance 82 | # will be created from the given options or block. 83 | # @param config [Hash] Attributes for {Faulty::Options} 84 | # @yield [Faulty::Options] For setting options in a block 85 | # @return [Faulty, nil] The previously-registered instance of that name if 86 | # it already existed, otherwise nil. 87 | def register(name, instance = nil, **config, &block) 88 | raise UninitializedError unless @instances 89 | 90 | if instance 91 | raise ArgumentError, 'Do not give config options if an instance is given' if !config.empty? || block 92 | else 93 | instance = new(**config, &block) 94 | end 95 | 96 | @instances.put_if_absent(name.to_s, instance) 97 | end 98 | 99 | # Get the options for the default instance 100 | # 101 | # @raise MissingDefaultInstanceError If the default instance has not been created 102 | # @return [Faulty::Options] 103 | def options 104 | default.options 105 | end 106 | 107 | # Get or create a circuit for the default instance 108 | # 109 | # @raise UninitializedError If the default instance has not been created 110 | # @param (see Faulty#circuit) 111 | # @yield (see Faulty#circuit) 112 | # @return (see Faulty#circuit) 113 | def circuit(name, **config, &block) 114 | default.circuit(name, **config, &block) 115 | end 116 | 117 | # Get a list of all circuit names for the default instance 118 | # 119 | # @see #list_circuits 120 | # @return [Array] The circuit names 121 | def list_circuits 122 | options.storage.list 123 | end 124 | 125 | # The current time 126 | # 127 | # Used by Faulty wherever the current time is needed. Can be overridden 128 | # for testing 129 | # 130 | # @return [Time] The current time 131 | def current_time 132 | Time.now.to_f 133 | end 134 | 135 | # Disable Faulty circuits 136 | # 137 | # This allows circuits to run as if they were always closed. Does 138 | # not disable caching. 139 | # 140 | # Intended for use in tests, or to disable Faulty entirely for an 141 | # environment. 142 | # 143 | # @return [void] 144 | def disable! 145 | @disabled = true 146 | end 147 | 148 | # Re-enable Faulty if disabled with {.disable!} 149 | # 150 | # @return [void] 151 | def enable! 152 | @disabled = false 153 | end 154 | 155 | # Check whether Faulty was disabled with {.disable!} 156 | # 157 | # @return [Boolean] True if disabled 158 | def disabled? 159 | @disabled == true 160 | end 161 | 162 | # Reset all circuits for the default instance 163 | # 164 | # @see #clear 165 | # @return [void] 166 | def clear! 167 | default.clear 168 | end 169 | end 170 | 171 | attr_reader :options 172 | 173 | # Options for {Faulty} 174 | # 175 | # @!attribute [r] cache 176 | # @see Cache::AutoWire 177 | # @return [Cache::Interface] A cache backend if you want 178 | # to use Faulty's cache support. Automatically wrapped in a 179 | # {Cache::AutoWire}. Default `Cache::AutoWire.new`. 180 | # @!attribute [r] circuit_defaults 181 | # @see Circuit::Options 182 | # @return [Hash] A hash of default options to be used when creating 183 | # new circuits. See {Circuit::Options} for a full list. 184 | # @!attribute [r] storage 185 | # @see Storage::AutoWire 186 | # @return [Storage::Interface, Array] The storage 187 | # backend. Automatically wrapped in a {Storage::AutoWire}, so this can also 188 | # be given an array of prioritized backends. Default `Storage::AutoWire.new`. 189 | # @!attribute [r] listeners 190 | # @see Events::ListenerInterface 191 | # @return [Array] listeners Faulty event listeners 192 | # @!attribute [r] notifier 193 | # @return [Events::Notifier] A Faulty notifier. If given, listeners are 194 | # ignored. 195 | Options = Struct.new( 196 | :cache, 197 | :circuit_defaults, 198 | :storage, 199 | :listeners, 200 | :notifier 201 | ) do 202 | include ImmutableOptions 203 | 204 | private 205 | 206 | def finalize 207 | self.notifier ||= Events::Notifier.new(listeners || []) 208 | self.storage = Storage::AutoWire.wrap(storage, notifier: notifier) 209 | self.cache = Cache::AutoWire.wrap(cache, notifier: notifier) 210 | end 211 | 212 | def required 213 | %i[cache circuit_defaults storage notifier] 214 | end 215 | 216 | def defaults 217 | { 218 | circuit_defaults: {}, 219 | listeners: [Events::LogListener.new] 220 | } 221 | end 222 | end 223 | 224 | # Create a new {Faulty} instance 225 | # 226 | # Note, the process of creating a new instance is not thread safe, 227 | # so make sure instances are setup during your application's initialization 228 | # phase. 229 | # 230 | # For the most part, {Faulty} instances are independent, however for some 231 | # cache and storage backends, you will need to ensure that the cache keys 232 | # and circuit names don't overlap between instances. For example, if using the 233 | # {Storage::Redis} storage backend, you should specify different key 234 | # prefixes for each instance. 235 | # 236 | # @see Options 237 | # @param options [Hash] Attributes for {Options} 238 | # @yield [Options] For setting options in a block 239 | def initialize(**options, &block) 240 | @options = Options.new(options, &block) 241 | @registry = CircuitRegistry.new(circuit_options) 242 | end 243 | 244 | # Create or retrieve a circuit 245 | # 246 | # Within an instance, circuit instances have unique names, so if the given circuit 247 | # name already exists, then the existing circuit will be returned, otherwise 248 | # a new circuit will be created. If an existing circuit is returned, then 249 | # the {options} param and block are ignored. 250 | # 251 | # @param name [String] The name of the circuit 252 | # @param options [Hash] Attributes for {Circuit::Options} 253 | # @yield [Circuit::Options] For setting options in a block 254 | # @return [Circuit] The new circuit or the existing circuit if it already exists 255 | def circuit(name, **options, &block) 256 | name = name.to_s 257 | @registry.retrieve(name, options, &block) 258 | end 259 | 260 | # Get a list of all circuit names 261 | # 262 | # @return [Array] The circuit names 263 | def list_circuits 264 | options.storage.list 265 | end 266 | 267 | # Reset all circuits 268 | # 269 | # Intended for use in tests. This can be expensive and is not appropriate 270 | # to call in production code 271 | # 272 | # See the documentation for your chosen backend for specific semantics and 273 | # safety concerns. For example, the Redis backend resets all circuits, but 274 | # it does not clear the circuit list to maintain thread-safety. 275 | # 276 | # @return [void] 277 | def clear! 278 | options.storage.clear 279 | end 280 | 281 | private 282 | 283 | # Get circuit options from the {Faulty} options 284 | # 285 | # @return [Hash] The circuit options 286 | def circuit_options 287 | @options.to_h 288 | .select { |k, _v| %i[cache storage notifier].include?(k) } 289 | .merge(options.circuit_defaults) 290 | end 291 | end 292 | -------------------------------------------------------------------------------- /lib/faulty/cache.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # The namespace for Faulty caching 5 | module Cache 6 | end 7 | end 8 | 9 | require 'faulty/cache/auto_wire' 10 | require 'faulty/cache/default' 11 | require 'faulty/cache/circuit_proxy' 12 | require 'faulty/cache/fault_tolerant_proxy' 13 | require 'faulty/cache/mock' 14 | require 'faulty/cache/null' 15 | require 'faulty/cache/rails' 16 | -------------------------------------------------------------------------------- /lib/faulty/cache/auto_wire.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # Automatically configure a cache backend 6 | # 7 | # Used by {Faulty#initialize} to setup sensible cache defaults 8 | class AutoWire 9 | # Options for {AutoWire} 10 | # 11 | # @!attribute [r] circuit 12 | # @return [Circuit] A circuit for {CircuitProxy} if one is created. 13 | # When modifying this, be careful to use only a reliable circuit 14 | # storage backend so that you don't introduce cascading failures. 15 | # @!attribute [r] notifier 16 | # @return [Events::Notifier] A Faulty notifier. If given, listeners are 17 | # ignored. 18 | Options = Struct.new( 19 | :circuit, 20 | :notifier 21 | ) do 22 | include ImmutableOptions 23 | 24 | def required 25 | %i[notifier] 26 | end 27 | end 28 | 29 | class << self 30 | # Wrap a cache backend with sensible defaults 31 | # 32 | # If the cache is `nil`, create a new {Default}. 33 | # 34 | # If the backend is not fault tolerant, wrap it in {CircuitProxy} and 35 | # {FaultTolerantProxy}. 36 | # 37 | # @param cache [Interface] A cache backend 38 | # @param options [Hash] Attributes for {Options} 39 | # @yield [Options] For setting options in a block 40 | def wrap(cache, **options, &block) 41 | options = Options.new(options, &block) 42 | if cache.nil? 43 | Cache::Default.new 44 | elsif cache.fault_tolerant? 45 | cache 46 | else 47 | Cache::FaultTolerantProxy.new( 48 | Cache::CircuitProxy.new(cache, circuit: options.circuit, notifier: options.notifier), 49 | notifier: options.notifier 50 | ) 51 | end 52 | end 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/faulty/cache/circuit_proxy.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # A circuit wrapper for cache backends 6 | # 7 | # This class uses an internal {Circuit} to prevent the cache backend from 8 | # causing application issues. If the backend fails continuously, this 9 | # circuit will trip to prevent cascading failures. This internal circuit 10 | # uses an independent in-memory backend by default. 11 | class CircuitProxy 12 | attr_reader :options 13 | 14 | # Options for {CircuitProxy} 15 | # 16 | # @!attribute [r] circuit 17 | # @return [Circuit] A replacement for the internal circuit. When 18 | # modifying this, be careful to use only a reliable circuit storage 19 | # backend so that you don't introduce cascading failures. 20 | # @!attribute [r] notifier 21 | # @return [Events::Notifier] A Faulty notifier to use for failure 22 | # notifications. If `circuit` is given, this is ignored. 23 | Options = Struct.new( 24 | :circuit, 25 | :notifier 26 | ) do 27 | include ImmutableOptions 28 | 29 | def finalize 30 | raise ArgumentError, 'The circuit or notifier option must be given' unless notifier || circuit 31 | 32 | self.circuit ||= Circuit.new( 33 | Faulty::Storage::CircuitProxy.name, 34 | notifier: Events::FilterNotifier.new(notifier, exclude: %i[circuit_success]), 35 | cache: Cache::Null.new 36 | ) 37 | end 38 | end 39 | 40 | # @param cache [Cache::Interface] The cache backend to wrap 41 | # @param options [Hash] Attributes for {Options} 42 | # @yield [Options] For setting options in a block 43 | def initialize(cache, **options, &block) 44 | @cache = cache 45 | @options = Options.new(options, &block) 46 | end 47 | 48 | %i[read write].each do |method| 49 | define_method(method) do |*args| 50 | options.circuit.run { @cache.public_send(method, *args) } 51 | end 52 | end 53 | 54 | def fault_tolerant? 55 | @cache.fault_tolerant? 56 | end 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/faulty/cache/default.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # The default cache implementation 6 | # 7 | # It tries to make a logical decision of what cache implementation to use 8 | # based on the current environment. 9 | # 10 | # - If Rails is loaded, it will use Rails.cache 11 | # - If ActiveSupport is available, it will use an `ActiveSupport::Cache::MemoryStore` 12 | # - Otherwise it will use a {Faulty::Cache::Null} 13 | class Default 14 | extend Forwardable 15 | 16 | def initialize 17 | @cache = if defined?(::Rails) 18 | Cache::Rails.new(::Rails.cache) 19 | elsif defined?(::ActiveSupport::Cache::MemoryStore) 20 | Cache::Rails.new(ActiveSupport::Cache::MemoryStore.new, fault_tolerant: true) 21 | else 22 | Cache::Null.new 23 | end 24 | end 25 | 26 | # @!method read(key) 27 | # (see Faulty::Cache::Interface#read) 28 | # 29 | # @!method write(key, value, expires_in: expires_in) 30 | # (see Faulty::Cache::Interface#write) 31 | # 32 | # @!method fault_tolerant 33 | # (see Faulty::Cache::Interface#fault_tolerant?) 34 | def_delegators :@cache, :read, :write, :fault_tolerant? 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/faulty/cache/fault_tolerant_proxy.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # A wrapper for cache backends that may raise errors 6 | # 7 | # {Faulty#initialize} automatically wraps all non-fault-tolerant cache backends with 8 | # this class. 9 | # 10 | # If the cache backend raises a `StandardError`, it will be captured and 11 | # sent to the notifier. Reads errors will return `nil`, and writes will be 12 | # a no-op. 13 | class FaultTolerantProxy 14 | attr_reader :options 15 | 16 | # Options for {FaultTolerantProxy} 17 | # 18 | # @!attribute [r] notifier 19 | # @return [Events::Notifier] A Faulty notifier 20 | Options = Struct.new( 21 | :notifier 22 | ) do 23 | include ImmutableOptions 24 | 25 | def required 26 | %i[notifier] 27 | end 28 | end 29 | 30 | # @param cache [Cache::Interface] The cache backend to wrap 31 | # @param options [Hash] Attributes for {Options} 32 | # @yield [Options] For setting options in a block 33 | def initialize(cache, **options, &block) 34 | @cache = cache 35 | @options = Options.new(options, &block) 36 | end 37 | 38 | # Wrap a cache in a FaultTolerantProxy unless it's already fault tolerant 39 | # 40 | # @param cache [Cache::Interface] The cache to maybe wrap 41 | # @return [Cache::Interface] The original cache or a {FaultTolerantProxy} 42 | def self.wrap(cache, **options, &block) 43 | return cache if cache.fault_tolerant? 44 | 45 | new(cache, **options, &block) 46 | end 47 | 48 | # Read from the cache safely 49 | # 50 | # If the backend raises a `StandardError`, this will return `nil`. 51 | # 52 | # @param (see Cache::Interface#read) 53 | # @return [Object, nil] The value if found, or nil if not found or if an 54 | # error was raised. 55 | def read(key) 56 | @cache.read(key) 57 | rescue StandardError => e 58 | options.notifier.notify(:cache_failure, key: key, action: :read, error: e) 59 | nil 60 | end 61 | 62 | # Write to the cache safely 63 | # 64 | # If the backend raises a `StandardError`, the write will be ignored 65 | # 66 | # @param (see Cache::Interface#write) 67 | # @return [void] 68 | def write(key, value, expires_in: nil) 69 | @cache.write(key, value, expires_in: expires_in) 70 | rescue StandardError => e 71 | options.notifier.notify(:cache_failure, key: key, action: :write, error: e) 72 | nil 73 | end 74 | 75 | # This cache makes any cache fault tolerant, so this is always `true` 76 | # 77 | # @return [true] 78 | def fault_tolerant? 79 | true 80 | end 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /lib/faulty/cache/interface.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # The interface required for a cache backend implementation 6 | # 7 | # This is for documentation only and is not loaded 8 | class Interface 9 | # Retrieve a value from the cache if available 10 | # 11 | # @param key [String] The cache key 12 | # @raise If the cache backend encounters a failure 13 | # @return [Object, nil] The object if present, otherwise nil 14 | def read(key) 15 | raise NotImplementedError 16 | end 17 | 18 | # Write a value to the cache 19 | # 20 | # This may be any object. It's up to the cache implementation to 21 | # serialize if necessary or raise an error if unsupported. 22 | # 23 | # @param key [String] The cache key 24 | # @param expires_in [Integer, nil] The number of seconds until this cache 25 | # entry expires. If nil, no expiration is set. 26 | # @param value [Object] The value to write to the cache 27 | # @raise If the cache backend encounters a failure 28 | # @return [void] 29 | def write(key, value, expires_in: nil) 30 | raise NotImplementedError 31 | end 32 | 33 | # Can this cache backend raise an error? 34 | # 35 | # If the cache backend returns false from this method, it will be wrapped 36 | # in a {FaultTolerantProxy}, otherwise it will be used as-is. 37 | # 38 | # @return [Boolean] True if this cache backend is fault tolerant 39 | def fault_tolerant? 40 | raise NotImplementedError 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/faulty/cache/mock.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # A mock cache for testing 6 | # 7 | # This never clears expired values from memory, and should not be used 8 | # in production applications. Instead, use a more robust implementation like 9 | # `ActiveSupport::Cache::MemoryStore`. 10 | class Mock 11 | def initialize 12 | @cache = {} 13 | @expires = {} 14 | end 15 | 16 | # Read `key` from the cache 17 | # 18 | # @return [Object, nil] The value if present and not expired 19 | def read(key) 20 | return if @expires[key] && @expires[key] < Faulty.current_time 21 | 22 | @cache[key] 23 | end 24 | 25 | # Write `key` to the cache with an optional expiration 26 | # 27 | # @return [void] 28 | def write(key, value, expires_in: nil) 29 | @cache[key] = value 30 | @expires[key] = Faulty.current_time + expires_in unless expires_in.nil? 31 | end 32 | 33 | # @return [true] 34 | def fault_tolerant? 35 | true 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/faulty/cache/null.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # A cache backend that does nothing 6 | # 7 | # All methods are stubs and do no caching 8 | class Null 9 | # @return [nil] 10 | def read(_key) 11 | end 12 | 13 | # @return [void] 14 | def write(_key, _value, expires_in: nil) 15 | end 16 | 17 | # @return [true] 18 | def fault_tolerant? 19 | true 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/faulty/cache/rails.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Cache 5 | # A wrapper for a Rails or ActiveSupport cache 6 | # 7 | class Rails 8 | extend Forwardable 9 | 10 | # @param cache The Rails cache to wrap 11 | # @param fault_tolerant [Boolean] Whether the Rails cache is 12 | # fault_tolerant. See {#fault_tolerant?} for more details 13 | def initialize(cache = ::Rails.cache, fault_tolerant: false) 14 | @cache = cache 15 | @fault_tolerant = fault_tolerant 16 | end 17 | 18 | # @!method read(key) 19 | # (see Faulty::Cache::Interface#read) 20 | # 21 | # @!method write(key, value, expires_in: expires_in) 22 | # (see Faulty::Cache::Interface#write) 23 | def_delegators :@cache, :read, :write 24 | 25 | # Although ActiveSupport cache implementations are fault-tolerant, 26 | # Rails.cache is not guranteed to be fault tolerant. For this reason, 27 | # we require the user of this class to explicitly mark this cache as 28 | # fault-tolerant using the {#initialize} parameter. 29 | # 30 | # @return [Boolean] 31 | def fault_tolerant? 32 | @fault_tolerant 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/faulty/circuit_registry.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # Used by Faulty instances to track and memoize Circuits 5 | # 6 | # Whenever a circuit is requested by `Faulty#circuit`, it calls 7 | # `#retrieve`. That will return a resolved circuit if there is one, or 8 | # otherwise, it will create a new circuit instance. 9 | # 10 | # Once any circuit is run, the circuit calls `#resolve`. That saves 11 | # the instance into the registry. Any calls to `#retrieve` after 12 | # the circuit is resolved will result in the same instance being returned. 13 | # 14 | # However, before a circuit is resolved, calling `Faulty#circuit` will result 15 | # in a new Circuit instance being created for every call. If multiples of 16 | # these call `resolve`, only the first one will "win" and be memoized. 17 | class CircuitRegistry 18 | def initialize(circuit_options) 19 | @circuit_options = circuit_options 20 | @circuit_options[:registry] = self 21 | @circuits = Concurrent::Map.new 22 | end 23 | 24 | # Retrieve a memoized circuit with the same name, or if none is yet 25 | # resolved, create a new one. 26 | # 27 | # @param name [String] The name of the circuit 28 | # @param options [Hash] Options for {Circuit::Options} 29 | # @yield [Circuit::Options] For setting options in a block 30 | # @return [Circuit] The new or memoized circuit 31 | def retrieve(name, options, &block) 32 | @circuits.fetch(name) do 33 | options = @circuit_options.merge(options) 34 | Circuit.new(name, **options, &block) 35 | end 36 | end 37 | 38 | # Save and memoize the given circuit as the "canonical" instance for 39 | # the circuit name 40 | # 41 | # If the name is already resolved, this will be ignored 42 | # 43 | # @return [Circuit, nil] If this circuit name is already resolved, the 44 | # already-resolved circuit 45 | def resolve(circuit) 46 | @circuits.put_if_absent(circuit.name, circuit) 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/faulty/deprecation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # Support deprecating Faulty features 5 | module Deprecation 6 | class << self 7 | # Call to raise errors instead of logging warnings for Faulty deprecations 8 | def raise_errors!(enabled = true) # rubocop:disable Style/OptionalBooleanParameter 9 | @raise_errors = (enabled == true) 10 | end 11 | 12 | def silenced 13 | @silence = true 14 | yield 15 | ensure 16 | @silence = false 17 | end 18 | 19 | # @private 20 | def method(klass, name, note: nil, sunset: nil) 21 | deprecate("#{klass}##{name}", note: note, sunset: sunset) 22 | end 23 | 24 | # @private 25 | def deprecate(subject, note: nil, sunset: nil) 26 | return if @silence 27 | 28 | message = "#{subject} is deprecated" 29 | message += " and will be removed in #{sunset}" if sunset 30 | message += " (#{note})" if note 31 | raise DeprecationError, message if @raise_errors 32 | 33 | Kernel.warn("DEPRECATION: #{message}") 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/faulty/error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # The base error for all Faulty errors 5 | class FaultyError < StandardError; end 6 | 7 | # Raised if using the global Faulty object without initializing it 8 | class UninitializedError < FaultyError 9 | def initialize(message = nil) 10 | message ||= 'Faulty is not initialized' 11 | super(message) 12 | end 13 | end 14 | 15 | # Raised if {Faulty.init} is called multiple times 16 | class AlreadyInitializedError < FaultyError 17 | def initialize(message = nil) 18 | message ||= 'Faulty is already initialized' 19 | super(message) 20 | end 21 | end 22 | 23 | # Raised if getting the default instance without initializing one 24 | class MissingDefaultInstanceError < FaultyError 25 | def initialize(message = nil) 26 | message ||= 'No default instance. Create one with init or get your instance with Faulty[:name]' 27 | super(message) 28 | end 29 | end 30 | 31 | class DeprecationError < FaultyError 32 | end 33 | 34 | # Included in faulty circuit errors to provide common features for 35 | # native and patched errors 36 | module CircuitErrorBase 37 | attr_reader :circuit 38 | 39 | # @param message [String] 40 | # @param circuit [Circuit] The circuit that raised the error 41 | def initialize(message, circuit) 42 | full_message = %(circuit error for "#{circuit.name}") 43 | full_message = %(#{full_message}: #{message}) if message 44 | 45 | @circuit = circuit 46 | super(full_message) 47 | end 48 | end 49 | 50 | # The base error for all errors raised during circuit runs 51 | # 52 | class CircuitError < FaultyError 53 | include CircuitErrorBase 54 | end 55 | 56 | # Raised when running a circuit that is already open 57 | class OpenCircuitError < CircuitError; end 58 | 59 | # Raised when an error occurred while running a circuit 60 | # 61 | # The `cause` will always be set and will be the internal error 62 | # 63 | # @see CircuitTrippedError For when the circuit is tripped 64 | class CircuitFailureError < CircuitError; end 65 | 66 | # Raised when an error occurred causing a circuit to close 67 | # 68 | # The `cause` will always be set and will be the internal error 69 | class CircuitTrippedError < CircuitError; end 70 | 71 | # Raised if calling get or error on a result without checking it 72 | class UncheckedResultError < FaultyError; end 73 | 74 | # An error that wraps multiple other errors 75 | class FaultyMultiError < FaultyError 76 | def initialize(message, errors) 77 | message = "#{message}: #{errors.map(&:message).join(', ')}" 78 | super(message) 79 | end 80 | end 81 | 82 | # Raised if getting the wrong result type. 83 | # 84 | # For example, calling get on an error result will raise this 85 | class WrongResultError < FaultyError; end 86 | 87 | # Raised if a FallbackChain partially fails 88 | class PartialFailureError < FaultyMultiError; end 89 | 90 | # Raised if all FallbackChain backends fail 91 | class AllFailedError < FaultyMultiError; end 92 | end 93 | -------------------------------------------------------------------------------- /lib/faulty/events.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # The namespace for Faulty events and event listeners 5 | module Events 6 | # All possible events that can be raised by Faulty 7 | EVENTS = %i[ 8 | cache_failure 9 | circuit_cache_hit 10 | circuit_cache_miss 11 | circuit_cache_write 12 | circuit_closed 13 | circuit_failure 14 | circuit_opened 15 | circuit_reopened 16 | circuit_skipped 17 | circuit_success 18 | storage_failure 19 | ].freeze 20 | 21 | EVENT_SET = Set.new(EVENTS) 22 | end 23 | end 24 | 25 | require 'faulty/events/callback_listener' 26 | require 'faulty/events/honeybadger_listener' 27 | require 'faulty/events/log_listener' 28 | require 'faulty/events/notifier' 29 | require 'faulty/events/filter_notifier' 30 | -------------------------------------------------------------------------------- /lib/faulty/events/callback_listener.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Events 5 | # A simple listener implementation that uses callback blocks as handlers 6 | # 7 | # Each event in {EVENTS} has a method on this class that can be used 8 | # to register a callback for that event. 9 | # 10 | # @example 11 | # listener = CallbackListener.new 12 | # listener.circuit_opened do |payload| 13 | # logger.error( 14 | # "Circuit #{payload[:circuit].name} opened: #{payload[:error].message}" 15 | # ) 16 | # end 17 | class CallbackListener 18 | def initialize 19 | @handlers = {} 20 | yield self if block_given? 21 | end 22 | 23 | # @param (see ListenerInterface#handle) 24 | # @return [void] 25 | def handle(event, payload) 26 | return unless EVENT_SET.include?(event) 27 | return unless @handlers.key?(event) 28 | 29 | @handlers[event].each do |handler| 30 | handler.call(payload) 31 | end 32 | end 33 | 34 | EVENTS.each do |event| 35 | define_method(event) do |&block| 36 | @handlers[event] ||= [] 37 | @handlers[event] << block 38 | end 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/faulty/events/filter_notifier.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Events 5 | # Wraps a Notifier and filters events by name 6 | class FilterNotifier 7 | # @param notifier [Notifier] The internal notifier to filter events for 8 | # @param events [Array, nil] An array of events to allow. If nil, all 9 | # {EVENTS} will be used 10 | # @param exclude [Array, nil] An array of events to disallow. If nil, 11 | # no events will be disallowed. Takes priority over `events`. 12 | def initialize(notifier, events: nil, exclude: nil) 13 | @notifier = notifier 14 | @events = Set.new(events || EVENTS) 15 | exclude&.each { |e| @events.delete(e) } 16 | end 17 | 18 | # Notify all listeners of an event 19 | # 20 | # If a listener raises an error while handling an event, that error will 21 | # be captured and written to STDERR. 22 | # 23 | # @param (see Notifier) 24 | def notify(event, payload) 25 | return unless @events.include?(event) 26 | 27 | @notifier.notify(event, payload) 28 | end 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/faulty/events/honeybadger_listener.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Events 5 | # Reports circuit errors to Honeybadger 6 | # 7 | # https://www.honeybadger.io/ 8 | # 9 | # The honeybadger gem must be available. 10 | class HoneybadgerListener 11 | HONEYBADGER_EVENTS = Set[ 12 | :circuit_failure, 13 | :circuit_opened, 14 | :circuit_reopened, 15 | :cache_failure, 16 | :storage_failure 17 | ].freeze 18 | 19 | # (see ListenerInterface#handle) 20 | def handle(event, payload) 21 | return unless HONEYBADGER_EVENTS.include?(event) 22 | 23 | send(event, payload) 24 | end 25 | 26 | private 27 | 28 | def circuit_failure(payload) 29 | _circuit_error(payload) 30 | end 31 | 32 | def circuit_opened(payload) 33 | _circuit_error(payload) 34 | end 35 | 36 | def circuit_reopened(payload) 37 | _circuit_error(payload) 38 | end 39 | 40 | def cache_failure(payload) 41 | Honeybadger.notify(payload[:error], context: { 42 | action: payload[:action], 43 | key: payload[:key] 44 | }) 45 | end 46 | 47 | def storage_failure(payload) 48 | Honeybadger.notify(payload[:error], context: { 49 | action: payload[:action], 50 | circuit: payload[:circuit]&.name 51 | }) 52 | end 53 | 54 | def _circuit_error(payload) 55 | Honeybadger.notify(payload[:error], context: { 56 | circuit: payload[:circuit].name 57 | }) 58 | end 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/faulty/events/listener_interface.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Events 5 | # The interface required to implement a event listener 6 | # 7 | # This is for documentation only and is not loaded 8 | class ListenerInterface 9 | # Handle an event raised by Faulty 10 | # 11 | # @param event [Symbol] The event name. Will be a member of {EVENTS}. 12 | # @param payload [Hash] A hash with keys based on the event type 13 | # @return [void] 14 | def handle(event, payload) 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/faulty/events/log_listener.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Events 5 | # A default listener that logs Faulty events 6 | class LogListener 7 | attr_reader :logger 8 | 9 | # @param logger A logger similar to stdlib `Logger`. Uses the Rails logger 10 | # by default if available, otherwise it creates a new `Logger` to 11 | # stderr. 12 | def initialize(logger = nil) 13 | logger ||= defined?(Rails) ? Rails.logger : ::Logger.new($stderr) 14 | @logger = logger 15 | end 16 | 17 | # (see ListenerInterface#handle) 18 | def handle(event, payload) 19 | return unless EVENT_SET.include?(event) 20 | 21 | send(event, payload) 22 | end 23 | 24 | private 25 | 26 | def circuit_cache_hit(payload) 27 | log(:debug, 'Circuit cache hit', payload[:circuit].name, key: payload[:key]) 28 | end 29 | 30 | def circuit_cache_miss(payload) 31 | log(:debug, 'Circuit cache miss', payload[:circuit].name, key: payload[:key]) 32 | end 33 | 34 | def circuit_cache_write(payload) 35 | log(:debug, 'Circuit cache write', payload[:circuit].name, key: payload[:key]) 36 | end 37 | 38 | def circuit_success(payload) 39 | log(:debug, 'Circuit succeeded', payload[:circuit].name) 40 | end 41 | 42 | def circuit_failure(payload) 43 | log( 44 | :error, 'Circuit failed', payload[:circuit].name, 45 | state: payload[:status].state, 46 | error: payload[:error].message 47 | ) 48 | end 49 | 50 | def circuit_skipped(payload) 51 | log(:warn, 'Circuit skipped', payload[:circuit].name) 52 | end 53 | 54 | def circuit_opened(payload) 55 | log(:error, 'Circuit opened', payload[:circuit].name, error: payload[:error].message) 56 | end 57 | 58 | def circuit_reopened(payload) 59 | log(:error, 'Circuit reopened', payload[:circuit].name, error: payload[:error].message) 60 | end 61 | 62 | def circuit_closed(payload) 63 | log(:info, 'Circuit closed', payload[:circuit].name) 64 | end 65 | 66 | def cache_failure(payload) 67 | log( 68 | :error, 'Cache failure', payload[:action], 69 | key: payload[:key], 70 | error: payload[:error].message 71 | ) 72 | end 73 | 74 | def storage_failure(payload) 75 | extra = {} 76 | extra[:circuit] = payload[:circuit].name if payload.key?(:circuit) 77 | extra[:error] = payload[:error].message 78 | log(:error, 'Storage failure', payload[:action], extra) 79 | end 80 | 81 | def log(level, msg, action, extra = {}) 82 | @logger.public_send(level) do 83 | extra_str = extra.map { |k, v| "#{k}=#{v}" }.join(' ') 84 | extra_str = " #{extra_str}" unless extra_str.empty? 85 | 86 | "#{msg}: #{action}#{extra_str}" 87 | end 88 | end 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/faulty/events/notifier.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Events 5 | # The default event dispatcher for Faulty 6 | class Notifier 7 | # @param listeners [Array] An array of event listeners 8 | def initialize(listeners = []) 9 | @listeners = listeners.freeze 10 | end 11 | 12 | # Notify all listeners of an event 13 | # 14 | # If a listener raises an error while handling an event, that error will 15 | # be captured and written to STDERR. 16 | # 17 | # @param event [Symbol] The event name 18 | # @param payload [Hash] A hash of event payload data. The payload keys 19 | # differ between events, but should be consistent across calls for a 20 | # single event 21 | def notify(event, payload) 22 | raise ArgumentError, "Unknown event #{event}" unless EVENTS.include?(event) 23 | 24 | @listeners.each do |listener| 25 | begin 26 | listener.handle(event, payload) 27 | rescue StandardError => e 28 | warn "Faulty listener #{listener.class.name} crashed: #{e.message}" 29 | end 30 | end 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/faulty/immutable_options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # A struct that cannot be modified after initialization 5 | module ImmutableOptions 6 | # @param hash [Hash] A hash of attributes to initialize with 7 | # @yield [self] Yields itself to the block to set options before freezing 8 | def initialize(hash, &block) 9 | setup(defaults.merge(hash), &block) 10 | end 11 | 12 | def dup_with(hash, &block) 13 | dup.setup(hash, &block) 14 | end 15 | 16 | def setup(hash) 17 | hash&.each { |key, value| self[key] = value } 18 | yield self if block_given? 19 | finalize 20 | guard_required! 21 | freeze 22 | self 23 | end 24 | 25 | # A hash of default values to set before yielding to the block 26 | # 27 | # @return [Hash] 28 | def defaults 29 | {} 30 | end 31 | 32 | # An array of required attributes 33 | # 34 | # @return [Array] 35 | def required 36 | [] 37 | end 38 | 39 | # Runs before freezing to finalize attribute initialization 40 | # 41 | # @return [void] 42 | def finalize 43 | end 44 | 45 | private 46 | 47 | # Raise an error if required options are missing 48 | def guard_required! 49 | required.each do |key| 50 | raise ArgumentError, "Missing required attribute #{key}" if self[key].nil? 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/faulty/patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'faulty/patch/base' 4 | 5 | class Faulty 6 | # Automatic wrappers for common core dependencies like database connections 7 | # or caches 8 | module Patch 9 | class << self 10 | # Create a circuit from a configuration hash 11 | # 12 | # This is intended to be used in contexts where the user passes in 13 | # something like a connection hash to a third-party library. For example 14 | # the Redis patch hooks into the normal Redis connection options to add 15 | # a `:faulty` key, which is a hash of faulty circuit options. This is 16 | # slightly different from the normal Faulty circuit options because 17 | # we also accept an `:instance` key which is a faulty instance. 18 | # 19 | # @example 20 | # # We pass in a faulty instance along with some circuit options 21 | # Patch.circuit_from_hash( 22 | # :mysql, 23 | # { host: 'localhost', faulty: { 24 | # name: 'my_mysql', # A custom circuit name can be included 25 | # instance: Faulty.new, 26 | # sample_threshold: 5 27 | # } 28 | # } 29 | # ) 30 | # 31 | # @example 32 | # # instance can be a registered faulty instance referenced by a string 33 | # or symbol 34 | # Faulty.register(:db_faulty, Faulty.new) 35 | # Patch.circuit_from_hash( 36 | # :mysql, 37 | # { host: 'localhost', faulty: { instance: :db_faulty } } 38 | # ) 39 | # @example 40 | # # If instance is a hash with the key :constant, the value can be 41 | # # a global constant name containing a Faulty instance 42 | # DB_FAULTY = Faulty.new 43 | # Patch.circuit_from_hash( 44 | # :mysql, 45 | # { host: 'localhost', faulty: { instance: { constant: 'DB_FAULTY' } } } 46 | # ) 47 | # 48 | # @example 49 | # # Certain patches may want to enforce certain options like :errors 50 | # # This can be done via hash or the usual block syntax 51 | # Patch.circuit_from_hash(:mysql, 52 | # { host: 'localhost', faulty: {} } 53 | # errors: [Mysql2::Error] 54 | # ) 55 | # 56 | # Patch.circuit_from_hash(:mysql, 57 | # { host: 'localhost', faulty: {} } 58 | # ) do |conf| 59 | # conf.errors = [Mysql2::Error] 60 | # end 61 | # 62 | # @param default_name [String] The default name for the circuit 63 | # @param hash [Hash] A hash of user-provided options. Supports any circuit 64 | # option and these additional options 65 | # @option hash [String] :name The circuit name. Defaults to `default_name` 66 | # @option hash [Boolean] :patch_errors By default, circuit errors will be 67 | # subclasses of `options[:patched_error_mapper]`. The user can disable 68 | # this by setting this option to false. 69 | # @option hash [Faulty, String, Symbol, Hash{ constant: String }] :instance 70 | # A reference to a faulty instance. See examples. 71 | # @param options [Hash] Additional override options. Supports any circuit 72 | # option and these additional ones. 73 | # @option options [Module] :patched_error_mapper The namespace module 74 | # for patched errors or a mapping proc. See {Faulty::Circuit::Options} 75 | # `:error_mapper` 76 | # @yield [Circuit::Options] For setting override options in a block 77 | # @return [Circuit, nil] The circuit if one was created 78 | def circuit_from_hash(default_name, hash, **options, &block) 79 | return unless hash 80 | 81 | hash = symbolize_keys(hash) 82 | name = hash.delete(:name) || default_name 83 | patch_errors = hash.delete(:patch_errors) != false 84 | error_mapper = options.delete(:patched_error_mapper) 85 | hash[:error_mapper] ||= error_mapper if error_mapper && patch_errors 86 | faulty = resolve_instance(hash.delete(:instance)) 87 | faulty.circuit(name, **hash, **options, &block) 88 | end 89 | 90 | # Create a full set of {CircuitError}s with a given base error class 91 | # 92 | # For patches that need their errors to be subclasses of a common base. 93 | # 94 | # @param namespace [Module] The module to define the error classes in 95 | # @param base [Class] The base class for the error classes 96 | # @return [void] 97 | def define_circuit_errors(namespace, base) 98 | circuit_error = Class.new(base) { include CircuitErrorBase } 99 | namespace.const_set('CircuitError', circuit_error) 100 | namespace.const_set('OpenCircuitError', Class.new(circuit_error)) 101 | namespace.const_set('CircuitFailureError', Class.new(circuit_error)) 102 | namespace.const_set('CircuitTrippedError', Class.new(circuit_error)) 103 | end 104 | 105 | private 106 | 107 | # Resolves a constant from a constant name or returns a default 108 | # 109 | # - If value is a string or symbol, gets a registered Faulty instance with that name 110 | # - If value is a Hash with a key `:constant`, resolves the value to a global constant 111 | # - If value is nil, gets Faulty.default 112 | # - Otherwise, return value directly 113 | # 114 | # @param value [String, Symbol, Faulty, nil] The object or constant name to resolve 115 | # @return [Object] The resolved Faulty instance 116 | def resolve_instance(value) 117 | case value 118 | when String, Symbol 119 | result = Faulty[value] 120 | raise NameError, "No Faulty instance for #{value}" unless result 121 | 122 | result 123 | when Hash 124 | const_name = value[:constant] 125 | raise ArgumentError 'Missing hash key :constant for Faulty instance' unless const_name 126 | 127 | Kernel.const_get(const_name) 128 | when nil 129 | Faulty.default 130 | else 131 | value 132 | end 133 | end 134 | 135 | # Some config files may not suport symbol keys, so we convert the hash 136 | # to use symbols so that users can pass in strings 137 | # 138 | # We cannot use transform_keys since we support Ruby < 2.5 139 | # 140 | # @param hash [Hash] A hash to convert 141 | # @return [Hash] The hash with keys as symbols 142 | def symbolize_keys(hash) 143 | result = {} 144 | hash.each do |key, val| 145 | result[key.to_sym] = if val.is_a?(Hash) 146 | symbolize_keys(val) 147 | else 148 | val 149 | end 150 | end 151 | result 152 | end 153 | end 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /lib/faulty/patch/base.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Patch 5 | # Can be included in patch modules to provide common functionality 6 | # 7 | # The patch needs to set `@faulty_circuit` 8 | # 9 | # @example 10 | # module ThingPatch 11 | # include Faulty::Patch::Base 12 | # 13 | # def initialize(options = {}) 14 | # @faulty_circuit = Faulty::Patch.circuit_from_hash('thing', options[:faulty]) 15 | # end 16 | # 17 | # def do_something 18 | # faulty_run { super } 19 | # end 20 | # end 21 | # 22 | # Thing.prepend(ThingPatch) 23 | module Base 24 | # Run a block wrapped by `@faulty_circuit` 25 | # 26 | # If `@faulty_circuit` is not set, the block will be run with no 27 | # circuit. 28 | # 29 | # Nested calls to this method will only cause the circuit to be triggered 30 | # once. 31 | # 32 | # @yield A block to run inside the circuit 33 | # @return The block return value 34 | def faulty_run(&block) 35 | faulty_running_key = "faulty_running_#{object_id}" 36 | return yield unless @faulty_circuit 37 | return yield if Thread.current[faulty_running_key] 38 | 39 | Thread.current[faulty_running_key] = true 40 | @faulty_circuit.run(&block) 41 | ensure 42 | Thread.current[faulty_running_key] = nil 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/faulty/patch/elasticsearch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Patch 5 | # Patch Elasticsearch to run requests in a circuit 6 | # 7 | # This module is not required by default 8 | # 9 | # Pass a `:faulty` key into your Elasticsearch client options to enable 10 | # circuit protection. See {Patch.circuit_from_hash} for the available 11 | # options. 12 | # 13 | # By default, all circuit errors raised by this patch inherit from 14 | # `::Elasticsearch::Transport::Transport::Error`. One side effect of the way 15 | # this patch wraps errors is that `host_unreachable_exceptions` raised by 16 | # the inner transport adapters are converted into 17 | # `Elasticsearch::Transport::Transport::Error` instead of the transport 18 | # error type such as `Faraday::ConnectionFailed`. 19 | # 20 | # @example 21 | # require 'faulty/patch/elasticsearch' 22 | # 23 | # es = Elasticsearch::Client.new(url: 'http://localhost:9200', faulty: {}) 24 | # es.search(q: 'test') # raises Faulty::CircuitError if connection fails 25 | # 26 | # # If the faulty key is not given, no circuit is used 27 | # es = Elasticsearch::Client.new(url: 'http://localhost:9200', faulty: {}) 28 | # es.search(q: 'test') # not protected by a circuit 29 | # 30 | # # With Searchkick 31 | # Searchkick.client_options[:faulty] = {} 32 | # 33 | # @see Patch.circuit_from_hash 34 | module Elasticsearch 35 | include Base 36 | 37 | module Error; end 38 | module SnifferTimeoutError; end 39 | module ServerError; end 40 | 41 | PATCHED_MODULE = if Gem.loaded_specs['opensearch-ruby'] 42 | require 'opensearch' 43 | ::OpenSearch 44 | else 45 | require 'elasticsearch' 46 | ::Elasticsearch 47 | end 48 | 49 | # We will freeze this after adding the dynamic error classes 50 | MAPPED_ERRORS = { # rubocop:disable Style/MutableConstant 51 | PATCHED_MODULE::Transport::Transport::Error => Error, 52 | PATCHED_MODULE::Transport::Transport::SnifferTimeoutError => SnifferTimeoutError, 53 | PATCHED_MODULE::Transport::Transport::ServerError => ServerError 54 | } 55 | 56 | module Errors 57 | PATCHED_MODULE::Transport::Transport::ERRORS.each do |_code, klass| 58 | MAPPED_ERRORS[klass] = const_set(klass.name.split('::').last, Module.new) 59 | end 60 | end 61 | 62 | MAPPED_ERRORS.freeze 63 | MAPPED_ERRORS.each do |klass, mod| 64 | Patch.define_circuit_errors(mod, klass) 65 | end 66 | 67 | ERROR_MAPPER = lambda do |error_name, cause, circuit| 68 | MAPPED_ERRORS.fetch(cause&.class, Error).const_get(error_name).new(cause&.message, circuit) 69 | end 70 | private_constant :ERROR_MAPPER, :MAPPED_ERRORS 71 | 72 | def initialize(arguments = {}, &block) 73 | super 74 | 75 | errors = [PATCHED_MODULE::Transport::Transport::Error] 76 | errors.concat(@transport.host_unreachable_exceptions) 77 | 78 | @faulty_circuit = Patch.circuit_from_hash( 79 | 'elasticsearch', 80 | arguments[:faulty], 81 | errors: errors, 82 | exclude: PATCHED_MODULE::Transport::Transport::Errors::NotFound, 83 | patched_error_mapper: ERROR_MAPPER 84 | ) 85 | end 86 | 87 | # Protect all elasticsearch requests 88 | def perform_request(*args) 89 | faulty_run { super } 90 | end 91 | end 92 | end 93 | end 94 | 95 | if Gem.loaded_specs['opensearch-ruby'] 96 | module OpenSearch 97 | module Transport 98 | class Client 99 | prepend(Faulty::Patch::Elasticsearch) 100 | end 101 | end 102 | end 103 | else 104 | module Elasticsearch 105 | module Transport 106 | class Client 107 | prepend(Faulty::Patch::Elasticsearch) 108 | end 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /lib/faulty/patch/mysql2.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'mysql2' 4 | 5 | if Gem::Version.new(Mysql2::VERSION) < Gem::Version.new('0.5.0') 6 | raise NotImplementedError, 'The faulty mysql2 patch requires mysql2 0.5.0 or later' 7 | end 8 | 9 | class Faulty 10 | module Patch 11 | # Patch Mysql2 to run connections and queries in a circuit 12 | # 13 | # This module is not required by default 14 | # 15 | # Pass a `:faulty` key into your MySQL connection options to enable 16 | # circuit protection. See {Patch.circuit_from_hash} for the available 17 | # options. 18 | # 19 | # COMMIT, ROLLBACK, and RELEASE SAVEPOINT queries are intentionally not 20 | # protected by the circuit. This is to allow open transactions to be closed 21 | # if possible. 22 | # 23 | # By default, all circuit errors raised by this patch inherit from 24 | # `::Mysql2::Error::ConnectionError` 25 | # 26 | # @example 27 | # require 'faulty/patch/mysql2' 28 | # 29 | # mysql = Mysql2::Client.new(host: '127.0.0.1', faulty: {}) 30 | # mysql.query('SELECT * FROM users') # raises Faulty::CircuitError if connection fails 31 | # 32 | # # If the faulty key is not given, no circuit is used 33 | # mysql = Mysql2::Client.new(host: '127.0.0.1') 34 | # mysql.query('SELECT * FROM users') # not protected by a circuit 35 | # 36 | # @see Patch.circuit_from_hash 37 | module Mysql2 38 | include Base 39 | 40 | Patch.define_circuit_errors(self, ::Mysql2::Error::ConnectionError) 41 | 42 | QUERY_WHITELIST = [ 43 | %r{\A(?:/\*.*?\*/)?\s*ROLLBACK}i, 44 | %r{\A(?:/\*.*?\*/)?\s*COMMIT}i, 45 | %r{\A(?:/\*.*?\*/)?\s*RELEASE\s+SAVEPOINT}i 46 | ].freeze 47 | 48 | def initialize(opts = {}) 49 | @faulty_circuit = Patch.circuit_from_hash( 50 | 'mysql2', 51 | opts[:faulty], 52 | errors: [ 53 | ::Mysql2::Error::ConnectionError, 54 | ::Mysql2::Error::TimeoutError 55 | ], 56 | patched_error_mapper: Faulty::Patch::Mysql2 57 | ) 58 | 59 | super 60 | end 61 | 62 | # Protect manual connection pings 63 | def ping 64 | faulty_run { super } 65 | rescue Faulty::Patch::Mysql2::FaultyError 66 | false 67 | end 68 | 69 | # Protect the initial connnection 70 | def connect(*args) 71 | faulty_run { super } 72 | end 73 | 74 | # Protect queries unless they are whitelisted 75 | def query(*args) 76 | return super if QUERY_WHITELIST.any? { |r| !r.match(args.first).nil? } 77 | 78 | faulty_run { super } 79 | end 80 | end 81 | end 82 | end 83 | 84 | module Mysql2 85 | class Client 86 | prepend(Faulty::Patch::Mysql2) 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/faulty/patch/redis.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'redis' 4 | 5 | class Faulty 6 | module Patch 7 | # Patch Redis to run all network IO in a circuit 8 | # 9 | # This module is not required by default 10 | # 11 | # Redis <= 4 12 | # --------------------- 13 | # Pass a `:faulty` key into your Redis connection options to enable 14 | # circuit protection. See {Patch.circuit_from_hash} for the available 15 | # options. On Redis 5+, the faulty key should be passed in the `:custom` hash 16 | # instead of the top-level options. See example. 17 | # 18 | # By default, all circuit errors raised by this patch inherit from 19 | # `::Redis::BaseConnectionError` 20 | # 21 | # @example 22 | # require 'faulty/patch/redis' 23 | # 24 | # # Redis <= 4 25 | # redis = Redis.new(url: 'redis://localhost:6379', faulty: {}) 26 | # # Or for Redis 5+ 27 | # redis = Redis.new(url: 'redis://localhost:6379', custom: { faulty: {} }) 28 | # 29 | # redis.connect # raises Faulty::CircuitError if connection fails 30 | # 31 | # # If the faulty key is not given, no circuit is used 32 | # redis = Redis.new(url: 'redis://localhost:6379') 33 | # redis.connect # not protected by a circuit 34 | # 35 | # @see Patch.circuit_from_hash 36 | module Redis 37 | end 38 | end 39 | end 40 | 41 | if Redis::VERSION.to_f < 5 42 | require 'faulty/patch/redis/patch' 43 | else 44 | require 'faulty/patch/redis/middleware' 45 | end 46 | -------------------------------------------------------------------------------- /lib/faulty/patch/redis/middleware.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Patch 5 | module Redis 6 | Patch.define_circuit_errors(self, ::RedisClient::ConnectionError) 7 | 8 | class BusyError < ::RedisClient::CommandError 9 | end 10 | 11 | module Middleware 12 | include Base 13 | 14 | def initialize(client) 15 | @faulty_circuit = Patch.circuit_from_hash( 16 | 'redis', 17 | client.config.custom[:faulty], 18 | errors: [ 19 | ::RedisClient::ConnectionError, 20 | BusyError 21 | ], 22 | patched_error_mapper: Faulty::Patch::Redis 23 | ) 24 | 25 | super 26 | end 27 | 28 | def connect(redis_config) 29 | faulty_run { super } 30 | end 31 | 32 | def call(commands, redis_config) 33 | faulty_run { wrap_command { super } } 34 | end 35 | 36 | def call_pipelined(commands, redis_config) 37 | faulty_run { wrap_command { super } } 38 | end 39 | 40 | private 41 | 42 | def wrap_command 43 | yield 44 | rescue ::RedisClient::CommandError => e 45 | raise BusyError, e.message if e.message.start_with?('BUSY') 46 | 47 | raise 48 | end 49 | end 50 | 51 | ::RedisClient.register(Middleware) 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/faulty/patch/redis/patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'redis' 4 | 5 | class Faulty 6 | module Patch 7 | module Redis 8 | include Base 9 | 10 | Patch.define_circuit_errors(self, ::Redis::BaseConnectionError) 11 | 12 | class BusyError < ::Redis::CommandError 13 | end 14 | 15 | # Patches Redis to add the `:faulty` key 16 | def initialize(options = {}) 17 | @faulty_circuit = Patch.circuit_from_hash( 18 | 'redis', 19 | options[:faulty], 20 | errors: [ 21 | ::Redis::BaseConnectionError, 22 | BusyError 23 | ], 24 | patched_error_mapper: Faulty::Patch::Redis 25 | ) 26 | 27 | super 28 | end 29 | 30 | # The initial connection is protected by a circuit 31 | def connect 32 | faulty_run { super } 33 | end 34 | 35 | # Protect command calls 36 | def call(command) 37 | faulty_run { super } 38 | end 39 | 40 | # Protect command_loop calls 41 | def call_loop(command, timeout = 0) 42 | faulty_run { super } 43 | end 44 | 45 | # Protect pipelined commands 46 | def call_pipelined(commands) 47 | faulty_run { super } 48 | end 49 | 50 | # Inject specific error classes if client is patched 51 | # 52 | # This method does not raise errors, it returns them 53 | # as exception objects, so we simply modify that error if necessary and 54 | # return it. 55 | # 56 | # The call* methods above will then raise that error, so we are able to 57 | # capture it with faulty_run. 58 | def io(&block) 59 | return super unless @faulty_circuit 60 | 61 | reply = super 62 | if reply.is_a?(::Redis::CommandError) && reply.message.start_with?('BUSY') 63 | reply = BusyError.new(reply.message) 64 | end 65 | 66 | reply 67 | end 68 | end 69 | end 70 | end 71 | 72 | class Redis 73 | class Client 74 | prepend(Faulty::Patch::Redis) 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/faulty/result.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # An approximation of the `Result` type from some strongly-typed languages. 5 | # 6 | # F#: https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/results 7 | # 8 | # Rust: https://doc.rust-lang.org/std/result/enum.Result.html 9 | # 10 | # Since we can't enforce the type at compile-time, we use runtime errors to 11 | # check the result for consistency as early as possible. This means we 12 | # enforce runtime checks of the result type. This approach does not eliminate 13 | # issues, but it does help remind the user to check the result in most cases. 14 | # 15 | # This is returned from {Circuit#try_run} to allow error handling without 16 | # needing to rescue from errors. 17 | # 18 | # @example 19 | # result = Result.new(ok: 'foo') 20 | # 21 | # # Check the result before calling get 22 | # if result.ok? 23 | # puts result.get 24 | # else 25 | # puts result.error.message 26 | # end 27 | # 28 | # @example 29 | # result = Result.new(error: StandardError.new) 30 | # puts result.or_default('fallback') # prints "fallback" 31 | # 32 | # @example 33 | # result = Result.new(ok: 'foo') 34 | # result.get # raises UncheckedResultError 35 | # 36 | # @example 37 | # result = Result.new(ok: 'foo') 38 | # if result.ok? 39 | # result.error.message # raises WrongResultError 40 | # end 41 | class Result 42 | # The constant used to designate that a value is empty 43 | # 44 | # This is needed to differentiate between an ok `nil` value and 45 | # an empty value. 46 | # 47 | # @private 48 | NOTHING = {}.freeze 49 | 50 | # Create a new `Result` with either an ok or error value 51 | # 52 | # Exactly one parameter must be given, and not both. 53 | # 54 | # @param ok An ok value 55 | # @param error [Error] An error instance 56 | def initialize(ok: NOTHING, error: NOTHING) 57 | if ok.equal?(NOTHING) && error.equal?(NOTHING) 58 | raise ArgumentError, 'Result must have an ok or error value' 59 | end 60 | if !ok.equal?(NOTHING) && !error.equal?(NOTHING) 61 | raise ArgumentError, 'Result must not have both an ok and error value' 62 | end 63 | 64 | @ok = ok 65 | @error = error 66 | @checked = false 67 | end 68 | 69 | # Check if the value is an ok value 70 | # 71 | # @return [Boolean] True if this result is ok 72 | def ok? 73 | @checked = true 74 | ok_unchecked? 75 | end 76 | 77 | # Check if the value is an error value 78 | # 79 | # @return [Boolean] True if this result is an error 80 | def error? 81 | !ok? 82 | end 83 | 84 | # Get the ok value 85 | # 86 | # @raise UncheckedResultError if this result was not checked using {#ok?} or {#error?} 87 | # @raise WrongResultError if this result is an error 88 | # @return The ok value 89 | def get 90 | validate_checked!('get') 91 | unsafe_get 92 | end 93 | 94 | # Get the ok value without checking whether it's safe to do so 95 | # 96 | # @raise WrongResultError if this result is an error 97 | # @return The ok value 98 | def unsafe_get 99 | raise WrongResultError, 'Tried to get value for error result' unless ok_unchecked? 100 | 101 | @ok 102 | end 103 | 104 | # Get the error value 105 | # 106 | # @raise UncheckedResultError if this result was not checked using {#ok?} or {#error?} 107 | # @raise WrongResultError if this result is ok 108 | def error 109 | validate_checked!('error') 110 | unsafe_error 111 | end 112 | 113 | # Get the error value without checking whether it's safe to do so 114 | # 115 | # @raise WrongResultError if this result is ok 116 | # @return [Error] The error 117 | def unsafe_error 118 | raise WrongResultError, 'Tried to get error for ok result' if ok_unchecked? 119 | 120 | @error 121 | end 122 | 123 | # Get the ok value if this result is ok, otherwise return a default 124 | # 125 | # @param default The default value. Ignored if a block is given 126 | # @yield A block returning the default value 127 | # @return The ok value or the default if this result is an error 128 | def or_default(default = nil) 129 | if ok_unchecked? 130 | @ok 131 | elsif block_given? 132 | yield @error 133 | else 134 | default 135 | end 136 | end 137 | 138 | private 139 | 140 | def ok_unchecked? 141 | !@ok.equal?(NOTHING) 142 | end 143 | 144 | def validate_checked!(method) 145 | unless @checked 146 | raise UncheckedResultError, "Result: Called #{method} without checking ok? or error?" 147 | end 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /lib/faulty/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # The status of a circuit 5 | # 6 | # Includes information like the state and locks. Also calculates 7 | # whether a circuit can be run, or if it has failed a threshold. 8 | # 9 | # @!attribute [r] state 10 | # @return [:open, :closed] The stored circuit state. This is always open 11 | # or closed. Half-open is calculated from the current time. For that 12 | # reason, calling state directly should be avoided. Instead use the 13 | # status methods {#open?}, {#closed?}, and {#half_open?}. 14 | # Default `:closed` 15 | # @!attribute [r] lock 16 | # @return [:open, :closed, nil] If the circuit is locked, the state that 17 | # it is locked in. Default `nil`. 18 | # @!attribute [r] opened_at 19 | # @return [Integer, nil] If the circuit is open, the timestamp that it was 20 | # opened. This is not necessarily reset when the circuit is closed. 21 | # Default `nil`. 22 | # @!attribute [r] failure_rate 23 | # @return [Float] A number from 0 to 1 representing the percentage of 24 | # failures for the circuit. For exmaple 0.5 represents a 50% failure rate. 25 | # @!attribute [r] sample_size 26 | # @return [Integer] The number of samples used to calculate the failure rate. 27 | # @!attribute [r] options 28 | # @return [Circuit::Options] The options for the circuit 29 | # @!attribute [r] stub 30 | # @return [Boolean] True if this status is a stub and not calculated from 31 | # the storage backend. Used by {Storage::FaultTolerantProxy} when 32 | # returning the status for an offline storage backend. Default `false`. 33 | Status = Struct.new( 34 | :state, 35 | :lock, 36 | :opened_at, 37 | :failure_rate, 38 | :sample_size, 39 | :options, 40 | :stub 41 | ) 42 | 43 | class Status 44 | include ImmutableOptions 45 | 46 | # The allowed state values 47 | STATES = %i[ 48 | open 49 | closed 50 | ].freeze 51 | 52 | # The allowed lock values 53 | LOCKS = %i[ 54 | open 55 | closed 56 | ].freeze 57 | 58 | # Create a new `Status` from a list of circuit runs 59 | # 60 | # For storage backends that store entries, this automatically calculates 61 | # failure_rate and sample size. 62 | # 63 | # @param entries [Array] An array of entry tuples. See 64 | # {Circuit#history} for details 65 | # @param hash [Hash] The status attributes minus failure_rate and 66 | # sample_size 67 | # @return [Status] 68 | def self.from_entries(entries, **hash) 69 | window_start = Faulty.current_time - hash[:options].evaluation_window 70 | size = entries.size 71 | i = 0 72 | failures = 0 73 | sample_size = 0 74 | 75 | # This is a hot loop, and while is slightly faster than each 76 | while i < size 77 | time, success = entries[i] 78 | i += 1 79 | next unless time > window_start 80 | 81 | sample_size += 1 82 | failures += 1 unless success 83 | end 84 | 85 | new(hash.merge( 86 | sample_size: sample_size, 87 | failure_rate: sample_size.zero? ? 0.0 : failures.to_f / sample_size 88 | )) 89 | end 90 | 91 | # Whether the circuit is open 92 | # 93 | # This is mutually exclusive with {#closed?} and {#half_open?} 94 | # 95 | # @return [Boolean] True if open 96 | def open? 97 | state == :open && opened_at + options.cool_down > Faulty.current_time 98 | end 99 | 100 | # Whether the circuit is closed 101 | # 102 | # This is mutually exclusive with {#open?} and {#half_open?} 103 | # 104 | # @return [Boolean] True if closed 105 | def closed? 106 | state == :closed 107 | end 108 | 109 | # Whether the circuit is half-open 110 | # 111 | # This is mutually exclusive with {#open?} and {#closed?} 112 | # 113 | # @return [Boolean] True if half-open 114 | def half_open? 115 | state == :open && opened_at + options.cool_down <= Faulty.current_time 116 | end 117 | 118 | # Whether the circuit is locked open 119 | # 120 | # @return [Boolean] True if locked open 121 | def locked_open? 122 | lock == :open 123 | end 124 | 125 | # Whether the circuit is locked closed 126 | # 127 | # @return [Boolean] True if locked closed 128 | def locked_closed? 129 | lock == :closed 130 | end 131 | 132 | # Whether the circuit can be run 133 | # 134 | # Takes the circuit state, locks and cooldown into account 135 | # 136 | # @return [Boolean] True if the circuit can be run 137 | def can_run? 138 | return false if locked_open? 139 | 140 | closed? || locked_closed? || half_open? 141 | end 142 | 143 | # Whether the circuit fails the sample size and rate thresholds 144 | # 145 | # @return [Boolean] True if the circuit fails the thresholds 146 | def fails_threshold? 147 | return false if sample_size < options.sample_threshold 148 | 149 | failure_rate >= options.rate_threshold 150 | end 151 | 152 | def finalize 153 | raise ArgumentError, "state must be a symbol in #{self.class}::STATES" unless STATES.include?(state) 154 | unless lock.nil? || LOCKS.include?(lock) 155 | raise ArgumentError, "lock must be a symbol in #{self.class}::LOCKS or nil" 156 | end 157 | raise ArgumentError, 'opened_at is required if state is open' if state == :open && opened_at.nil? 158 | end 159 | 160 | def required 161 | %i[state failure_rate sample_size options stub] 162 | end 163 | 164 | def defaults 165 | { 166 | state: :closed, 167 | failure_rate: 0.0, 168 | sample_size: 0, 169 | stub: false 170 | } 171 | end 172 | end 173 | end 174 | -------------------------------------------------------------------------------- /lib/faulty/storage.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # The namespace for Faulty storage 5 | module Storage 6 | end 7 | end 8 | 9 | require 'faulty/storage/auto_wire' 10 | require 'faulty/storage/circuit_proxy' 11 | require 'faulty/storage/fallback_chain' 12 | require 'faulty/storage/fault_tolerant_proxy' 13 | require 'faulty/storage/null' 14 | require 'faulty/storage/memory' 15 | require 'faulty/storage/redis' 16 | -------------------------------------------------------------------------------- /lib/faulty/storage/auto_wire.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # Automatically configure a storage backend 6 | # 7 | # Used by {Faulty#initialize} to setup sensible storage defaults 8 | class AutoWire 9 | # Options for {AutoWire} 10 | # 11 | # @!attribute [r] circuit 12 | # @return [Circuit] A circuit for {CircuitProxy} if one is created. 13 | # When modifying this, be careful to use only a reliable circuit 14 | # storage backend so that you don't introduce cascading failures. 15 | # @!attribute [r] notifier 16 | # @return [Events::Notifier] A Faulty notifier. If given, listeners are 17 | # ignored. 18 | Options = Struct.new( 19 | :circuit, 20 | :notifier 21 | ) do 22 | include ImmutableOptions 23 | 24 | def required 25 | %i[notifier] 26 | end 27 | end 28 | 29 | class << self 30 | # Wrap storage backends with sensible defaults 31 | # 32 | # If the cache is `nil`, create a new {Memory} storage. 33 | # 34 | # If a single storage backend is given and is fault tolerant, leave it 35 | # unmodified. 36 | # 37 | # If a single storage backend is given and is not fault tolerant, wrap it 38 | # in a {CircuitProxy} and a {FaultTolerantProxy}. 39 | # 40 | # If an array of storage backends is given, wrap each non-fault-tolerant 41 | # entry in a {CircuitProxy} and create a {FallbackChain}. If none of the 42 | # backends in the array are fault tolerant, also wrap the {FallbackChain} 43 | # in a {FaultTolerantProxy}. 44 | # 45 | # @todo Consider using a {FallbackChain} for non-fault-tolerant storages 46 | # by default. This would fallback to a {Memory} storage. It would 47 | # require a more conservative implementation of {Memory} that could 48 | # limit the number of circuits stored. For now, users need to manually 49 | # configure fallbacks. 50 | # 51 | # @param storage [Interface, Array] A storage backed or array 52 | # of storage backends to setup. 53 | # @param options [Hash] Attributes for {Options} 54 | # @yield [Options] For setting options in a block 55 | def wrap(storage, **options, &block) 56 | options = Options.new(options, &block) 57 | if storage.nil? 58 | Memory.new 59 | elsif storage.is_a?(Array) 60 | wrap_array(storage, options) 61 | elsif !storage.fault_tolerant? 62 | wrap_one(storage, options) 63 | else 64 | storage 65 | end 66 | end 67 | 68 | private 69 | 70 | # Wrap an array of storage backends in a fault-tolerant FallbackChain 71 | # 72 | # @param array [Array] The array to wrap 73 | # @param options [Options] 74 | # @return [Storage::Interface] A fault-tolerant fallback chain 75 | def wrap_array(array, options) 76 | FaultTolerantProxy.wrap(FallbackChain.new( 77 | array.map { |s| s.fault_tolerant? ? s : circuit_proxy(s, options) }, 78 | notifier: options.notifier 79 | ), notifier: options.notifier) 80 | end 81 | 82 | # Wrap one storage backend in fault-tolerant backends 83 | # 84 | # @param storage [Storage::Interface] The storage to wrap 85 | # @param options [Options] 86 | # @return [Storage::Interface] A fault-tolerant storage backend 87 | def wrap_one(storage, options) 88 | FaultTolerantProxy.new( 89 | circuit_proxy(storage, options), 90 | notifier: options.notifier 91 | ) 92 | end 93 | 94 | # Wrap storage in a CircuitProxy 95 | # 96 | # @param storage [Storage::Interface] The storage to wrap 97 | # @param options [Options] 98 | # @return [CircuitProxy] 99 | def circuit_proxy(storage, options) 100 | CircuitProxy.new(storage, circuit: options.circuit, notifier: options.notifier) 101 | end 102 | end 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /lib/faulty/storage/circuit_proxy.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # A circuit wrapper for storage backends 6 | # 7 | # This class uses an internal {Circuit} to prevent the storage backend from 8 | # causing application issues. If the backend fails continuously, this 9 | # circuit will trip to prevent cascading failures. This internal circuit 10 | # uses an independent in-memory backend by default. 11 | class CircuitProxy 12 | attr_reader :options 13 | 14 | # Options for {CircuitProxy} 15 | # 16 | # @!attribute [r] circuit 17 | # @return [Circuit] A replacement for the internal circuit. When 18 | # modifying this, be careful to use only a reliable storage backend 19 | # so that you don't introduce cascading failures. 20 | # @!attribute [r] notifier 21 | # @return [Events::Notifier] A Faulty notifier to use for circuit 22 | # notifications. If `circuit` is given, this is ignored. 23 | Options = Struct.new( 24 | :circuit, 25 | :notifier 26 | ) do 27 | include ImmutableOptions 28 | 29 | def finalize 30 | raise ArgumentError, 'The circuit or notifier option must be given' unless notifier || circuit 31 | 32 | self.circuit ||= Circuit.new( 33 | Faulty::Storage::CircuitProxy.name, 34 | notifier: Events::FilterNotifier.new(notifier, exclude: %i[circuit_success]), 35 | cache: Cache::Null.new 36 | ) 37 | end 38 | end 39 | 40 | # @param storage [Storage::Interface] The storage backend to wrap 41 | # @param options [Hash] Attributes for {Options} 42 | # @yield [Options] For setting options in a block 43 | def initialize(storage, **options, &block) 44 | @storage = storage 45 | @options = Options.new(options, &block) 46 | end 47 | 48 | %i[ 49 | get_options 50 | set_options 51 | entry 52 | open 53 | reopen 54 | close 55 | lock 56 | unlock 57 | reset 58 | status 59 | history 60 | list 61 | clear 62 | ].each do |method| 63 | define_method(method) do |*args| 64 | options.circuit.run { @storage.public_send(method, *args) } 65 | end 66 | end 67 | 68 | # This cache makes any storage fault tolerant, so this is always `true` 69 | # 70 | # @return [true] 71 | def fault_tolerant? 72 | @storage.fault_tolerant? 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/faulty/storage/fallback_chain.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # An prioritized list of storage backends 6 | # 7 | # If any backend fails, the next will be tried until one succeeds. This 8 | # should typically be used when using a fault-prone backend such as 9 | # {Storage::Redis}. 10 | # 11 | # This is used by {Faulty#initialize} if the `storage` option is set to an 12 | # array. 13 | # 14 | # @example 15 | # # This storage will try Redis first, then fallback to memory storage 16 | # # if Redis is unavailable. 17 | # storage = Faulty::Storage::FallbackChain.new([ 18 | # Faulty::Storage::Redis.new, 19 | # Faulty::Storage::Memory.new 20 | # ]) 21 | class FallbackChain 22 | attr_reader :options 23 | 24 | # Options for {FallbackChain} 25 | # 26 | # @!attribute [r] notifier 27 | # @return [Events::Notifier] A Faulty notifier 28 | Options = Struct.new( 29 | :notifier 30 | ) do 31 | include ImmutableOptions 32 | 33 | def required 34 | %i[notifier] 35 | end 36 | end 37 | 38 | # Create a new {FallbackChain} to automatically fallback to reliable storage 39 | # 40 | # @param storages [Array] An array of storage backends. 41 | # The primary storage should be specified first. If that one fails, 42 | # additional entries will be tried in sequence until one succeeds. 43 | # @param options [Hash] Attributes for {Options} 44 | # @yield [Options] For setting options in a block 45 | def initialize(storages, **options, &block) 46 | @storages = storages 47 | @options = Options.new(options, &block) 48 | end 49 | 50 | # Get options from the first available storage backend 51 | # 52 | # @param (see Interface#get_options) 53 | # @return (see Interface#get_options) 54 | def get_options(circuit) 55 | send_chain(:get_options, circuit) do |e| 56 | options.notifier.notify(:storage_failure, circuit: circuit, action: :get_options, error: e) 57 | end 58 | end 59 | 60 | # Try to set circuit options on all backends 61 | # 62 | # @param (see Interface#set_options) 63 | # @return (see Interface#set_options) 64 | def set_options(circuit, stored_options) 65 | send_all(:set_options, circuit, stored_options) 66 | end 67 | 68 | # Create a circuit entry in the first available storage backend 69 | # 70 | # @param (see Interface#entry) 71 | # @return (see Interface#entry) 72 | def entry(circuit, time, success, status) 73 | send_chain(:entry, circuit, time, success, status) do |e| 74 | options.notifier.notify(:storage_failure, circuit: circuit, action: :entry, error: e) 75 | end 76 | end 77 | 78 | # Open a circuit in the first available storage backend 79 | # 80 | # @param (see Interface#open) 81 | # @return (see Interface#open) 82 | def open(circuit, opened_at) 83 | send_chain(:open, circuit, opened_at) do |e| 84 | options.notifier.notify(:storage_failure, circuit: circuit, action: :open, error: e) 85 | end 86 | end 87 | 88 | # Reopen a circuit in the first available storage backend 89 | # 90 | # @param (see Interface#reopen) 91 | # @return (see Interface#reopen) 92 | def reopen(circuit, opened_at, previous_opened_at) 93 | send_chain(:reopen, circuit, opened_at, previous_opened_at) do |e| 94 | options.notifier.notify(:storage_failure, circuit: circuit, action: :reopen, error: e) 95 | end 96 | end 97 | 98 | # Close a circuit in the first available storage backend 99 | # 100 | # @param (see Interface#close) 101 | # @return (see Interface#close) 102 | def close(circuit) 103 | send_chain(:close, circuit) do |e| 104 | options.notifier.notify(:storage_failure, circuit: circuit, action: :close, error: e) 105 | end 106 | end 107 | 108 | # Lock a circuit in all storage backends 109 | # 110 | # @param (see Interface#lock) 111 | # @return (see Interface#lock) 112 | def lock(circuit, state) 113 | send_all(:lock, circuit, state) 114 | end 115 | 116 | # Unlock a circuit in all storage backends 117 | # 118 | # @param (see Interface#unlock) 119 | # @return (see Interface#unlock) 120 | def unlock(circuit) 121 | send_all(:unlock, circuit) 122 | end 123 | 124 | # Reset a circuit in all storage backends 125 | # 126 | # @param (see Interface#reset) 127 | # @return (see Interface#reset) 128 | def reset(circuit) 129 | send_all(:reset, circuit) 130 | end 131 | 132 | # Get the status of a circuit from the first available storage backend 133 | # 134 | # @param (see Interface#status) 135 | # @return (see Interface#status) 136 | def status(circuit) 137 | send_chain(:status, circuit) do |e| 138 | options.notifier.notify(:storage_failure, circuit: circuit, action: :status, error: e) 139 | end 140 | end 141 | 142 | # Get the history of a circuit from the first available storage backend 143 | # 144 | # @param (see Interface#history) 145 | # @return (see Interface#history) 146 | def history(circuit) 147 | send_chain(:history, circuit) do |e| 148 | options.notifier.notify(:storage_failure, circuit: circuit, action: :history, error: e) 149 | end 150 | end 151 | 152 | # Get the list of circuits from the first available storage backend 153 | # 154 | # @param (see Interface#list) 155 | # @return (see Interface#list) 156 | def list 157 | send_chain(:list) do |e| 158 | options.notifier.notify(:storage_failure, action: :list, error: e) 159 | end 160 | end 161 | 162 | # Clears circuits in all storage backends 163 | # 164 | # @param (see Interface#clear) 165 | # @return (see Interface#clear) 166 | def clear 167 | send_all(:clear) 168 | end 169 | 170 | # This is fault tolerant if any of the available backends are fault tolerant 171 | # 172 | # @param (see Interface#fault_tolerant?) 173 | # @return (see Interface#fault_tolerant?) 174 | def fault_tolerant? 175 | @storages.any?(&:fault_tolerant?) 176 | end 177 | 178 | private 179 | 180 | # Call a method on the backend and return the first successful result 181 | # 182 | # Short-circuits, so that if a call succeeds, no additional backends are 183 | # called. 184 | # 185 | # @param method [Symbol] The method to call 186 | # @param args [Array] The arguments to send 187 | # @raise [AllFailedError] AllFailedError if all backends fail 188 | # @return The return value from the first successful call 189 | def send_chain(method, *args) 190 | errors = [] 191 | @storages.each do |s| 192 | begin 193 | return s.public_send(method, *args) 194 | rescue StandardError => e 195 | errors << e 196 | yield e 197 | end 198 | end 199 | 200 | raise AllFailedError.new("#{self.class}##{method} failed for all storage backends", errors) 201 | end 202 | 203 | # Call a method on every backend 204 | # 205 | # @param method [Symbol] The method to call 206 | # @param args [Array] The arguments to send 207 | # @raise [AllFailedError] AllFailedError if all backends fail 208 | # @raise [PartialFailureError] PartialFailureError if some but not all 209 | # backends fail 210 | # @return [nil] 211 | def send_all(method, *args) 212 | errors = [] 213 | @storages.each do |s| 214 | begin 215 | s.public_send(method, *args) 216 | rescue StandardError => e 217 | errors << e 218 | end 219 | end 220 | 221 | if errors.empty? 222 | nil 223 | elsif errors.size < @storages.size 224 | raise PartialFailureError.new("#{self.class}##{method} failed for some storage backends", errors) 225 | else 226 | raise AllFailedError.new("#{self.class}##{method} failed for all storage backends", errors) 227 | end 228 | end 229 | end 230 | end 231 | end 232 | -------------------------------------------------------------------------------- /lib/faulty/storage/fault_tolerant_proxy.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # A wrapper for storage backends that may raise errors 6 | # 7 | # {Faulty#initialize} automatically wraps all non-fault-tolerant storage backends with 8 | # this class. 9 | # 10 | # If the storage backend raises a `StandardError`, it will be captured and 11 | # sent to the notifier. 12 | class FaultTolerantProxy 13 | extend Forwardable 14 | 15 | attr_reader :options 16 | 17 | # Options for {FaultTolerantProxy} 18 | # 19 | # @!attribute [r] notifier 20 | # @return [Events::Notifier] A Faulty notifier 21 | Options = Struct.new( 22 | :notifier 23 | ) do 24 | include ImmutableOptions 25 | 26 | def required 27 | %i[notifier] 28 | end 29 | end 30 | 31 | # @param storage [Storage::Interface] The storage backend to wrap 32 | # @param options [Hash] Attributes for {Options} 33 | # @yield [Options] For setting options in a block 34 | def initialize(storage, **options, &block) 35 | @storage = storage 36 | @options = Options.new(options, &block) 37 | end 38 | 39 | # Wrap a storage backend in a FaultTolerantProxy unless it's already 40 | # fault tolerant 41 | # 42 | # @param storage [Storage::Interface] The storage to maybe wrap 43 | # @return [Storage::Interface] The original storage or a {FaultTolerantProxy} 44 | def self.wrap(storage, **options, &block) 45 | return storage if storage.fault_tolerant? 46 | 47 | new(storage, **options, &block) 48 | end 49 | 50 | # @!method lock(circuit, state) 51 | # Lock is not called in normal operation, so it doesn't capture errors 52 | # 53 | # @see Interface#lock 54 | # @param (see Interface#lock) 55 | # @return (see Interface#lock) 56 | # 57 | # @!method unlock(circuit) 58 | # Unlock is not called in normal operation, so it doesn't capture errors 59 | # 60 | # @see Interface#unlock 61 | # @param (see Interface#unlock) 62 | # @return (see Interface#unlock) 63 | # 64 | # @!method reset(circuit) 65 | # Reset is not called in normal operation, so it doesn't capture errors 66 | # 67 | # @see Interface#reset 68 | # @param (see Interface#reset) 69 | # @return (see Interface#reset) 70 | # 71 | # @!method history(circuit) 72 | # History is not called in normal operation, so it doesn't capture errors 73 | # 74 | # @see Interface#history 75 | # @param (see Interface#history) 76 | # @return (see Interface#history) 77 | # 78 | # @!method list 79 | # List is not called in normal operation, so it doesn't capture errors 80 | # 81 | # @see Interface#list 82 | # @param (see Interface#list) 83 | # @return (see Interface#list) 84 | # 85 | # @!method clear 86 | # Clear is not called in normal operation, so it doesn't capture errors 87 | # 88 | # @see Interface#list 89 | # @param (see Interface#list) 90 | # @return (see Interface#list) 91 | def_delegators :@storage, :lock, :unlock, :reset, :history, :list, :clear 92 | 93 | # Get circuit options safely 94 | # 95 | # @see Interface#get_options 96 | # @param (see Interface#get_options) 97 | # @return (see Interface#get_options) 98 | def get_options(circuit) 99 | @storage.get_options(circuit) 100 | rescue StandardError => e 101 | options.notifier.notify(:storage_failure, circuit: circuit, action: :get_options, error: e) 102 | nil 103 | end 104 | 105 | # Set circuit options safely 106 | # 107 | # @see Interface#get_options 108 | # @param (see Interface#set_options) 109 | # @return (see Interface#set_options) 110 | def set_options(circuit, stored_options) 111 | @storage.set_options(circuit, stored_options) 112 | rescue StandardError => e 113 | options.notifier.notify(:storage_failure, circuit: circuit, action: :set_options, error: e) 114 | nil 115 | end 116 | 117 | # Add a history entry safely 118 | # 119 | # @see Interface#entry 120 | # @param (see Interface#entry) 121 | # @return (see Interface#entry) 122 | def entry(circuit, time, success, status) 123 | @storage.entry(circuit, time, success, status) 124 | rescue StandardError => e 125 | options.notifier.notify(:storage_failure, circuit: circuit, action: :entry, error: e) 126 | stub_status(circuit) if status 127 | end 128 | 129 | # Safely mark a circuit as open 130 | # 131 | # @see Interface#open 132 | # @param (see Interface#open) 133 | # @return (see Interface#open) 134 | def open(circuit, opened_at) 135 | @storage.open(circuit, opened_at) 136 | rescue StandardError => e 137 | options.notifier.notify(:storage_failure, circuit: circuit, action: :open, error: e) 138 | false 139 | end 140 | 141 | # Safely mark a circuit as reopened 142 | # 143 | # @see Interface#reopen 144 | # @param (see Interface#reopen) 145 | # @return (see Interface#reopen) 146 | def reopen(circuit, opened_at, previous_opened_at) 147 | @storage.reopen(circuit, opened_at, previous_opened_at) 148 | rescue StandardError => e 149 | options.notifier.notify(:storage_failure, circuit: circuit, action: :reopen, error: e) 150 | false 151 | end 152 | 153 | # Safely mark a circuit as closed 154 | # 155 | # @see Interface#close 156 | # @param (see Interface#close) 157 | # @return (see Interface#close) 158 | def close(circuit) 159 | @storage.close(circuit) 160 | rescue StandardError => e 161 | options.notifier.notify(:storage_failure, circuit: circuit, action: :close, error: e) 162 | false 163 | end 164 | 165 | # Safely get the status of a circuit 166 | # 167 | # If the backend is unavailable, this returns a stub status that 168 | # indicates that the circuit is closed. 169 | # 170 | # @see Interface#status 171 | # @param (see Interface#status) 172 | # @return (see Interface#status) 173 | def status(circuit) 174 | @storage.status(circuit) 175 | rescue StandardError => e 176 | options.notifier.notify(:storage_failure, circuit: circuit, action: :status, error: e) 177 | stub_status(circuit) 178 | end 179 | 180 | # This cache makes any storage fault tolerant, so this is always `true` 181 | # 182 | # @return [true] 183 | def fault_tolerant? 184 | true 185 | end 186 | 187 | private 188 | 189 | # Create a stub status object to close the circuit by default 190 | # 191 | # @return [Status] The stub status 192 | def stub_status(circuit) 193 | Faulty::Status.new( 194 | options: circuit.options, 195 | stub: true 196 | ) 197 | end 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /lib/faulty/storage/interface.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # The interface required for a storage backend implementation 6 | # 7 | # This is for documentation only and is not loaded 8 | class Interface 9 | # Get the options stored for circuit 10 | # 11 | # They should be returned exactly as given by {#set_options} 12 | # 13 | # @return [Hash] A hash of the options stored by {#set_options}. The keys 14 | # must be symbols. 15 | def get_options(circuit) 16 | raise NotImplementedError 17 | end 18 | 19 | # Store the options for a circuit 20 | # 21 | # They should be returned exactly as given by {#set_options} 22 | # 23 | # @param circuit [Circuit] The circuit to set options for 24 | # @param stored_options [Hash] A hash of symbol option names to 25 | # circuit options. These option values are guranteed to be primive 26 | # values. 27 | # @return [void] 28 | def set_options(circuit, stored_options) 29 | raise NotImplementedError 30 | end 31 | 32 | # Add a circuit run entry to storage 33 | # 34 | # The backend may choose to store this in whatever manner it chooses as 35 | # long as it can implement the other read methods. 36 | # 37 | # @param circuit [Circuit] The circuit that ran 38 | # @param time [Integer] The unix timestamp for the run 39 | # @param success [Boolean] True if the run succeeded 40 | # @param status [Status, nil] The previous status. If given, this method must 41 | # return an updated status object from the new entry data. 42 | # @return [Status, nil] If `status` is not nil, the updated status object. 43 | def entry(circuit, time, success, status) 44 | raise NotImplementedError 45 | end 46 | 47 | # Set the circuit state to open 48 | # 49 | # If multiple parallel processes open the circuit simultaneously, open 50 | # may be called more than once. If so, this method should return true 51 | # only once, when the circuit transitions from closed to open. 52 | # 53 | # If the backend does not support locking or atomic operations, then 54 | # it may always return true, but that could result in duplicate open 55 | # notifications. 56 | # 57 | # If returning true, this method also updates opened_at to the 58 | # current time. 59 | # 60 | # @param circuit [Circuit] The circuit to open 61 | # @param opened_at [Integer] The timestmp the circuit was opened at 62 | # @return [Boolean] True if the circuit transitioned from closed to open 63 | def open(circuit, opened_at) 64 | raise NotImplementedError 65 | end 66 | 67 | # Reset the opened_at time for a half_open circuit 68 | # 69 | # If multiple parallel processes open the circuit simultaneously, reopen 70 | # may be called more than once. If so, this method should return true 71 | # only once, when the circuit updates the opened_at value. It can use the 72 | # value from previous_opened_at to do a compare-and-set operation. 73 | # 74 | # If the backend does not support locking or atomic operations, then 75 | # it may always return true, but that could result in duplicate reopen 76 | # notifications. 77 | # 78 | # @param circuit [Circuit] The circuit to reopen 79 | # @param opened_at [Integer] The timestmp the circuit was opened at 80 | # @param previous_opened_at [Integer] The last known value of opened_at. 81 | # Can be used to comare-and-set. 82 | # @return [Boolean] True if the opened_at time was updated 83 | def reopen(circuit, opened_at, previous_opened_at) 84 | raise NotImplementedError 85 | end 86 | 87 | # Set the circuit state to closed 88 | # 89 | # If multiple parallel processes close the circuit simultaneously, close 90 | # may be called more than once. If so, this method should return true 91 | # only once, when the circuit transitions from open to closed. 92 | # 93 | # If the backend does not support locking or atomic operations, then 94 | # it may always return true, but that could result in duplicate close 95 | # notifications. 96 | # 97 | # @return [Boolean] True if the circuit transitioned from open to closed 98 | def close(circuit) 99 | raise NotImplementedError 100 | end 101 | 102 | # Lock the circuit in a given state 103 | # 104 | # No concurrency gurantees are provided for locking 105 | # 106 | # @param circuit [Circuit] The circuit to lock 107 | # @param state [:open, :closed] The state to lock the circuit in 108 | # @return [void] 109 | def lock(circuit, state) 110 | raise NotImplementedError 111 | end 112 | 113 | # Unlock the circuit from any state 114 | # 115 | # No concurrency gurantees are provided for locking 116 | # 117 | # @param circuit [Circuit] The circuit to unlock 118 | # @return [void] 119 | def unlock(circuit) 120 | raise NotImplementedError 121 | end 122 | 123 | # Reset the circuit to a fresh state 124 | # 125 | # Clears all circuit status including entries, state, locks, 126 | # opened_at, options, and any other values that would affect Status. 127 | # 128 | # No concurrency gurantees are provided for resetting 129 | # 130 | # @param circuit [Circuit] The circuit to unlock 131 | # @return [void] 132 | def reset(circuit) 133 | raise NotImplementedError 134 | end 135 | 136 | # Get the status object for a circuit 137 | # 138 | # No concurrency gurantees are provided for getting status. It's possible 139 | # that status may represent a circuit in the middle of modification. 140 | # 141 | # @param circuit [Circuit] The circuit to get status for 142 | # @return [Status] The current status 143 | def status(circuit) 144 | raise NotImplementedError 145 | end 146 | 147 | # Get the entry history of a circuit 148 | # 149 | # No concurrency gurantees are provided for getting status. It's possible 150 | # that status may represent a circuit in the middle of modification. 151 | # 152 | # A storage backend may choose not to implement this method and instead 153 | # return an empty array. 154 | # 155 | # Each item in the history array is an array of two items (a tuple) of 156 | # `[run_time, succeeded]`, where `run_time` is a unix timestamp, and 157 | # `succeeded` is a boolean, true if the run succeeded. 158 | # 159 | # @param circuit [Circuit] The circuit to get history for 160 | # @return [Array] An array of history tuples 161 | def history(circuit) 162 | raise NotImplementedError 163 | end 164 | 165 | # Get a list of all circuit names 166 | # 167 | # If the storage backend does not support listing circuits, this may 168 | # return an empty array. 169 | # 170 | # @return [Array] 171 | def list 172 | raise NotImplementedError 173 | end 174 | 175 | # Reset all circuits 176 | # 177 | # Some implementions may clear circuits on a best-effort basis since 178 | # all circuits may not be known. 179 | # 180 | # @raise NotImplementedError If the storage backend does not support clearing. 181 | # @return [void] 182 | def clear 183 | raise NotImplementedError 184 | end 185 | 186 | # Can this storage backend raise an error? 187 | # 188 | # If the storage backend returns false from this method, it will be wrapped 189 | # in a {FaultTolerantProxy}, otherwise it will be used as-is. 190 | # 191 | # @return [Boolean] True if this cache backend is fault tolerant 192 | def fault_tolerant? 193 | raise NotImplementedError 194 | end 195 | end 196 | end 197 | end 198 | -------------------------------------------------------------------------------- /lib/faulty/storage/memory.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # The default in-memory storage for circuits 6 | # 7 | # This implementation is thread-safe and circuit state is shared across 8 | # threads. Since state is stored in-memory, this state is not shared across 9 | # processes, or persisted across application restarts. 10 | # 11 | # Circuit state and runs are stored in memory. Although runs have a maximum 12 | # size within a circuit, there is no limit on the number of circuits that 13 | # can be stored. This means the user should be careful about the number of 14 | # circuits that are created. To that end, it's a good idea to avoid 15 | # dynamically-named circuits with this backend. 16 | # 17 | # For a more robust distributed implementation, use the {Redis} storage 18 | # backend. 19 | # 20 | # This can be used as a reference implementation for storage backends that 21 | # store a list of circuit run entries. 22 | # 23 | # @todo Add a more sophsticated implmentation that can limit the number of 24 | # circuits stored. 25 | class Memory 26 | attr_reader :options 27 | 28 | # Options for {Memory} 29 | # 30 | # @!attribute [r] max_sample_size 31 | # @return [Integer] The number of cache run entries to keep in memory 32 | # for each circuit. Default `100`. 33 | Options = Struct.new(:max_sample_size) do 34 | include ImmutableOptions 35 | 36 | def defaults 37 | { max_sample_size: 100 } 38 | end 39 | end 40 | 41 | # The internal object for storing a circuit 42 | # 43 | # @private 44 | MemoryCircuit = Struct.new(:state, :runs, :opened_at, :lock, :options) do 45 | def initialize 46 | self.state = Concurrent::Atom.new(:closed) 47 | self.runs = Concurrent::MVar.new([], dup_on_deref: true) 48 | self.opened_at = Concurrent::Atom.new(nil) 49 | self.lock = nil 50 | end 51 | 52 | # Create a status object from the current circuit state 53 | # 54 | # @param circuit_options [Circuit::Options] The circuit options object 55 | # @return [Status] The newly created status 56 | def status(circuit_options) 57 | status = nil 58 | runs.borrow do |locked_runs| 59 | status = Faulty::Status.from_entries( 60 | locked_runs, 61 | state: state.value, 62 | lock: lock, 63 | opened_at: opened_at.value, 64 | options: circuit_options 65 | ) 66 | end 67 | 68 | status 69 | end 70 | end 71 | 72 | # @param options [Hash] Attributes for {Options} 73 | # @yield [Options] For setting options in a block 74 | def initialize(**options, &block) 75 | @circuits = Concurrent::Map.new 76 | @options = Options.new(options, &block) 77 | end 78 | 79 | # Get the options stored for circuit 80 | # 81 | # @see Interface#get_options 82 | # @param (see Interface#get_options) 83 | # @return (see Interface#get_options) 84 | def get_options(circuit) 85 | fetch(circuit).options 86 | end 87 | 88 | # Store the options for a circuit 89 | # 90 | # @see Interface#set_options 91 | # @param (see Interface#set_options) 92 | # @return (see Interface#set_options) 93 | def set_options(circuit, stored_options) 94 | fetch(circuit).options = stored_options 95 | end 96 | 97 | # Add an entry to storage 98 | # 99 | # @see Interface#entry 100 | # @param (see Interface#entry) 101 | # @return (see Interface#entry) 102 | def entry(circuit, time, success, status) 103 | memory = fetch(circuit) 104 | memory.runs.borrow do |runs| 105 | runs.push([time, success]) 106 | runs.shift if runs.size > options.max_sample_size 107 | end 108 | 109 | Status.from_entries(memory.runs.value, **status.to_h) if status 110 | end 111 | 112 | # Mark a circuit as open 113 | # 114 | # @see Interface#open 115 | # @param (see Interface#open) 116 | # @return (see Interface#open) 117 | def open(circuit, opened_at) 118 | memory = fetch(circuit) 119 | opened = memory.state.compare_and_set(:closed, :open) 120 | memory.opened_at.reset(opened_at) if opened 121 | opened 122 | end 123 | 124 | # Mark a circuit as reopened 125 | # 126 | # @see Interface#reopen 127 | # @param (see Interface#reopen) 128 | # @return (see Interface#reopen) 129 | def reopen(circuit, opened_at, previous_opened_at) 130 | memory = fetch(circuit) 131 | memory.opened_at.compare_and_set(previous_opened_at, opened_at) 132 | end 133 | 134 | # Mark a circuit as closed 135 | # 136 | # @see Interface#close 137 | # @param (see Interface#close) 138 | # @return (see Interface#close) 139 | def close(circuit) 140 | memory = fetch(circuit) 141 | memory.runs.modify { |_old| [] } 142 | memory.state.compare_and_set(:open, :closed) 143 | end 144 | 145 | # Lock a circuit open or closed 146 | # 147 | # @see Interface#lock 148 | # @param (see Interface#lock) 149 | # @return (see Interface#lock) 150 | def lock(circuit, state) 151 | memory = fetch(circuit) 152 | memory.lock = state 153 | end 154 | 155 | # Unlock a circuit 156 | # 157 | # @see Interface#unlock 158 | # @param (see Interface#unlock) 159 | # @return (see Interface#unlock) 160 | def unlock(circuit) 161 | memory = fetch(circuit) 162 | memory.lock = nil 163 | end 164 | 165 | # Reset a circuit 166 | # 167 | # @see Interface#reset 168 | # @param (see Interface#reset) 169 | # @return (see Interface#reset) 170 | def reset(circuit) 171 | @circuits.delete(circuit.name) 172 | end 173 | 174 | # Get the status of a circuit 175 | # 176 | # @see Interface#status 177 | # @param (see Interface#status) 178 | # @return (see Interface#status) 179 | def status(circuit) 180 | fetch(circuit).status(circuit.options) 181 | end 182 | 183 | # Get the circuit history up to `max_sample_size` 184 | # 185 | # @see Interface#history 186 | # @param (see Interface#history) 187 | # @return (see Interface#history) 188 | def history(circuit) 189 | fetch(circuit).runs.value 190 | end 191 | 192 | # Get a list of circuit names 193 | # 194 | # @return [Array] The circuit names 195 | def list 196 | @circuits.keys 197 | end 198 | 199 | # Clears all circuits 200 | # 201 | # @return [void] 202 | def clear 203 | @circuits.clear 204 | end 205 | 206 | # Memory storage is fault-tolerant by default 207 | # 208 | # @return [true] 209 | def fault_tolerant? 210 | true 211 | end 212 | 213 | private 214 | 215 | # Fetch circuit storage safely or create it if it doesn't exist 216 | # 217 | # @return [MemoryCircuit] 218 | def fetch(circuit) 219 | @circuits.compute_if_absent(circuit.name) { MemoryCircuit.new } 220 | end 221 | end 222 | end 223 | end 224 | -------------------------------------------------------------------------------- /lib/faulty/storage/null.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Storage 5 | # A no-op backend for disabling circuits 6 | class Null 7 | # Define a single global instance 8 | @instance = new 9 | 10 | def self.new 11 | @instance 12 | end 13 | 14 | # @param (see Interface#get_options) 15 | # @return (see Interface#get_options) 16 | def get_options(_circuit) 17 | {} 18 | end 19 | 20 | # @param (see Interface#set_options) 21 | # @return (see Interface#set_options) 22 | def set_options(_circuit, _stored_options) 23 | end 24 | 25 | # @param (see Interface#entry) 26 | # @return (see Interface#entry) 27 | def entry(circuit, _time, _success, status) 28 | stub_status(circuit) if status 29 | end 30 | 31 | # @param (see Interface#open) 32 | # @return (see Interface#open) 33 | def open(_circuit, _opened_at) 34 | true 35 | end 36 | 37 | # @param (see Interface#reopen) 38 | # @return (see Interface#reopen) 39 | def reopen(_circuit, _opened_at, _previous_opened_at) 40 | true 41 | end 42 | 43 | # @param (see Interface#close) 44 | # @return (see Interface#close) 45 | def close(_circuit) 46 | true 47 | end 48 | 49 | # @param (see Interface#lock) 50 | # @return (see Interface#lock) 51 | def lock(_circuit, _state) 52 | end 53 | 54 | # @param (see Interface#unlock) 55 | # @return (see Interface#unlock) 56 | def unlock(_circuit) 57 | end 58 | 59 | # @param (see Interface#reset) 60 | # @return (see Interface#reset) 61 | def reset(_circuit) 62 | end 63 | 64 | # @param (see Interface#status) 65 | # @return (see Interface#status) 66 | def status(circuit) 67 | stub_status(circuit) 68 | end 69 | 70 | # @param (see Interface#history) 71 | # @return (see Interface#history) 72 | def history(_circuit) 73 | [] 74 | end 75 | 76 | # @param (see Interface#list) 77 | # @return (see Interface#list) 78 | def list 79 | [] 80 | end 81 | 82 | # @param (see Interface#clear) 83 | # @return (see Interface#clear) 84 | def clear 85 | end 86 | 87 | # This backend is fault tolerant 88 | # 89 | # @param (see Interface#fault_tolerant?) 90 | # @return (see Interface#fault_tolerant?) 91 | def fault_tolerant? 92 | true 93 | end 94 | 95 | private 96 | 97 | def stub_status(circuit) 98 | Faulty::Status.new( 99 | options: circuit.options, 100 | stub: true 101 | ) 102 | end 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /lib/faulty/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | # The current Faulty version 5 | def self.version 6 | Gem::Version.new('0.11.0') 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /spec/cache/auto_wire_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::AutoWire do 4 | subject(:auto_wire) { described_class.wrap(backend, circuit: circuit, notifier: notifier) } 5 | 6 | let(:circuit) { Faulty::Circuit.new('test') } 7 | let(:notifier) { Faulty::Events::Notifier.new } 8 | let(:backend) { nil } 9 | 10 | context 'with a fault-tolerant backend' do 11 | let(:backend) { Faulty::Cache::Mock.new } 12 | 13 | it 'delegates directly if a fault-tolerant backend is given' do 14 | expect(auto_wire).to eq(backend) 15 | end 16 | end 17 | 18 | context 'with a non-fault-tolerant backend' do 19 | let(:backend) { Faulty::Cache::Rails.new(nil, fault_tolerant: false) } 20 | 21 | it 'is fault tolerant' do 22 | expect(auto_wire).to be_fault_tolerant 23 | end 24 | 25 | it 'wraps in FaultTolerantProxy and CircuitProxy' do 26 | expect(auto_wire).to be_a(Faulty::Cache::FaultTolerantProxy) 27 | 28 | circuit_proxy = auto_wire.instance_variable_get(:@cache) 29 | expect(circuit_proxy).to be_a(Faulty::Cache::CircuitProxy) 30 | expect(circuit_proxy.options.circuit).to eq(circuit) 31 | 32 | original = circuit_proxy.instance_variable_get(:@cache) 33 | expect(original).to eq(backend) 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/cache/circuit_proxy_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::CircuitProxy do 4 | let(:notifier) { Faulty::Events::Notifier.new } 5 | let(:circuit) { Faulty::Circuit.new('test', sample_threshold: 2) } 6 | 7 | let(:failing_cache) do 8 | Class.new do 9 | def method_missing(*_args) 10 | raise 'fail' 11 | end 12 | 13 | def respond_to_missing?(*_args) 14 | true 15 | end 16 | end 17 | end 18 | 19 | it 'trips its internal circuit when storage fails repeatedly' do 20 | backend = failing_cache.new 21 | proxy = described_class.new(backend, notifier: notifier, circuit: circuit) 22 | begin 23 | 2.times { proxy.read('foo') } 24 | rescue Faulty::CircuitFailureError 25 | nil 26 | end 27 | 28 | expect { proxy.read('foo') }.to raise_error(Faulty::CircuitTrippedError) 29 | end 30 | 31 | it 'does not notify for circuit sucesses by default' do 32 | expect(notifier).not_to receive(:notify) 33 | backend = Faulty::Cache::Mock.new 34 | proxy = described_class.new(backend, notifier: notifier) 35 | proxy.read('foo') 36 | end 37 | 38 | it 'delegates fault_tolerant? directly' do 39 | backend = instance_double(Faulty::Cache::Mock) 40 | marker = Object.new 41 | allow(backend).to receive(:fault_tolerant?).and_return(marker) 42 | expect(described_class.new(backend, notifier: notifier).fault_tolerant?).to eq(marker) 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/cache/default_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::Default do 4 | subject(:cache) { described_class.new } 5 | 6 | it 'creates Null when neither Rails nor ActiveSupport is defined' do 7 | expect(cache.instance_variable_get(:@cache)).to be_a(Faulty::Cache::Null) 8 | end 9 | 10 | context 'when ActiveSupport is defined' do 11 | before do 12 | stub_const('ActiveSupport::Cache::MemoryStore', Faulty::Cache::Mock) 13 | end 14 | 15 | it 'forwards methods to internal cache' do 16 | cache.write('foo', 'bar') 17 | expect(cache.read('foo')).to eq('bar') 18 | end 19 | 20 | it 'uses ActiveSupport::Cache::MemoryStore' do 21 | wrapper = cache.instance_variable_get(:@cache) 22 | expect(wrapper).to be_a(Faulty::Cache::Rails) 23 | expect(wrapper.instance_variable_get(:@cache)).to be_a(ActiveSupport::Cache::MemoryStore) 24 | end 25 | 26 | context 'when Rails is defined' do 27 | before do 28 | stub_const( 29 | 'Rails', 30 | Class.new do 31 | def self.cache 32 | @cache ||= Object.new 33 | end 34 | end 35 | ) 36 | end 37 | 38 | it 'uses Rails.cache' do 39 | wrapper = cache.instance_variable_get(:@cache) 40 | expect(wrapper).to be_a(Faulty::Cache::Rails) 41 | expect(wrapper.instance_variable_get(:@cache)).to eq(::Rails.cache) 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/cache/fault_tolerant_proxy_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::FaultTolerantProxy do 4 | let(:notifier) { Faulty::Events::Notifier.new } 5 | 6 | let(:failing_cache_class) do 7 | Class.new do 8 | def method_missing(*_args) 9 | raise 'fail' 10 | end 11 | 12 | def respond_to_missing?(*_args) 13 | true 14 | end 15 | end 16 | end 17 | 18 | let(:failing_cache) do 19 | failing_cache_class.new 20 | end 21 | 22 | let(:mock_cache) { Faulty::Cache::Mock.new } 23 | 24 | it 'delegates to backend when reading succeeds' do 25 | mock_cache.write('foo', 'val') 26 | value = described_class.new(mock_cache, notifier: notifier).read('foo') 27 | expect(value).to eq('val') 28 | end 29 | 30 | it 'returns nil when reading fails' do 31 | expect(notifier).to receive(:notify) 32 | .with(:cache_failure, key: 'foo', action: :read, error: instance_of(RuntimeError)) 33 | result = described_class.new(failing_cache, notifier: notifier).read('foo') 34 | expect(result).to be_nil 35 | end 36 | 37 | it 'delegates to backend when writing succeeds' do 38 | described_class.new(mock_cache, notifier: notifier).write('foo', 'val') 39 | expect(mock_cache.read('foo')).to eq('val') 40 | end 41 | 42 | it 'skips writing when backend fails' do 43 | expect(notifier).to receive(:notify) 44 | .with(:cache_failure, key: 'foo', action: :write, error: instance_of(RuntimeError)) 45 | proxy = described_class.new(failing_cache, notifier: notifier) 46 | proxy.write('foo', 'val') 47 | end 48 | 49 | it 'is always fault tolerant' do 50 | expect(described_class.new(Object.new, notifier: notifier)).to be_fault_tolerant 51 | end 52 | 53 | describe '.wrap' do 54 | it 'returns fault-tolerant cache unmodified' do 55 | expect(described_class.wrap(mock_cache, notifier: notifier)).to eq(mock_cache) 56 | end 57 | 58 | it 'wraps fault-tolerant cache' do 59 | expect(described_class.wrap(Faulty::Cache::Rails.new(nil), notifier: notifier)) 60 | .to be_a(described_class) 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /spec/cache/mock_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::Mock do 4 | subject(:cache) { described_class.new } 5 | 6 | it 'writes and reads a value' do 7 | cache.write('a key', 'a value') 8 | expect(cache.read('a key')).to eq('a value') 9 | end 10 | 11 | it 'expires values after expires_in' do 12 | cache.write('a key', 'a value', expires_in: 10) 13 | Timecop.travel(Time.now + 5) 14 | expect(cache.read('a key')).to eq('a value') 15 | Timecop.travel(Time.now + 6) 16 | expect(cache.read('a key')).to be_nil 17 | end 18 | 19 | it 'is fault tolerant' do 20 | expect(cache).to be_fault_tolerant 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/cache/null_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::Null do 4 | subject(:cache) { described_class.new } 5 | 6 | it 'reads nothing after writing' do 7 | cache.write('foo', 'bar') 8 | expect(cache.read('foo')).to be_nil 9 | end 10 | 11 | it 'is fault_tolerant' do 12 | expect(cache.fault_tolerant?).to be(true) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /spec/cache/rails_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Cache::Rails do 4 | before do 5 | stub_const( 6 | 'Rails', 7 | Class.new do 8 | def self.cache 9 | @cache ||= Object.new 10 | end 11 | end 12 | ) 13 | end 14 | 15 | it 'uses global Rails.cache by default' do 16 | cache = described_class.new 17 | expect(cache.instance_variable_get(:@cache)).to eq(::Rails.cache) 18 | expect(cache).not_to be_fault_tolerant 19 | end 20 | 21 | it 'accepts a custom cache backend' do 22 | backend = Faulty::Cache::Mock.new 23 | cache = described_class.new(backend) 24 | cache.write('foo', 'bar') 25 | expect(backend.read('foo')).to eq('bar') 26 | end 27 | 28 | it 'can be marked as fault tolerant' do 29 | cache = described_class.new(fault_tolerant: true) 30 | expect(cache).to be_fault_tolerant 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/deprecation_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Deprecation do 4 | it 'prints note and sunset version' do 5 | expect(Kernel).to receive(:warn) 6 | .with('DEPRECATION: foo is deprecated and will be removed in 1.0 (Use bar)') 7 | described_class.deprecate(:foo, note: 'Use bar', sunset: '1.0') 8 | end 9 | 10 | it 'prints only subject' do 11 | expect(Kernel).to receive(:warn) 12 | .with('DEPRECATION: blah is deprecated') 13 | described_class.deprecate('blah') 14 | end 15 | 16 | it 'prints method deprecation' do 17 | expect(Kernel).to receive(:warn) 18 | .with('DEPRECATION: Faulty::Circuit#foo is deprecated and will be removed in 1.0 (Use bar)') 19 | described_class.method(Faulty::Circuit, :foo, note: 'Use bar', sunset: '1.0') 20 | end 21 | 22 | context 'with raise_errors!' do 23 | before { described_class.raise_errors! } 24 | 25 | after { described_class.raise_errors!(false) } 26 | 27 | it 'raises DeprecationError' do 28 | expect { described_class.deprecate('blah') } 29 | .to raise_error(Faulty::DeprecationError, 'blah is deprecated') 30 | end 31 | end 32 | 33 | context 'when silenced' do 34 | it 'does not surface deprecations' do 35 | expect(Kernel).not_to receive(:warn) 36 | described_class.silenced do 37 | described_class.deprecate('blah') 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/events/callback_listener_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Events::CallbackListener do 4 | subject(:listener) { described_class.new } 5 | 6 | it 'calls handler with event payload' do 7 | result = nil 8 | listener.circuit_opened { |payload| result = payload } 9 | listener.handle(:circuit_opened, circuit: 'test') 10 | expect(result[:circuit]).to eq('test') 11 | end 12 | 13 | it 'does nothing for unknown event' do 14 | listener.handle(:fake_event, circuit: 'test') 15 | end 16 | 17 | it 'allows event with no handlers' do 18 | listener.handle(:circuit_opened, circuit: 'test') 19 | end 20 | 21 | it 'calls multiple handlers' do 22 | results = [] 23 | listener.circuit_opened { |payload| results << payload } 24 | listener.circuit_opened { |payload| results << payload } 25 | listener.handle(:circuit_opened, circuit: 'test') 26 | expect(results).to match_array([{ circuit: 'test' }, { circuit: 'test' }]) 27 | end 28 | 29 | it 'can register listeners in initialize block' do 30 | result = nil 31 | listener = described_class.new do |events| 32 | events.circuit_closed do |payload| 33 | result = payload 34 | end 35 | end 36 | 37 | listener.handle(:circuit_closed, circuit: 'test') 38 | expect(result[:circuit]).to eq('test') 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /spec/events/filter_notifier_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Events::FilterNotifier do 4 | let(:backend) { Faulty::Events::Notifier.new } 5 | 6 | it 'forwards all events by default' do 7 | filter = described_class.new(backend) 8 | expect(backend).to receive(:notify).with(:circuit_success, {}) 9 | filter.notify(:circuit_success, {}) 10 | end 11 | 12 | it 'forwards only given events' do 13 | filter = described_class.new(backend, events: %i[circuit_failure]) 14 | expect(backend).to receive(:notify).with(:circuit_failure, {}) 15 | filter.notify(:circuit_success, {}) 16 | filter.notify(:circuit_failure, {}) 17 | end 18 | 19 | it 'forwards all except exluded events' do 20 | filter = described_class.new(backend, exclude: %i[circuit_success]) 21 | expect(backend).to receive(:notify).with(:circuit_failure, {}) 22 | filter.notify(:circuit_success, {}) 23 | filter.notify(:circuit_failure, {}) 24 | end 25 | 26 | it 'forwards given events except excluded' do 27 | filter = described_class.new(backend, events: %i[circuit_failure circuit_success], exclude: %i[circuit_success]) 28 | expect(backend).to receive(:notify).with(:circuit_failure, {}) 29 | filter.notify(:circuit_success, {}) 30 | filter.notify(:circuit_failure, {}) 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/events/honeybadger_listener_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Events::HoneybadgerListener do 4 | let(:circuit) { Faulty::Circuit.new('test_circuit') } 5 | let(:error) { StandardError.new('fail') } 6 | let(:notice) do 7 | Honeybadger.flush 8 | Honeybadger::Backend::Test.notifications[:notices].first 9 | end 10 | 11 | before do 12 | skip 'Honeybadger only supports >= Ruby 2.4' unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.4') 13 | require 'honeybadger/ruby' 14 | 15 | Honeybadger.configure do |c| 16 | c.backend = 'test' 17 | c.api_key = 'test' 18 | end 19 | Honeybadger::Backend::Test.notifications[:notices] = [] 20 | end 21 | 22 | it 'notifies honeybadger of a circuit failure' do 23 | described_class.new.handle(:circuit_failure, { error: error, circuit: circuit }) 24 | expect(notice.error_message).to eq('StandardError: fail') 25 | expect(notice.context[:circuit]).to eq('test_circuit') 26 | end 27 | 28 | it 'notifies honeybadger of a circuit open' do 29 | described_class.new.handle(:circuit_opened, { error: error, circuit: circuit }) 30 | expect(notice.error_message).to eq('StandardError: fail') 31 | expect(notice.context[:circuit]).to eq('test_circuit') 32 | end 33 | 34 | it 'notifies honeybadger of a circuit reopen' do 35 | described_class.new.handle(:circuit_reopened, { error: error, circuit: circuit }) 36 | expect(notice.error_message).to eq('StandardError: fail') 37 | expect(notice.context[:circuit]).to eq('test_circuit') 38 | end 39 | 40 | it 'notifies honeybadger of a cache failure' do 41 | described_class.new.handle(:cache_failure, { error: error, action: :read, key: 'test' }) 42 | expect(notice.error_message).to eq('StandardError: fail') 43 | expect(notice.context[:action]).to eq(:read) 44 | expect(notice.context[:key]).to eq('test') 45 | end 46 | 47 | it 'notifies honeybadger of a storage failure' do 48 | described_class.new.handle(:storage_failure, { error: error, action: :open, circuit: circuit }) 49 | expect(notice.error_message).to eq('StandardError: fail') 50 | expect(notice.context[:action]).to eq(:open) 51 | expect(notice.context[:circuit]).to eq('test_circuit') 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /spec/events/log_listener_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Events::LogListener do 4 | subject(:listener) { described_class.new(logger) } 5 | 6 | let(:logger) do 7 | logger = ::Logger.new(io) 8 | logger.level = :debug 9 | logger 10 | end 11 | let(:io) { StringIO.new } 12 | let(:circuit) { Faulty::Circuit.new('test', notifier: notifier, cache: cache) } 13 | let(:error) { StandardError.new('fail') } 14 | let(:notifier) { Faulty::Events::Notifier.new } 15 | let(:cache) { Faulty::Cache::Null.new } 16 | let(:status) { Faulty::Status.new({ options: circuit.options }) } 17 | let(:logs) do 18 | io.rewind 19 | io.read.strip 20 | end 21 | 22 | before do 23 | require 'logger' 24 | end 25 | 26 | context 'when Rails is available' do 27 | before do 28 | l = logger 29 | stub_const( 30 | 'Rails', 31 | Class.new do 32 | define_singleton_method(:logger) do 33 | l 34 | end 35 | end 36 | ) 37 | end 38 | 39 | it 'logs to Rails logger by default' do 40 | described_class.new.handle(:circuit_success, circuit: circuit) 41 | expect(logs).to end_with('DEBUG -- : Circuit succeeded: test') 42 | end 43 | end 44 | 45 | it 'logs to stderr by default if Rails is not present' do 46 | expect do 47 | described_class.new.handle(:circuit_success, circuit: circuit) 48 | end.to output(/DEBUG -- : Circuit succeeded: test$/).to_stderr 49 | end 50 | 51 | # cache_failure 52 | # circuit_cache_hit 53 | # circuit_cache_miss 54 | # circuit_cache_write 55 | # circuit_closed 56 | # circuit_failure 57 | # circuit_opened 58 | # circuit_reopened 59 | # circuit_skipped 60 | # circuit_success 61 | # storage_failure 62 | 63 | it 'logs cache_failure' do 64 | listener.handle(:cache_failure, key: 'foo', action: :read, error: error) 65 | expect(logs).to end_with('ERROR -- : Cache failure: read key=foo error=fail') 66 | end 67 | 68 | it 'logs circuit_cache_hit' do 69 | listener.handle(:circuit_cache_hit, circuit: circuit, key: 'foo') 70 | expect(logs).to end_with('DEBUG -- : Circuit cache hit: test key=foo') 71 | end 72 | 73 | it 'logs circuit_cache_miss' do 74 | listener.handle(:circuit_cache_miss, circuit: circuit, key: 'foo') 75 | expect(logs).to end_with('DEBUG -- : Circuit cache miss: test key=foo') 76 | end 77 | 78 | it 'logs circuit_cache_write' do 79 | listener.handle(:circuit_cache_write, circuit: circuit, key: 'foo') 80 | expect(logs).to end_with('DEBUG -- : Circuit cache write: test key=foo') 81 | end 82 | 83 | it 'logs circuit_closed' do 84 | listener.handle(:circuit_closed, circuit: circuit) 85 | expect(logs).to end_with('INFO -- : Circuit closed: test') 86 | end 87 | 88 | it 'logs circuit_failure' do 89 | listener.handle(:circuit_failure, circuit: circuit, status: status, error: error) 90 | expect(logs).to end_with('ERROR -- : Circuit failed: test state=closed error=fail') 91 | end 92 | 93 | it 'logs circuit_opened' do 94 | listener.handle(:circuit_opened, circuit: circuit, error: error) 95 | expect(logs).to end_with('ERROR -- : Circuit opened: test error=fail') 96 | end 97 | 98 | it 'logs circuit_reopened' do 99 | listener.handle(:circuit_reopened, circuit: circuit, error: error) 100 | expect(logs).to end_with('ERROR -- : Circuit reopened: test error=fail') 101 | end 102 | 103 | it 'logs circuit_skipped' do 104 | listener.handle(:circuit_skipped, circuit: circuit) 105 | expect(logs).to end_with('WARN -- : Circuit skipped: test') 106 | end 107 | 108 | it 'logs circuit_success' do 109 | listener.handle(:circuit_success, circuit: circuit) 110 | expect(logs).to end_with('DEBUG -- : Circuit succeeded: test') 111 | end 112 | 113 | it 'logs storage_failure with circuit' do 114 | listener.handle(:storage_failure, circuit: circuit, action: :entry, error: error) 115 | expect(logs).to end_with('ERROR -- : Storage failure: entry circuit=test error=fail') 116 | end 117 | 118 | it 'logs storage_failure without circuit' do 119 | listener.handle(:storage_failure, action: :list, error: error) 120 | expect(logs).to end_with('ERROR -- : Storage failure: list error=fail') 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /spec/events/notifier_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Events::Notifier do 4 | let(:listener_class) do 5 | Class.new do 6 | attr_reader :events 7 | 8 | def initialize 9 | @events = [] 10 | end 11 | 12 | def handle(event, payload) 13 | @events << [event, payload] 14 | end 15 | end 16 | end 17 | 18 | let(:failing_class) do 19 | Class.new do 20 | def self.name 21 | 'Failing' 22 | end 23 | 24 | def handle(_event, _payload) 25 | raise 'fail' 26 | end 27 | end 28 | end 29 | 30 | it 'calls handle for each listener' do 31 | listeners = [listener_class.new, listener_class.new] 32 | notifier = described_class.new(listeners) 33 | notifier.notify(:circuit_closed, {}) 34 | expect(listeners[0].events).to eq([[:circuit_closed, {}]]) 35 | expect(listeners[1].events).to eq([[:circuit_closed, {}]]) 36 | end 37 | 38 | it 'suppresses and prints errors' do 39 | notifier = described_class.new([failing_class.new]) 40 | expect { notifier.notify(:circuit_opened, {}) } 41 | .to output("Faulty listener Failing crashed: fail\n").to_stderr 42 | end 43 | 44 | it 'raises error for incorrect event' do 45 | notifier = described_class.new 46 | expect { notifier.notify(:foo, {}) }.to raise_error(ArgumentError) 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /spec/faulty_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty do 4 | subject(:instance) { described_class.new(listeners: []) } 5 | 6 | after do 7 | # Reset the global Faulty instance 8 | # We don't want to expose a public method to do this 9 | # because it could cause concurrency errors, and confusion about what 10 | # exactly gets reset 11 | described_class.instance_variable_set(:@instances, nil) 12 | described_class.instance_variable_set(:@default_instance, nil) 13 | end 14 | 15 | it 'can be initialized with no args' do 16 | described_class.init 17 | expect(described_class.default).to be_a(described_class) 18 | end 19 | 20 | it 'gets options from the default instance' do 21 | described_class.init 22 | expect(described_class.options).to eq(described_class.default.options) 23 | end 24 | 25 | it '#default raises uninitialized error if #init not called' do 26 | expect { described_class.default }.to raise_error(Faulty::UninitializedError) 27 | end 28 | 29 | it 'raises error when initialized twice' do 30 | described_class.init 31 | expect { described_class.init }.to raise_error(Faulty::AlreadyInitializedError) 32 | end 33 | 34 | it '#default raises missing instance error if default not created' do 35 | described_class.init(nil) 36 | expect { described_class.default }.to raise_error(Faulty::MissingDefaultInstanceError) 37 | end 38 | 39 | it 'can rename the default instance on #init' do 40 | described_class.init(:foo) 41 | expect(described_class.default).to be_a(described_class) 42 | expect(described_class[:foo]).to eq(described_class.default) 43 | end 44 | 45 | it 'can be reinitialized if initialization fails' do 46 | expect { described_class.init(not_an_option: true) }.to raise_error(NameError) 47 | described_class.init 48 | end 49 | 50 | it 'registers a named instance' do 51 | described_class.init 52 | instance = described_class.new 53 | described_class.register(:new_instance, instance) 54 | expect(described_class[:new_instance]).to eq(instance) 55 | end 56 | 57 | it 'accesses intances by string or symbol' do 58 | described_class.init 59 | instance = described_class.new 60 | described_class.register(:symbol, instance) 61 | expect(described_class['symbol']).to eq(instance) 62 | described_class.register(:string, instance) 63 | expect(described_class['string']).to eq(instance) 64 | end 65 | 66 | it 'registers a named instance without default' do 67 | described_class.init(nil) 68 | instance = described_class.new 69 | described_class.register(:new_instance, instance) 70 | expect(described_class[:new_instance]).to eq(instance) 71 | end 72 | 73 | it 'registers a named instance with a block' do 74 | described_class.init(nil) 75 | described_class.register(:new_instance) { |c| c.circuit_defaults = { sample_threshold: 6 } } 76 | expect(described_class[:new_instance].options.circuit_defaults[:sample_threshold]).to eq(6) 77 | end 78 | 79 | it 'registers a named instance with options' do 80 | described_class.init(nil) 81 | described_class.register(:new_instance, circuit_defaults: { sample_threshold: 7 }) 82 | expect(described_class[:new_instance].options.circuit_defaults[:sample_threshold]).to eq(7) 83 | end 84 | 85 | it 'raises an error when passed instance with options' do 86 | described_class.init(nil) 87 | instance = described_class.new 88 | expect do 89 | described_class.register(:new_instance, instance, circuit_defaults: { sample_threshold: 7 }) 90 | end.to raise_error(ArgumentError, 'Do not give config options if an instance is given') 91 | end 92 | 93 | it 'memoizes named instances' do 94 | described_class.init 95 | instance1 = described_class.new 96 | instance2 = described_class.new 97 | expect(described_class.register(:named, instance1)).to be_nil 98 | expect(described_class.register(:named, instance2)).to eq(instance1) 99 | expect(described_class[:named]).to eq(instance1) 100 | end 101 | 102 | it 'delegates circuit to the default instance' do 103 | described_class.init(listeners: []) 104 | described_class.circuit('test').run { 'ok' } 105 | expect(described_class.default.list_circuits).to eq(['test']) 106 | end 107 | 108 | it 'lists the circuits from the default instance' do 109 | described_class.init(listeners: []) 110 | described_class.circuit('test').run { 'ok' } 111 | expect(described_class.list_circuits).to eq(['test']) 112 | end 113 | 114 | it 'gets the current timestamp' do 115 | Timecop.freeze(Time.new(2020, 1, 1, 0, 0, 0, '+00:00')) 116 | expect(described_class.current_time).to eq(1_577_836_800) 117 | end 118 | 119 | it 'does not memoize circuits before they are run' do 120 | expect(instance.circuit('test')).not_to eq(instance.circuit('test')) 121 | end 122 | 123 | it 'memoizes circuits once run' do 124 | circuit = instance.circuit('test') 125 | circuit.run { 'ok' } 126 | expect(instance.circuit('test')).to eq(circuit) 127 | end 128 | 129 | it 'keeps options passed to the first memoized instance and ignores others' do 130 | instance.circuit('test', cool_down: 404).run { 'ok' } 131 | expect(instance.circuit('test', cool_down: 302).options.cool_down).to eq(404) 132 | end 133 | 134 | it 'replaces own circuit options from the first-run circuit' do 135 | test1 = instance.circuit('test', cool_down: 123) 136 | test2 = instance.circuit('test', cool_down: 456) 137 | test1.run { 'ok' } 138 | test2.run { 'ok' } 139 | expect(test2.options.cool_down).to eq(123) 140 | end 141 | 142 | it 'passes options from itself to new circuits' do 143 | instance = described_class.new( 144 | circuit_defaults: { sample_threshold: 14, cool_down: 30 } 145 | ) 146 | circuit = instance.circuit('test', cool_down: 10) 147 | expect(circuit.options.cache).to eq(instance.options.cache) 148 | expect(circuit.options.storage).to eq(instance.options.storage) 149 | expect(circuit.options.notifier).to eq(instance.options.notifier) 150 | expect(circuit.options.sample_threshold).to eq(14) 151 | expect(circuit.options.cool_down).to eq(10) 152 | end 153 | 154 | it 'converts symbol names to strings' do 155 | circuit = instance.circuit(:test) 156 | circuit.run { 'ok' } 157 | expect(instance.circuit('test')).to eq(circuit) 158 | end 159 | 160 | it 'lists circuit names' do 161 | instance.circuit('test1').run { 'ok' } 162 | instance.circuit('test2').run { 'ok' } 163 | expect(instance.list_circuits).to match_array(%w[test1 test2]) 164 | end 165 | 166 | it 'wraps non-fault-tolerant storage in FaultTolerantProxy' do 167 | instance = described_class.new(storage: Faulty::Storage::Redis.new) 168 | expect(instance.options.storage).to be_a(Faulty::Storage::FaultTolerantProxy) 169 | end 170 | 171 | it 'wraps non-fault-tolerant cache in FaultTolerantProxy' do 172 | instance = described_class.new(cache: Faulty::Cache::Rails.new(nil)) 173 | expect(instance.options.cache).to be_a(Faulty::Cache::FaultTolerantProxy) 174 | end 175 | 176 | it 'can be disabled and enabled' do 177 | described_class.disable! 178 | expect(described_class.disabled?).to be(true) 179 | described_class.enable! 180 | expect(described_class.disabled?).to be(false) 181 | end 182 | 183 | it 'clears circuits' do 184 | instance.circuit('test').run { 'ok' } 185 | instance.clear! 186 | expect(instance.circuit('test').history).to eq([]) 187 | end 188 | end 189 | -------------------------------------------------------------------------------- /spec/immutable_options_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::ImmutableOptions do 4 | let(:example_class) do 5 | Struct.new(:name, :cache, :storage) do 6 | include Faulty::ImmutableOptions 7 | 8 | def defaults 9 | { cache: 'default_cache' } 10 | end 11 | 12 | def required 13 | %i[name] 14 | end 15 | 16 | def finalize 17 | self.storage = 'finalized' 18 | end 19 | end 20 | end 21 | 22 | it 'applies a default if an option is not present' do 23 | opts = example_class.new(name: 'foo') 24 | expect(opts.cache).to eq('default_cache') 25 | end 26 | 27 | it 'overrides defaults with given hash' do 28 | opts = example_class.new(name: 'foo', cache: 'special_cache') 29 | expect(opts.cache).to eq('special_cache') 30 | end 31 | 32 | it 'overrides defaults with block' do 33 | opts = example_class.new(name: 'foo') { |o| o.cache = 'special_cache' } 34 | expect(opts.cache).to eq('special_cache') 35 | end 36 | 37 | it 'calls finalize after options are set' do 38 | opts = example_class.new(name: 'foo', storage: 'from_hash') { |o| o.storage = 'from_block' } 39 | expect(opts.storage).to eq('finalized') 40 | end 41 | 42 | it 'raises error if required option is missing' do 43 | expect { example_class.new({}) }.to raise_error(ArgumentError, /Missing required attribute name/) 44 | end 45 | 46 | it 'raises error if required option is nil' do 47 | expect { example_class.new(name: nil) }.to raise_error(ArgumentError, /Missing required attribute name/) 48 | end 49 | 50 | # truffleruby does not freeze objects 51 | if defined?(RUBY_ENGINE) && RUBY_ENGINE != 'truffleruby' 52 | it 'freezes options after initialization' do 53 | opts = example_class.new(name: 'foo') 54 | expect { opts.name = 'bar' }.to raise_error(/can't modify frozen/) 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /spec/patch/base_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Patch::Base do 4 | include described_class 5 | 6 | let(:circuit) { Faulty::Circuit.new('test', **options) } 7 | let(:options) { { cache: cache, storage: storage } } 8 | let(:storage) { Faulty::Storage::Memory.new } 9 | let(:cache) { Faulty::Cache::Mock.new } 10 | 11 | before { @faulty_circuit = circuit } 12 | 13 | it 'wraps block in a circuit' do 14 | expect { faulty_run { raise 'fail' } }.to raise_error(Faulty::CircuitFailureError) 15 | end 16 | 17 | it 'does not double wrap errors' do 18 | expect do 19 | faulty_run { faulty_run { raise 'fail' } } 20 | end.to raise_error(Faulty::CircuitFailureError) 21 | expect(circuit.status.sample_size).to eq(1) 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/patch/elasticsearch_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Patch::Elasticsearch do 4 | let(:faulty) { Faulty.new(listeners: []) } 5 | 6 | let(:patched_module) { Faulty::Patch::Elasticsearch::PATCHED_MODULE } 7 | let(:good_url) { ENV.fetch('ELASTICSEARCH_URL', nil) } 8 | let(:bad_url) { 'localhost:9876' } 9 | let(:patched_good_client) { build_client(url: good_url, faulty: { instance: faulty }) } 10 | let(:patched_bad_client) { build_client(url: bad_url, faulty: { instance: faulty }) } 11 | let(:unpatched_good_client) { build_client(url: good_url) } 12 | let(:unpatched_bad_client) { build_client(url: bad_url) } 13 | let(:bad_client_unpatched_errors) do 14 | build_client(url: bad_url, faulty: { instance: faulty, patch_errors: false }) 15 | end 16 | 17 | def build_client(**options) 18 | patched_module::Client.new(options) 19 | end 20 | 21 | it 'captures patched transport error' do 22 | expect { patched_bad_client.perform_request('GET', '_cluster/state') } 23 | .to raise_error do |error| 24 | expect(error).to be_a(patched_module::Transport::Transport::Error) 25 | expect(error.class).to eq(Faulty::Patch::Elasticsearch::Error::CircuitFailureError) 26 | expect(error).to be_a(Faulty::CircuitErrorBase) 27 | expect(error.cause).to be_a(Faraday::ConnectionFailed) 28 | end 29 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(1) 30 | end 31 | 32 | it 'performs normal request for patched client' do 33 | expect(patched_good_client.perform_request('GET', '_cluster/health').body) 34 | .to have_key('status') 35 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(0) 36 | end 37 | 38 | it 'performs normal request for unpatched client' do 39 | expect(unpatched_good_client.perform_request('GET', '_cluster/health').body) 40 | .to have_key('status') 41 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(0) 42 | end 43 | 44 | it 'does not capture transport error for unpatched client' do 45 | expect { unpatched_bad_client.perform_request('GET', '_cluster/state') } 46 | .to raise_error(Faraday::ConnectionFailed) 47 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(0) 48 | end 49 | 50 | it 'raises unpatched errors if configured to' do 51 | expect { bad_client_unpatched_errors.perform_request('GET', '_cluster/state') } 52 | .to raise_error do |error| 53 | expect(error.class).to eq(Faulty::CircuitFailureError) 54 | expect(error.cause).to be_a(Faraday::ConnectionFailed) 55 | end 56 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(1) 57 | end 58 | 59 | it 'raises case-specific Elasticsearch errors' do 60 | # Force the client validation request 61 | patched_good_client.perform_request('GET', '/') 62 | faulty.circuit('elasticsearch').reset! 63 | 64 | expect { patched_good_client.perform_request('PUT', '') } 65 | .to raise_error do |error| 66 | expect(error).to be_a(patched_module::Transport::Transport::Errors::MethodNotAllowed) 67 | expect(error.class).to eq(Faulty::Patch::Elasticsearch::Errors::MethodNotAllowed::CircuitFailureError) 68 | expect(error).to be_a(Faulty::CircuitErrorBase) 69 | expect(error.cause.class).to eq(patched_module::Transport::Transport::Errors::MethodNotAllowed) 70 | end 71 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(1) 72 | end 73 | 74 | it 'ignores 404 errors' do 75 | expect { patched_good_client.perform_request('GET', 'not_an_index') } 76 | .to raise_error do |error| 77 | expect(error.class).to eq(patched_module::Transport::Transport::Errors::NotFound) 78 | end 79 | expect(faulty.circuit('elasticsearch').status.failure_rate).to eq(0) 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/patch/mysql2_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe 'Faulty::Patch::Mysql2', if: defined?(Mysql2) do 4 | def new_client(opts = {}) 5 | Mysql2::Client.new({ 6 | username: ENV.fetch('MYSQL_USER', nil), 7 | password: ENV.fetch('MYSQL_PASSWORD', nil), 8 | host: ENV.fetch('MYSQL_HOST', nil), 9 | port: ENV.fetch('MYSQL_PORT', nil), 10 | socket: ENV.fetch('MYSQL_SOCKET', nil) 11 | }.merge(opts)) 12 | end 13 | 14 | def create_table(client, name) 15 | client.query("CREATE TABLE `#{db_name}`.`#{name}`(id INT NOT NULL)") 16 | end 17 | 18 | def trip_circuit 19 | client 20 | 4.times do 21 | begin 22 | new_client(host: '127.0.0.1', port: 9999, faulty: { instance: faulty }) 23 | rescue Mysql2::Error 24 | # Expect connection failure 25 | end 26 | end 27 | end 28 | 29 | let(:client) { new_client(database: db_name, faulty: { instance: faulty }) } 30 | let(:bad_client) { new_client(host: '127.0.0.1', port: 9999, faulty: { instance: faulty }) } 31 | let(:bad_unpatched_client) { new_client(host: '127.0.0.1', port: 9999) } 32 | let(:db_name) { SecureRandom.hex(6) } 33 | let(:faulty) { Faulty.new(listeners: [], circuit_defaults: { sample_threshold: 2 }) } 34 | 35 | before do 36 | new_client.query("CREATE DATABASE `#{db_name}`") 37 | end 38 | 39 | after do 40 | new_client.query("DROP DATABASE `#{db_name}`") 41 | end 42 | 43 | it 'captures connection error' do 44 | expect { bad_client.query('SELECT 1 FROM dual') }.to raise_error do |error| 45 | expect(error).to be_a(Faulty::Patch::Mysql2::CircuitError) 46 | expect(error.cause).to be_a(Mysql2::Error::ConnectionError) 47 | end 48 | expect(faulty.circuit('mysql2').status.failure_rate).to eq(1) 49 | end 50 | 51 | it 'does not capture unpatched client errors' do 52 | expect { bad_unpatched_client.query('SELECT 1 FROM dual') }.to raise_error(Mysql2::Error::ConnectionError) 53 | expect(faulty.circuit('mysql2').status.failure_rate).to eq(0) 54 | end 55 | 56 | it 'does not capture application errors' do 57 | expect { client.query('SELECT * FROM not_a_table') }.to raise_error(Mysql2::Error) 58 | expect(faulty.circuit('mysql2').status.failure_rate).to eq(0) 59 | end 60 | 61 | it 'successfully executes query' do 62 | create_table(client, 'test') 63 | client.query('INSERT INTO test VALUES(1)') 64 | expect(client.query('SELECT * FROM test').to_a).to eq([{ 'id' => 1 }]) 65 | expect(faulty.circuit('mysql2').status.failure_rate).to eq(0) 66 | end 67 | 68 | it 'prevents additional queries when tripped' do 69 | trip_circuit 70 | expect { client.query('SELECT 1 FROM dual') } 71 | .to raise_error(Faulty::Patch::Mysql2::OpenCircuitError) 72 | end 73 | 74 | it 'allows COMMIT when tripped' do 75 | create_table(client, 'test') 76 | client.query('BEGIN') 77 | client.query('INSERT INTO test VALUES(1)') 78 | trip_circuit 79 | expect(client.query('COMMIT')).to be_nil 80 | expect { client.query('SELECT * FROM test') } 81 | .to raise_error(Faulty::Patch::Mysql2::OpenCircuitError) 82 | faulty.circuit('mysql2').reset! 83 | expect(client.query('SELECT * FROM test').to_a).to eq([{ 'id' => 1 }]) 84 | end 85 | 86 | it 'allows ROLLBACK with leading comment when tripped' do 87 | create_table(client, 'test') 88 | client.query('BEGIN') 89 | client.query('INSERT INTO test VALUES(1)') 90 | trip_circuit 91 | expect(client.query('/* hi there */ ROLLBACK')).to be_nil 92 | expect { client.query('SELECT * FROM test') } 93 | .to raise_error(Faulty::Patch::Mysql2::OpenCircuitError) 94 | faulty.circuit('mysql2').reset! 95 | expect(client.query('SELECT * FROM test').to_a).to eq([]) 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/patch/redis_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Patch::Redis do 4 | let(:faulty) { Faulty.new(listeners: []) } 5 | 6 | let(:bad_url) { 'redis://127.0.0.1:9876' } 7 | let(:bad_redis) { ::Redis.new(opts(url: bad_url, faulty: { instance: faulty })) } 8 | let(:good_redis) { ::Redis.new(opts(faulty: { instance: faulty })) } 9 | let(:bad_unpatched_redis) { ::Redis.new(opts(url: bad_url)) } 10 | let(:bad_redis_unpatched_errors) do 11 | ::Redis.new(opts(url: bad_url, faulty: { instance: faulty, patch_errors: false })) 12 | end 13 | let(:timeout) { 1 } 14 | 15 | def opts(faulty: nil, **opts) 16 | if Redis::VERSION.to_f >= 5 17 | { custom: { faulty: faulty }, timeout: timeout, **opts } 18 | else 19 | { faulty: faulty, timeout: timeout, **opts } 20 | end 21 | end 22 | 23 | def connect(redis) 24 | if Redis::VERSION.to_f >= 5 25 | redis.connect 26 | else 27 | redis.client.connect 28 | end 29 | end 30 | 31 | def faulty_cause(error) 32 | if Redis::VERSION.to_f >= 5 33 | error.cause.cause 34 | else 35 | error.cause 36 | end 37 | end 38 | 39 | it 'captures connection error' do 40 | expect { connect(bad_redis) }.to raise_error do |error| 41 | expect(error).to be_a(::Redis::BaseConnectionError) 42 | if Redis::VERSION.to_f >= 5 43 | expect(faulty_cause(error)).to be_a(Faulty::Patch::Redis::CircuitError) 44 | else 45 | expect(error).to be_a(Faulty::Patch::Redis::CircuitError) 46 | end 47 | end 48 | expect(faulty.circuit('redis').status.failure_rate).to eq(1) 49 | end 50 | 51 | it 'does not capture connection error if no circuit' do 52 | expect { connect(bad_unpatched_redis) }.to raise_error(::Redis::BaseConnectionError) 53 | expect(faulty.circuit('redis').status.failure_rate).to eq(0) 54 | end 55 | 56 | it 'captures connection error during command' do 57 | expect { bad_redis.ping }.to raise_error do |error| 58 | expect(error).to be_a(::Redis::BaseConnectionError) 59 | expect(faulty_cause(error)).to be_a(Faulty::Patch::Redis::CircuitError) 60 | end 61 | expect(faulty.circuit('redis').status.failure_rate).to eq(1) 62 | end 63 | 64 | it 'does not capture command error' do 65 | expect { good_redis.foo }.to raise_error(Redis::CommandError) 66 | expect(faulty.circuit('redis').status.failure_rate).to eq(0) 67 | end 68 | 69 | it 'raises unpatched errors if specified' do 70 | expect { bad_redis_unpatched_errors.ping }.to raise_error(Faulty::CircuitError) 71 | expect(faulty.circuit('redis').status.failure_rate).to eq(1) 72 | end 73 | 74 | context 'with busy Redis instance' do 75 | let(:busy_thread) do 76 | event = Concurrent::Event.new 77 | thread = Thread.new do 78 | begin 79 | event.wait(1) 80 | # This thread will block here until killed 81 | ::Redis.new(timeout: 10).eval("while true do\n end") 82 | rescue Redis::CommandError 83 | # Ok when script is killed 84 | end 85 | end 86 | # Wait for the new thread to be scheduled 87 | # and for the Redis command to be executed 88 | event.set 89 | sleep(0.5) 90 | thread 91 | end 92 | let(:timeout) { 3 } 93 | 94 | before do 95 | good_redis 96 | busy_thread 97 | end 98 | 99 | after do 100 | begin 101 | ::Redis.new(timeout: 10).call(%w[SCRIPT KILL]) 102 | rescue Redis::CommandError 103 | # Ok if no script is running 104 | end 105 | busy_thread.join 106 | end 107 | 108 | it 'captures busy command error' do 109 | expect { good_redis.ping }.to raise_error do |error| 110 | expect(error).to be_a(::Redis::BaseConnectionError) 111 | expect(faulty_cause(error)).to be_a(Faulty::Patch::Redis::BusyError) 112 | expect(faulty_cause(error).message).to match( 113 | /BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE/ 114 | ) 115 | end 116 | 117 | expect(faulty.circuit('redis').status.failure_rate).to be > 0 118 | end 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /spec/patch_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Patch do 4 | let(:error_base) do 5 | stub_const('TestErrorBase', Class.new(RuntimeError)) 6 | end 7 | 8 | describe '.circuit_from_hash' do 9 | let(:faulty) { Faulty.new(listeners: []) } 10 | 11 | let(:error_mapper) do 12 | stub_const('TestErrors', Module.new) 13 | described_class.define_circuit_errors(TestErrors, error_base) 14 | TestErrors 15 | end 16 | 17 | after do 18 | described_class.instance_variable_set(:@instances, nil) 19 | described_class.instance_variable_set(:@default_instance, nil) 20 | end 21 | 22 | it 'can specify an instance' do 23 | circuit = described_class.circuit_from_hash('test', { instance: faulty }) 24 | circuit.run { 'ok' } 25 | expect(faulty.circuit('test')).to eq(circuit) 26 | end 27 | 28 | it 'returns nil if hash is nil' do 29 | circuit = described_class.circuit_from_hash('test', nil) 30 | expect(circuit).to be_nil 31 | end 32 | 33 | it 'can specify a custom name' do 34 | circuit = described_class.circuit_from_hash('test', { instance: faulty, name: 'my_test' }) 35 | circuit.run { 'ok' } 36 | expect(faulty.circuit('my_test')).to eq(circuit) 37 | end 38 | 39 | it 'passes circuit options to the circuit' do 40 | circuit = described_class.circuit_from_hash('test', { instance: faulty, sample_threshold: 10 }) 41 | circuit.run { 'ok' } 42 | expect(circuit.options.sample_threshold).to eq(10) 43 | end 44 | 45 | it 'overrides hash options with keyword arg' do 46 | circuit = described_class.circuit_from_hash( 47 | 'test', 48 | { 49 | instance: faulty, 50 | sample_threshold: 10 51 | }, 52 | sample_threshold: 20 53 | ) 54 | expect(circuit.options.sample_threshold).to eq(20) 55 | end 56 | 57 | it 'overrides hash options with block' do 58 | circuit = described_class.circuit_from_hash('test', { instance: faulty, sample_threshold: 10 }) do |config| 59 | config.sample_threshold = 30 60 | end 61 | expect(circuit.options.sample_threshold).to eq(30) 62 | end 63 | 64 | context 'when patch_errors is enabled' do 65 | it 'sets error_mapper' do 66 | circuit = described_class.circuit_from_hash( 67 | 'test', 68 | { instance: faulty }, 69 | patched_error_mapper: error_mapper 70 | ) 71 | expect(circuit.options.error_mapper).to eq(error_mapper) 72 | end 73 | end 74 | 75 | context 'when patch_errors is enabled but patched_error_mapper is missing' do 76 | it 'uses Faulty error module' do 77 | circuit = described_class.circuit_from_hash( 78 | 'test', 79 | { instance: faulty, patch_errors: true } 80 | ) 81 | expect(circuit.options.error_mapper).to eq(Faulty) 82 | end 83 | end 84 | 85 | context 'when user sets error_mapper manually' do 86 | it 'overrides patched_error_mapper' do 87 | circuit = described_class.circuit_from_hash( 88 | 'test', 89 | { instance: faulty, error_mapper: Faulty } 90 | ) 91 | expect(circuit.options.error_mapper).to eq(Faulty) 92 | end 93 | end 94 | 95 | context 'when patch_errors is disabled' do 96 | it 'uses Faulty error module' do 97 | circuit = described_class.circuit_from_hash( 98 | 'test', 99 | { instance: faulty, patch_errors: false } 100 | ) 101 | expect(circuit.options.error_mapper).to eq(Faulty) 102 | end 103 | end 104 | 105 | context 'with Faulty.default' do 106 | before { Faulty.init(listeners: []) } 107 | 108 | it 'can be run with empty hash' do 109 | circuit = described_class.circuit_from_hash('test', {}) 110 | circuit.run { 'ok' } 111 | expect(Faulty.circuit('test')).to eq(circuit) 112 | end 113 | end 114 | 115 | context 'with constant name' do 116 | before { stub_const('MY_FAULTY', faulty) } 117 | 118 | it 'gets instance by constant name' do 119 | circuit = described_class.circuit_from_hash('test', { instance: { constant: :MY_FAULTY } }) 120 | circuit.run { 'ok' } 121 | expect(faulty.circuit('test')).to eq(circuit) 122 | end 123 | 124 | it 'can pass in string keys and constant name' do 125 | circuit = described_class.circuit_from_hash('test', { 'instance' => { 'constant' => 'MY_FAULTY' } }) 126 | circuit.run { 'ok' } 127 | expect(faulty.circuit('test')).to eq(circuit) 128 | end 129 | end 130 | 131 | context 'with symbol name' do 132 | it 'gets registered instance by symbol' do 133 | Faulty.register(:my_faulty, faulty) 134 | circuit = described_class.circuit_from_hash('test', { instance: :my_faulty }) 135 | circuit.run { 'ok' } 136 | expect(faulty.circuit('test')).to eq(circuit) 137 | end 138 | end 139 | end 140 | 141 | describe '.define_circuit_errors' do 142 | let(:namespace) do 143 | stub_const('TestErrors', Module.new) 144 | end 145 | 146 | it 'creates all circuit error classes in the namespace' do 147 | described_class.define_circuit_errors(namespace, error_base) 148 | expect(TestErrors::CircuitError.superclass).to eq(TestErrorBase) 149 | expect(TestErrors::CircuitError.ancestors).to include(Faulty::CircuitErrorBase) 150 | expect(TestErrors::OpenCircuitError.superclass).to eq(TestErrors::CircuitError) 151 | expect(TestErrors::CircuitFailureError.superclass).to eq(TestErrors::CircuitError) 152 | expect(TestErrors::CircuitTrippedError.superclass).to eq(TestErrors::CircuitError) 153 | end 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /spec/result_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Result do 4 | let(:ok) { described_class.new(ok: 'foo') } 5 | let(:error) { described_class.new(error: StandardError.new) } 6 | 7 | it 'can be constructed with an ok' do 8 | expect(ok.ok?).to be(true) 9 | end 10 | 11 | it 'can be constructed with an error' do 12 | expect(error.error?).to be(true) 13 | end 14 | 15 | it 'raises an error if unchecked get is called' do 16 | expect { ok.get }.to raise_error(Faulty::UncheckedResultError) 17 | end 18 | 19 | it 'raises an error if unchecked error is called' do 20 | expect { error.error }.to raise_error(Faulty::UncheckedResultError) 21 | end 22 | 23 | it 'raises an error if get is called on error' do 24 | error.ok? 25 | expect { error.get }.to raise_error(Faulty::WrongResultError) 26 | end 27 | 28 | it 'raises an error if error is called on ok' do 29 | ok.ok? 30 | expect { ok.error }.to raise_error(Faulty::WrongResultError) 31 | end 32 | 33 | it 'raises an error if constructed with nothing' do 34 | expect { described_class.new }.to raise_error(ArgumentError) 35 | end 36 | 37 | it 'raises an error if constructed with both' do 38 | expect { described_class.new(ok: 'foo', error: StandardError.new) }.to raise_error(ArgumentError) 39 | end 40 | 41 | it 'does not confuse NOTHING with empty object' do 42 | expect(described_class.new(ok: {}).ok?).to be(true) 43 | end 44 | 45 | it 'with ok #or_default passes value through' do 46 | expect(ok.or_default('fallback')).to eq('foo') 47 | end 48 | 49 | it 'with error #or_default returns alternate from argument' do 50 | expect(error.or_default('fallback')).to eq('fallback') 51 | end 52 | 53 | it 'with error #or_default returns alternate from block' do 54 | expect(error.or_default { 'fallback' }).to eq('fallback') 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'byebug' if Gem.loaded_specs['byebug'] 4 | 5 | if Gem.loaded_specs['simplecov'] && (ENV.fetch('COVERAGE', nil) || ENV.fetch('CI', nil)) 6 | require 'simplecov' 7 | if ENV['CI'] 8 | require 'simplecov-cobertura' 9 | SimpleCov.formatter = SimpleCov::Formatter::CoberturaFormatter 10 | end 11 | 12 | SimpleCov.start do 13 | enable_coverage :branch 14 | add_filter '/spec/' 15 | add_filter '/vendor/' 16 | end 17 | end 18 | 19 | require 'faulty' 20 | require 'faulty/patch/redis' 21 | require 'faulty/patch/elasticsearch' 22 | require 'timecop' 23 | require 'redis' 24 | require 'json' 25 | require 'connection_pool' 26 | 27 | begin 28 | # We don't test Mysql2 on Ruby 2.3 since that would require 29 | # installing an old EOL version of OpenSSL 30 | require 'faulty/patch/mysql2' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.4') 31 | rescue LoadError 32 | # Ok if mysql2 isn't available 33 | end 34 | 35 | require_relative 'support/concurrency' 36 | 37 | RSpec.configure do |config| 38 | config.expect_with :rspec do |expectations| 39 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 40 | end 41 | 42 | config.mock_with :rspec do |mocks| 43 | mocks.verify_partial_doubles = true 44 | end 45 | 46 | config.disable_monkey_patching! 47 | config.warnings = false 48 | 49 | config.after do 50 | Timecop.return 51 | Faulty.enable! 52 | end 53 | 54 | config.include Faulty::Specs::Concurrency 55 | end 56 | -------------------------------------------------------------------------------- /spec/status_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Status do 4 | let(:options) { Faulty::Circuit::Options.new({}) } 5 | 6 | it 'is closed by default' do 7 | expect(described_class.new(options: options)).to be_closed 8 | end 9 | 10 | context 'when state is closed' do 11 | subject(:status) { described_class.new(options: options, state: :closed) } 12 | 13 | it('is closed') { expect(status).to be_closed } 14 | it('is not open') { expect(status).not_to be_open } 15 | it('is not half_open') { expect(status).not_to be_half_open } 16 | it('can run') { expect(status.can_run?).to be(true) } 17 | it('is not locked_open') { expect(status).not_to be_locked_open } 18 | it('is not locked_closed') { expect(status).not_to be_locked_closed } 19 | end 20 | 21 | context 'when state is open and cool_down is not passed' do 22 | subject(:status) do 23 | described_class.new(options: options, state: :open, opened_at: Faulty.current_time) 24 | end 25 | 26 | it('is open') { expect(status).to be_open } 27 | it('is not closed') { expect(status).not_to be_closed } 28 | it('is not half_open') { expect(status).not_to be_half_open } 29 | it('cannot run') { expect(status.can_run?).to be(false) } 30 | end 31 | 32 | context 'when state is open and cool_down is passed' do 33 | subject(:status) do 34 | described_class.new(options: options, state: :open, opened_at: Faulty.current_time - 500) 35 | end 36 | 37 | it('is half_open') { expect(status).to be_half_open } 38 | it('is not open') { expect(status).not_to be_open } 39 | it('is not closed') { expect(status).not_to be_closed } 40 | it('can run') { expect(status.can_run?).to be(true) } 41 | end 42 | 43 | context 'when locked open' do 44 | subject(:status) { described_class.new(options: options, state: :closed, lock: :open) } 45 | 46 | it('is locked_open') { expect(status).to be_locked_open } 47 | it('cannot run') { expect(status.can_run?).to be(false) } 48 | end 49 | 50 | context 'when locked closed' do 51 | subject(:status) do 52 | described_class.new( 53 | options: options, 54 | state: :open, 55 | opened_at: Faulty.current_time, 56 | lock: :closed 57 | ) 58 | end 59 | 60 | it('is locked_closed') { expect(status).to be_locked_closed } 61 | it('can run') { expect(status.can_run?).to be(true) } 62 | end 63 | 64 | context 'when sample size is too small' do 65 | subject(:status) { described_class.new(options: options, sample_size: 1, failure_rate: 0.99) } 66 | 67 | it('passes threshold') { expect(status.fails_threshold?).to be(false) } 68 | end 69 | 70 | context 'when failure rate is below rate_threshold' do 71 | subject(:status) { described_class.new(options: options, sample_size: 4, failure_rate: 0.4) } 72 | 73 | it('passes threshold') { expect(status.fails_threshold?).to be(false) } 74 | end 75 | 76 | context 'when failure rate is above rate_threshold' do 77 | subject(:status) { described_class.new(options: options, sample_size: 4, failure_rate: 0.6) } 78 | 79 | it('fails threshold') { expect(status.fails_threshold?).to be(true) } 80 | end 81 | 82 | context 'when failure rate equals rate_threshold' do 83 | subject(:status) { described_class.new(options: options, sample_size: 4, failure_rate: 0.5) } 84 | 85 | it('fails threshold') { expect(status.fails_threshold?).to be(true) } 86 | end 87 | 88 | it 'rejects invalid state' do 89 | expect { described_class.new(options: options, state: :blah) } 90 | .to raise_error(ArgumentError, /state must be a symbol in Faulty::Status::STATES/) 91 | end 92 | 93 | it 'rejects invalid lock' do 94 | expect { described_class.new(options: options, lock: :blah) } 95 | .to raise_error(ArgumentError, /lock must be a symbol in Faulty::Status::LOCKS/) 96 | end 97 | 98 | it 'requires opened_at if state is open' do 99 | expect { described_class.new(options: options, state: :open) } 100 | .to raise_error(ArgumentError, /opened_at is required if state is open/) 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /spec/storage/auto_wire_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Storage::AutoWire do 4 | subject(:auto_wire) { described_class.wrap(backend, circuit: circuit, notifier: notifier) } 5 | 6 | let(:circuit) { Faulty::Circuit.new('test') } 7 | let(:notifier) { Faulty::Events::Notifier.new } 8 | let(:backend) { nil } 9 | 10 | context 'with a fault-tolerant backend' do 11 | let(:backend) { Faulty::Storage::Memory.new } 12 | 13 | it 'delegates directly if a fault-tolerant backend is given' do 14 | expect(auto_wire).to eq(backend) 15 | end 16 | end 17 | 18 | context 'with a non-fault-tolerant backend' do 19 | let(:backend) { Faulty::Storage::Redis.new } 20 | 21 | it 'is fault tolerant' do 22 | expect(auto_wire).to be_fault_tolerant 23 | end 24 | 25 | it 'wraps in FaultTolerantProxy and CircuitProxy' do 26 | expect(auto_wire).to be_a(Faulty::Storage::FaultTolerantProxy) 27 | 28 | circuit_proxy = auto_wire.instance_variable_get(:@storage) 29 | expect(circuit_proxy).to be_a(Faulty::Storage::CircuitProxy) 30 | expect(circuit_proxy.options.circuit).to eq(circuit) 31 | 32 | original = circuit_proxy.instance_variable_get(:@storage) 33 | expect(original).to eq(backend) 34 | end 35 | end 36 | 37 | context 'with a fault-tolerant array' do 38 | let(:redis_storage) { Faulty::Storage::Redis.new } 39 | let(:mem_storage) { Faulty::Storage::Memory.new } 40 | let(:backend) { [redis_storage, mem_storage] } 41 | 42 | it 'creates a FallbackChain' do 43 | expect(auto_wire).to be_a(Faulty::Storage::FallbackChain) 44 | 45 | storages = auto_wire.instance_variable_get(:@storages) 46 | expect(storages[0]).to be_a(Faulty::Storage::CircuitProxy) 47 | expect(storages[0].instance_variable_get(:@storage)).to eq(redis_storage) 48 | expect(storages[1]).to eq(mem_storage) 49 | end 50 | end 51 | 52 | context 'with a non-fault-tolerant array' do 53 | let(:redis_storage1) { Faulty::Storage::Redis.new } 54 | let(:redis_storage2) { Faulty::Storage::Redis.new } 55 | let(:backend) { [redis_storage1, redis_storage2] } 56 | 57 | it 'creates a FallbackChain inside a FaultTolerantProxy' do 58 | expect(auto_wire).to be_a(Faulty::Storage::FaultTolerantProxy) 59 | 60 | chain = auto_wire.instance_variable_get(:@storage) 61 | expect(chain).to be_a(Faulty::Storage::FallbackChain) 62 | 63 | storages = chain.instance_variable_get(:@storages) 64 | expect(storages[0]).to be_a(Faulty::Storage::CircuitProxy) 65 | expect(storages[0].instance_variable_get(:@storage)).to eq(redis_storage1) 66 | expect(storages[1]).to be_a(Faulty::Storage::CircuitProxy) 67 | expect(storages[1].instance_variable_get(:@storage)).to eq(redis_storage2) 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/storage/circuit_proxy_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Storage::CircuitProxy do 4 | let(:notifier) { Faulty::Events::Notifier.new } 5 | let(:circuit) { Faulty::Circuit.new('test') } 6 | let(:internal_circuit) { Faulty::Circuit.new('internal', sample_threshold: 2) } 7 | 8 | let(:failing_storage) do 9 | Class.new do 10 | def method_missing(*_args) 11 | raise 'fail' 12 | end 13 | 14 | def respond_to_missing?(*_args) 15 | true 16 | end 17 | end 18 | end 19 | 20 | it 'trips its internal circuit when storage fails repeatedly' do 21 | backend = failing_storage.new 22 | proxy = described_class.new(backend, notifier: notifier, circuit: internal_circuit) 23 | 24 | begin 25 | 2.times { proxy.entry(circuit, Faulty.current_time, true) } 26 | rescue Faulty::CircuitFailureError 27 | nil 28 | end 29 | 30 | expect { proxy.entry(circuit, Faulty.current_time, true) } 31 | .to raise_error(Faulty::CircuitTrippedError) 32 | end 33 | 34 | it 'does not notify for circuit sucesses by default' do 35 | expect(notifier).not_to receive(:notify) 36 | backend = Faulty::Storage::Null.new 37 | proxy = described_class.new(backend, notifier: notifier) 38 | proxy.entry(circuit, Faulty.current_time, true, nil) 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /spec/storage/fallback_chain_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Storage::FallbackChain do 4 | let(:failing_class) do 5 | Class.new do 6 | def method_missing(_method, *_args) 7 | raise 'fail' 8 | end 9 | 10 | def respond_to_missing?(_method, _include_all = false) 11 | true 12 | end 13 | end 14 | end 15 | 16 | let(:failing) { failing_class.new } 17 | let(:memory) { Faulty::Storage::Memory.new } 18 | let(:memory2) { Faulty::Storage::Memory.new } 19 | let(:notifier) { Faulty::Events::Notifier.new } 20 | let(:circuit) { Faulty::Circuit.new('test') } 21 | let(:init_status) { Faulty::Status.new(options: circuit.options) } 22 | let(:succeeding_chain) { described_class.new([memory, memory2], notifier: notifier) } 23 | let(:partially_failing_chain) { described_class.new([failing, memory], notifier: notifier) } 24 | let(:midway_failure_chain) { described_class.new([memory, failing, memory2], notifier: notifier) } 25 | let(:long_chain) { described_class.new([failing, failing_class.new, memory], notifier: notifier) } 26 | let(:failing_chain) { described_class.new([failing, failing_class.new], notifier: notifier) } 27 | 28 | context 'with #entry' do 29 | it 'calls only first storage when successful' do 30 | status = succeeding_chain.entry(circuit, Faulty.current_time, false, init_status) 31 | expect(status.sample_size).to eq(1) 32 | expect(memory.history(circuit).size).to eq(1) 33 | expect(memory2.history(circuit).size).to eq(0) 34 | end 35 | 36 | it 'falls back to next storage after failure' do 37 | expect(notifier).to receive(:notify) 38 | .with(:storage_failure, circuit: circuit, action: :entry, error: be_a(RuntimeError)) 39 | status = partially_failing_chain.entry(circuit, Faulty.current_time, false, init_status) 40 | expect(status.sample_size).to eq(1) 41 | expect(memory.history(circuit).size).to eq(1) 42 | 43 | expect(notifier).to receive(:notify) 44 | .with(:storage_failure, circuit: circuit, action: :history, error: be_a(RuntimeError)) 45 | expect(partially_failing_chain.history(circuit).size).to eq(1) 46 | end 47 | 48 | it 'chains fallbacks for multiple failures' do 49 | expect(notifier).to receive(:notify) 50 | .with(:storage_failure, circuit: circuit, action: :entry, error: be_a(RuntimeError)) 51 | .twice 52 | status = long_chain.entry(circuit, Faulty.current_time, false, init_status) 53 | expect(status.sample_size).to eq(1) 54 | expect(memory.history(circuit).size).to eq(1) 55 | 56 | expect(notifier).to receive(:notify) 57 | .with(:storage_failure, circuit: circuit, action: :history, error: be_a(RuntimeError)) 58 | .twice 59 | expect(long_chain.history(circuit).size).to eq(1) 60 | end 61 | 62 | it 'raises error if all storages fail' do 63 | expect do 64 | failing_chain.entry(circuit, Faulty.current_time, true, nil) 65 | end.to raise_error( 66 | Faulty::AllFailedError, 67 | 'Faulty::Storage::FallbackChain#entry failed for all storage backends: fail, fail' 68 | ) 69 | end 70 | end 71 | 72 | context 'with #lock' do 73 | it 'delegates to all when successful' do 74 | succeeding_chain.lock(circuit, :open) 75 | expect(memory.status(circuit).locked_open?).to be(true) 76 | expect(memory2.status(circuit).locked_open?).to be(true) 77 | end 78 | 79 | it 'continues delegating after failure and raises' do 80 | expect do 81 | midway_failure_chain.lock(circuit, :open) 82 | end.to raise_error( 83 | Faulty::PartialFailureError, 84 | 'Faulty::Storage::FallbackChain#lock failed for some storage backends: fail' 85 | ) 86 | 87 | expect(memory.status(circuit).locked_open?).to be(true) 88 | expect(memory2.status(circuit).locked_open?).to be(true) 89 | end 90 | 91 | it 'raises error if all storages fail' do 92 | expect do 93 | failing_chain.lock(circuit, :open) 94 | end.to raise_error( 95 | Faulty::AllFailedError, 96 | 'Faulty::Storage::FallbackChain#lock failed for all storage backends: fail, fail' 97 | ) 98 | end 99 | end 100 | 101 | shared_examples 'chained method' do 102 | it 'calls only first storage when successful' do 103 | chain = described_class.new([memory, instance_double(Faulty::Storage::Memory)], notifier: notifier) 104 | marker = Object.new 105 | expected = receive(action).and_return(marker) 106 | args.empty? ? expected.with(no_args) : expected.with(*args) 107 | expect(memory).to expected 108 | expect(chain.public_send(action, *args)).to eq(marker) 109 | end 110 | 111 | it 'falls back to next storage after failure' do 112 | event_payload = { action: action, error: be_a(RuntimeError) } 113 | event_payload[:circuit] = circuit unless action == :list 114 | expect(notifier).to receive(:notify).with(:storage_failure, event_payload) 115 | marker = Object.new 116 | expected = receive(action).and_return(marker) 117 | args.empty? ? expected.with(no_args) : expected.with(*args) 118 | expect(memory).to expected 119 | expect(partially_failing_chain.public_send(action, *args)).to eq(marker) 120 | end 121 | end 122 | 123 | shared_examples 'fan-out method' do 124 | it 'calls all backends' do 125 | chain = described_class.new([memory, memory2], notifier: notifier) 126 | expected = receive(action) 127 | args.empty? ? expected.with(no_args) : expected.with(*args) 128 | expect(memory).to expected 129 | expect(memory2).to expected 130 | expect(chain.public_send(action, *args)).to be_nil 131 | end 132 | end 133 | 134 | describe '#get_options' do 135 | let(:action) { :get_options } 136 | let(:args) { [circuit] } 137 | 138 | it_behaves_like 'chained method' 139 | end 140 | 141 | describe '#set_options' do 142 | let(:action) { :set_options } 143 | let(:args) { [circuit, { cool_down: 5 }] } 144 | 145 | it_behaves_like 'fan-out method' 146 | end 147 | 148 | describe '#open' do 149 | let(:action) { :open } 150 | let(:args) { [circuit, Faulty.current_time] } 151 | 152 | it_behaves_like 'chained method' 153 | end 154 | 155 | describe '#reopen' do 156 | let(:action) { :reopen } 157 | let(:args) { [circuit, Faulty.current_time, Faulty.current_time - 300] } 158 | 159 | it_behaves_like 'chained method' 160 | end 161 | 162 | describe '#close' do 163 | let(:action) { :close } 164 | let(:args) { [circuit] } 165 | 166 | it_behaves_like 'chained method' 167 | end 168 | 169 | describe '#lock' do 170 | let(:action) { :lock } 171 | let(:args) { [circuit, :open] } 172 | 173 | it_behaves_like 'fan-out method' 174 | end 175 | 176 | describe '#unlock' do 177 | let(:action) { :unlock } 178 | let(:args) { [circuit] } 179 | 180 | it_behaves_like 'fan-out method' 181 | end 182 | 183 | describe '#reset' do 184 | let(:action) { :reset } 185 | let(:args) { [circuit] } 186 | 187 | it_behaves_like 'fan-out method' 188 | end 189 | 190 | describe '#status' do 191 | let(:action) { :status } 192 | let(:args) { [circuit] } 193 | 194 | it_behaves_like 'chained method' 195 | end 196 | 197 | describe '#history' do 198 | let(:action) { :history } 199 | let(:args) { [circuit] } 200 | 201 | it_behaves_like 'chained method' 202 | end 203 | 204 | describe '#list' do 205 | let(:action) { :list } 206 | let(:args) { [] } 207 | 208 | it_behaves_like 'chained method' 209 | end 210 | 211 | it 'is fault tolerant if any storage is fault tolerant' do 212 | expect(described_class.new([Faulty::Storage::Redis.new, memory], notifier: notifier)) 213 | .to be_fault_tolerant 214 | end 215 | 216 | it 'is not fault tolerant if no storage is fault tolerant' do 217 | expect(described_class.new( 218 | [Faulty::Storage::Redis.new, Faulty::Storage::Redis.new], 219 | notifier: notifier 220 | )).not_to be_fault_tolerant 221 | end 222 | end 223 | -------------------------------------------------------------------------------- /spec/storage/fault_tolerant_proxy_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Storage::FaultTolerantProxy do 4 | let(:notifier) { Faulty::Events::Notifier.new } 5 | 6 | let(:failing_storage_class) do 7 | Class.new do 8 | def method_missing(*_args) 9 | raise 'fail' 10 | end 11 | 12 | def respond_to_missing?(*_args) 13 | true 14 | end 15 | end 16 | end 17 | 18 | let(:failing_storage) { failing_storage_class.new } 19 | let(:inner_storage) { Faulty::Storage::Memory.new } 20 | let(:circuit) { Faulty::Circuit.new('test') } 21 | 22 | it 'delegates to storage when adding entry succeeds' do 23 | described_class.new(inner_storage, notifier: notifier) 24 | .entry(circuit, Faulty.current_time, true, nil) 25 | expect(inner_storage.history(circuit).size).to eq(1) 26 | end 27 | 28 | it 'returns stub status when adding entry fails' do 29 | expect(notifier).to receive(:notify) 30 | .with(:storage_failure, circuit: circuit, action: :entry, error: instance_of(RuntimeError)) 31 | status = described_class.new(failing_storage, notifier: notifier) 32 | .entry(circuit, Faulty.current_time, false, Faulty::Status.new(options: circuit.options)) 33 | expect(status.stub).to be(true) 34 | end 35 | 36 | it 'returns stub status when getting #status' do 37 | expect(notifier).to receive(:notify) 38 | .with(:storage_failure, circuit: circuit, action: :status, error: instance_of(RuntimeError)) 39 | status = described_class.new(failing_storage, notifier: notifier) 40 | .status(circuit) 41 | expect(status.stub).to be(true) 42 | end 43 | 44 | shared_examples 'delegated action' do 45 | it 'delegates success to inner storage' do 46 | marker = Object.new 47 | expected = receive(action).and_return(marker) 48 | args.empty? ? expected.with(no_args) : expected.with(*args) 49 | expect(inner_storage).to expected 50 | result = described_class.new(inner_storage, notifier: notifier) 51 | .public_send(action, *args) 52 | expect(result).to eq(marker) 53 | end 54 | end 55 | 56 | shared_examples 'unsafe action' do 57 | it 'raises error on failure' do 58 | expect do 59 | described_class.new(failing_storage, notifier: notifier).public_send(action, *args) 60 | end.to raise_error('fail') 61 | end 62 | 63 | it_behaves_like 'delegated action' 64 | end 65 | 66 | shared_examples 'safely wrapped action' do 67 | it 'catches error and returns false' do 68 | expect(notifier).to receive(:notify) 69 | .with(:storage_failure, circuit: circuit, action: action, error: instance_of(RuntimeError)) 70 | result = described_class.new(failing_storage, notifier: notifier) 71 | .public_send(action, *args) 72 | expect(result).to eq(retval) 73 | end 74 | 75 | it_behaves_like 'delegated action' 76 | end 77 | 78 | describe '#get_options' do 79 | let(:action) { :get_options } 80 | let(:args) { [circuit] } 81 | let(:retval) { nil } 82 | 83 | it_behaves_like 'safely wrapped action' 84 | end 85 | 86 | describe '#set_options' do 87 | let(:action) { :set_options } 88 | let(:args) { [circuit, { cool_down: 3 }] } 89 | let(:retval) { nil } 90 | 91 | it_behaves_like 'safely wrapped action' 92 | end 93 | 94 | describe '#open' do 95 | let(:action) { :open } 96 | let(:args) { [circuit, Faulty.current_time] } 97 | let(:retval) { false } 98 | 99 | it_behaves_like 'safely wrapped action' 100 | end 101 | 102 | describe '#reopen' do 103 | let(:action) { :reopen } 104 | let(:args) { [circuit, Faulty.current_time, Faulty.current_time - 300] } 105 | let(:retval) { false } 106 | 107 | it_behaves_like 'safely wrapped action' 108 | end 109 | 110 | describe '#close' do 111 | let(:action) { :close } 112 | let(:args) { [circuit] } 113 | let(:retval) { false } 114 | 115 | it_behaves_like 'safely wrapped action' 116 | end 117 | 118 | describe '#lock' do 119 | let(:action) { :lock } 120 | let(:args) { [circuit, :open] } 121 | 122 | it_behaves_like 'unsafe action' 123 | end 124 | 125 | describe '#unlock' do 126 | let(:action) { :unlock } 127 | let(:args) { [circuit] } 128 | 129 | it_behaves_like 'unsafe action' 130 | end 131 | 132 | describe '#reset' do 133 | let(:action) { :reset } 134 | let(:args) { [circuit] } 135 | 136 | it_behaves_like 'unsafe action' 137 | end 138 | 139 | describe '#history' do 140 | let(:action) { :history } 141 | let(:args) { [circuit] } 142 | 143 | it_behaves_like 'unsafe action' 144 | end 145 | 146 | describe '#list' do 147 | let(:action) { :list } 148 | let(:args) { [] } 149 | 150 | it_behaves_like 'unsafe action' 151 | end 152 | 153 | it 'raises when storage fails while getting list' do 154 | expect do 155 | described_class.new(failing_storage, notifier: notifier).list 156 | end.to raise_error('fail') 157 | end 158 | 159 | it 'is fault tolerant for non-fault-tolerant storage' do 160 | fault_tolerant = described_class.new(Faulty::Storage::Redis.new, notifier: notifier) 161 | expect(fault_tolerant).to be_fault_tolerant 162 | end 163 | 164 | describe '.wrap' do 165 | it 'returns fault-tolerant storage unmodified' do 166 | memory = Faulty::Storage::Memory.new 167 | expect(described_class.wrap(memory, notifier: notifier)).to eq(memory) 168 | end 169 | 170 | it 'wraps fault-tolerant cache' do 171 | redis = Faulty::Storage::Redis.new 172 | expect(described_class.wrap(redis, notifier: notifier)).to be_a(described_class) 173 | end 174 | end 175 | end 176 | -------------------------------------------------------------------------------- /spec/storage/memory_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Faulty::Storage::Memory do 4 | let(:circuit) { Faulty::Circuit.new('test') } 5 | 6 | it 'rotates entries after max_sample_size' do 7 | storage = described_class.new(max_sample_size: 3) 8 | 3.times { |i| storage.entry(circuit, i, true, nil) } 9 | expect(storage.history(circuit).map { |h| h[0] }).to eq([0, 1, 2]) 10 | storage.entry(circuit, 9, true, nil) 11 | expect(storage.history(circuit).map { |h| h[0] }).to eq([1, 2, 9]) 12 | end 13 | 14 | it 'clears circuits and list' do 15 | storage = described_class.new 16 | storage.entry(circuit, Faulty.current_time, true, nil) 17 | storage.clear 18 | expect(storage.list).to eq([]) 19 | expect(storage.history(circuit)).to eq([]) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/storage/redis_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'connection_pool' 4 | require 'redis' 5 | 6 | RSpec.describe Faulty::Storage::Redis do 7 | subject(:storage) { described_class.new(**options.merge(client: client)) } 8 | 9 | let(:options) { {} } 10 | let(:client) { Redis.new(timeout: 1) } 11 | let(:circuit) { Faulty::Circuit.new('test', storage: storage) } 12 | 13 | after { circuit&.reset! } 14 | 15 | context 'with default options' do 16 | subject(:storage) { described_class.new } 17 | 18 | it 'can add an entry' do 19 | storage.entry(circuit, Faulty.current_time, true, nil) 20 | expect(storage.history(circuit).size).to eq(1) 21 | end 22 | 23 | it 'clears circuits and list' do 24 | storage.entry(circuit, Faulty.current_time, true, nil) 25 | storage.clear 26 | expect(storage.list).to eq(%w[test]) 27 | expect(storage.history(circuit)).to eq([]) 28 | end 29 | end 30 | 31 | context 'with connection pool' do 32 | let(:pool_size) { 100 } 33 | 34 | let(:client) do 35 | ConnectionPool.new(size: pool_size, timeout: 1) { Redis.new(timeout: 1) } 36 | end 37 | 38 | it 'adds an entry' do 39 | storage.entry(circuit, Faulty.current_time, true, nil) 40 | expect(storage.history(circuit).size).to eq(1) 41 | end 42 | 43 | it 'opens the circuit once when called concurrently', concurrency: true do 44 | concurrent_warmup do 45 | # Do something small just to get a connection from the pool 46 | storage.unlock(circuit) 47 | end 48 | 49 | result = concurrently(pool_size) do 50 | storage.open(circuit, Faulty.current_time) 51 | end 52 | expect(result.count { |r| r }).to eq(1) 53 | end 54 | end 55 | 56 | context 'when Redis has high timeout' do 57 | let(:client) { Redis.new(timeout: 5.0) } 58 | 59 | it 'prints timeout warning' do 60 | timeouts = { connect_timeout: 5.0, read_timeout: 5.0, write_timeout: 5.0 } 61 | expect { storage }.to output(/Your options are:\n#{timeouts}/).to_stderr 62 | end 63 | end 64 | 65 | context 'when Redis has high reconnect_attempts' do 66 | let(:client) { Redis.new(timeout: 1, reconnect_attempts: 2) } 67 | 68 | it 'prints reconnect_attempts warning' do 69 | expect { storage }.to output(/Your setting is larger/).to_stderr 70 | end 71 | end 72 | 73 | context 'when ConnectionPool has high timeout' do 74 | let(:client) do 75 | ConnectionPool.new(timeout: 6) { Redis.new(timeout: 1) } 76 | end 77 | 78 | it 'prints timeout warning' do 79 | expect { storage }.to output(/Your setting is 6/).to_stderr 80 | end 81 | end 82 | 83 | context 'when ConnectionPool Redis client has high timeout' do 84 | let(:client) do 85 | ConnectionPool.new(timeout: 1) { Redis.new(timeout: 7.0) } 86 | end 87 | 88 | it 'prints Redis timeout warning' do 89 | timeouts = { connect_timeout: 7.0, read_timeout: 7.0, write_timeout: 7.0 } 90 | expect { storage }.to output(/Your options are:\n#{timeouts}/).to_stderr 91 | end 92 | end 93 | 94 | context 'when an error is raised while checking settings' do 95 | let(:circuit) { nil } 96 | let(:client) do 97 | ConnectionPool.new(timeout: 1) { raise 'fail' } 98 | end 99 | 100 | it 'warns and continues' do 101 | expect { storage }.to output(/while checking client options: fail/).to_stderr 102 | end 103 | end 104 | 105 | context 'when opened_at is missing and status is open' do 106 | it 'sets opened_at to the maximum' do 107 | Timecop.freeze 108 | storage.open(circuit, Faulty.current_time) 109 | client.del('faulty:circuit:test:opened_at') 110 | status = storage.status(circuit) 111 | expect(status.opened_at).to eq(Faulty.current_time - storage.options.circuit_ttl) 112 | end 113 | end 114 | 115 | context 'when history entries are integers and floats' do 116 | it 'gets floats' do 117 | client.lpush('faulty:circuit:test:entries', '1660865630:1') 118 | client.lpush('faulty:circuit:test:entries', '1660865646.897674:1') 119 | expect(storage.history(circuit)).to eq([[1_660_865_630.0, true], [1_660_865_646.897674, true]]) 120 | end 121 | end 122 | 123 | context 'when ConnectionPool is not present' do 124 | before { hide_const('ConnectionPool') } 125 | 126 | it 'can construct a storage' do 127 | storage 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /spec/support/concurrency.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Faulty 4 | module Specs 5 | module Concurrency 6 | def concurrent_warmup(&block) 7 | @concurrent_warmup = block 8 | end 9 | 10 | def concurrently(times = 100, timeout: 3) 11 | barrier = Concurrent::CyclicBarrier.new(times) 12 | 13 | execute = lambda do 14 | @concurrent_warmup&.call 15 | barrier.wait(timeout) 16 | error = nil 17 | result = begin 18 | yield 19 | rescue StandardError => e 20 | error = e 21 | end 22 | 23 | barrier.wait(timeout) 24 | raise error if error 25 | 26 | result 27 | end 28 | 29 | threads = (0...(times - 1)).map do 30 | Thread.new do 31 | Thread.current.report_on_exception = false if Thread.current.respond_to?(:report_on_exception=) 32 | execute.call 33 | end 34 | end 35 | main_result = execute.call 36 | threads.map(&:value) + [main_result] 37 | end 38 | end 39 | end 40 | end 41 | --------------------------------------------------------------------------------