├── .github └── workflows │ ├── stale.yml │ └── ubuntu.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin ├── console └── setup ├── lib └── minitest │ ├── retry.rb │ └── retry │ └── version.rb ├── minitest-retry.gemspec └── test ├── minitest └── retry_test.rb └── test_helper.rb /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | # This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. 2 | # 3 | # You can adjust the behavior by modifying this file. 4 | # For more information, see: 5 | # https://github.com/actions/stale 6 | name: Mark stale issues and pull requests 7 | 8 | on: 9 | schedule: 10 | - cron: '28 20 * * *' 11 | 12 | jobs: 13 | stale: 14 | 15 | runs-on: ubuntu-latest 16 | permissions: 17 | issues: write 18 | pull-requests: write 19 | 20 | steps: 21 | - uses: actions/stale@v9 22 | with: 23 | repo-token: ${{ secrets.GITHUB_TOKEN }} 24 | stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' 25 | close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' 26 | stale-issue-label: 'no-issue-activity' 27 | days-before-issue-stale: 30 28 | days-before-issue-close: 5 29 | -------------------------------------------------------------------------------- /.github/workflows/ubuntu.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0', '2.7', '2.6', '2.5', '2.4', '2.3' ] 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: ruby/setup-ruby@v1 14 | with: 15 | ruby-version: ${{ matrix.ruby }} 16 | - name: Install dependencies 17 | run: | 18 | bundle install 19 | - name: Run test 20 | run: bundle exec rake 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.2.5 2 | 3 | * Add `exceptions_to_skip` option #48 [y-yagi] 4 | 5 | ## 0.2.4 6 | 7 | * Add `methods_to_skip` option #44 [y-yagi] 8 | 9 | ## 0.2.3 10 | 11 | * Add `classes_to_retry` option #42 [y-yagi] 12 | 13 | ## 0.2.2 14 | 15 | * Add `methods_to_retry` option #33 [edudepetris] 16 | 17 | ## 0.2.1 18 | 19 | * Pass a test result to consistent failure callback #30 [rmacklin] 20 | 21 | ## 0.2.0 22 | 23 | * Pass a test result to failure and retry callbacks #29 [aabrahamian] 24 | 25 | ## 0.1.9 26 | 27 | * Add on_consistent_failure callback #20 [staugaard] 28 | 29 | ## 0.1.8 30 | 31 | * Add retry callback #17 [julien-meichelbeck] 32 | * Pass test class and test name to failure callback #15, #16 [julien-meichelbeck] 33 | 34 | ## 0.1.7 35 | 36 | * Run a callback on each test failure #15 [julien-meichelbeck] 37 | 38 | ## 0.1.6 39 | 40 | * Add exceptions_to_retry option #13 [panjan] 41 | 42 | ## 0.1.5 43 | 44 | * Improve message information when an unexpected error occurs #9 [abarre] 45 | 46 | ## 0.1.4 47 | 48 | * Fix typo in verbose output #6 [semanticart] 49 | 50 | ## 0.1.3 51 | 52 | * modify so as not to retry it a had gone to retry the skip test. 53 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'debug' 4 | # Specify your gem's dependencies in minitest-retry.gemspec 5 | gemspec 6 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Yuji Yaginuma 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Minitest::Retry 2 | 3 | Re-run the test when the test fails. 4 | 5 | ![](https://github.com/y-yagi/minitest-retry/workflows/CI/badge.svg) 6 | [![Gem Version](https://badge.fury.io/rb/minitest-retry.svg)](http://badge.fury.io/rb/minitest-retry) 7 | 8 | ## Installation 9 | 10 | Add this line to your application's Gemfile: 11 | 12 | ```ruby 13 | gem 'minitest-retry' 14 | ``` 15 | 16 | And then execute: 17 | 18 | $ bundle 19 | 20 | Or install it yourself as: 21 | 22 | $ gem install minitest-retry 23 | 24 | ## Usage 25 | 26 | In your `test_helper.rb` file, add the following lines: 27 | 28 | ```ruby 29 | require 'minitest/retry' 30 | Minitest::Retry.use! 31 | ``` 32 | 33 | Options can be specified to `use!` method. Can specify options are as follows: 34 | 35 | ```ruby 36 | Minitest::Retry.use!( 37 | retry_count: 3, # The number of times to retry. The default is 3. 38 | verbose: true, # Whether or not to display the message at the time of retry. The default is true. 39 | io: $stdout, # Display destination of retry when the message. The default is stdout. 40 | exceptions_to_retry: [], # List of exceptions that will trigger a retry (when empty, all exceptions will). 41 | methods_to_retry: [], # List of methods that will trigger a retry (when empty, all methods will). 42 | classes_to_retry: [], # List of classes that will trigger a retry (when empty, all classes will). 43 | methods_to_skip: [], # List of methods that will skip a retry (when empty, all methods will retry). 44 | exceptions_to_skip: [] # List of exceptions that will skip a retry (when empty, all exceptions will retry). 45 | ) 46 | ``` 47 | 48 | #### Callbacks 49 | The `on_failure` callback is executed each time a test fails: 50 | ```ruby 51 | Minitest::Retry.on_failure do |klass, test_name, result| 52 | # code omitted 53 | end 54 | ``` 55 | 56 | The `on_consistent_failure` callback is executed when a test consistently fails: 57 | ```ruby 58 | Minitest::Retry.on_consistent_failure do |klass, test_name, result| 59 | # code omitted 60 | end 61 | ``` 62 | 63 | The `on_retry` callback is executed each time a test is retried: 64 | ```ruby 65 | Minitest::Retry.on_retry do |klass, test_name, retry_count, result| 66 | # code omitted 67 | end 68 | ``` 69 | 70 | ## Example 71 | 72 | ```ruby 73 | 74 | Minitest::Retry.use! 75 | 76 | class Minitest::RetryTest < Minitest::Test 77 | def test_fail 78 | assert false, 'test fail' 79 | end 80 | end 81 | ``` 82 | 83 | ```console 84 | # Running: 85 | 86 | [MinitestRetry] retry 'test_fail' count: 1, msg: test fail 87 | [MinitestRetry] retry 'test_fail' count: 2, msg: test fail 88 | [MinitestRetry] retry 'test_fail' count: 3, msg: test fail 89 | F 90 | 91 | Finished in 0.002479s, 403.4698 runs/s, 403.4698 assertions/s. 92 | 93 | 1) Failure: 94 | Minitest::RetryTest#test_fail [test/minitest/sample_test.rb:6]: 95 | test fail 96 | ``` 97 | 98 | ## Contributing 99 | 100 | Bug reports and pull requests are welcome on GitHub at https://github.com/y-yagi/minitest-retry. 101 | 102 | 103 | ## License 104 | 105 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 106 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | Rake::TestTask.new(:test) do |t| 5 | t.libs << "test" 6 | t.libs << "lib" 7 | t.test_files = FileList['test/**/*_test.rb'] 8 | t.verbose = true 9 | t.warning = true 10 | end 11 | 12 | task :default => :test 13 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "minitest/retry" 5 | 6 | require "irb" 7 | IRB.start(__FILE__) 8 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | bundle install 6 | 7 | # Do any other automated setup that you need to do here 8 | -------------------------------------------------------------------------------- /lib/minitest/retry.rb: -------------------------------------------------------------------------------- 1 | require "minitest/retry/version" 2 | 3 | module Minitest 4 | module Retry 5 | class << self 6 | def use!(retry_count: 3, io: $stdout, verbose: true, exceptions_to_retry: [], methods_to_retry: [], classes_to_retry: [], methods_to_skip: [], exceptions_to_skip: []) 7 | @retry_count, @io, @verbose, @exceptions_to_retry, @methods_to_retry, @classes_to_retry, @methods_to_skip, @exceptions_to_skip = retry_count, io, verbose, exceptions_to_retry, methods_to_retry, classes_to_retry, methods_to_skip, exceptions_to_skip 8 | @failure_callback, @consistent_failure_callback, @retry_callback = nil, nil, nil 9 | Minitest.prepend(self) 10 | end 11 | 12 | def on_failure(&block) 13 | return unless block_given? 14 | @failure_callback = block 15 | end 16 | 17 | def on_consistent_failure(&block) 18 | return unless block_given? 19 | @consistent_failure_callback = block 20 | end 21 | 22 | def on_retry(&block) 23 | return unless block_given? 24 | @retry_callback = block 25 | end 26 | 27 | def retry_count 28 | @retry_count 29 | end 30 | 31 | def io 32 | @io 33 | end 34 | 35 | def verbose 36 | @verbose 37 | end 38 | 39 | def exceptions_to_retry 40 | @exceptions_to_retry 41 | end 42 | 43 | def methods_to_retry 44 | @methods_to_retry 45 | end 46 | 47 | def classes_to_retry 48 | @classes_to_retry 49 | end 50 | 51 | def failure_callback 52 | @failure_callback 53 | end 54 | 55 | def consistent_failure_callback 56 | @consistent_failure_callback 57 | end 58 | 59 | def retry_callback 60 | @retry_callback 61 | end 62 | 63 | def methods_to_skip 64 | @methods_to_skip 65 | end 66 | 67 | def exceptions_to_skip 68 | @exceptions_to_skip 69 | end 70 | 71 | def failure_to_retry?(failures = [], klass_method_name, klass) 72 | return false if failures.empty? 73 | 74 | if methods_to_retry.any? 75 | return methods_to_retry.include?(klass_method_name) 76 | end 77 | 78 | if exceptions_to_retry.any? 79 | errors = failures.map(&:error).map(&:class) 80 | return (errors & exceptions_to_retry).any? 81 | end 82 | 83 | if methods_to_skip.any? 84 | return !methods_to_skip.include?(klass_method_name) 85 | end 86 | 87 | if exceptions_to_skip.any? 88 | errors = failures.map(&:error).map(&:class) 89 | return !(errors & exceptions_to_skip).any? 90 | end 91 | 92 | return true if classes_to_retry.empty? 93 | ancestors = klass.ancestors.map(&:to_s) 94 | return classes_to_retry.any? { |class_to_retry| ancestors.include?(class_to_retry) } 95 | end 96 | end 97 | 98 | module ClassMethods 99 | def run_one_method(klass, method_name) 100 | result = super(klass, method_name) 101 | 102 | klass_method_name = "#{klass.name}##{method_name}" 103 | return result unless Minitest::Retry.failure_to_retry?(result.failures, klass_method_name, klass) 104 | if !result.skipped? 105 | Minitest::Retry.failure_callback.call(klass, method_name, result) if Minitest::Retry.failure_callback 106 | Minitest::Retry.retry_count.times do |count| 107 | Minitest::Retry.retry_callback.call(klass, method_name, count + 1, result) if Minitest::Retry.retry_callback 108 | if Minitest::Retry.verbose && Minitest::Retry.io 109 | msg = "[MinitestRetry] retry '%s' count: %s, msg: %s\n" % 110 | [method_name, count + 1, result.failures.map(&:message).join(",")] 111 | Minitest::Retry.io.puts(msg) 112 | end 113 | 114 | result = super(klass, method_name) 115 | break if result.failures.empty? 116 | end 117 | 118 | if Minitest::Retry.consistent_failure_callback && !result.failures.empty? 119 | Minitest::Retry.consistent_failure_callback.call(klass, method_name, result) 120 | end 121 | end 122 | result 123 | end 124 | end 125 | 126 | def self.prepended(base) 127 | class << base 128 | prepend ClassMethods 129 | end 130 | end 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /lib/minitest/retry/version.rb: -------------------------------------------------------------------------------- 1 | module Minitest 2 | module Retry 3 | VERSION = "0.2.5" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /minitest-retry.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require 'minitest/retry/version' 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "minitest-retry" 7 | spec.version = Minitest::Retry::VERSION 8 | spec.authors = ["Yuji Yaginuma"] 9 | spec.email = ["yuuji.yaginuma@gmail.com"] 10 | 11 | spec.summary = %q{re-run the test when the test fails} 12 | spec.description = %q{re-run the test when the test fails} 13 | spec.homepage = "https://github.com/y-yagi/minitest-retry" 14 | spec.license = "MIT" 15 | 16 | # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or 17 | # delete this section to allow pushing this gem to any host. 18 | spec.metadata['allowed_push_host'] = "https://rubygems.org" 19 | spec.metadata["rubygems_mfa_required"] = "true" 20 | 21 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 22 | spec.bindir = "exe" 23 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 24 | spec.require_paths = ["lib"] 25 | 26 | spec.add_dependency 'minitest', '>= 5.0' 27 | 28 | spec.add_development_dependency "bundler" 29 | spec.add_development_dependency "rake" 30 | end 31 | -------------------------------------------------------------------------------- /test/minitest/retry_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class Minitest::RetryTest < Minitest::Test 4 | attr_accessor :reporter 5 | 6 | def setup 7 | self.reporter = Minitest::CompositeReporter.new 8 | self.reporter << Minitest::SummaryReporter.new 9 | end 10 | 11 | def capture_stdout 12 | out = StringIO.new 13 | $stdout = out 14 | yield 15 | out.string 16 | ensure 17 | $stdout = STDOUT 18 | end 19 | 20 | def test_display_retry_msg 21 | output = capture_stdout do 22 | retry_test = Class.new(Minitest::Test) do 23 | Minitest::Retry.use! 24 | def fail 25 | assert false, 'fail test' 26 | end 27 | end 28 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 29 | end 30 | expect = <<-EOS 31 | [MinitestRetry] retry 'fail' count: 1, msg: fail test 32 | [MinitestRetry] retry 'fail' count: 2, msg: fail test 33 | [MinitestRetry] retry 'fail' count: 3, msg: fail test 34 | EOS 35 | 36 | refute reporter.passed? 37 | assert_equal expect, output 38 | end 39 | 40 | def test_display_retry_msg_for_unexpected_exception 41 | output = capture_stdout do 42 | retry_test = Class.new(Minitest::Test) do 43 | Minitest::Retry.use! 44 | def fail 45 | raise 'parsing error' 46 | end 47 | end 48 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 49 | end 50 | 51 | failed_method_name = RUBY_VERSION >= "3.4" ? "'fail'" : "`fail'" 52 | path = Gem::Version.new(Minitest::VERSION) >= Gem::Version.new("5.21.0") ? "test/minitest/retry_test.rb" : __FILE__ 53 | expect = <<-EOS 54 | [MinitestRetry] retry 'fail' count: 1, msg: RuntimeError: parsing error\n #{path}:45:in #{failed_method_name} 55 | [MinitestRetry] retry 'fail' count: 2, msg: RuntimeError: parsing error\n #{path}:45:in #{failed_method_name} 56 | [MinitestRetry] retry 'fail' count: 3, msg: RuntimeError: parsing error\n #{path}:45:in #{failed_method_name} 57 | EOS 58 | 59 | refute reporter.passed? 60 | assert_equal expect, output 61 | end 62 | 63 | def test_if_test_is_successful_in_middle_of_retry 64 | output = capture_stdout do 65 | retry_test = Class.new(Minitest::Test) do 66 | @@counter = 0 67 | Minitest::Retry.use! 68 | def fail 69 | @@counter += 1 70 | assert_equal 3, @@counter 71 | end 72 | end 73 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 74 | end 75 | expect = <<-EOS 76 | [MinitestRetry] retry 'fail' count: 1, msg: Expected: 3 77 | Actual: 1 78 | [MinitestRetry] retry 'fail' count: 2, msg: Expected: 3 79 | Actual: 2 80 | EOS 81 | 82 | assert reporter.passed? 83 | assert_equal expect, output 84 | end 85 | 86 | def test_having_to_only_specified_count_retry 87 | output = capture_stdout do 88 | retry_test = Class.new(Minitest::Test) do 89 | Minitest::Retry.use!(retry_count: 5) 90 | def fail 91 | assert false, 'fail test' 92 | end 93 | end 94 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 95 | end 96 | expect = <<-EOS 97 | [MinitestRetry] retry 'fail' count: 1, msg: fail test 98 | [MinitestRetry] retry 'fail' count: 2, msg: fail test 99 | [MinitestRetry] retry 'fail' count: 3, msg: fail test 100 | [MinitestRetry] retry 'fail' count: 4, msg: fail test 101 | [MinitestRetry] retry 'fail' count: 5, msg: fail test 102 | EOS 103 | 104 | refute reporter.passed? 105 | assert_equal expect, output 106 | end 107 | 108 | def test_msg_does_not_display_when_verbose_false 109 | output = capture_stdout do 110 | retry_test = Class.new(Minitest::Test) do 111 | @@counter = 0 112 | Minitest::Retry.use!(verbose: false) 113 | def fail 114 | @@counter += 1 115 | assert_equal 3, @@counter 116 | end 117 | end 118 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 119 | end 120 | 121 | assert reporter.passed? 122 | assert_empty output 123 | end 124 | 125 | def test_msg_does_not_display_when_do_not_use_retry 126 | output = capture_stdout do 127 | retry_test = Class.new(Minitest::Test) do 128 | def fail 129 | assert false, 'fail test' 130 | end 131 | end 132 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 133 | end 134 | 135 | refute reporter.passed? 136 | assert_empty output 137 | end 138 | 139 | def test_donot_retry_skipped_Test 140 | output = capture_stdout do 141 | retry_test = Class.new(Minitest::Test) do 142 | Minitest::Retry.use! 143 | def skip_test 144 | skip 'skip test' 145 | end 146 | end 147 | Minitest::Runnable.run_one_method(retry_test, :skip_test, self.reporter) 148 | end 149 | 150 | assert reporter.passed? 151 | assert_empty output 152 | end 153 | 154 | def test_retry_when_error_in_exceptions_to_retry 155 | capture_stdout do 156 | retry_test = Class.new(Minitest::Test) do 157 | @@counter = 0 158 | def self.counter 159 | @@counter 160 | end 161 | Minitest::Retry.use! exceptions_to_retry: [TestError] 162 | def raise_test_error 163 | @@counter += 1 164 | raise TestError, 'This triggers a retry.' 165 | end 166 | end 167 | Minitest::Runnable.run_one_method(retry_test, :raise_test_error, self.reporter) 168 | 169 | assert_equal 4, retry_test.counter 170 | end 171 | end 172 | 173 | def test_donot_retry_when_not_in_exceptions_to_retry 174 | capture_stdout do 175 | retry_test = Class.new(Minitest::Test) do 176 | @@counter = 0 177 | def self.counter 178 | @@counter 179 | end 180 | Minitest::Retry.use! exceptions_to_retry: [TestError] 181 | def raise_test_error 182 | @@counter += 1 183 | raise ArgumentError, 'This does not trigger a retry.' 184 | end 185 | end 186 | Minitest::Runnable.run_one_method(retry_test, :raise_test_error, reporter) 187 | 188 | assert_equal 1, retry_test.counter 189 | end 190 | end 191 | 192 | def test_retry_when_method_in_methods_to_retry 193 | capture_stdout do 194 | retry_test = Class.new(Minitest::Test) do 195 | @@counter = 0 196 | 197 | class << self 198 | def name 199 | 'TestClass' 200 | end 201 | end 202 | 203 | def self.counter 204 | @@counter 205 | end 206 | Minitest::Retry.use! methods_to_retry: ["TestClass#fail"] 207 | def fail 208 | @@counter += 1 209 | assert false, 'fail test' 210 | end 211 | end 212 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 213 | 214 | assert_equal 4, retry_test.counter 215 | end 216 | end 217 | 218 | def test_donot_retry_when_not_in_methods_to_retry 219 | capture_stdout do 220 | retry_test = Class.new(Minitest::Test) do 221 | @@counter = 0 222 | 223 | class << self 224 | def name 225 | 'TestClass' 226 | end 227 | end 228 | 229 | def self.counter 230 | @@counter 231 | end 232 | Minitest::Retry.use! methods_to_retry: ["TestClass#fail"] 233 | def another_fail 234 | @@counter += 1 235 | assert false, 'fail test' 236 | end 237 | end 238 | Minitest::Runnable.run_one_method(retry_test, :another_fail, self.reporter) 239 | 240 | assert_equal 1, retry_test.counter 241 | end 242 | end 243 | 244 | def test_retry_when_class_in_classes_to_retry 245 | capture_stdout do 246 | retry_test = Class.new(Minitest::Test) do 247 | @@counter = 0 248 | 249 | class << self 250 | def name 251 | 'TestClass' 252 | end 253 | end 254 | 255 | def self.counter 256 | @@counter 257 | end 258 | Minitest::Retry.use! classes_to_retry: ["Minitest::Test"] 259 | def fail 260 | @@counter += 1 261 | assert false, 'fail test' 262 | end 263 | end 264 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 265 | 266 | assert_equal 4, retry_test.counter 267 | end 268 | end 269 | 270 | def test_donot_retry_when_not_in_classes_to_retry 271 | capture_stdout do 272 | retry_test = Class.new(Minitest::Test) do 273 | @@counter = 0 274 | 275 | class << self 276 | def name 277 | 'TestClass' 278 | end 279 | end 280 | 281 | def self.counter 282 | @@counter 283 | end 284 | Minitest::Retry.use! classes_to_retry: ["OtherClass"] 285 | def another_fail 286 | @@counter += 1 287 | assert false, 'fail test' 288 | end 289 | end 290 | Minitest::Runnable.run_one_method(retry_test, :another_fail, self.reporter) 291 | 292 | assert_equal 1, retry_test.counter 293 | end 294 | end 295 | 296 | def test_retry_when_method_in_methods_to_skip 297 | capture_stdout do 298 | retry_test = Class.new(Minitest::Test) do 299 | @@counter = 0 300 | 301 | class << self 302 | def name 303 | 'TestClass' 304 | end 305 | end 306 | 307 | def self.counter 308 | @@counter 309 | end 310 | Minitest::Retry.use! methods_to_skip: ["TestClass#fail"] 311 | def fail 312 | @@counter += 1 313 | assert false, 'fail test' 314 | end 315 | end 316 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 317 | 318 | assert_equal 1, retry_test.counter 319 | end 320 | end 321 | 322 | def test_retry_when_error_in_exceptions_to_skip 323 | capture_stdout do 324 | retry_test = Class.new(Minitest::Test) do 325 | @@counter = 0 326 | def self.counter 327 | @@counter 328 | end 329 | Minitest::Retry.use! exceptions_to_skip: [TestError] 330 | def raise_test_error 331 | @@counter += 1 332 | raise TestError, 'This does not trigger a retry.' 333 | end 334 | end 335 | Minitest::Runnable.run_one_method(retry_test, :raise_test_error, self.reporter) 336 | 337 | assert_equal 1, retry_test.counter 338 | end 339 | end 340 | 341 | def test_retry_when_error_not_in_exceptions_to_skip 342 | capture_stdout do 343 | retry_test = Class.new(Minitest::Test) do 344 | @@counter = 0 345 | def self.counter 346 | @@counter 347 | end 348 | Minitest::Retry.use! exceptions_to_skip: [TestError] 349 | def raise_test_error 350 | @@counter += 1 351 | raise ArgumentError, 'This triggers a retry.' 352 | end 353 | end 354 | Minitest::Runnable.run_one_method(retry_test, :raise_test_error, self.reporter) 355 | 356 | assert_equal 4, retry_test.counter 357 | end 358 | end 359 | 360 | def test_run_failure_callback_on_failure 361 | on_failure_block_has_ran = false 362 | test_name, test_class, retry_test, result_in_callback = nil 363 | capture_stdout do 364 | retry_test = Class.new(Minitest::Test) do 365 | Minitest::Retry.use! 366 | Minitest::Retry.on_failure do |klass, failed_test, result| 367 | on_failure_block_has_ran = true 368 | test_class = klass 369 | test_name = failed_test 370 | result_in_callback = result 371 | end 372 | 373 | def fail 374 | assert false, 'fail test' 375 | end 376 | end 377 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 378 | end 379 | assert_equal :fail, test_name 380 | assert_equal retry_test, test_class 381 | assert on_failure_block_has_ran 382 | refute_nil result_in_callback 383 | assert_instance_of Minitest::Assertion, result_in_callback.failures[0] 384 | end 385 | 386 | def test_do_not_run_failure_callback_on_success 387 | on_failure_block_has_ran = false 388 | capture_stdout do 389 | retry_test = Class.new(Minitest::Test) do 390 | Minitest::Retry.use! 391 | Minitest::Retry.on_failure do 392 | on_failure_block_has_ran = true 393 | end 394 | 395 | def success 396 | assert true, 'success test' 397 | end 398 | end 399 | Minitest::Runnable.run_one_method(retry_test, :success, self.reporter) 400 | end 401 | refute on_failure_block_has_ran 402 | end 403 | 404 | def test_run_retry_callback_on_each_retry 405 | retry_counts, test_names, test_classes, results_in_callbacks = [], [], [], [] 406 | retry_test = nil 407 | capture_stdout do 408 | retry_test = Class.new(Minitest::Test) do 409 | Minitest::Retry.use! 410 | Minitest::Retry.on_retry do |klass, test_name, retry_count, result| 411 | retry_counts << retry_count 412 | test_names << test_name 413 | test_classes << klass 414 | results_in_callbacks << result 415 | end 416 | 417 | def fail_sometimes 418 | assert_equal 3, 0 419 | end 420 | end 421 | Minitest::Runnable.run_one_method(retry_test, :fail_sometimes, self.reporter) 422 | end 423 | assert_equal [1, 2, 3], retry_counts 424 | assert_equal [:fail_sometimes] * 3, test_names 425 | assert_equal [retry_test] * 3, test_classes 426 | refute_empty results_in_callbacks 427 | assert_equal [Minitest::Assertion] * 3, results_in_callbacks.map{|x| x.failures[0].class} 428 | end 429 | 430 | def test_run_consistent_failure_callback_on_failure 431 | on_consistent_failure_block_call_count = 0 432 | test_name, test_class, retry_test, result_in_callback = nil 433 | capture_stdout do 434 | retry_test = Class.new(Minitest::Test) do 435 | Minitest::Retry.use! 436 | Minitest::Retry.on_consistent_failure do |klass, failed_test, result| 437 | on_consistent_failure_block_call_count += 1 438 | test_class = klass 439 | test_name = failed_test 440 | result_in_callback = result 441 | end 442 | 443 | def fail 444 | assert false, 'fail test' 445 | end 446 | end 447 | Minitest::Runnable.run_one_method(retry_test, :fail, self.reporter) 448 | end 449 | assert_equal :fail, test_name 450 | assert_equal retry_test, test_class 451 | assert_equal 1, on_consistent_failure_block_call_count 452 | refute_nil result_in_callback 453 | assert_instance_of Minitest::Assertion, result_in_callback.failures[0] 454 | end 455 | 456 | class TestError < StandardError; end 457 | end 458 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'debug' 3 | require 'minitest/retry' 4 | require 'minitest/autorun' 5 | --------------------------------------------------------------------------------