├── lib └── minitest │ ├── retry │ └── version.rb │ └── retry.rb ├── Gemfile ├── bin ├── console └── setup ├── gemfiles ├── minitest_5.gemfile └── minitest_6.gemfile ├── test ├── test_helper.rb └── minitest │ └── retry_test.rb ├── .gitignore ├── Rakefile ├── .github └── workflows │ ├── push_gem.yml │ ├── stale.yml │ └── ubuntu.yml ├── LICENSE.txt ├── CHANGELOG.md ├── minitest-retry.gemspec └── README.md /lib/minitest/retry/version.rb: -------------------------------------------------------------------------------- 1 | module Minitest 2 | module Retry 3 | VERSION = "0.3.0" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'debug' 4 | # Specify your gem's dependencies in minitest-retry.gemspec 5 | gemspec 6 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "minitest/retry" 5 | 6 | require "irb" 7 | IRB.start(__FILE__) 8 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | bundle install 6 | 7 | # Do any other automated setup that you need to do here 8 | -------------------------------------------------------------------------------- /gemfiles/minitest_5.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | eval_gemfile File.expand_path("../Gemfile", __dir__) 4 | 5 | gem "minitest", "< 6" 6 | -------------------------------------------------------------------------------- /gemfiles/minitest_6.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | eval_gemfile File.expand_path("../Gemfile", __dir__) 4 | 5 | gem "minitest", ">= 6", "< 7" 6 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'debug' 3 | require 'minitest/retry' 4 | require 'minitest/autorun' 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | /gemfiles/*.lock 11 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | Rake::TestTask.new(:test) do |t| 5 | t.libs << "test" 6 | t.libs << "lib" 7 | t.test_files = FileList['test/**/*_test.rb'] 8 | t.verbose = true 9 | t.warning = true 10 | end 11 | 12 | task :default => :test 13 | -------------------------------------------------------------------------------- /.github/workflows/push_gem.yml: -------------------------------------------------------------------------------- 1 | name: Push Gem 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | push: 13 | if: github.repository == 'y-yagi/minitest-retry' 14 | runs-on: ubuntu-latest 15 | 16 | permissions: 17 | contents: write 18 | id-token: write 19 | 20 | steps: 21 | - name: Harden Runner 22 | uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 23 | with: 24 | egress-policy: audit 25 | 26 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 27 | - name: Set up Ruby 28 | uses: ruby/setup-ruby@2e007403fc1ec238429ecaa57af6f22f019cc135 # v1.234.0 29 | with: 30 | bundler-cache: true 31 | ruby-version: ruby 32 | 33 | - uses: rubygems/release-gem@9e85cb11501bebc2ae661c1500176316d3987059 # v1.1.0 34 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | # This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. 2 | # 3 | # You can adjust the behavior by modifying this file. 4 | # For more information, see: 5 | # https://github.com/actions/stale 6 | name: Mark stale issues and pull requests 7 | 8 | on: 9 | schedule: 10 | - cron: '28 20 * * *' 11 | 12 | jobs: 13 | stale: 14 | 15 | runs-on: ubuntu-latest 16 | permissions: 17 | issues: write 18 | pull-requests: write 19 | 20 | steps: 21 | - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 22 | with: 23 | repo-token: ${{ secrets.GITHUB_TOKEN }} 24 | stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' 25 | close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' 26 | stale-issue-label: 'no-issue-activity' 27 | days-before-issue-stale: 30 28 | days-before-issue-close: 5 29 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Yuji Yaginuma 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.3.0 2 | 3 | * Support minitest v6 #50 [y-yagi] 4 | 5 | ## 0.2.5 6 | 7 | * Add `exceptions_to_skip` option #48 [y-yagi] 8 | 9 | ## 0.2.4 10 | 11 | * Add `methods_to_skip` option #44 [y-yagi] 12 | 13 | ## 0.2.3 14 | 15 | * Add `classes_to_retry` option #42 [y-yagi] 16 | 17 | ## 0.2.2 18 | 19 | * Add `methods_to_retry` option #33 [edudepetris] 20 | 21 | ## 0.2.1 22 | 23 | * Pass a test result to consistent failure callback #30 [rmacklin] 24 | 25 | ## 0.2.0 26 | 27 | * Pass a test result to failure and retry callbacks #29 [aabrahamian] 28 | 29 | ## 0.1.9 30 | 31 | * Add on_consistent_failure callback #20 [staugaard] 32 | 33 | ## 0.1.8 34 | 35 | * Add retry callback #17 [julien-meichelbeck] 36 | * Pass test class and test name to failure callback #15, #16 [julien-meichelbeck] 37 | 38 | ## 0.1.7 39 | 40 | * Run a callback on each test failure #15 [julien-meichelbeck] 41 | 42 | ## 0.1.6 43 | 44 | * Add exceptions_to_retry option #13 [panjan] 45 | 46 | ## 0.1.5 47 | 48 | * Improve message information when an unexpected error occurs #9 [abarre] 49 | 50 | ## 0.1.4 51 | 52 | * Fix typo in verbose output #6 [semanticart] 53 | 54 | ## 0.1.3 55 | 56 | * modify so as not to retry it a had gone to retry the skip test. 57 | -------------------------------------------------------------------------------- /.github/workflows/ubuntu.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | minitest-5: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0', '2.7', '2.6', '2.5', '2.4', '2.3' ] 11 | env: 12 | BUNDLE_GEMFILE: gemfiles/minitest_5.gemfile 13 | steps: 14 | - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 15 | - uses: ruby/setup-ruby@a25f1e45f0e65a92fcb1e95e8847f78fb0a7197a # v1.273.0 16 | with: 17 | ruby-version: ${{ matrix.ruby }} 18 | - name: Install dependencies 19 | run: bundle install 20 | - name: Run test 21 | run: bundle exec rake 22 | 23 | minitest-6: 24 | runs-on: ubuntu-latest 25 | strategy: 26 | matrix: 27 | ruby: [ '3.4', '3.3', '3.2' ] 28 | env: 29 | BUNDLE_GEMFILE: gemfiles/minitest_6.gemfile 30 | steps: 31 | - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 32 | - uses: ruby/setup-ruby@a25f1e45f0e65a92fcb1e95e8847f78fb0a7197a # v1.273.0 33 | with: 34 | ruby-version: ${{ matrix.ruby }} 35 | - name: Install dependencies 36 | run: bundle install 37 | - name: Run test 38 | run: bundle exec rake 39 | -------------------------------------------------------------------------------- /minitest-retry.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require 'minitest/retry/version' 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "minitest-retry" 7 | spec.version = Minitest::Retry::VERSION 8 | spec.authors = ["Yuji Yaginuma"] 9 | spec.email = ["yuuji.yaginuma@gmail.com"] 10 | 11 | spec.summary = %q{re-run the test when the test fails} 12 | spec.description = %q{re-run the test when the test fails} 13 | spec.homepage = "https://github.com/y-yagi/minitest-retry" 14 | spec.license = "MIT" 15 | 16 | # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or 17 | # delete this section to allow pushing this gem to any host. 18 | spec.metadata['allowed_push_host'] = "https://rubygems.org" 19 | spec.metadata["rubygems_mfa_required"] = "true" 20 | 21 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 22 | spec.bindir = "exe" 23 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 24 | spec.require_paths = ["lib"] 25 | 26 | spec.add_dependency 'minitest', '>= 5.0' 27 | 28 | spec.add_development_dependency "bundler" 29 | spec.add_development_dependency "rake" 30 | end 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Minitest::Retry 2 | 3 | Re-run the test when the test fails. 4 | 5 | ![](https://github.com/y-yagi/minitest-retry/workflows/CI/badge.svg) 6 | [![Gem Version](https://badge.fury.io/rb/minitest-retry.svg)](http://badge.fury.io/rb/minitest-retry) 7 | 8 | ## Installation 9 | 10 | Add this line to your application's Gemfile: 11 | 12 | ```ruby 13 | gem 'minitest-retry' 14 | ``` 15 | 16 | And then execute: 17 | 18 | $ bundle 19 | 20 | Or install it yourself as: 21 | 22 | $ gem install minitest-retry 23 | 24 | ## Usage 25 | 26 | In your `test_helper.rb` file, add the following lines: 27 | 28 | ```ruby 29 | require 'minitest/retry' 30 | Minitest::Retry.use! 31 | ``` 32 | 33 | Options can be specified to `use!` method. Can specify options are as follows: 34 | 35 | ```ruby 36 | Minitest::Retry.use!( 37 | retry_count: 3, # The number of times to retry. The default is 3. 38 | verbose: true, # Whether or not to display the message at the time of retry. The default is true. 39 | io: $stdout, # Display destination of retry when the message. The default is stdout. 40 | exceptions_to_retry: [], # List of exceptions that will trigger a retry (when empty, all exceptions will). 41 | methods_to_retry: [], # List of methods that will trigger a retry (when empty, all methods will). 42 | classes_to_retry: [], # List of classes that will trigger a retry (when empty, all classes will). 43 | methods_to_skip: [], # List of methods that will skip a retry (when empty, all methods will retry). 44 | exceptions_to_skip: [] # List of exceptions that will skip a retry (when empty, all exceptions will retry). 45 | ) 46 | ``` 47 | 48 | #### Callbacks 49 | The `on_failure` callback is executed each time a test fails: 50 | ```ruby 51 | Minitest::Retry.on_failure do |klass, test_name, result| 52 | # code omitted 53 | end 54 | ``` 55 | 56 | The `on_consistent_failure` callback is executed when a test consistently fails: 57 | ```ruby 58 | Minitest::Retry.on_consistent_failure do |klass, test_name, result| 59 | # code omitted 60 | end 61 | ``` 62 | 63 | The `on_retry` callback is executed each time a test is retried: 64 | ```ruby 65 | Minitest::Retry.on_retry do |klass, test_name, retry_count, result| 66 | # code omitted 67 | end 68 | ``` 69 | 70 | ## Example 71 | 72 | ```ruby 73 | 74 | Minitest::Retry.use! 75 | 76 | class Minitest::RetryTest < Minitest::Test 77 | def test_fail 78 | assert false, 'test fail' 79 | end 80 | end 81 | ``` 82 | 83 | ```console 84 | # Running: 85 | 86 | [MinitestRetry] retry 'test_fail' count: 1, msg: test fail 87 | [MinitestRetry] retry 'test_fail' count: 2, msg: test fail 88 | [MinitestRetry] retry 'test_fail' count: 3, msg: test fail 89 | F 90 | 91 | Finished in 0.002479s, 403.4698 runs/s, 403.4698 assertions/s. 92 | 93 | 1) Failure: 94 | Minitest::RetryTest#test_fail [test/minitest/sample_test.rb:6]: 95 | test fail 96 | ``` 97 | 98 | ## Contributing 99 | 100 | Bug reports and pull requests are welcome on GitHub at https://github.com/y-yagi/minitest-retry. 101 | 102 | 103 | ## License 104 | 105 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 106 | -------------------------------------------------------------------------------- /lib/minitest/retry.rb: -------------------------------------------------------------------------------- 1 | require "minitest/retry/version" 2 | 3 | module Minitest 4 | module Retry 5 | class << self 6 | def use!(retry_count: 3, io: $stdout, verbose: true, exceptions_to_retry: [], methods_to_retry: [], classes_to_retry: [], methods_to_skip: [], exceptions_to_skip: []) 7 | @retry_count, @io, @verbose, @exceptions_to_retry, @methods_to_retry, @classes_to_retry, @methods_to_skip, @exceptions_to_skip = retry_count, io, verbose, exceptions_to_retry, methods_to_retry, classes_to_retry, methods_to_skip, exceptions_to_skip 8 | @failure_callback, @consistent_failure_callback, @retry_callback = nil, nil, nil 9 | Minitest.prepend(self) 10 | end 11 | 12 | def on_failure(&block) 13 | return unless block_given? 14 | @failure_callback = block 15 | end 16 | 17 | def on_consistent_failure(&block) 18 | return unless block_given? 19 | @consistent_failure_callback = block 20 | end 21 | 22 | def on_retry(&block) 23 | return unless block_given? 24 | @retry_callback = block 25 | end 26 | 27 | def retry_count 28 | @retry_count 29 | end 30 | 31 | def io 32 | @io 33 | end 34 | 35 | def verbose 36 | @verbose 37 | end 38 | 39 | def exceptions_to_retry 40 | @exceptions_to_retry 41 | end 42 | 43 | def methods_to_retry 44 | @methods_to_retry 45 | end 46 | 47 | def classes_to_retry 48 | @classes_to_retry 49 | end 50 | 51 | def failure_callback 52 | @failure_callback 53 | end 54 | 55 | def consistent_failure_callback 56 | @consistent_failure_callback 57 | end 58 | 59 | def retry_callback 60 | @retry_callback 61 | end 62 | 63 | def methods_to_skip 64 | @methods_to_skip 65 | end 66 | 67 | def exceptions_to_skip 68 | @exceptions_to_skip 69 | end 70 | 71 | def failure_to_retry?(failures = [], klass_method_name, klass) 72 | return false if failures.empty? 73 | 74 | if methods_to_retry.any? 75 | return methods_to_retry.include?(klass_method_name) 76 | end 77 | 78 | if exceptions_to_retry.any? 79 | errors = failures.map(&:error).map(&:class) 80 | return (errors & exceptions_to_retry).any? 81 | end 82 | 83 | if methods_to_skip.any? 84 | return !methods_to_skip.include?(klass_method_name) 85 | end 86 | 87 | if exceptions_to_skip.any? 88 | errors = failures.map(&:error).map(&:class) 89 | return !(errors & exceptions_to_skip).any? 90 | end 91 | 92 | return true if classes_to_retry.empty? 93 | ancestors = klass.ancestors.map(&:to_s) 94 | return classes_to_retry.any? { |class_to_retry| ancestors.include?(class_to_retry) } 95 | end 96 | 97 | def run_with_retry(klass, method_name) 98 | klass_method_name = "#{klass.name}##{method_name}" 99 | result = yield 100 | 101 | return result unless failure_to_retry?(result.failures, klass_method_name, klass) 102 | return result if result.skipped? 103 | 104 | failure_callback&.call(klass, method_name, result) 105 | 106 | retry_count.times do |count| 107 | retry_callback&.call(klass, method_name, count + 1, result) 108 | 109 | if verbose && io 110 | msg = "[MinitestRetry] retry '%s' count: %s, msg: %s\n" % 111 | [method_name, count + 1, result.failures.map(&:message).join(",")] 112 | io.puts(msg) 113 | end 114 | 115 | result = yield 116 | break if result.failures.empty? 117 | end 118 | 119 | if consistent_failure_callback && !result.failures.empty? 120 | consistent_failure_callback.call(klass, method_name, result) 121 | end 122 | 123 | result 124 | end 125 | end 126 | 127 | module ClassMethods 128 | def run_one_method(klass, method_name) 129 | Minitest::Retry.run_with_retry(klass, method_name) do 130 | super(klass, method_name) 131 | end 132 | end 133 | end 134 | 135 | module RunnableMethods 136 | def run(klass, method_name, reporter) 137 | reporter.prerecord klass, method_name 138 | result = Minitest::Retry.run_with_retry(klass, method_name) do 139 | klass.new(method_name).run 140 | end 141 | reporter.record result 142 | end 143 | end 144 | 145 | def self.prepended(base) 146 | if Minitest::VERSION > "6" 147 | class << Minitest::Runnable 148 | prepend RunnableMethods 149 | end 150 | else 151 | class << base 152 | prepend ClassMethods 153 | end 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /test/minitest/retry_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class Minitest::RetryTest < Minitest::Test 4 | attr_accessor :reporter 5 | 6 | def setup 7 | self.reporter = Minitest::CompositeReporter.new 8 | self.reporter << Minitest::SummaryReporter.new 9 | end 10 | 11 | def capture_stdout 12 | out = StringIO.new 13 | $stdout = out 14 | yield 15 | out.string 16 | ensure 17 | $stdout = STDOUT 18 | end 19 | 20 | def run_test(klass, method_name) 21 | if Minitest::Runnable.respond_to?(:run_one_method) 22 | Minitest::Runnable.run_one_method(klass, method_name, reporter) 23 | else 24 | Minitest::Runnable.run(klass, method_name, reporter) 25 | end 26 | end 27 | 28 | def test_display_retry_msg 29 | output = capture_stdout do 30 | retry_test = Class.new(Minitest::Test) do 31 | Minitest::Retry.use! 32 | def fail 33 | assert false, 'fail test' 34 | end 35 | end 36 | run_test(retry_test, :fail) 37 | end 38 | expect = <<-EOS 39 | [MinitestRetry] retry 'fail' count: 1, msg: fail test 40 | [MinitestRetry] retry 'fail' count: 2, msg: fail test 41 | [MinitestRetry] retry 'fail' count: 3, msg: fail test 42 | EOS 43 | 44 | refute reporter.passed? 45 | assert_equal expect, output 46 | end 47 | 48 | def test_display_retry_msg_for_unexpected_exception 49 | output = capture_stdout do 50 | retry_test = Class.new(Minitest::Test) do 51 | Minitest::Retry.use! 52 | def fail 53 | raise 'parsing error' 54 | end 55 | end 56 | run_test(retry_test, :fail) 57 | end 58 | 59 | failed_method_name = RUBY_VERSION >= "3.4" ? "'fail'" : "`fail'" 60 | path = Gem::Version.new(Minitest::VERSION) >= Gem::Version.new("5.21.0") ? "test/minitest/retry_test.rb" : __FILE__ 61 | expect = <<-EOS 62 | [MinitestRetry] retry 'fail' count: 1, msg: RuntimeError: parsing error\n #{path}:53:in #{failed_method_name} 63 | [MinitestRetry] retry 'fail' count: 2, msg: RuntimeError: parsing error\n #{path}:53:in #{failed_method_name} 64 | [MinitestRetry] retry 'fail' count: 3, msg: RuntimeError: parsing error\n #{path}:53:in #{failed_method_name} 65 | EOS 66 | 67 | refute reporter.passed? 68 | assert_equal expect, output 69 | end 70 | 71 | def test_if_test_is_successful_in_middle_of_retry 72 | output = capture_stdout do 73 | retry_test = Class.new(Minitest::Test) do 74 | @@counter = 0 75 | Minitest::Retry.use! 76 | def fail 77 | @@counter += 1 78 | assert_equal 3, @@counter 79 | end 80 | end 81 | run_test(retry_test, :fail) 82 | end 83 | expect = <<-EOS 84 | [MinitestRetry] retry 'fail' count: 1, msg: Expected: 3 85 | Actual: 1 86 | [MinitestRetry] retry 'fail' count: 2, msg: Expected: 3 87 | Actual: 2 88 | EOS 89 | 90 | assert reporter.passed? 91 | assert_equal expect, output 92 | end 93 | 94 | def test_having_to_only_specified_count_retry 95 | output = capture_stdout do 96 | retry_test = Class.new(Minitest::Test) do 97 | Minitest::Retry.use!(retry_count: 5) 98 | def fail 99 | assert false, 'fail test' 100 | end 101 | end 102 | run_test(retry_test, :fail) 103 | end 104 | expect = <<-EOS 105 | [MinitestRetry] retry 'fail' count: 1, msg: fail test 106 | [MinitestRetry] retry 'fail' count: 2, msg: fail test 107 | [MinitestRetry] retry 'fail' count: 3, msg: fail test 108 | [MinitestRetry] retry 'fail' count: 4, msg: fail test 109 | [MinitestRetry] retry 'fail' count: 5, msg: fail test 110 | EOS 111 | 112 | refute reporter.passed? 113 | assert_equal expect, output 114 | end 115 | 116 | def test_msg_does_not_display_when_verbose_false 117 | output = capture_stdout do 118 | retry_test = Class.new(Minitest::Test) do 119 | @@counter = 0 120 | Minitest::Retry.use!(verbose: false) 121 | def fail 122 | @@counter += 1 123 | assert_equal 3, @@counter 124 | end 125 | end 126 | run_test(retry_test, :fail) 127 | end 128 | 129 | assert reporter.passed? 130 | assert_empty output 131 | end 132 | 133 | def test_msg_does_not_display_when_do_not_use_retry 134 | output = capture_stdout do 135 | retry_test = Class.new(Minitest::Test) do 136 | def fail 137 | assert false, 'fail test' 138 | end 139 | end 140 | run_test(retry_test, :fail) 141 | end 142 | 143 | refute reporter.passed? 144 | assert_empty output 145 | end 146 | 147 | def test_donot_retry_skipped_Test 148 | output = capture_stdout do 149 | retry_test = Class.new(Minitest::Test) do 150 | Minitest::Retry.use! 151 | def skip_test 152 | skip 'skip test' 153 | end 154 | end 155 | run_test(retry_test, :skip_test) 156 | end 157 | 158 | assert reporter.passed? 159 | assert_empty output 160 | end 161 | 162 | def test_retry_when_error_in_exceptions_to_retry 163 | capture_stdout do 164 | retry_test = Class.new(Minitest::Test) do 165 | @@counter = 0 166 | def self.counter 167 | @@counter 168 | end 169 | Minitest::Retry.use! exceptions_to_retry: [TestError] 170 | def raise_test_error 171 | @@counter += 1 172 | raise TestError, 'This triggers a retry.' 173 | end 174 | end 175 | run_test(retry_test, :raise_test_error) 176 | 177 | assert_equal 4, retry_test.counter 178 | end 179 | end 180 | 181 | def test_donot_retry_when_not_in_exceptions_to_retry 182 | capture_stdout do 183 | retry_test = Class.new(Minitest::Test) do 184 | @@counter = 0 185 | def self.counter 186 | @@counter 187 | end 188 | Minitest::Retry.use! exceptions_to_retry: [TestError] 189 | def raise_test_error 190 | @@counter += 1 191 | raise ArgumentError, 'This does not trigger a retry.' 192 | end 193 | end 194 | run_test(retry_test, :raise_test_error) 195 | 196 | assert_equal 1, retry_test.counter 197 | end 198 | end 199 | 200 | def test_retry_when_method_in_methods_to_retry 201 | capture_stdout do 202 | retry_test = Class.new(Minitest::Test) do 203 | @@counter = 0 204 | 205 | class << self 206 | def name 207 | 'TestClass' 208 | end 209 | end 210 | 211 | def self.counter 212 | @@counter 213 | end 214 | Minitest::Retry.use! methods_to_retry: ["TestClass#fail"] 215 | def fail 216 | @@counter += 1 217 | assert false, 'fail test' 218 | end 219 | end 220 | run_test(retry_test, :fail) 221 | 222 | assert_equal 4, retry_test.counter 223 | end 224 | end 225 | 226 | def test_donot_retry_when_not_in_methods_to_retry 227 | capture_stdout do 228 | retry_test = Class.new(Minitest::Test) do 229 | @@counter = 0 230 | 231 | class << self 232 | def name 233 | 'TestClass' 234 | end 235 | end 236 | 237 | def self.counter 238 | @@counter 239 | end 240 | Minitest::Retry.use! methods_to_retry: ["TestClass#fail"] 241 | def another_fail 242 | @@counter += 1 243 | assert false, 'fail test' 244 | end 245 | end 246 | run_test(retry_test, :another_fail) 247 | 248 | assert_equal 1, retry_test.counter 249 | end 250 | end 251 | 252 | def test_retry_when_class_in_classes_to_retry 253 | capture_stdout do 254 | retry_test = Class.new(Minitest::Test) do 255 | @@counter = 0 256 | 257 | class << self 258 | def name 259 | 'TestClass' 260 | end 261 | end 262 | 263 | def self.counter 264 | @@counter 265 | end 266 | Minitest::Retry.use! classes_to_retry: ["Minitest::Test"] 267 | def fail 268 | @@counter += 1 269 | assert false, 'fail test' 270 | end 271 | end 272 | run_test(retry_test, :fail) 273 | 274 | assert_equal 4, retry_test.counter 275 | end 276 | end 277 | 278 | def test_donot_retry_when_not_in_classes_to_retry 279 | capture_stdout do 280 | retry_test = Class.new(Minitest::Test) do 281 | @@counter = 0 282 | 283 | class << self 284 | def name 285 | 'TestClass' 286 | end 287 | end 288 | 289 | def self.counter 290 | @@counter 291 | end 292 | Minitest::Retry.use! classes_to_retry: ["OtherClass"] 293 | def another_fail 294 | @@counter += 1 295 | assert false, 'fail test' 296 | end 297 | end 298 | run_test(retry_test, :another_fail) 299 | 300 | assert_equal 1, retry_test.counter 301 | end 302 | end 303 | 304 | def test_retry_when_method_in_methods_to_skip 305 | capture_stdout do 306 | retry_test = Class.new(Minitest::Test) do 307 | @@counter = 0 308 | 309 | class << self 310 | def name 311 | 'TestClass' 312 | end 313 | end 314 | 315 | def self.counter 316 | @@counter 317 | end 318 | Minitest::Retry.use! methods_to_skip: ["TestClass#fail"] 319 | def fail 320 | @@counter += 1 321 | assert false, 'fail test' 322 | end 323 | end 324 | run_test(retry_test, :fail) 325 | 326 | assert_equal 1, retry_test.counter 327 | end 328 | end 329 | 330 | def test_retry_when_error_in_exceptions_to_skip 331 | capture_stdout do 332 | retry_test = Class.new(Minitest::Test) do 333 | @@counter = 0 334 | def self.counter 335 | @@counter 336 | end 337 | Minitest::Retry.use! exceptions_to_skip: [TestError] 338 | def raise_test_error 339 | @@counter += 1 340 | raise TestError, 'This does not trigger a retry.' 341 | end 342 | end 343 | run_test(retry_test, :raise_test_error) 344 | 345 | assert_equal 1, retry_test.counter 346 | end 347 | end 348 | 349 | def test_retry_when_error_not_in_exceptions_to_skip 350 | capture_stdout do 351 | retry_test = Class.new(Minitest::Test) do 352 | @@counter = 0 353 | def self.counter 354 | @@counter 355 | end 356 | Minitest::Retry.use! exceptions_to_skip: [TestError] 357 | def raise_test_error 358 | @@counter += 1 359 | raise ArgumentError, 'This triggers a retry.' 360 | end 361 | end 362 | run_test(retry_test, :raise_test_error) 363 | 364 | assert_equal 4, retry_test.counter 365 | end 366 | end 367 | 368 | def test_run_failure_callback_on_failure 369 | on_failure_block_has_ran = false 370 | test_name, test_class, retry_test, result_in_callback = nil 371 | capture_stdout do 372 | retry_test = Class.new(Minitest::Test) do 373 | Minitest::Retry.use! 374 | Minitest::Retry.on_failure do |klass, failed_test, result| 375 | on_failure_block_has_ran = true 376 | test_class = klass 377 | test_name = failed_test 378 | result_in_callback = result 379 | end 380 | 381 | def fail 382 | assert false, 'fail test' 383 | end 384 | end 385 | run_test(retry_test, :fail) 386 | end 387 | assert_equal :fail, test_name 388 | assert_equal retry_test, test_class 389 | assert on_failure_block_has_ran 390 | refute_nil result_in_callback 391 | assert_instance_of Minitest::Assertion, result_in_callback.failures[0] 392 | end 393 | 394 | def test_do_not_run_failure_callback_on_success 395 | on_failure_block_has_ran = false 396 | capture_stdout do 397 | retry_test = Class.new(Minitest::Test) do 398 | Minitest::Retry.use! 399 | Minitest::Retry.on_failure do 400 | on_failure_block_has_ran = true 401 | end 402 | 403 | def success 404 | assert true, 'success test' 405 | end 406 | end 407 | run_test(retry_test, :success) 408 | end 409 | refute on_failure_block_has_ran 410 | end 411 | 412 | def test_run_retry_callback_on_each_retry 413 | retry_counts, test_names, test_classes, results_in_callbacks = [], [], [], [] 414 | retry_test = nil 415 | capture_stdout do 416 | retry_test = Class.new(Minitest::Test) do 417 | Minitest::Retry.use! 418 | Minitest::Retry.on_retry do |klass, test_name, retry_count, result| 419 | retry_counts << retry_count 420 | test_names << test_name 421 | test_classes << klass 422 | results_in_callbacks << result 423 | end 424 | 425 | def fail_sometimes 426 | assert_equal 3, 0 427 | end 428 | end 429 | run_test(retry_test, :fail_sometimes) 430 | end 431 | assert_equal [1, 2, 3], retry_counts 432 | assert_equal [:fail_sometimes] * 3, test_names 433 | assert_equal [retry_test] * 3, test_classes 434 | refute_empty results_in_callbacks 435 | assert_equal [Minitest::Assertion] * 3, results_in_callbacks.map{|x| x.failures[0].class} 436 | end 437 | 438 | def test_run_consistent_failure_callback_on_failure 439 | on_consistent_failure_block_call_count = 0 440 | test_name, test_class, retry_test, result_in_callback = nil 441 | capture_stdout do 442 | retry_test = Class.new(Minitest::Test) do 443 | Minitest::Retry.use! 444 | Minitest::Retry.on_consistent_failure do |klass, failed_test, result| 445 | on_consistent_failure_block_call_count += 1 446 | test_class = klass 447 | test_name = failed_test 448 | result_in_callback = result 449 | end 450 | 451 | def fail 452 | assert false, 'fail test' 453 | end 454 | end 455 | run_test(retry_test, :fail) 456 | end 457 | assert_equal :fail, test_name 458 | assert_equal retry_test, test_class 459 | assert_equal 1, on_consistent_failure_block_call_count 460 | refute_nil result_in_callback 461 | assert_instance_of Minitest::Assertion, result_in_callback.failures[0] 462 | end 463 | 464 | class TestError < StandardError; end 465 | end 466 | --------------------------------------------------------------------------------