├── .circleci └── config.yml ├── .gitignore ├── .rspec ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE.md ├── README.md ├── Rakefile ├── bin ├── ci ├── console ├── mk_circle_config ├── rake ├── rspec └── setup ├── confidence-check.gemspec ├── lib ├── confidence-check.rb └── confidence_check │ ├── check_method.rb │ ├── confidence_check_failed.rb │ ├── for_minitest.rb │ └── for_rspec.rb └── spec ├── check_method_spec.rb ├── confidence_check_failed_spec.rb ├── for_minitest_spec.rb ├── for_rspec_spec.rb └── spec_helper.rb /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # THIS IS GENERATED - DO NOT EDIT 2 | # regenerate with bin/mk_circle_config 3 | # You are very welcome 4 | --- 5 | version: '2.1' 6 | jobs: 7 | ruby__2_6: 8 | docker: 9 | - image: cimg/ruby:2.6 10 | steps: 11 | - checkout 12 | - run: 13 | name: Setup for build 14 | command: bin/setup 15 | - run: 16 | name: Ensure bin/setup is idempotent 17 | command: bin/setup 18 | - run: 19 | name: Create the test results dir 20 | command: mkdir -p /tmp/test-results/2.6 21 | - run: 22 | name: Run all tests 23 | command: bin/ci /tmp/test-results/2.6/rspec_results.xml 24 | - store_test_results: 25 | path: "/tmp/test-results/2.6" 26 | - store_artifacts: 27 | path: "/tmp/test-results/2.6" 28 | ruby__2_7: 29 | docker: 30 | - image: cimg/ruby:2.7 31 | steps: 32 | - checkout 33 | - run: 34 | name: Setup for build 35 | command: bin/setup 36 | - run: 37 | name: Ensure bin/setup is idempotent 38 | command: bin/setup 39 | - run: 40 | name: Create the test results dir 41 | command: mkdir -p /tmp/test-results/2.7 42 | - run: 43 | name: Run all tests 44 | command: bin/ci /tmp/test-results/2.7/rspec_results.xml 45 | - store_test_results: 46 | path: "/tmp/test-results/2.7" 47 | - store_artifacts: 48 | path: "/tmp/test-results/2.7" 49 | ruby__3_0: 50 | docker: 51 | - image: cimg/ruby:3.0 52 | steps: 53 | - checkout 54 | - run: 55 | name: Setup for build 56 | command: bin/setup 57 | - run: 58 | name: Ensure bin/setup is idempotent 59 | command: bin/setup 60 | - run: 61 | name: Create the test results dir 62 | command: mkdir -p /tmp/test-results/3.0 63 | - run: 64 | name: Run all tests 65 | command: bin/ci /tmp/test-results/3.0/rspec_results.xml 66 | - store_test_results: 67 | path: "/tmp/test-results/3.0" 68 | - store_artifacts: 69 | path: "/tmp/test-results/3.0" 70 | ruby__3_1: 71 | docker: 72 | - image: cimg/ruby:3.1 73 | steps: 74 | - checkout 75 | - run: 76 | name: Setup for build 77 | command: bin/setup 78 | - run: 79 | name: Ensure bin/setup is idempotent 80 | command: bin/setup 81 | - run: 82 | name: Create the test results dir 83 | command: mkdir -p /tmp/test-results/3.1 84 | - run: 85 | name: Run all tests 86 | command: bin/ci /tmp/test-results/3.1/rspec_results.xml 87 | - store_test_results: 88 | path: "/tmp/test-results/3.1" 89 | - store_artifacts: 90 | path: "/tmp/test-results/3.1" 91 | workflows: 92 | version: 2 93 | all_rubies: 94 | jobs: 95 | - ruby__2_6 96 | - ruby__2_7 97 | - ruby__3_0 98 | - ruby__3_1 99 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | Gemfile.lock 10 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at davetron5000 (at) gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | * Code changes for style, cleanliness, eleganance, or other aesthetic stuff will not be accepted. 4 | * All proposed changes must have a clear problem statement and be based on a real-world scenario. Imagined problems will not be solved. 5 | * Tests are appreciated with all pull requests. 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Specify your gem's dependencies in confidence-check.gemspec 4 | gemspec 5 | 6 | gem "rake", "~> 12.0" 7 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | [confidence-check] Copyright (2021) (David Copeland)(“Licensor”) 2 | 3 | Hippocratic License Version Number: 2.1. 4 | 5 | Purpose. The purpose of this License is for the Licensor named above to permit the Licensee (as defined below) broad permission, if consistent with Human Rights Laws and Human Rights Principles (as each is defined below), to use and work with the Software (as defined below) within the full scope of Licensor’s copyright and patent rights, if any, in the Software, while ensuring attribution and protecting the Licensor from liability. 6 | 7 | Permission and Conditions. The Licensor grants permission by this license (“License”), free of charge, to the extent of Licensor’s rights under applicable copyright and patent law, to any person or entity (the “Licensee”) obtaining a copy of this software and associated documentation files (the “Software”), to do everything with the Software that would otherwise infringe (i) the Licensor’s copyright in the Software or (ii) any patent claims to the Software that the Licensor can license or becomes able to license, subject to all of the following terms and conditions: 8 | 9 | * Acceptance. This License is automatically offered to every person and entity subject to its terms and conditions. Licensee accepts this License and agrees to its terms and conditions by taking any action with the Software that, absent this License, would infringe any intellectual property right held by Licensor. 10 | 11 | * Notice. Licensee must ensure that everyone who gets a copy of any part of this Software from Licensee, with or without changes, also receives the License and the above copyright notice (and if included by the Licensor, patent, trademark and attribution notice). Licensee must cause any modified versions of the Software to carry prominent notices stating that Licensee changed the Software. For clarity, although Licensee is free to create modifications of the Software and distribute only the modified portion created by Licensee with additional or different terms, the portion of the Software not modified must be distributed pursuant to this License. If anyone notifies Licensee in writing that Licensee has not complied with this Notice section, Licensee can keep this License by taking all practical steps to comply within 30 days after the notice. If Licensee does not do so, Licensee’s License (and all rights licensed hereunder) shall end immediately. 12 | 13 | * Compliance with Human Rights Principles and Human Rights Laws. 14 | 15 | 1. Human Rights Principles. 16 | 17 | (a) Licensee is advised to consult the articles of the United Nations Universal Declaration of Human Rights and the United Nations Global Compact that define recognized principles of international human rights (the “Human Rights Principles”). Licensee shall use the Software in a manner consistent with Human Rights Principles. 18 | 19 | (b) Unless the Licensor and Licensee agree otherwise, any dispute, controversy, or claim arising out of or relating to (i) Section 1(a) regarding Human Rights Principles, including the breach of Section 1(a), termination of this License for breach of the Human Rights Principles, or invalidity of Section 1(a) or (ii) a determination of whether any Law is consistent or in conflict with Human Rights Principles pursuant to Section 2, below, shall be settled by arbitration in accordance with the Hague Rules on Business and Human Rights Arbitration (the “Rules”); provided, however, that Licensee may elect not to participate in such arbitration, in which event this License (and all rights licensed hereunder) shall end immediately. The number of arbitrators shall be one unless the Rules require otherwise. 20 | 21 | Unless both the Licensor and Licensee agree to the contrary: (1) All documents and information concerning the arbitration shall be public and may be disclosed by any party; (2) The repository referred to under Article 43 of the Rules shall make available to the public in a timely manner all documents concerning the arbitration which are communicated to it, including all submissions of the parties, all evidence admitted into the record of the proceedings, all transcripts or other recordings of hearings and all orders, decisions and awards of the arbitral tribunal, subject only to the arbitral tribunal's powers to take such measures as may be necessary to safeguard the integrity of the arbitral process pursuant to Articles 18, 33, 41 and 42 of the Rules; and (3) Article 26(6) of the Rules shall not apply. 22 | 23 | 2. Human Rights Laws. The Software shall not be used by any person or entity for any systems, activities, or other uses that violate any Human Rights Laws. “Human Rights Laws” means any applicable laws, regulations, or rules (collectively, “Laws”) that protect human, civil, labor, privacy, political, environmental, security, economic, due process, or similar rights; provided, however, that such Laws are consistent and not in conflict with Human Rights Principles (a dispute over the consistency or a conflict between Laws and Human Rights Principles shall be determined by arbitration as stated above). Where the Human Rights Laws of more than one jurisdiction are applicable or in conflict with respect to the use of the Software, the Human Rights Laws that are most protective of the individuals or groups harmed shall apply. 24 | 25 | 3. Indemnity. Licensee shall hold harmless and indemnify Licensor (and any other contributor) against all losses, damages, liabilities, deficiencies, claims, actions, judgments, settlements, interest, awards, penalties, fines, costs, or expenses of whatever kind, including Licensor’s reasonable attorneys’ fees, arising out of or relating to Licensee’s use of the Software in violation of Human Rights Laws or Human Rights Principles. 26 | 27 | * Failure to Comply. Any failure of Licensee to act according to the terms and conditions of this License is both a breach of the License and an infringement of the intellectual property rights of the Licensor (subject to exceptions under Laws, e.g., fair use). In the event of a breach or infringement, the terms and conditions of this License may be enforced by Licensor under the Laws of any jurisdiction to which Licensee is subject. Licensee also agrees that the Licensor may enforce the terms and conditions of this License against Licensee through specific performance (or similar remedy under Laws) to the extent permitted by Laws. For clarity, except in the event of a breach of this License, infringement, or as otherwise stated in this License, Licensor may not terminate this License with Licensee. 28 | 29 | * Enforceability and Interpretation. If any term or provision of this License is determined to be invalid, illegal, or unenforceable by a court of competent jurisdiction, then such invalidity, illegality, or unenforceability shall not affect any other term or provision of this License or invalidate or render unenforceable such term or provision in any other jurisdiction; provided, however, subject to a court modification pursuant to the immediately following sentence, if any term or provision of this License pertaining to Human Rights Laws or Human Rights Principles is deemed invalid, illegal, or unenforceable against Licensee by a court of competent jurisdiction, all rights in the Software granted to Licensee shall be deemed null and void as between Licensor and Licensee. Upon a determination that any term or provision is invalid, illegal, or unenforceable, to the extent permitted by Laws, the court may modify this License to affect the original purpose that the Software be used in compliance with Human Rights Principles and Human Rights Laws as closely as possible. The language in this License shall be interpreted as to its fair meaning and not strictly for or against any party. 30 | 31 | * Disclaimer. TO THE FULL EXTENT ALLOWED BY LAW, THIS SOFTWARE COMES “AS IS,” WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED, AND LICENSOR AND ANY OTHER CONTRIBUTOR SHALL NOT BE LIABLE TO ANYONE FOR ANY DAMAGES OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THIS LICENSE, UNDER ANY KIND OF LEGAL CLAIM. 32 | 33 | This Hippocratic License is an Ethical Source license (https://ethicalsource.dev) and is offered for use by licensors and licensees at their own risk, on an “AS IS” basis, and with no warranties express or implied, to the maximum extent permitted by Laws. 34 | 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # confidence\_check - Get confidence in your test setup 2 | 3 | [![](https://circleci.com/gh/sustainable-rails/confidence-check.svg?style=shield)](https://app.circleci.com/pipelines/github/sustainable-rails/confidence-check) 4 | 5 | Sometimes tests have some setup required before you run the test, and sometimes that setup is complicated, and sometimes that 6 | setup can break if other parts of the system aren't working. When that happens, you can a test failure that doesn't mean your 7 | code is broken, just that some other code is broken. 8 | 9 | ```ruby 10 | visit "/" 11 | click_on "sign_up" 12 | 13 | fill_in :username, with: "davetron5000" 14 | expect { 15 | click_on "Create an Account" 16 | }.to change { 17 | Account.count 18 | }.by(1) 19 | ``` 20 | 21 | What if the `sign_up` link is broken, but your account registration code is working perfectly? You'll get a test failure on 22 | that call to `fill_in`. This is confusing. 23 | 24 | ```ruby 25 | confidence_check do 26 | visit "/" 27 | click_on "sign_up" 28 | end 29 | 30 | fill_in :username, with: "davetron5000" 31 | expect { 32 | click_on "Create an Account" 33 | }.to change { 34 | Account.count 35 | }.by(1) 36 | ``` 37 | 38 | Now, if something goes wrong getting to the page, i.e. setting up for your test, you'll get a different error: CONFIDENCE CHECK 39 | FAILED. 40 | 41 | This can help tremendously with isolating the root cause of a test failure. Instead of digging into the code you are testing, you can quickly see that some *other* code that is needed just to run a test is broken. 42 | 43 | ## Install 44 | 45 | ``` 46 | > gem install confidence-check 47 | ``` 48 | 49 | Or, if using a `Gemfile`: 50 | 51 | ```ruby 52 | gem "confidence-check" 53 | ``` 54 | 55 | ### Setup - RSpec 56 | 57 | 1. Include `ConfidenceCheck::ForRSpec` in your test. The most common way is in your `spec_helper.rb`: 58 | 59 | ```ruby 60 | require "confidence_check/for_rspec" 61 | RSpec.configure do |c| 62 | c.include ConfidenceCheck::ForRSpec 63 | end 64 | ``` 65 | 2. You can now call `confidence_check` anywhere you need to assert your test is set up: 66 | 67 | ```ruby 68 | RSpec.describe Person do 69 | include ConfidenceCheck 70 | describe "#age" do 71 | it "is based on birthdate" do 72 | person = create(:person) 73 | 74 | confidence_check do 75 | expect(person.birthdate).not_to be_nil 76 | end 77 | 78 | expect(person.age).to eq(47) 79 | end 80 | end 81 | end 82 | ``` 83 | 84 | ## Setup - Minitest 85 | 86 | 1. Include `ConfidenceCheck::ForMinitest` in your test. For Rails, you'd include this in `ActiveSupport::TestCase`: 87 | 88 | ```ruby 89 | # test/test_helper.rb 90 | require "confidence_check/for_minitest" 91 | 92 | class ActiveSupport::TestCase 93 | include ConfidenceCheck::ForMinitest 94 | 95 | # ... 96 | end 97 | ``` 98 | 2. You can now call `confidence_check` anywhere you need to assert your test is set up: 99 | 100 | ```ruby 101 | class PersonTest < MiniTest::Test 102 | include ConfidenceCheck 103 | def test_age 104 | person = create(:person) 105 | 106 | confidence_check do 107 | refute_nil person.birthdate 108 | end 109 | 110 | assert_equal 47,person.age 111 | end 112 | end 113 | ``` 114 | 115 | ### Setup - Custom 116 | 117 | 1. The module `ConfidenceCheck::CheckMethod` makes a call to `exception_klasses`, which returns an array of exception classes you want to rescue inside a `confidence_check` call. You'll need to implement this yourself: 118 | 119 | ```ruby 120 | require "confidence_check/check_method" 121 | module MyCustomConfidenceCheck 122 | include ConfidenceCheck::CheckMethod 123 | def exception_klasses 124 | [ MyCustomError ] 125 | end 126 | end 127 | ``` 128 | 2. Note that you can include any of the other modules as a base. For example, if you want to use RSpec but add your own 129 | additional method: 130 | ```ruby 131 | module MyCustomConfidenceCheck 132 | include ConfidenceCheck::ForRSpec 133 | include ConfidenceCheck::CheckMethod 134 | def exception_klasses 135 | super + [ MyCustomError ] 136 | end 137 | end 138 | ``` 139 | 3. Now, you need to include this module in your tests. HOw to do this dependson how you are writing tests, but hopefully it's 140 | as simple as `include MyCustomConfidenceCheck` 141 | 142 | ### Setup - with Capybara 143 | 144 | Capybara raises several exceptions if navigation or page manipulation fails. These are almost always the types of failures 145 | a confidence check should notify you about because they usually mean a page or pages aren't even working enough for you to 146 | execute a test. 147 | 148 | If you use the `*WithCapybara` versions of the modules, you can wrap your Capybara navigation commands in a `confidence_check`: 149 | 150 | #### RSpec 151 | 152 | ```ruby 153 | require "confidence_check/for_rspec" 154 | RSpec.configure do |c| 155 | c.include ConfidenceCheck::ForRSpec::WithCapybara 156 | end 157 | ``` 158 | 159 | #### Minitest 160 | 161 | ```ruby 162 | # test/test_helper.rb 163 | require "confidence_check/for_minitest" 164 | 165 | class ActiveSupport::TestCase 166 | include ConfidenceCheck::ForMinitest::WithCapybara 167 | 168 | # ... 169 | end 170 | ``` 171 | 172 | #### Custom 173 | 174 | ```ruby 175 | require "confidence_check/check_method" 176 | module MyCustomConfidenceCheck 177 | include ConfidenceCheck::CheckMethod 178 | def exception_klasses 179 | [ MyCustomError, Capybara::CapybaraError ] 180 | end 181 | end 182 | 183 | # OR, to e.g. re-use RSpec's 184 | 185 | require "confidence_check/for_rspec" 186 | module MyCustomConfidenceCheck 187 | include ConfidenceCheck::ForRSpec::WithCapybara 188 | include ConfidenceCheck::CheckMethod 189 | def exception_klasses 190 | super + [ MyCustomError ] 191 | end 192 | end 193 | ``` 194 | 195 | ## Developing 196 | 197 | * Set up with `bin/setup` 198 | * Run tests with `bin/ci` 199 | 200 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require 'rspec/core/rake_task' 3 | RSpec::Core::RakeTask.new(:spec) 4 | task :default => :spec 5 | -------------------------------------------------------------------------------- /bin/ci: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | echo "[bin/ci] Running tests" 5 | if [ -z $1 ]; then 6 | bin/rspec --format=doc 7 | else 8 | echo "[bin/ci] Generating JUnit XML output to $1" 9 | bin/rspec --format RspecJunitFormatter --out $1 --format=doc 10 | fi 11 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "confidence-check" 5 | 6 | require "irb" 7 | IRB.start(__FILE__) 8 | -------------------------------------------------------------------------------- /bin/mk_circle_config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "yaml" 4 | require "pathname" 5 | 6 | circle_config = { 7 | "version" => "2.1", 8 | "jobs" => {}, 9 | "workflows" => { 10 | "version" => 2, 11 | "all_rubies" => { 12 | "jobs" => [ 13 | ], 14 | }, 15 | } 16 | } 17 | 18 | supported_rubies = [ 19 | "2.6", 20 | "2.7", 21 | "3.0", 22 | "3.1", 23 | ] 24 | 25 | supported_rubies.each do |ruby_verison| 26 | 27 | test_results_dir = "/tmp/test-results/#{ruby_verison}" 28 | job_name = "ruby__#{ruby_verison.gsub(/\./,"_")}" 29 | 30 | job = { 31 | "docker" => [ 32 | { 33 | "image" => "cimg/ruby:#{ruby_verison}", 34 | } 35 | ], 36 | "steps" => [ 37 | "checkout", 38 | { 39 | "run" => { 40 | "name" => "Setup for build", 41 | "command" => "bin/setup", 42 | } 43 | }, 44 | { 45 | "run" => { 46 | "name" => "Ensure bin/setup is idempotent", 47 | "command" => "bin/setup", 48 | } 49 | }, 50 | { 51 | "run" => { 52 | "name" => "Create the test results dir", 53 | "command" => "mkdir -p #{test_results_dir}", 54 | } 55 | }, 56 | { 57 | "run" => { 58 | "name" => "Run all tests", 59 | "command" => "bin/ci #{test_results_dir}/rspec_results.xml", 60 | } 61 | }, 62 | { 63 | "store_test_results" => { 64 | "path" => test_results_dir, 65 | } 66 | }, 67 | { 68 | "store_artifacts" => { 69 | "path" => test_results_dir, 70 | } 71 | }, 72 | ] 73 | } 74 | circle_config["jobs"][job_name] = job 75 | circle_config["workflows"]["all_rubies"]["jobs"] << job_name 76 | end 77 | 78 | circle_config_file = (Pathname(__FILE__).dirname / ".." / ".circleci" / "config.yml").expand_path 79 | File.open(circle_config_file,"w") do |file| 80 | file.puts "# THIS IS GENERATED - DO NOT EDIT" 81 | file.puts "# regenerate with bin/mk_circle_config" 82 | file.puts "# You are very welcome" 83 | file.puts circle_config.to_yaml 84 | end 85 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rake' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rake", "rake") 30 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rspec' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rspec-core", "rspec") 30 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle check || bundle install 7 | -------------------------------------------------------------------------------- /confidence-check.gemspec: -------------------------------------------------------------------------------- 1 | require_relative "lib/confidence-check" 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "confidence-check" 5 | spec.version = ConfidenceCheck::VERSION 6 | spec.authors = ["Dave Copeland"] 7 | spec.email = ["davec@naildrivin5.com"] 8 | spec.summary = %q{Assert that the conditions of your test are correct before testing} 9 | spec.homepage = "https://sustainable-rails.com" 10 | spec.license = "Hippocratic" 11 | 12 | spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0") 13 | 14 | spec.metadata["homepage_uri"] = spec.homepage 15 | spec.metadata["source_code_uri"] = "https://github.com/sustainable-rails/confidence-check" 16 | spec.metadata["changelog_uri"] = "https://github.com/sustainable-rails/confidence-check/releases" 17 | 18 | spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do 19 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 20 | end 21 | spec.bindir = "exe" 22 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 23 | spec.require_paths = ["lib"] 24 | 25 | spec.add_development_dependency("rspec") 26 | spec.add_development_dependency("minitest") 27 | spec.add_development_dependency("capybara") 28 | spec.add_development_dependency("rspec_junit_formatter") 29 | end 30 | -------------------------------------------------------------------------------- /lib/confidence-check.rb: -------------------------------------------------------------------------------- 1 | module ConfidenceCheck 2 | VERSION = "1.0.1" 3 | end 4 | -------------------------------------------------------------------------------- /lib/confidence_check/check_method.rb: -------------------------------------------------------------------------------- 1 | require_relative "confidence_check_failed" 2 | 3 | module ConfidenceCheck 4 | module CheckMethod 5 | def confidence_check(context=nil, &block) 6 | if block.nil? 7 | raise "#confidence_check requires a block" 8 | end 9 | block.() 10 | rescue Exception => ex 11 | $stdout.puts context.inspect if context 12 | if exception_klasses.any? {|_| ex.kind_of?(_) } 13 | raise ConfidenceCheckedFailed.new(ex) 14 | else 15 | raise 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/confidence_check/confidence_check_failed.rb: -------------------------------------------------------------------------------- 1 | require "forwardable" 2 | module ConfidenceCheck 3 | class ConfidenceCheckedFailed < Exception 4 | extend Forwardable 5 | def initialize(exception) 6 | super("CONFIDENCE CHECK FAILED: #{exception.message}") 7 | @exception = exception 8 | end 9 | 10 | def_delegators :@exception, :backtrace, :cause 11 | end 12 | end 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/confidence_check/for_minitest.rb: -------------------------------------------------------------------------------- 1 | require_relative "check_method" 2 | 3 | module ConfidenceCheck 4 | module ForMinitest 5 | include ConfidenceCheck::CheckMethod 6 | def exception_klasses 7 | [::Minitest::Assertion] 8 | end 9 | 10 | module WithCapybara 11 | include ConfidenceCheck::CheckMethod 12 | include ConfidenceCheck::ForMinitest 13 | def exception_klasses 14 | super + [ ::Capybara::CapybaraError ] 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/confidence_check/for_rspec.rb: -------------------------------------------------------------------------------- 1 | require_relative "check_method" 2 | 3 | module ConfidenceCheck 4 | module ForRSpec 5 | include ConfidenceCheck::CheckMethod 6 | def exception_klasses 7 | [::RSpec::Expectations::ExpectationNotMetError ] 8 | end 9 | 10 | module WithCapybara 11 | include ConfidenceCheck::CheckMethod 12 | include ConfidenceCheck::ForRSpec 13 | def exception_klasses 14 | super + [ ::Capybara::CapybaraError ] 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/check_method_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | require "confidence_check/check_method" 3 | 4 | RSpec.describe ConfidenceCheck::CheckMethod do 5 | class MyError < StandardError 6 | end 7 | 8 | module MyCustomChecks 9 | def exception_klasses 10 | [ MyError, ArgumentError ] 11 | end 12 | include ConfidenceCheck::CheckMethod 13 | end 14 | 15 | class IsolateCheckMethod 16 | extend MyCustomChecks 17 | end 18 | 19 | describe "#confidence_check" do 20 | context "when a block is given" do 21 | context "and the code inside the block raises an error" do 22 | [ 23 | MyError, 24 | ArgumentError, 25 | ].each do |exception_klass| 26 | context "and that error is #{exception_klass}" do 27 | it "raises a ConfidenceCheckedFailed" do 28 | expect { 29 | IsolateCheckMethod.confidence_check do 30 | raise exception_klass, "OH NO" 31 | end 32 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 33 | end 34 | context "when a context was provided" do 35 | it "prints the context before raising" do 36 | allow($stdout).to receive(:puts) 37 | expect { 38 | IsolateCheckMethod.confidence_check("some context") do 39 | raise exception_klass, "OH NO" 40 | end 41 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 42 | expect($stdout).to have_received(:puts).with("some context".inspect) 43 | end 44 | end 45 | end 46 | end 47 | context "but the error is not one we are expecting" do 48 | it "re-raises the error" do 49 | expect { 50 | IsolateCheckMethod.confidence_check do 51 | raise "WTF" 52 | end 53 | }.to raise_error(StandardError,"WTF") 54 | end 55 | context "when a context was provided" do 56 | it "prints the context before raising" do 57 | allow($stdout).to receive(:puts) 58 | expect { 59 | IsolateCheckMethod.confidence_check("some context") do 60 | raise "WTF" 61 | end 62 | }.to raise_error(StandardError,"WTF") 63 | expect($stdout).to have_received(:puts).with("some context".inspect) 64 | end 65 | end 66 | end 67 | context "but the code inside the block does not raise an error" do 68 | it "does not raise an error" do 69 | expect { 70 | IsolateCheckMethod.confidence_check do 71 | expect(false).to eq(false) 72 | end 73 | }.not_to raise_error 74 | end 75 | end 76 | end 77 | context "when no block is given" do 78 | it "raises an error" do 79 | expect { 80 | IsolateCheckMethod.confidence_check 81 | }.to raise_error(/requires a block/i) 82 | end 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/confidence_check_failed_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | require "confidence_check/confidence_check_failed" 3 | 4 | RSpec.describe ConfidenceCheck::ConfidenceCheckedFailed do 5 | def raised_exception 6 | expect(false).to eq(true) 7 | rescue Exception => ex 8 | ex 9 | end 10 | 11 | describe "#message" do 12 | it "indicates the failure is due to a confidence check not a test failure" do 13 | actual_exception = raised_exception 14 | exception = ConfidenceCheck::ConfidenceCheckedFailed.new(actual_exception) 15 | expect(exception.message).to eq("CONFIDENCE CHECK FAILED: #{actual_exception.message}") 16 | end 17 | end 18 | describe "#backtrace" do 19 | it "delegates to the caught exception" do 20 | actual_exception = raised_exception 21 | exception = ConfidenceCheck::ConfidenceCheckedFailed.new(actual_exception) 22 | expect(exception.backtrace).to be(actual_exception.backtrace) 23 | end 24 | end 25 | describe "#cause" do 26 | it "delegates to the caught exception" do 27 | actual_exception = raised_exception 28 | exception = ConfidenceCheck::ConfidenceCheckedFailed.new(actual_exception) 29 | expect(exception.cause).to be(actual_exception.cause) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/for_minitest_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | require "minitest" 3 | require "capybara" 4 | require "confidence_check/for_minitest" 5 | 6 | RSpec.describe ConfidenceCheck::ForMinitest do 7 | class IsolateForMinitest 8 | extend ConfidenceCheck::ForMinitest 9 | extend Minitest::Assertions 10 | @assertions = 0 11 | def self.assertions 12 | @assertions || 0 13 | end 14 | def self.assertions=(new_value) 15 | @assertions = new_value 16 | end 17 | end 18 | describe "#confidence_check" do 19 | it "handles ExpectationNotMetError" do 20 | expect { 21 | IsolateForMinitest.confidence_check do 22 | IsolateForMinitest.assert false 23 | end 24 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 25 | end 26 | end 27 | describe ConfidenceCheck::ForMinitest::WithCapybara do 28 | class IsolateForMinitestWithCapybara < IsolateForMinitest 29 | extend ConfidenceCheck::ForMinitest::WithCapybara 30 | end 31 | describe "#confidence_check" do 32 | it "handles ExpectationNotMetError" do 33 | expect { 34 | IsolateForMinitestWithCapybara.confidence_check do 35 | IsolateForMinitest.assert false 36 | end 37 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 38 | end 39 | it "handles Capybara::CapybaraError" do 40 | expect { 41 | IsolateForMinitestWithCapybara.confidence_check do 42 | raise Capybara::ElementNotFound 43 | end 44 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /spec/for_rspec_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | require "capybara" 3 | require "confidence_check/for_rspec" 4 | 5 | RSpec.describe ConfidenceCheck::ForRSpec do 6 | class IsolateForRSpec 7 | extend ConfidenceCheck::ForRSpec 8 | end 9 | describe "#confidence_check" do 10 | it "handles ExpectationNotMetError" do 11 | expect { 12 | IsolateForRSpec.confidence_check do 13 | expect(false).to eq(true) 14 | end 15 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 16 | end 17 | it "handles MultipleExpectationsNotMetError" do 18 | expect { 19 | IsolateForRSpec.confidence_check do 20 | aggregate_failures do 21 | expect(false).to eq(true) 22 | expect(true).to eq(false) 23 | end 24 | end 25 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 26 | end 27 | end 28 | describe "with Capybara" do 29 | class IsolateForRSpecWithCapybara 30 | extend ConfidenceCheck::ForRSpec::WithCapybara 31 | end 32 | describe "#confidence_check" do 33 | it "handles Capybara::CapybaraError" do 34 | expect { 35 | IsolateForRSpecWithCapybara.confidence_check do 36 | raise Capybara::ElementNotFound 37 | end 38 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 39 | end 40 | it "handles ExpectationNotMetError" do 41 | expect { 42 | IsolateForRSpecWithCapybara.confidence_check do 43 | expect(false).to eq(true) 44 | end 45 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 46 | end 47 | it "handles MultipleExpectationsNotMetError" do 48 | expect { 49 | IsolateForRSpecWithCapybara.confidence_check do 50 | aggregate_failures do 51 | expect(false).to eq(true) 52 | expect(true).to eq(false) 53 | end 54 | end 55 | }.to raise_error(ConfidenceCheck::ConfidenceCheckedFailed) 56 | end 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | RSpec.configure do |config| 2 | config.expect_with :rspec do |expectations| 3 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 4 | end 5 | 6 | config.mock_with :rspec do |mocks| 7 | mocks.verify_partial_doubles = true 8 | end 9 | 10 | config.shared_context_metadata_behavior = :apply_to_host_groups 11 | 12 | config.disable_monkey_patching! 13 | 14 | config.warnings = true 15 | 16 | if config.files_to_run.one? 17 | config.default_formatter = "doc" 18 | end 19 | 20 | config.order = :random 21 | 22 | Kernel.srand config.seed 23 | end 24 | --------------------------------------------------------------------------------