├── .circleci └── config.yml ├── .codecov.yml ├── .flake8 ├── .gitignore ├── .readthedocs.yaml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── dev-requirements.txt ├── docs ├── changelog.rst ├── conf.py └── index.rst ├── invoke.yml ├── pytest.ini ├── pytest_relaxed ├── __init__.py ├── _version.py ├── classes.py ├── fixtures.py ├── plugin.py ├── raises.py ├── reporter.py └── trap.py ├── setup.py ├── tasks.py └── tests ├── test_collection.py ├── test_display.py └── test_raises.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | orb: invocations/orb@1.3.1 5 | 6 | 7 | jobs: 8 | confirm-loaded: 9 | executor: 10 | name: orb/default 11 | version: "3.6" 12 | steps: 13 | - orb/setup 14 | - run: pytest -VV | grep pytest-relaxed 15 | - orb/debug 16 | 17 | 18 | workflows: 19 | main: 20 | jobs: 21 | - orb/lint: 22 | name: Lint 23 | - orb/format: 24 | name: Style check 25 | - orb/coverage: 26 | name: Test 27 | - confirm-loaded: 28 | name: Confirm plugin loads into pytest 29 | - orb/docs: 30 | name: Docs 31 | task: "docs --nitpick" 32 | - orb/test-release: 33 | name: Release test 34 | - orb/test: 35 | name: << matrix.version >> 36 | # It's not worth testing on other interpreters if the baseline one 37 | # failed. Can't run >4 jobs at a time anyhow! 38 | requires: ["Test"] 39 | matrix: 40 | parameters: 41 | version: ["3.7", "3.8", "3.9", "3.10", "3.11"] 42 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | coverage: 3 | precision: 0 4 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,build,dist 3 | ignore = E124,E125,E128,E261,E301,E302,E303,W503 4 | max-line-length = 79 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .coverage 3 | htmlcov 4 | docs/_build 5 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-22.04" 5 | tools: 6 | python: "3.7" 7 | 8 | python: 9 | install: 10 | - requirements: dev-requirements.txt 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Jeff Forcier. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 17 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include tasks.py 4 | recursive-include docs * 5 | recursive-exclude docs/_build * 6 | include dev-requirements.txt 7 | recursive-include tests * 8 | recursive-exclude tests *.pyc *.pyo 9 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |version| |python| |license| |ci| |coverage| 2 | 3 | .. |version| image:: https://img.shields.io/pypi/v/pytest-relaxed 4 | :target: https://pypi.org/project/pytest-relaxed/ 5 | :alt: PyPI - Package Version 6 | .. |python| image:: https://img.shields.io/pypi/pyversions/pytest-relaxed 7 | :target: https://pypi.org/project/pytest-relaxed/ 8 | :alt: PyPI - Python Version 9 | .. |license| image:: https://img.shields.io/pypi/l/pytest-relaxed 10 | :target: https://github.com/bitprophet/pytest-relaxed/blob/main/LICENSE 11 | :alt: PyPI - License 12 | .. |ci| image:: https://img.shields.io/circleci/build/github/bitprophet/pytest-relaxed/main 13 | :target: https://app.circleci.com/pipelines/github/bitprophet/pytest-relaxed 14 | :alt: CircleCI 15 | .. |coverage| image:: https://img.shields.io/codecov/c/gh/bitprophet/pytest-relaxed 16 | :target: https://app.codecov.io/gh/bitprophet/pytest-relaxed 17 | :alt: Codecov 18 | 19 | ============== 20 | pytest-relaxed 21 | ============== 22 | 23 | ``pytest-relaxed`` provides 'relaxed' test discovery for pytest. 24 | 25 | It is the spiritual successor to https://pypi.python.org/pypi/spec, but is 26 | built for ``pytest`` instead of ``nosetests``, and rethinks some aspects of 27 | the design (such as increased ability to opt-in to various behaviors.) 28 | 29 | For a development roadmap, see the maintainer's `roadmap page 30 | `_. 31 | 32 | 33 | Rationale 34 | ========= 35 | 36 | Has it ever felt strange to you that we put our tests in ``tests/``, then name 37 | the files ``test_foo.py``, name the test classes ``TestFoo``, and finally 38 | name the test methods ``test_foo_bar``? Especially when almost all of the code 39 | inside of ``tests/`` is, well, *tests*? 40 | 41 | This pytest plugin takes a page from the rest of Python, where you don't have 42 | to explicitly note public module/class members, but only need to hint as to 43 | which ones are private. By default, all files and objects pytest is told to 44 | scan will be considered tests; to mark something as not-a-test, simply prefix 45 | it with an underscore. 46 | 47 | 48 | Relaxed discovery 49 | ================= 50 | 51 | The "it's a test by default unless underscored" approach works for files:: 52 | 53 | tests 54 | ├── _util.py 55 | ├── one_module.py 56 | └── another_module.py 57 | 58 | It's applied to module members:: 59 | 60 | def _helper(): 61 | pass 62 | 63 | def one_thing(): 64 | assert True 65 | 66 | def another_thing(): 67 | assert False 68 | 69 | def yet_another(): 70 | assert _helper() == 'something' 71 | 72 | And to class members:: 73 | 74 | class SomeObject: 75 | def behavior_one(self): 76 | assert True 77 | 78 | def another_behavior(self): 79 | assert False 80 | 81 | def _helper(self): 82 | pass 83 | 84 | def it_does_things(self): 85 | assert self._helper() == 'whatever' 86 | 87 | Special cases 88 | ------------- 89 | 90 | As you might expect, there are a few more special cases around discovery to 91 | avoid fouling up common test extensions: 92 | 93 | - Files named ``conftest.py`` aren't treated as tests, because they do special 94 | pytest things; 95 | - Module and class members named ``setup_(module|class|method|function)`` are 96 | not considered tests, as they are how pytest implements classic/xunit style 97 | setup and teardown; 98 | - Objects decorated as fixtures with ``@pytest.fixture`` are, of course, 99 | also skipped. 100 | 101 | Backwards compatibility 102 | ----------------------- 103 | 104 | If you like the idea of pytest-relaxed but have a large test suite, it may be 105 | daunting to think about "upgrading" it all in one go. It's relatively simple to 106 | arrive at a 'hybrid' test suite where your legacy tests still run normally (as 107 | long as they're already pytest-compatible, which is true for most unittest 108 | suites) but 'relaxed' style tests also work as expected. 109 | 110 | - The only change you'll still have to make is renaming 'helper' files (any 111 | whose name doesn't start with ``test_``) so their names begin with an 112 | underscore; then, of course, search and replace any imports of such files. 113 | - ``pytest-relaxed`` explicitly sidesteps around anything that looks like 114 | "classic" test files (i.e. named ``test_*``), allowing pytest's native 115 | collection to take effect. Such files should not need any alteration. 116 | - Our reporter (display) functionality still works pretty well with legacy 117 | style tests; test prefixes and suffixes are stripped at display time, so 118 | ``TestMyThing.test_something`` still shows up as if it was written in relaxed 119 | style: ``MyThing`` w/ nested ``something``. 120 | 121 | - However, because we don't *collect* such tests, nesting and other 122 | features we offer won't work until you've renamed the files to not start 123 | with ``test_``, and changed any classes to not inherit from 124 | ``unittest.TestCase`` or similar. 125 | 126 | 127 | Nested class organization 128 | ========================= 129 | 130 | On top of the relaxed discovery algorithm, ``pytest-relaxed`` also lets you 131 | organize tests in a nested fashion, again like the ``spec`` nose plugin or the 132 | tools that inspired it, such as Ruby's ``rspec``. 133 | 134 | This is purely optional, but we find it's a nice middle ground between having a 135 | proliferation of files or suffering a large, flat test namespace making it hard 136 | to see which feature areas have been impacted by a bug (or whatnot). 137 | 138 | The feature is enabled by using nested/inner classes, like so:: 139 | 140 | class SomeObject: 141 | def basic_behavior(self): 142 | assert True 143 | 144 | class init: 145 | "__init__" 146 | 147 | def no_args_required(self): 148 | assert True 149 | 150 | def accepts_some_arg(self): 151 | assert True 152 | 153 | def sets_up_config(self): 154 | assert False 155 | 156 | class some_method: 157 | def accepts_whatever_params(self): 158 | assert False 159 | 160 | def base_behavior(self): 161 | assert True 162 | 163 | class when_config_says_foo: 164 | def it_behaves_like_this(self): 165 | assert False 166 | 167 | class when_config_says_bar: 168 | def it_behaves_like_this(self): 169 | assert True 170 | 171 | Test discovery on these inner classes is recursive, so you *can* nest them as 172 | deeply as you like. Naturally, as with all Python code, sometimes you can have 173 | too much of a good thing...but that's up to you. 174 | 175 | 176 | Nested class attributes 177 | ----------------------- 178 | 179 | If you're namespacing your tests via nested classes, you may find yourself 180 | wanting to reference the enclosing "scope" of the outer classes they live in, 181 | such as class attributes. pytest-relaxed automatically copies such attributes 182 | onto inner classes during the test collection phase, allowing you to write code 183 | like this:: 184 | 185 | class Outer: 186 | behavior_one = True 187 | 188 | def outer_test(self): 189 | assert self.behavior_one 190 | 191 | class Inner: 192 | behavior_two = True 193 | 194 | def inner_test(self): 195 | assert self.behavior_one and self.behavior_two 196 | 197 | Notably: 198 | 199 | - The behavior is nested, infinitely, as you might expect; 200 | - Attributes that look like test classes or methods themselves, are not copied 201 | (though others, i.e. ones named with a leading underscore, are); 202 | - Only attributes _not_ already present on the inner class are copied; thus 203 | inner classes may naturally "override" attributes, just as with class 204 | inheritance. 205 | 206 | 207 | Other test helpers 208 | ================== 209 | 210 | ``pytest-relaxed`` offers a few other random lightweight test-related utilities 211 | that don't merit their own PyPI entries (most ported from ``spec``), such as: 212 | 213 | - ``trap``, a decorator for use on test functions and/or test 214 | helpers/subroutines which is similar to pytest's own ``capsys``/``capfd`` 215 | fixtures in that it allows capture of stdout/err. 216 | 217 | - It offers a slightly simpler API: it replaces ``sys.(stdout|stderr)`` with 218 | ``IO`` objects which can be ``getvalue()``'d as needed. 219 | - More importantly, it can wrap arbitrary callables, which is useful for 220 | code-sharing use cases that don't easily fit into the design of fixtures. 221 | 222 | - ``raises``, a wrapper around ``pytest.raises`` which works as a decorator, 223 | similar to the Nose testing tool of the same name. 224 | 225 | 226 | Nested output display 227 | ===================== 228 | 229 | Continuing in the "port of ``spec`` / inspired by RSpec and friends" vein, 230 | ``pytest-relaxed`` greatly enhances pytest's verbose display mode: 231 | 232 | - Tests are shown in a nested, tree-like fashion, with 'header' lines shown for 233 | modules, classes (including nested classes) and so forth. 234 | - The per-test-result lines thus consist of just the test names, and are 235 | colorized (similar to the built-in verbose mode) based on 236 | success/failure/skip. 237 | - Headers and test names are massaged to look more human-readable, such as 238 | replacing underscores with spaces. 239 | 240 | *Unlike* ``spec``, this functionality doesn't affect normal/non-verbose output 241 | at all, and can be disabled entirely, allowing you to use the relaxed test 242 | discovery alongside normal pytest verbose display or your favorite pytest 243 | output plugins (such as ``pytest-sugar``.) 244 | 245 | 246 | Installation & use 247 | ================== 248 | 249 | As with most pytest plugins, it's quite simple: 250 | 251 | - ``pip install pytest-relaxed``; 252 | - Tell pytest where your tests live via the ``testpaths`` option; otherwise 253 | pytest-relaxed will cause pytest to load all of your non-test code as tests! 254 | - Not required, but **strongly recommended**: configure pytest's default 255 | filename pattern (``python_files``) to be an unqualified glob (``*``). 256 | 257 | - This doesn't impact (our) test discovery, but pytest's assertion 258 | 'rewriting' (the feature that turns ``assert var == othervar`` into 259 | ``assert 17 == 2`` during error display) reuses this setting when 260 | determining which files to manipulate. 261 | 262 | - Thus, a recommended ``setup.cfg`` (or ``pytest.ini``, sans the header) is:: 263 | 264 | [tool:pytest] 265 | testpaths = tests 266 | python_files = * 267 | 268 | - Write some tests, as exampled above; 269 | - ``pytest`` to run the tests, and you're done! 270 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Direct dev dependencies 3 | # 4 | 5 | # Linting/formatting 6 | black==22.8.0 7 | flake8==5.0.4 8 | # Packaging 9 | twine==3.8.0 10 | setuptools>=56.0.0 11 | # Test coverage (see note in tasks.py::coverage) 12 | coverage==6.2 13 | # Task running 14 | invoke>=2 15 | invocations>=3.3 16 | # Docs (also transitively via invocations; consider nuking again) 17 | releases>=2 18 | 19 | # Ourselves! 20 | -e . 21 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Changelog 3 | ========= 4 | 5 | - :release:`2.0.2 <2024-03-29>` 6 | - :bug:`32` Fix dangling compatibility issues with pytest version 8.x. Thanks 7 | to Alex Gaynor for the patch! 8 | - :release:`2.0.1 <2023-05-22>` 9 | - :bug:`9` Don't try loading Pytest fixture functions as if they were test 10 | functions. Classifying this as a bug even though it's a moderately sized 11 | change in behavior; it's vanishingly unlikely anybody was relying on this 12 | somehow! Thanks to ``@cajopa`` for the report. 13 | - :release:`2.0.0 <2022-12-31>` 14 | - :bug:`- major` Prior to version 2, we failed to correctly support true Pytest 15 | setup/teardown methods (i.e. ``setup_method`` and ``teardown_method``) and 16 | these would not get copied to inner class scopes. This has been fixed. We 17 | still support old nose-style ``setup``/``teardown`` for now, despite them 18 | going away in Pytest 8. 19 | - :support:`-` Modernize codebase/project a bunch: 20 | 21 | - Dropped support for Python <3.6 (including 2.7) 22 | - Pytest support upgraded to support, **and require**, Pytest >=7. 23 | 24 | - This plugin never worked on Pytests 5 and 6 anyways, and supporting 5-7 25 | appears to require a lot more effort than just 7. 26 | 27 | - Behavioral changes in Pytest internals have fixed a handful of sorta-bugs 28 | present in pytest-relaxed under Pytest versions 3 and 4: 29 | 30 | - The order of nested test display may change slightly, typically for the 31 | better; eg under older versions, tests defined on a class might have been 32 | displayed after subclasses/nested tests - now they're more likely to be 33 | listed first, which was the original intent. 34 | - These bugs sometimes enabled "state bleed", such as outer scopes 35 | appearing to grant inner ones attributes set at runtime (eg by the outer 36 | scope's ``setup``, even when the inner scope redefined ``setup``). 37 | 38 | - If you encounter odd bugs after upgrading, please take a close look at 39 | your code and make sure you weren't accidentally using such a 40 | "feature". One good way to test for this is to run the "newly failing" 41 | tests by themselves on the old dependencies -- they will likely also 42 | fail there. 43 | 44 | - :release:`1.1.5 <2019-06-14>` 45 | - :bug:`2` Fix compatibility with pytest versions 3.3 and above. 46 | - :release:`1.1.4 <2018-07-24>` 47 | - :release:`1.0.2 <2018-07-24>` 48 | - :support:`- backported` Add missing universal wheel indicator in setup 49 | metadata. 50 | - :release:`1.1.3 <2018-07-24>` 51 | - :release:`1.0.1 <2018-07-24>` 52 | - :bug:`-` Fix the ``@raises`` helper decorator so it actually raises an 53 | exception when the requested exception is not raised by the decorated 54 | function. That's definitely not a confusing sentence. 55 | - :release:`1.1.2 <2018-04-16>` 56 | - :bug:`-` Neglected to update setup metadata when setting up a tiny Read The 57 | Docs instance. Homepage link now fixed! 58 | - :release:`1.1.1 <2018-04-16>` 59 | - :bug:`-` Installation and other ``setup.py`` activities implicitly assumed 60 | native Unicode support due to naively opening ``README.rst``. ``setup.py`` now 61 | explicitly opens that file with a ``utf-8`` encoding argument. Thanks to 62 | Ondřej Súkup for catch & patch. 63 | - :bug:`-` Bypass ``pytestmark`` objects and attributes during our custom 64 | collection phase; we don't need to process them ourselves, pytest is already 65 | picking up the original top level copies, and having them percolate into 66 | nested classes was causing loud pytest collection-step warnings. 67 | - :release:`1.1.0 <2017-11-21>` 68 | - :feature:`-` Add support for collecting/displaying hybrid/legacy test suites 69 | -- specifically, by getting out of pytest's way on collection of 70 | ``test_named_files`` and stripping test prefixes/suffixes when displaying 71 | tests in verbose mode. This makes it easier to take an existing test suite 72 | and slowly port it to 'relaxed' style. 73 | - :release:`1.0.0 <2017-11-06>` 74 | - :support:`-` Drop Python 2.6 and 3.3 support. 75 | - :feature:`-` Implement early drafts of Spec-like nested test display (which 76 | fires only when verbose output is enabled, unlike Spec which completely took 77 | over all output of nosetests.) 78 | - :support:`-` Revert internal tests to *not* eat our own dogfood; typical TDD 79 | lifecycles don't work very well when partly-implemented new features cause 80 | all of the older tests to fail as well! 81 | - :feature:`-` Create a ``@raises`` decorator which wraps ``pytest.raises`` 82 | (we're not sure why it's not natively offered as a decorator!) and thus ends 83 | up appearing very similar to Nose's API member of same name. 84 | - :feature:`-` Port ``@trap`` from Spec as it's currently a lot more natural to 85 | use than pytest's builtin capture fixtures. May back it out again later if 86 | we can make better sense of the latter / fit it into how our existing suites 87 | are organized. 88 | - :support:`-` Basic Travis and CodeCov support. 89 | - :bug:`- major` Various and sundry bugfixes, including "didn't skip 90 | underscore-named directories." 91 | - :release:`0.1.0 <2017-04-08>` 92 | - :feature:`-` Early draft functionality (test discovery only; zero display 93 | features.) This includes "releases" 0.0.1-0.0.4. 94 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import os 3 | import sys 4 | 5 | 6 | extensions = ["releases"] 7 | templates_path = ["_templates"] 8 | source_suffix = ".rst" 9 | master_doc = "index" 10 | exclude_patterns = ["_build"] 11 | 12 | project = "pytest-relaxed" 13 | year = datetime.now().year 14 | copyright = f"{year} Jeff Forcier" 15 | 16 | # Ensure project directory is on PYTHONPATH for version, autodoc access 17 | sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), ".."))) 18 | 19 | # Extension/theme settings 20 | html_theme_options = { 21 | "description": "Relaxed pytest discovery", 22 | "github_user": "bitprophet", 23 | "github_repo": "pytest-relaxed", 24 | "fixed_sidebar": True, 25 | } 26 | html_sidebars = { 27 | "**": [ 28 | "about.html", 29 | "navigation.html", 30 | "relations.html", 31 | "searchbox.html", 32 | "donate.html", 33 | ] 34 | } 35 | # TODO: make it easier to configure this and alabaster at same time, jeez 36 | releases_github_path = "bitprophet/pytest-relaxed" 37 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | .. toctree:: 4 | :hidden: 5 | :glob: 6 | 7 | * 8 | -------------------------------------------------------------------------------- /invoke.yml: -------------------------------------------------------------------------------- 1 | packaging: 2 | wheel: true 3 | # TODO: this wants to really be 'related' to the docs settings somehow; used 4 | # to use os.path.join() when it was in-python. 5 | changelog_file: docs/changelog.rst 6 | run: 7 | env: 8 | # Our ANSI color tests test against hardcoded codes appropriate for 9 | # this terminal, for now. 10 | TERM: screen-256color 11 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | -------------------------------------------------------------------------------- /pytest_relaxed/__init__.py: -------------------------------------------------------------------------------- 1 | # Convenience imports. 2 | # flake8: noqa 3 | from .trap import trap 4 | from .raises import raises 5 | -------------------------------------------------------------------------------- /pytest_relaxed/_version.py: -------------------------------------------------------------------------------- 1 | __version_info__ = (2, 0, 2) 2 | __version__ = ".".join(map(str, __version_info__)) 3 | -------------------------------------------------------------------------------- /pytest_relaxed/classes.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import logging 3 | import types 4 | 5 | from pytest import Class, Module 6 | 7 | # NOTE: don't see any other way to get access to pytest innards besides using 8 | # the underscored name :( 9 | from _pytest.python import PyCollector 10 | 11 | 12 | log = logging.getLogger("relaxed") 13 | 14 | 15 | # NOTE: these are defined here for reuse by both pytest's own machinery and our 16 | # internal bits. 17 | def istestclass(name): 18 | return not name.startswith("_") 19 | 20 | 21 | # NOTE: this is defined at top level due to a couple spots of reuse outside of 22 | # the mixin class itself. 23 | def istestfunction(obj, name): 24 | is_hidden_name = name.startswith("_") or name in ( 25 | "setup", 26 | "setup_method", 27 | "teardown", 28 | "teardown_method", 29 | ) 30 | # TODO: is this reliable? how about __pytest_wrapped__? 31 | is_fixture = hasattr(obj, "_pytestfixturefunction") 32 | return not (is_hidden_name or is_fixture) 33 | 34 | 35 | # All other classes in here currently inherit from PyCollector, and it is what 36 | # defines the default istestfunction/istestclass, so makes sense to inherit 37 | # from it for our mixin. (PyobjMixin, another commonly found class, offers 38 | # nothing of interest to us however.) 39 | class RelaxedMixin(PyCollector): 40 | """ 41 | A mixin applying collection rules to both modules and inner/nested classes. 42 | """ 43 | 44 | # TODO: 45 | # - worth calling super() in these? Difficult to know what to do with it; 46 | # it would say "no" to lots of stuff we want to say "yes" to. 47 | # - are there other tests to apply to 'obj' in a vacuum? so far only thing 48 | # we test 'obj' for is its membership in a module, which must happen inside 49 | # SpecModule's override. 50 | 51 | def istestclass(self, obj, name): 52 | return istestclass(name) 53 | 54 | def istestfunction(self, obj, name): 55 | return istestfunction(obj, name) 56 | 57 | 58 | class SpecModule(RelaxedMixin, Module): 59 | def _is_test_obj(self, test_func, obj, name): 60 | # First run our super() test, which should be RelaxedMixin's. 61 | good_name = getattr(super(), test_func)(obj, name) 62 | # If RelaxedMixin said no, we can't really say yes, as the name itself 63 | # was bad - private, other non test name like setup(), etc 64 | if not good_name: 65 | return False 66 | # Here, we dig further based on our own wrapped module obj, by 67 | # rejecting anything not defined locally. 68 | if inspect.getmodule(obj) is not self.obj: 69 | return False 70 | # No other complaints -> it's probably good 71 | return True 72 | 73 | def istestfunction(self, obj, name): 74 | return self._is_test_obj("istestfunction", obj, name) 75 | 76 | def istestclass(self, obj, name): 77 | return self._is_test_obj("istestclass", obj, name) 78 | 79 | def collect(self): 80 | # Given we've overridden naming constraints etc above, just use 81 | # superclass' collection logic for the rest of the necessary behavior. 82 | items = super().collect() 83 | collected = [] 84 | for item in items: 85 | # Replace Class objects with recursive SpecClasses 86 | # NOTE: we could explicitly skip unittest objects here (we'd want 87 | # them to be handled by pytest's own unittest support) but since 88 | # those are almost always in test_prefixed_filenames anyways...meh 89 | if isinstance(item, Class): 90 | item = SpecClass.from_parent(item.parent, name=item.name) 91 | collected.append(item) 92 | return collected 93 | 94 | 95 | class SpecClass(RelaxedMixin, Class): 96 | def _getobj(self): 97 | # Regular object-making first 98 | obj = super()._getobj() 99 | # Short circuit if this obj isn't a nested class (aka child): 100 | # - no parent attr: implies module-level obj definition 101 | # - parent attr, but isn't a class: implies method 102 | if not hasattr(self, "parent") or not isinstance( 103 | self.parent, SpecClass 104 | ): 105 | return obj 106 | # Then decorate it with our parent's extra attributes, allowing nested 107 | # test classes to appear as an aggregate of parents' "scopes". 108 | parent_obj = self.parent.obj 109 | # Obtain parent attributes, etc not found on our obj (serves as both a 110 | # useful identifier of "stuff added to an outer class" and a way of 111 | # ensuring that we can override such attrs), and set them on obj 112 | delta = set(dir(parent_obj)).difference(set(dir(obj))) 113 | for name in delta: 114 | value = getattr(parent_obj, name) 115 | # Pytest's pytestmark attributes always get skipped, we don't want 116 | # to spread that around where it's not wanted. (Besides, it can 117 | # cause a lot of collection level warnings.) 118 | if name == "pytestmark": 119 | continue 120 | # Classes get skipped; they'd always just be other 'inner' classes 121 | # that we don't want to copy elsewhere. 122 | if isinstance(value, type): 123 | continue 124 | # Functions (methods) may get skipped, or not, depending: 125 | # NOTE: as of pytest 7, for some reason the value appears as a 126 | # function and not a method (???) so covering both bases... 127 | if isinstance(value, (types.MethodType, types.FunctionType)): 128 | # If they look like tests, they get skipped; don't want to copy 129 | # tests around! 130 | if istestfunction(obj, name): 131 | continue 132 | # Non-test == they're probably lifecycle methods 133 | # (setup/teardown) or helpers (_do_thing). Rebind them to the 134 | # target instance, otherwise the 'self' in the setup/helper is 135 | # not the same 'self' as that in the actual test method it runs 136 | # around or within! 137 | setattr(obj, name, value) 138 | # Anything else should be some data-type attribute, which is copied 139 | # verbatim / by-value. 140 | else: 141 | setattr(obj, name, value) 142 | return obj 143 | 144 | def collect(self): 145 | ret = [] 146 | for item in super().collect(): 147 | # More pytestmark skipping. 148 | if item.name == "pytestmark": 149 | continue 150 | if isinstance(item, Class): 151 | item = SpecClass.from_parent( 152 | parent=item.parent, name=item.name, obj=item.obj 153 | ) 154 | ret.append(item) 155 | return ret 156 | -------------------------------------------------------------------------------- /pytest_relaxed/fixtures.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from pytest import fixture 4 | 5 | 6 | # TODO: consider making this a "no param/funcarg required" fixture (i.e. one 7 | # that gets decorated onto test classes instead of injected as magic kwargs) 8 | # and have uses of it simply call os.environ as normal. Pro: test code looks 9 | # less magical, con: removes any ability to do anything more interesting with 10 | # the yielded value (like proxying or whatever.) See the pytest 3.1.2 docs at: 11 | # /fixture.html#using-fixtures-from-classes-modules-or-projects 12 | @fixture 13 | def environ(): 14 | """ 15 | Enforce restoration of current shell environment after modifications. 16 | 17 | Yields the ``os.environ`` dict after snapshotting it; restores the original 18 | value (wholesale) during fixture teardown. 19 | """ 20 | current_environ = os.environ.copy() 21 | yield os.environ 22 | os.environ = current_environ 23 | -------------------------------------------------------------------------------- /pytest_relaxed/plugin.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from .classes import SpecModule 4 | from .reporter import RelaxedReporter 5 | 6 | # NOTE: fixtures must be present in the module listed under our setup.py's 7 | # pytest11 entry_points value (i.e., this one.) Just being in the import path 8 | # (e.g. package __init__.py) was not sufficient! 9 | from .fixtures import environ # noqa 10 | 11 | 12 | def pytest_ignore_collect(collection_path, config): 13 | # Ignore files and/or directories marked as private via Python convention. 14 | return collection_path.name.startswith("_") 15 | 16 | 17 | # We need to use collect_file, not pycollect_makemodule, as otherwise users 18 | # _must_ specify a config blob to use us, vs that being optional. 19 | # TODO: otoh, I personally use it all the time and we "strongly recommend it" 20 | # so maybe find a way to make that config bit default somehow (update 21 | # docs/changelog appropriately), and then switch hooks? 22 | def pytest_collect_file(file_path, parent): 23 | # Modify file selection to choose all .py files besides conftest.py. 24 | # (Skipping underscored names is handled up in pytest_ignore_collect, which 25 | # applies to directories too.) 26 | if ( 27 | file_path.suffix != ".py" 28 | or file_path.name == "conftest.py" 29 | # Also skip anything prefixed with test_; pytest's own native 30 | # collection will get that stuff, and we don't _want_ to try modifying 31 | # such files anyways. 32 | or file_path.name.startswith("test_") 33 | ): 34 | return 35 | # Then use our custom module class which performs modified 36 | # function/class selection as well as class recursion 37 | return SpecModule.from_parent(parent=parent, path=file_path) 38 | 39 | 40 | @pytest.mark.trylast # So we can be sure builtin terminalreporter exists 41 | def pytest_configure(config): 42 | # TODO: we _may_ sometime want to do the isatty/slaveinput/etc checks that 43 | # pytest-sugar does? 44 | builtin = config.pluginmanager.getplugin("terminalreporter") 45 | # Pass the configured, instantiated builtin terminal reporter to our 46 | # instance so it can refer to e.g. the builtin reporter's configuration 47 | ours = RelaxedReporter(builtin) 48 | # Unregister the builtin first so only our output appears 49 | config.pluginmanager.unregister(builtin) 50 | config.pluginmanager.register(ours, "terminalreporter") 51 | -------------------------------------------------------------------------------- /pytest_relaxed/raises.py: -------------------------------------------------------------------------------- 1 | from decorator import decorator 2 | 3 | 4 | # Thought pytest.raises was like nose.raises, but nooooooo. So let's make it 5 | # like that. 6 | def raises(klass): 7 | @decorator 8 | def inner(f, *args, **kwargs): 9 | try: 10 | f(*args, **kwargs) 11 | except klass: 12 | pass 13 | else: 14 | raise Exception( 15 | "Did not receive expected {}!".format(klass.__name__) 16 | ) 17 | 18 | return inner 19 | -------------------------------------------------------------------------------- /pytest_relaxed/reporter.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from _pytest.terminal import TerminalReporter 4 | 5 | 6 | # TODO: 7 | # - how can we be sure the tests are in the right order? 8 | # - aka how can we 'sort' them? in post-collection step? 9 | # - how to handle display of 'header' lines? Probably state tracking as w/ 10 | # spec? 11 | # - how to deal with flat modules vs nested classes? 12 | # - would be nice to examine all tests in a tree, but that requires waiting 13 | # till all results are in, which is no bueno. So we really do just need to 14 | # ensure a tree-based sort (which, assuming solid test ID strings, can be a 15 | # lexical sort.) 16 | # - sadly, what this means is that the parent/child relationship between test 17 | # objects doesn't really help us any, since we have to take action on a 18 | # per-report basis. Meh. (guess if we NEEDED to access attributes of a parent 19 | # in a child, that'd be possible, but...seems unlikely-ish? Maybe indent 20 | # based on parent relationships instead of across-the-run state tracking?) 21 | 22 | 23 | TEST_PREFIX = re.compile(r"^(Test|test_)") 24 | TEST_SUFFIX = re.compile(r"(Test|_test)$") 25 | 26 | 27 | # NOTE: much of the high level "replace default output bits" approach is 28 | # cribbed directly from pytest-sugar at 0.8.0 29 | class RelaxedReporter(TerminalReporter): 30 | def __init__(self, builtin): 31 | # Pass in the builtin reporter's config so we're not redoing all of its 32 | # initial setup/cli parsing/etc. NOTE: TerminalReporter is old-style :( 33 | TerminalReporter.__init__(self, builtin.config) 34 | # Which headers have already been displayed 35 | # TODO: faster data structure probably wise 36 | self.headers_displayed = [] 37 | # Size of indents. TODO: configuration 38 | self.indent = " " * 4 39 | 40 | def pytest_runtest_logstart(self, nodeid, location): 41 | # Non-verbose: do whatever normal pytest does. 42 | if not self.verbosity: 43 | return TerminalReporter.pytest_runtest_logstart( 44 | self, nodeid, location 45 | ) 46 | # Verbose: do nothing, preventing normal display of test location/id. 47 | # Leaves all display up to other hooks. 48 | 49 | def pytest_runtest_logreport(self, report): 50 | # TODO: if we _need_ access to the test item/node itself, we may want 51 | # to implement pytest_runtest_makereport instead? (Feels a little 52 | # 'off', but without other grody hax, no real way to get the obj so...) 53 | 54 | # Non-verbose: do whatever normal pytest does. 55 | # TODO: kinda want colors & per-module headers/indent though... 56 | if not self.verbosity: 57 | return TerminalReporter.pytest_runtest_logreport(self, report) 58 | 59 | # First, the default impl of this method seems to take care of updating 60 | # overall run stats; if we don't repeat that we lose all end-of-run 61 | # tallying and whether the run failed...kind of important. (Why that's 62 | # not a separate hook, no idea :() 63 | self.update_stats(report) 64 | # After that, short-circuit if it's not reporting the main call (i.e. 65 | # we don't want to display "the test" during its setup or teardown) 66 | if report.when != "call": 67 | return 68 | id_ = report.nodeid 69 | # First, make sure we display non-per-test data, i.e. 70 | # module/class/nested class headers (which by necessity also includes 71 | # tracking indentation state.) 72 | self.ensure_headers(id_) 73 | # Then we can display the test name/status itself. 74 | self.display_result(report) 75 | 76 | def update_stats(self, report): 77 | cat, letter, word = self.config.hook.pytest_report_teststatus( 78 | report=report, config=self.config 79 | ) 80 | self.stats.setdefault(cat, []).append(report) 81 | # For use later; apparently some other plugins can yield display markup 82 | # in the 'word' field of a report. 83 | self.report_word = word 84 | 85 | def split(self, id_): 86 | # Split on pytest's :: joiner, and strip out our intermediate 87 | # SpecInstance objects (appear as '()') 88 | headers = [x for x in id_.split("::")[1:]] 89 | # Last one is the actual test being reported on, not a header 90 | leaf = headers.pop() 91 | return headers, leaf 92 | 93 | def transform_name(self, name): 94 | """ 95 | Take a test class/module/function name and make it human-presentable. 96 | """ 97 | # TestPrefixes / test_prefixes -> stripped 98 | name = re.sub(TEST_PREFIX, "", name) 99 | # TestSuffixes / suffixed_test -> stripped 100 | name = re.sub(TEST_SUFFIX, "", name) 101 | # All underscores become spaces, for sentence-ishness 102 | name = name.replace("_", " ") 103 | return name 104 | 105 | def ensure_headers(self, id_): 106 | headers, _ = self.split(id_) 107 | printed = False 108 | # TODO: this works for class-based tests but needs love for module ones 109 | # TODO: worth displaying filename ever? 110 | # Make sure we print all not-yet-seen headers 111 | for i, header in enumerate(headers): 112 | # Need to semi-uniq headers by their 'path'. (This is a lot like 113 | # "the test id minus the last segment" but since we have to 114 | # split/join either way...whatever. I like dots.) 115 | header_path = ".".join(headers[: i + 1]) 116 | if header_path in self.headers_displayed: 117 | continue 118 | self.headers_displayed.append(header_path) 119 | indent = self.indent * i 120 | header = self.transform_name(header) 121 | self._tw.write("\n{}{}\n".format(indent, header)) 122 | printed = True 123 | # No trailing blank line after all headers; only the 'last' one (i.e. 124 | # before any actual test names are printed). And only if at least one 125 | # header was actually printed! (Otherwise one gets newlines between all 126 | # tests.) 127 | if printed: 128 | self._tw.write("\n") 129 | 130 | def display_result(self, report): 131 | headers, leaf = self.split(report.nodeid) 132 | indent = self.indent * len(headers) 133 | leaf = self.transform_name(leaf) 134 | # This _tw.write() stuff seems to be how vanilla pytest writes its 135 | # colorized verbose output. Bit clunky, but it means we automatically 136 | # honor things like `--color=no` and whatnot. 137 | self._tw.write(indent) 138 | self._tw.write(leaf, **self.report_markup(report)) 139 | self._tw.write("\n") 140 | 141 | def report_markup(self, report): 142 | # Basically preserved from parent implementation; if something caused 143 | # the 'word' field in the report to be a tuple, it's a (word, markup) 144 | # tuple. We don't care about the word (possibly bad, but it doesn't fit 145 | # with our display ethos right now) but the markup may be worth 146 | # preserving. 147 | if isinstance(self.report_word, tuple): 148 | return self.report_word[1] 149 | # Otherwise, assume ye olde pass/fail/skip. 150 | if report.passed: 151 | color = "green" 152 | elif report.failed: 153 | color = "red" 154 | elif report.skipped: 155 | color = "yellow" 156 | return {color: True} 157 | -------------------------------------------------------------------------------- /pytest_relaxed/trap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test decorator for capturing stdout/stderr/both. 3 | 4 | Based on original code from Fabric 1.x, specifically: 5 | 6 | * fabric/tests/utils.py 7 | * as of Git SHA 62abc4e17aab0124bf41f9c5f9c4bc86cc7d9412 8 | 9 | Though modifications have been made since. 10 | """ 11 | 12 | import io 13 | import sys 14 | from functools import wraps 15 | 16 | 17 | class CarbonCopy(io.BytesIO): 18 | """ 19 | An IO wrapper capable of multiplexing its writes to other buffer objects. 20 | """ 21 | 22 | def __init__(self, buffer=b"", cc=None): 23 | """ 24 | If ``cc`` is given and is a file-like object or an iterable of same, 25 | it/they will be written to whenever this instance is written to. 26 | """ 27 | super().__init__(buffer) 28 | if cc is None: 29 | cc = [] 30 | elif hasattr(cc, "write"): 31 | cc = [cc] 32 | self.cc = cc 33 | 34 | def write(self, s): 35 | # Ensure we always write bytes. 36 | if isinstance(s, str): 37 | s = s.encode("utf-8") 38 | # Write out to our capturing object & any CC's 39 | super().write(s) 40 | for writer in self.cc: 41 | writer.write(s) 42 | 43 | # Real sys.std(out|err) requires writing to a buffer attribute obj in some 44 | # situations. 45 | @property 46 | def buffer(self): 47 | return self 48 | 49 | # Make sure we always hand back strings 50 | def getvalue(self): 51 | ret = super().getvalue() 52 | if isinstance(ret, bytes): 53 | ret = ret.decode("utf-8") 54 | return ret 55 | 56 | 57 | def trap(func): 58 | """ 59 | Replace sys.std(out|err) with a wrapper during execution, restored after. 60 | 61 | In addition, a new combined-streams output (another wrapper) will appear at 62 | ``sys.stdall``. This stream will resemble what a user sees at a terminal, 63 | i.e. both out/err streams intermingled. 64 | """ 65 | 66 | @wraps(func) 67 | def wrapper(*args, **kwargs): 68 | # Use another CarbonCopy even though we're not cc'ing; for our "write 69 | # bytes, return strings" behavior. Meh. 70 | sys.stdall = CarbonCopy() 71 | my_stdout, sys.stdout = sys.stdout, CarbonCopy(cc=sys.stdall) 72 | my_stderr, sys.stderr = sys.stderr, CarbonCopy(cc=sys.stdall) 73 | try: 74 | return func(*args, **kwargs) 75 | finally: 76 | sys.stdout = my_stdout 77 | sys.stderr = my_stderr 78 | del sys.stdall 79 | 80 | return wrapper 81 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | from io import open 5 | 6 | # Version info -- read without importing 7 | _locals = {} 8 | with open("pytest_relaxed/_version.py") as fp: 9 | exec(fp.read(), None, _locals) 10 | version = _locals["__version__"] 11 | 12 | 13 | setup( 14 | name="pytest-relaxed", 15 | version=version, 16 | description="Relaxed test discovery/organization for pytest", 17 | license="BSD", 18 | url="https://pytest-relaxed.readthedocs.io/", 19 | project_urls={ 20 | "Source": "https://github.com/bitprophet/pytest-relaxed", 21 | "Changelog": "https://github.com/bitprophet/pytest-relaxed/blob/main/docs/changelog.rst", # noqa 22 | "CI": "https://app.circleci.com/pipelines/github/bitprophet/pytest-relaxed", # noqa 23 | }, 24 | author="Jeff Forcier", 25 | author_email="jeff@bitprophet.org", 26 | long_description="\n" + open("README.rst", encoding="utf-8").read(), 27 | packages=find_packages(), 28 | entry_points={ 29 | # TODO: do we need to name the LHS 'pytest_relaxed' too? meh 30 | "pytest11": ["relaxed = pytest_relaxed.plugin"] 31 | }, 32 | python_requires=">=3.6", 33 | install_requires=[ 34 | # Difficult to support Pytest<7 + Pytest>=7 at same time, and 35 | # pytest-relaxed never supported pytests 5 or 6, so...why bother! 36 | "pytest>=7", 37 | # For @raises, primarily. At press time, most available decorator 38 | # versions (including 5.x) should work for us / our Python interpreter 39 | # versions. 40 | "decorator", 41 | ], 42 | classifiers=[ 43 | "Development Status :: 5 - Production/Stable", 44 | "Framework :: Pytest", 45 | "Intended Audience :: Developers", 46 | "License :: OSI Approved :: BSD License", 47 | "Operating System :: OS Independent", 48 | "Programming Language :: Python", 49 | "Programming Language :: Python :: 3", 50 | "Programming Language :: Python :: 3 :: Only", 51 | "Programming Language :: Python :: 3.6", 52 | "Programming Language :: Python :: 3.7", 53 | "Programming Language :: Python :: 3.8", 54 | "Programming Language :: Python :: 3.9", 55 | "Programming Language :: Python :: 3.10", 56 | "Programming Language :: Python :: 3.11", 57 | "Topic :: Software Development :: Testing", 58 | ], 59 | ) 60 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | from invoke import task, Collection 2 | from invocations import checks 3 | from invocations.packaging import release 4 | from invocations import docs, pytest as pytests 5 | 6 | 7 | @task 8 | def coverage(c, html=True, codecov=False): 9 | """ 10 | Run coverage with coverage.py. 11 | """ 12 | # NOTE: this MUST use coverage itself, and not pytest-cov, because the 13 | # latter is apparently unable to prevent pytest plugins from being loaded 14 | # before pytest-cov itself is able to start up coverage.py! The result is 15 | # that coverage _always_ skips over all module level code, i.e. constants, 16 | # 'def' lines, etc. Running coverage as the "outer" layer avoids this 17 | # problem, thus no need for pytest-cov. 18 | # NOTE: this does NOT hold true for NON-PYTEST code, so 19 | # pytest-relaxed-USING modules can happily use pytest-cov. 20 | c.run( 21 | "coverage run --source=pytest_relaxed,tests --branch --module pytest" 22 | ) 23 | if html: 24 | c.run("coverage html") 25 | if codecov: 26 | # Generate XML report from that already-gathered data (otherwise 27 | # codecov generates it on its own and gets it wrong!) 28 | c.run("coverage xml") 29 | # Upload to Codecov 30 | c.run("codecov") 31 | 32 | 33 | # TODO: good candidate for builtin-to-invoke "just wrap with a 34 | # tiny bit of behavior", and/or args/kwargs style invocations 35 | @task 36 | def test( 37 | c, 38 | verbose=True, 39 | color=True, 40 | capture="sys", 41 | opts="", 42 | x=False, 43 | k=None, 44 | module=None, 45 | ): 46 | """ 47 | Run pytest with given options. 48 | 49 | Wraps ``invocations.pytests.test``. See its docs for details. 50 | """ 51 | # TODO: could invert this & have our entire test suite manually _enable_ 52 | # our own plugin, but given pytest's options around plugin setup, this 53 | # seems to be both easier and simpler. 54 | opts += " -p no:relaxed" 55 | pytests.test( 56 | c, 57 | verbose=verbose, 58 | color=color, 59 | capture=capture, 60 | opts=opts, 61 | x=x, 62 | k=k, 63 | module=module, 64 | ) 65 | 66 | 67 | ns = Collection(checks.blacken, checks, coverage, docs, test, release) 68 | ns.configure({"blacken": {"find_opts": "-and -not -path './build*'"}}) 69 | -------------------------------------------------------------------------------- /tests/test_collection.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from pytest import ExitCode, mark 4 | 5 | 6 | # For 'testdir' fixture, mostly 7 | pytest_plugins = "pytester" 8 | 9 | 10 | class Test_pytest_collect_file: 11 | def test_only_loads_dot_py_files(self, testdir): 12 | testdir.makepyfile( 13 | somefile=""" 14 | def hello_how_are_you(): 15 | pass 16 | """ 17 | ) 18 | testdir.makefile(".txt", someotherfile="whatever") 19 | stdout = testdir.runpytest().stdout.str() 20 | # TODO: find it hard to believe pytest lacks strong "x in y" string 21 | # testing, but I cannot find any outside of fnmatch_lines (which is 22 | # specific to this testdir stuff, and also lacks an opposite...) 23 | assert "somefile.py" in stdout 24 | # This wouldn't actually even happen; we'd get an ImportError instead 25 | # as pytest tries importing 'someotherfile'. But eh. 26 | assert "whatever.txt" not in stdout 27 | 28 | def test_skips_underscored_files(self, testdir): 29 | testdir.makepyfile( 30 | hastests=""" 31 | from _util import helper 32 | 33 | def hello_how_are_you(): 34 | helper() 35 | """ 36 | ) 37 | testdir.makepyfile( 38 | _util=""" 39 | def helper(): 40 | pass 41 | """ 42 | ) 43 | # TODO: why Result.str() and not str(Result)? Seems unPythonic 44 | stdout = testdir.runpytest().stdout.str() 45 | assert "hastests.py" in stdout 46 | assert "_util.py" not in stdout 47 | 48 | def test_skips_underscored_directories(self, testdir): 49 | testdir.makepyfile( 50 | hello=""" 51 | def hi_im_a_test_function(): 52 | pass 53 | """ 54 | ) 55 | # NOTE: this appears to work due to impl details of pytester._makefile; 56 | # namely that the kwarg keys are handed directly to tmpdir.join(), 57 | # where tmpdir is a py.path.LocalPath. 58 | testdir.makepyfile( 59 | **{ 60 | "_nope/yallo": """ 61 | def hi_im_not_a_test_function(): 62 | pass 63 | """ 64 | } 65 | ) 66 | stdout = testdir.runpytest("-v").stdout.str() 67 | assert "hi im a test function" in stdout 68 | assert "hi im not a test function" not in stdout 69 | 70 | def test_does_not_consume_conftest_files(self, testdir): 71 | testdir.makepyfile( 72 | actual_tests=""" 73 | def hello_how_are_you(): 74 | pass 75 | """ 76 | ) 77 | testdir.makepyfile( 78 | conftest=""" 79 | def this_does_nothing_useful(): 80 | pass 81 | """ 82 | ) 83 | stdout = testdir.runpytest().stdout.str() 84 | assert "actual_tests.py" in stdout 85 | assert "conftest.py" not in stdout 86 | 87 | 88 | class TestRelaxedMixin: 89 | def test_selects_all_non_underscored_members(self, testdir): 90 | testdir.makepyfile( 91 | foo=""" 92 | def hello_how_are_you(): 93 | pass 94 | 95 | def _help_me_understand(): 96 | pass 97 | 98 | class YupThisIsTests: 99 | def please_test_me_thx(self): 100 | pass 101 | 102 | def _helper_method_hi(self): 103 | pass 104 | 105 | class NestedTestClassAhoy: 106 | def hello_I_am_a_test_method(self): 107 | pass 108 | 109 | def _but_I_am_not(self): 110 | pass 111 | 112 | class _NotSureWhyYouWouldDoThisButWhatever: 113 | def this_should_not_appear(self): 114 | pass 115 | 116 | class _ForSomeReasonIAmDefinedHereButAmNotATest: 117 | def usually_you_would_just_import_this_but_okay(self): 118 | pass 119 | """ 120 | ) 121 | stdout = testdir.runpytest("-v").stdout.str() 122 | for substring in ( 123 | "hello how are you", 124 | "please test me thx", 125 | "hello I am a test method", 126 | ): 127 | assert substring in stdout 128 | for substring in ( 129 | "help me understand", 130 | "helper method hi", 131 | "NotSureWhyYouWouldDoThisButWhatever", 132 | "ForSomeReasonIAmDefinedHereButAmNotATest", 133 | ): 134 | assert substring not in stdout 135 | 136 | def test_skips_setup_and_teardown(self, testdir): 137 | testdir.makepyfile( 138 | foo=""" 139 | def setup(): 140 | pass 141 | 142 | def teardown(): 143 | pass 144 | 145 | def setup_method(): 146 | pass 147 | 148 | def teardown_method(): 149 | pass 150 | 151 | def actual_test_here(): 152 | pass 153 | 154 | class Outer: 155 | def setup(self): 156 | pass 157 | 158 | def teardown(self): 159 | pass 160 | 161 | def setup_method(self): 162 | pass 163 | 164 | def teardown_method(self): 165 | pass 166 | 167 | def actual_nested_test_here(self): 168 | pass 169 | """ 170 | ) 171 | stdout = testdir.runpytest("-v").stdout.str() 172 | # These skipped. Gotta regex them because the test name includes the 173 | # words 'setup' and 'teardown', heh. 174 | assert not re.match(r"^setup$", stdout) 175 | assert not re.match(r"^teardown$", stdout) 176 | assert not re.match(r"^setup_method$", stdout) 177 | assert not re.match(r"^teardown_method$", stdout) 178 | # Real tests not skipped 179 | assert "actual test here" in stdout 180 | assert "actual nested test here" in stdout 181 | 182 | def test_skips_pytest_fixtures(self, testdir): 183 | testdir.makepyfile( 184 | foo=""" 185 | from pytest import fixture 186 | 187 | @fixture 188 | def pls_noload(): 189 | yield 190 | 191 | def actual_test_here(): 192 | pass 193 | """ 194 | ) 195 | stdout = testdir.runpytest("-v").stdout.str() 196 | assert "actual test here" in stdout 197 | # will be in stdout as a failure and warning if bug present 198 | assert "pls_noload" not in stdout 199 | 200 | def test_setup_given_inner_class_instances_when_inherited(self, testdir): 201 | # NOTE: without this functionality in place, we still see setup() 202 | # called on a per-test-method basis, but where 'self' is the outer 203 | # class, not the inner class! so anything actually touching 'self' 204 | # breaks. 205 | # TODO: should this pattern change to be something like a pytest 206 | # per-class autouse fixture method? 207 | # (https://docs.pytest.org/en/latest/fixture.html#autouse-fixtures-xunit-setup-on-steroids) 208 | testdir.makepyfile( 209 | foo=""" 210 | class Outer: 211 | def setup_method(self): 212 | self.some_attr = 17 213 | 214 | class inner: 215 | def actual_nested_test(self): 216 | assert self.some_attr == 17 217 | """ 218 | ) 219 | assert testdir.runpytest().ret is ExitCode.OK 220 | 221 | def test_setup_method_given_inner_class_instances(self, testdir): 222 | testdir.makepyfile( 223 | foo=""" 224 | class Outer: 225 | def setup_method(self): 226 | self.some_attr = 17 227 | 228 | class inner: 229 | def actual_nested_test(self): 230 | assert self.some_attr == 17 231 | """ 232 | ) 233 | assert testdir.runpytest().ret is ExitCode.OK 234 | 235 | 236 | class TestSpecModule: 237 | def test_skips_non_callable_items(self, testdir): 238 | testdir.makepyfile( 239 | foo=""" 240 | some_uncallable = 17 241 | 242 | def some_callable(): 243 | pass 244 | """ 245 | ) 246 | stdout = testdir.runpytest("-v").stdout.str() 247 | assert "some_uncallable" not in stdout 248 | 249 | def test_skips_imported_objects(self, testdir): 250 | testdir.makepyfile( 251 | _util=""" 252 | def helper(): 253 | pass 254 | 255 | class Helper: 256 | pass 257 | 258 | class NewHelper: 259 | pass 260 | """ 261 | ) 262 | testdir.makepyfile( 263 | foo=""" 264 | from _util import helper, Helper, NewHelper 265 | 266 | def a_test_is_me(): 267 | pass 268 | """ 269 | ) 270 | stdout = testdir.runpytest("-v").stdout.str() 271 | assert "a test is me" in stdout 272 | assert "helper" not in stdout 273 | assert "Helper" not in stdout 274 | assert "NewHelper" not in stdout 275 | 276 | def test_does_not_warn_about_imported_names(self, testdir): 277 | # Trigger is something that appears callable but isn't a real function; 278 | # almost any callable class seems to suffice. (Real world triggers are 279 | # things like invoke/fabric Task objects.) 280 | # Can also be triggered if our collection is buggy and does not 281 | # explicitly reject imported classes (i.e. if we only reject funcs). 282 | testdir.makepyfile( 283 | _util=""" 284 | class Callable: 285 | def __call__(self): 286 | pass 287 | 288 | helper = Callable() 289 | 290 | class HelperClass: 291 | def __init__(self): 292 | pass 293 | """ 294 | ) 295 | testdir.makepyfile( 296 | foo=""" 297 | from _util import helper, HelperClass 298 | 299 | def a_test(): 300 | pass 301 | """ 302 | ) 303 | stdout = testdir.runpytest("-sv").stdout.str() 304 | # TODO: more flexible test in case text changes? eh. 305 | for warning in ( 306 | "cannot collect 'helper' because it is not a function", 307 | "cannot collect test class 'HelperClass'", 308 | ): 309 | assert warning not in stdout 310 | 311 | def test_replaces_class_tests_with_custom_recursing_classes(self, testdir): 312 | testdir.makepyfile( 313 | foo=""" 314 | class Outer: 315 | class Middle: 316 | class Inner: 317 | def oh_look_an_actual_test_method(self): 318 | pass 319 | """ 320 | ) 321 | stdout = testdir.runpytest("-v").stdout.str() 322 | expected = """ 323 | Outer 324 | 325 | Middle 326 | 327 | Inner 328 | 329 | oh look an actual test method 330 | """.lstrip() 331 | assert expected in stdout 332 | 333 | def test_does_not_collect_test_prefixed_files(self, testdir): 334 | # Incidentally also tests display stripping; the display test suite has 335 | # explicit tests for that too tho. 336 | testdir.makepyfile( 337 | test_something=""" 338 | import unittest 339 | 340 | class TestMyStuff(unittest.TestCase): 341 | def test_things(self): 342 | pass 343 | """ 344 | ) 345 | stdout = testdir.runpytest("-v").stdout.str() 346 | expected = """ 347 | MyStuff 348 | 349 | things 350 | 351 | """.lstrip() 352 | assert expected in stdout 353 | # Make sure no warnings were emitted; much of the time, our collection 354 | # bits will cause nasty warnings if they end up consuming unittest 355 | # stuff or otherwise doubling up on already-collected objects. 356 | assert "warnings summary" not in stdout 357 | 358 | @mark.skip 359 | def test_correctly_handles_marked_test_cases(self, testdir): 360 | # I.e. @pytest.mark.someflag objects at the class level...figure out 361 | # how real collectors handle these exactly? the "actual" test class we 362 | # normally care about is inside of it. 363 | pass 364 | 365 | 366 | class TestSpecClass: 367 | def test_methods_self_objects_exhibit_class_attributes(self, testdir): 368 | # Mostly a sanity test; pytest seems to get out of the way enough that 369 | # the test is truly a bound method & the 'self' is truly an instance of 370 | # the class. 371 | testdir.makepyfile( 372 | foo=""" 373 | class MyClass: 374 | an_attr = 5 375 | 376 | def some_test(self): 377 | assert hasattr(self, 'an_attr') 378 | assert self.an_attr == 5 379 | """ 380 | ) 381 | # TODO: first thought was "why is this not automatic?", then realized 382 | # "duh, it'd be annoying if you wanted to test failure related behavior 383 | # a lot"...but still want some slightly nicer helper I think 384 | assert testdir.runpytest().ret is ExitCode.OK 385 | 386 | def test_nested_self_objects_exhibit_parent_attributes(self, testdir): 387 | # TODO: really starting to think going back to 'real' fixture files 388 | # makes more sense; this is all real python code and is eval'd as such, 389 | # but it is only editable and viewable as a string. No highlighting. 390 | testdir.makepyfile( 391 | foo=""" 392 | class MyClass: 393 | an_attr = 5 394 | 395 | class Inner: 396 | def inner_test(self): 397 | assert hasattr(self, 'an_attr') 398 | assert self.an_attr == 5 399 | """ 400 | ) 401 | assert testdir.runpytest().ret is ExitCode.OK 402 | 403 | def test_nesting_is_infinite(self, testdir): 404 | testdir.makepyfile( 405 | foo=""" 406 | class MyClass: 407 | an_attr = 5 408 | 409 | class Inner: 410 | class Deeper: 411 | class EvenDeeper: 412 | def innermost_test(self): 413 | assert hasattr(self, 'an_attr') 414 | assert self.an_attr == 5 415 | """ 416 | ) 417 | assert testdir.runpytest().ret is ExitCode.OK 418 | 419 | def test_overriding_works_naturally(self, testdir): 420 | testdir.makepyfile( 421 | foo=""" 422 | class MyClass: 423 | an_attr = 5 424 | 425 | class Inner: 426 | an_attr = 7 427 | 428 | def inner_test(self): 429 | assert self.an_attr == 7 430 | """ 431 | ) 432 | assert testdir.runpytest().ret is ExitCode.OK 433 | 434 | def test_normal_methods_from_outer_classes_are_not_copied(self, testdir): 435 | testdir.makepyfile( 436 | foo=""" 437 | class MyClass: 438 | def outer_test(self): 439 | pass 440 | 441 | class Inner: 442 | def inner_test(self): 443 | assert not hasattr(self, 'outer_test') 444 | """ 445 | ) 446 | assert testdir.runpytest().ret is ExitCode.OK 447 | 448 | def test_private_methods_from_outer_classes_are_copied(self, testdir): 449 | testdir.makepyfile( 450 | foo=""" 451 | class MyClass: 452 | def outer_test(self): 453 | pass 454 | 455 | def _outer_helper(self): 456 | pass 457 | 458 | class Inner: 459 | def inner_test(self): 460 | assert not hasattr(self, 'outer_test') 461 | assert hasattr(self, '_outer_helper') 462 | """ 463 | ) 464 | assert testdir.runpytest().ret is ExitCode.OK 465 | 466 | def test_module_contents_are_not_copied_into_top_level_classes( 467 | self, testdir 468 | ): 469 | testdir.makepyfile( 470 | foo=""" 471 | module_constant = 17 472 | 473 | class MyClass: 474 | def outer_test(self): 475 | assert not hasattr(self, 'module_constant') 476 | """ 477 | ) 478 | assert testdir.runpytest().ret is ExitCode.OK 479 | -------------------------------------------------------------------------------- /tests/test_display.py: -------------------------------------------------------------------------------- 1 | from pytest import skip 2 | 3 | # Load some fixtures we expose, without actually loading our entire plugin 4 | from pytest_relaxed.fixtures import environ # noqa 5 | 6 | 7 | # TODO: how best to make all of this opt-in/out? Reporter as separate plugin? 8 | # (May not be feasible if it has to assume something about how our collection 9 | # works?) CLI option (99% sure we can hook into that as a plugin)? 10 | 11 | 12 | def _expect_regular_output(testdir): 13 | output = testdir.runpytest().stdout.str() 14 | # Regular results w/ status letters 15 | assert "behaviors.py .." in output 16 | assert "other_behaviors.py s.F." in output 17 | # Failure/traceback reporting 18 | assert "== FAILURES ==" in output 19 | assert "AssertionError" in output 20 | # Summary 21 | assert "== 1 failed, 4 passed, 1 skipped in " in output 22 | 23 | 24 | class TestRegularFunctions: 25 | """ 26 | Function-oriented test modules, normal display mode. 27 | """ 28 | 29 | def test_acts_just_like_normal_pytest(self, testdir): 30 | testdir.makepyfile( 31 | behaviors=""" 32 | def behavior_one(): 33 | pass 34 | 35 | def behavior_two(): 36 | pass 37 | """, 38 | other_behaviors=""" 39 | from pytest import skip 40 | 41 | def behavior_one(): 42 | skip() 43 | 44 | def behavior_two(): 45 | pass 46 | 47 | def behavior_three(): 48 | assert False 49 | 50 | def behavior_four(): 51 | pass 52 | """, 53 | ) 54 | _expect_regular_output(testdir) 55 | 56 | 57 | class TestVerboseFunctions: 58 | """ 59 | Function-oriented test modules, verbose display mode. 60 | """ 61 | 62 | def test_displays_tests_indented_under_module_header(self, testdir): 63 | # TODO: at least, that seems like a reasonable thing to do offhand 64 | skip() 65 | 66 | def test_test_prefixes_are_stripped(self, testdir): 67 | testdir.makepyfile( 68 | legacy=""" 69 | def test_some_things(): 70 | pass 71 | 72 | def test_other_things(): 73 | pass 74 | """ 75 | ) 76 | expected = """ 77 | some things 78 | other things 79 | """.lstrip() 80 | output = testdir.runpytest_subprocess("-v").stdout.str() 81 | assert expected in output 82 | 83 | 84 | class TestNormalClasses: 85 | """ 86 | Class-oriented test modules, normal display mode. 87 | """ 88 | 89 | def acts_just_like_normal_pytest(self, testdir): 90 | testdir.makepyfile( 91 | behaviors=""" 92 | class Behaviors: 93 | def behavior_one(self): 94 | pass 95 | 96 | def behavior_two(self): 97 | pass 98 | """, 99 | other_behaviors=""" 100 | from pytest import skip 101 | 102 | class OtherBehaviors: 103 | def behavior_one(self): 104 | skip() 105 | 106 | def behavior_two(self): 107 | pass 108 | 109 | def behavior_three(self): 110 | assert False 111 | 112 | def behavior_four(self): 113 | pass 114 | """, 115 | ) 116 | _expect_regular_output(testdir) 117 | 118 | 119 | class TestVerboseClasses: 120 | """ 121 | Class-oriented test modules, verbose display mode. 122 | """ 123 | 124 | def test_shows_tests_nested_under_classes_without_files(self, testdir): 125 | testdir.makepyfile( 126 | behaviors=""" 127 | class Behaviors: 128 | def behavior_one(self): 129 | pass 130 | 131 | def behavior_two(self): 132 | pass 133 | """, 134 | other_behaviors=""" 135 | from pytest import skip 136 | class OtherBehaviors: 137 | def behavior_one(self): 138 | pass 139 | 140 | def behavior_two(self): 141 | skip() 142 | 143 | def behavior_three(self): 144 | pass 145 | 146 | def behavior_four(self): 147 | assert False 148 | """, 149 | ) 150 | output = testdir.runpytest_subprocess("-v").stdout.str() 151 | results = """ 152 | Behaviors 153 | 154 | behavior one 155 | behavior two 156 | 157 | OtherBehaviors 158 | 159 | behavior one 160 | behavior two 161 | behavior three 162 | behavior four 163 | """.lstrip() 164 | assert results in output 165 | # Ensure we're not accidentally nixing failure, summary output 166 | assert "== FAILURES ==" in output 167 | assert "AssertionError" in output 168 | # Summary 169 | assert "== 1 failed, 4 passed, 1 skipped in " in output 170 | 171 | def test_tests_are_colorized_by_test_result( 172 | self, testdir, environ # noqa: F811,E501 173 | ): 174 | # Make sure py._io.TerminalWriters write colors despite pytest output 175 | # capturing, which would otherwise trigger a 'False' result for "should 176 | # markup output". 177 | environ["PY_COLORS"] = "1" 178 | testdir.makepyfile( 179 | behaviors=""" 180 | class Behaviors: 181 | def behavior_one(self): 182 | pass 183 | 184 | def behavior_two(self): 185 | pass 186 | """, 187 | other_behaviors=""" 188 | from pytest import skip 189 | class OtherBehaviors: 190 | def behavior_one(self): 191 | pass 192 | 193 | def behavior_two(self): 194 | skip() 195 | 196 | def behavior_three(self): 197 | pass 198 | 199 | def behavior_four(self): 200 | assert False 201 | """, 202 | ) 203 | output = testdir.runpytest_subprocess("-v").stdout.str() 204 | results = """ 205 | Behaviors 206 | 207 | \x1b[32mbehavior one\x1b[0m 208 | \x1b[32mbehavior two\x1b[0m 209 | 210 | OtherBehaviors 211 | 212 | \x1b[32mbehavior one\x1b[0m 213 | \x1b[33mbehavior two\x1b[0m 214 | \x1b[32mbehavior three\x1b[0m 215 | \x1b[31mbehavior four\x1b[0m 216 | """.lstrip() 217 | for chunk in ( 218 | # Our own special sauce 219 | results, 220 | # Failure summary still present 221 | "== FAILURES ==", 222 | # Ditto error class 223 | "AssertionError", 224 | # And summary chunks (now colorized apparently?) 225 | "1 failed", 226 | "4 passed", 227 | "1 skipped", 228 | ): 229 | assert chunk in output 230 | 231 | def test_nests_many_levels_deep_no_problem(self, testdir): 232 | testdir.makepyfile( 233 | behaviors=""" 234 | class Behaviors: 235 | def behavior_one(self): 236 | pass 237 | 238 | def behavior_two(self): 239 | pass 240 | 241 | class MoreBehaviors: 242 | def an_behavior(self): 243 | pass 244 | 245 | def another_behavior(self): 246 | pass 247 | 248 | class YetMore: 249 | class Behaviors: 250 | def yup(self): 251 | pass 252 | 253 | def still_works(self): 254 | pass 255 | """ 256 | ) 257 | expected = """ 258 | Behaviors 259 | 260 | behavior one 261 | behavior two 262 | 263 | MoreBehaviors 264 | 265 | an behavior 266 | another behavior 267 | 268 | YetMore 269 | 270 | Behaviors 271 | 272 | yup 273 | still works 274 | """.lstrip() 275 | assert expected in testdir.runpytest("-v").stdout.str() 276 | 277 | def test_headers_and_tests_have_underscores_turn_to_spaces(self, testdir): 278 | testdir.makepyfile( 279 | behaviors=""" 280 | class some_non_class_name_like_header: 281 | def a_test_sentence(self): 282 | pass 283 | """ 284 | ) 285 | expected = """ 286 | some non class name like header 287 | 288 | a test sentence 289 | """.lstrip() 290 | assert expected in testdir.runpytest("-v").stdout.str() 291 | 292 | def test_test_prefixes_are_stripped(self, testdir): 293 | testdir.makepyfile( 294 | stripping=""" 295 | class TestSomeStuff: 296 | def test_the_stuff(self): 297 | pass 298 | """ 299 | ) 300 | expected = """ 301 | SomeStuff 302 | 303 | the stuff 304 | 305 | """.lstrip() 306 | assert expected in testdir.runpytest("-v").stdout.str() 307 | 308 | def test_test_suffixes_are_stripped(self, testdir): 309 | testdir.makepyfile( 310 | stripping=""" 311 | class StuffTest: 312 | def test_yup(self): 313 | pass 314 | """ 315 | ) 316 | expected = """ 317 | Stuff 318 | 319 | yup 320 | 321 | """.lstrip() 322 | assert expected in testdir.runpytest("-v").stdout.str() 323 | 324 | 325 | class TestNormalMixed: 326 | """ 327 | Mixed function and class test modules, normal display mode. 328 | """ 329 | 330 | # TODO: currently undefined; spec never even really worked for this 331 | pass 332 | 333 | 334 | class TestVerboseMixed: 335 | """ 336 | Mixed function and class test modules, verbose display mode. 337 | """ 338 | 339 | # TODO: currently undefined; spec never even really worked for this 340 | pass 341 | -------------------------------------------------------------------------------- /tests/test_raises.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pytest_relaxed import raises 4 | 5 | 6 | class Boom(Exception): 7 | pass 8 | 9 | 10 | class OtherBoom(Exception): 11 | pass 12 | 13 | 14 | class Test_raises: 15 | def test_when_given_exception_raised_no_problem(self): 16 | @raises(Boom) 17 | def kaboom(): 18 | raise Boom 19 | 20 | kaboom() 21 | # If we got here, we're good... 22 | 23 | def test_when_given_exception_not_raised_it_raises_Exception(self): 24 | # TODO: maybe raise a custom exception later? HEH. 25 | @raises(Boom) 26 | def kaboom(): 27 | pass 28 | 29 | # Buffalo buffalo 30 | with pytest.raises(Exception) as exc: 31 | kaboom() 32 | assert "Did not receive expected Boom!" in str(exc.value) 33 | 34 | def test_when_some_other_exception_raised_it_is_untouched(self): 35 | @raises(Boom) 36 | def kaboom(): 37 | raise OtherBoom("sup") 38 | 39 | # Buffalo buffalo 40 | with pytest.raises(OtherBoom) as exc: 41 | kaboom() 42 | assert "sup" == str(exc.value) 43 | --------------------------------------------------------------------------------