├── pytest_mypy_plugins ├── py.typed ├── __init__.py ├── tests │ ├── test_configs │ │ ├── mypy2.ini │ │ ├── setup2.cfg │ │ ├── mypy1.ini │ │ ├── setup1.cfg │ │ ├── pyproject2.toml │ │ ├── pyproject1.toml │ │ ├── pyproject3.toml │ │ └── test_join_toml_configs.py │ ├── test-extension.yml │ ├── test-mypy-config.yml │ ├── reveal_type_hook.py │ ├── test-paths-from-env.yml │ ├── test-assert-type.yml │ ├── test_explicit_configs.py │ ├── test-parametrized.yml │ ├── test_input_schema.py │ ├── test-regex-assertions.yml │ ├── test-simple-cases.yml │ └── test_utils.py ├── configs.py ├── schema.json ├── collect.py ├── utils.py └── item.py ├── tox.ini ├── MANIFEST.in ├── requirements.txt ├── .github ├── dependabot.yml └── workflows │ └── test.yml ├── .editorconfig ├── pyproject.toml ├── release.sh ├── LICENSE ├── setup.py ├── CHANGELOG.md ├── .gitignore └── README.md /pytest_mypy_plugins/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/mypy2.ini: -------------------------------------------------------------------------------- 1 | # Empty 2 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/setup2.cfg: -------------------------------------------------------------------------------- 1 | # Empty 2 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/mypy1.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | show_traceback = true 3 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/setup1.cfg: -------------------------------------------------------------------------------- 1 | [mypy] 2 | show_traceback = true 3 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/pyproject2.toml: -------------------------------------------------------------------------------- 1 | # This file has no `[tool.mypy]` existing config 2 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [testenv] 2 | deps = 3 | -rrequirements.txt 4 | commands = 5 | python -m pytest {posargs} 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include tox.ini 3 | include pyproject.toml 4 | graft pytest_mypy_plugins/tests 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | black 2 | isort 3 | types-decorator 4 | types-PyYAML 5 | types-setuptools 6 | types-regex 7 | mypy==1.19.1 8 | -e . 9 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-extension.yml: -------------------------------------------------------------------------------- 1 | - case: reveal_type_extension_is_loaded 2 | main: | 3 | # if hook works, main should contain 'reveal_type(1)' 4 | reveal_type: 1 5 | out: | 6 | main:1: note: Revealed type is "Literal[1]?" 7 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/pyproject1.toml: -------------------------------------------------------------------------------- 1 | # This file has `[tool.mypy]` existing config 2 | 3 | [tool.mypy] 4 | warn_unused_ignores = true 5 | pretty = true 6 | show_error_codes = true 7 | 8 | [tool.other] 9 | # This section should not be copied: 10 | key = 'value' 11 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-mypy-config.yml: -------------------------------------------------------------------------------- 1 | # Also used in `test_explicit_configs.py` 2 | 3 | - case: custom_mypy_config_disallow_any_explicit_set 4 | expect_fail: yes 5 | main: | 6 | from typing import Any 7 | a: Any = None # should raise an error 8 | mypy_config: | 9 | disallow_any_explicit = true 10 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/pyproject3.toml: -------------------------------------------------------------------------------- 1 | # This file has `[tool.mypy]` existing config 2 | 3 | [tool.mypy] 4 | warn_unused_ignores = true 5 | pretty = true 6 | show_error_codes = true 7 | 8 | [[tool.mypy.overrides]] 9 | # This section should be copied 10 | module = "mymodule" 11 | ignore_missing_imports = true 12 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "02:00" 8 | open-pull-requests-limit: 10 9 | 10 | - package-ecosystem: github-actions 11 | directory: "/" 12 | schedule: 13 | interval: daily 14 | time: "02:00" 15 | open-pull-requests-limit: 10 16 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Check http://editorconfig.org for more information 2 | # This is the main config file for this project: 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | end_of_line = lf 9 | indent_style = space 10 | insert_final_newline = true 11 | indent_size = 2 12 | 13 | [*.py] 14 | indent_size = 4 15 | 16 | [*.pyi] 17 | indent_size = 4 18 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/reveal_type_hook.py: -------------------------------------------------------------------------------- 1 | from pytest_mypy_plugins.item import YamlTestItem 2 | 3 | 4 | def hook(item: YamlTestItem) -> None: 5 | parsed_test_data = item.parsed_test_data 6 | obj_to_reveal = parsed_test_data.get("reveal_type") 7 | if obj_to_reveal: 8 | for file in item.files: 9 | if file.path.endswith("main.py"): 10 | file.content = f"reveal_type({obj_to_reveal})" 11 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-paths-from-env.yml: -------------------------------------------------------------------------------- 1 | - case: add_mypypath_env_var_to_package_search 2 | main: | 3 | import extra_module 4 | extra_module.extra_fn() 5 | 6 | extra_module.missing() # E: Module has no attribute "missing" [attr-defined] 7 | env: 8 | - MYPYPATH=../extras 9 | files: 10 | - path: ../extras/extra_module.py 11 | content: | 12 | def extra_fn() -> None: 13 | pass 14 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.mypy] 2 | strict = true 3 | ignore_missing_imports = true 4 | warn_unreachable = true 5 | 6 | [tool.pytest.ini_options] 7 | python_files = "test_*.py" 8 | addopts = "-s --mypy-extension-hook pytest_mypy_plugins.tests.reveal_type_hook.hook" 9 | 10 | [tool.black] 11 | line-length = 120 12 | target-version = ["py39"] 13 | 14 | [tool.isort] 15 | include_trailing_comma = true 16 | multi_line_output = 3 17 | profile = "black" 18 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | if [[ -z $(git status -s) ]] 5 | then 6 | if [[ "$VIRTUAL_ENV" != "" ]] 7 | then 8 | pip install --upgrade setuptools wheel twine 9 | python setup.py sdist bdist_wheel 10 | twine upload dist/* 11 | rm -rf dist/ build/ 12 | else 13 | echo "this script must be executed inside an active virtual env, aborting" 14 | fi 15 | else 16 | echo "git working tree is not clean, aborting" 17 | fi 18 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-assert-type.yml: -------------------------------------------------------------------------------- 1 | - case: assert_type 2 | main: | 3 | from typing_extensions import assert_type 4 | 5 | def x() -> int: 6 | return 1 7 | 8 | assert_type(x(), int) 9 | 10 | - case: assert_type_error 11 | mypy_config: | 12 | warn_unused_ignores = true 13 | main: | 14 | from typing_extensions import assert_type 15 | 16 | def x() -> int: 17 | return 1 18 | 19 | assert_type(x(), str) # type: ignore 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Maksim Kurnikov 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_explicit_configs.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | from typing import Final 4 | 5 | import pytest 6 | 7 | _PYPROJECT1: Final = str(Path(__file__).parent / "test_configs" / "pyproject1.toml") 8 | _PYPROJECT2: Final = str(Path(__file__).parent / "test_configs" / "pyproject2.toml") 9 | _MYPYINI1: Final = str(Path(__file__).parent / "test_configs" / "mypy1.ini") 10 | _MYPYINI2: Final = str(Path(__file__).parent / "test_configs" / "mypy2.ini") 11 | _SETUPCFG1: Final = str(Path(__file__).parent / "test_configs" / "setup1.cfg") 12 | _SETUPCFG2: Final = str(Path(__file__).parent / "test_configs" / "setup2.cfg") 13 | 14 | _TEST_FILE: Final = str(Path(__file__).parent / "test-mypy-config.yml") 15 | 16 | 17 | @pytest.mark.parametrize("config_file", [_PYPROJECT1, _PYPROJECT2]) 18 | def test_pyproject_toml(config_file: str) -> None: 19 | subprocess.check_output( 20 | [ 21 | "pytest", 22 | "--mypy-pyproject-toml-file", 23 | config_file, 24 | _TEST_FILE, 25 | ] 26 | ) 27 | 28 | 29 | @pytest.mark.parametrize( 30 | "config_file", 31 | [ 32 | _MYPYINI1, 33 | _MYPYINI2, 34 | _SETUPCFG1, 35 | _SETUPCFG2, 36 | ], 37 | ) 38 | def test_ini_files(config_file: str) -> None: 39 | subprocess.check_output( 40 | [ 41 | "pytest", 42 | "--mypy-ini-file", 43 | config_file, 44 | _TEST_FILE, 45 | ] 46 | ) 47 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | permissions: 11 | contents: read 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | test: 19 | runs-on: ubuntu-latest 20 | 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 25 | pytest-version: ["~=7.2", "~=8.3"] 26 | 27 | steps: 28 | - uses: actions/checkout@v6 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v6 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | - name: Install dependencies 34 | run: | 35 | pip install -U pip setuptools wheel 36 | pip install -e . 37 | # Force correct `pytest` version for different envs: 38 | pip install -U "pytest${{ matrix.pytest-version }}" 39 | - name: Run tests 40 | run: pytest 41 | 42 | lint: 43 | runs-on: ubuntu-latest 44 | 45 | steps: 46 | - uses: actions/checkout@v6 47 | - name: Set up Python 48 | uses: actions/setup-python@v6 49 | with: 50 | python-version: 3.12 51 | - name: Install dependencies 52 | run: | 53 | pip install -U pip setuptools wheel 54 | pip install -r requirements.txt 55 | - name: Run linters 56 | run: | 57 | mypy . 58 | black --check . 59 | isort --check --diff . 60 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | with open("README.md") as f: 4 | readme = f.read() 5 | 6 | dependencies = [ 7 | "Jinja2", 8 | "decorator", 9 | "jsonschema", 10 | "mypy>=1.3", 11 | "packaging", 12 | "pytest>=7.0.0", 13 | "pyyaml", 14 | "regex", 15 | "tomlkit>=0.11", 16 | ] 17 | 18 | setup( 19 | name="pytest-mypy-plugins", 20 | version="3.2.0", 21 | description="pytest plugin for writing tests for mypy plugins", 22 | long_description=readme, 23 | long_description_content_type="text/markdown", 24 | license="MIT", 25 | url="https://github.com/TypedDjango/pytest-mypy-plugins", 26 | author="Maksim Kurnikov", 27 | author_email="maxim.kurnikov@gmail.com", 28 | maintainer="Nikita Sobolev", 29 | maintainer_email="mail@sobolevn.me", 30 | packages=["pytest_mypy_plugins"], 31 | # the following makes a plugin available to pytest 32 | entry_points={"pytest11": ["pytest-mypy-plugins = pytest_mypy_plugins.collect"]}, 33 | install_requires=dependencies, 34 | python_requires=">=3.9", 35 | package_data={ 36 | "pytest_mypy_plugins": ["py.typed", "schema.json"], 37 | }, 38 | classifiers=[ 39 | "Development Status :: 4 - Beta", 40 | "License :: OSI Approved :: MIT License", 41 | "Programming Language :: Python :: 3", 42 | "Programming Language :: Python :: 3.9", 43 | "Programming Language :: Python :: 3.10", 44 | "Programming Language :: Python :: 3.11", 45 | "Programming Language :: Python :: 3.12", 46 | "Programming Language :: Python :: 3.13", 47 | "Typing :: Typed", 48 | ], 49 | ) 50 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-parametrized.yml: -------------------------------------------------------------------------------- 1 | - case: only_main 2 | parametrized: 3 | - a: 1 4 | revealed_type: builtins.int 5 | - a: 1.0 6 | revealed_type: builtins.float 7 | main: | 8 | a = {{ a }} 9 | reveal_type(a) # N: Revealed type is "{{ revealed_type }}" 10 | 11 | 12 | - case: with_extra 13 | parametrized: 14 | - a: 2 15 | b: null 16 | rt: Any 17 | - a: 3 18 | b: 3 19 | rt: Any 20 | main: | 21 | import foo 22 | reveal_type(foo.test({{ a }}, {{ b }})) # N: Revealed type is "{{ rt }}" 23 | files: 24 | - path: foo.py 25 | content: | 26 | from typing import Any 27 | 28 | def test(a: Any, b: Any) -> Any: 29 | ... 30 | 31 | 32 | - case: with_out 33 | parametrized: 34 | - what: cat 35 | rt: builtins.str 36 | - what: dog 37 | rt: builtins.str 38 | main: | 39 | animal = '{{ what }}' 40 | reveal_type(animal) 41 | try: 42 | animal / 2 43 | except Exception: 44 | ... 45 | out: | 46 | main:2: note: Revealed type is "{{ rt }}" 47 | main:4: error: Unsupported operand types for / ("str" and "int") [operator] 48 | 49 | - case: parametrized_can_skip_mypy_config_section 50 | parametrized: 51 | - val: False 52 | - val: True 53 | mypy_config: | 54 | hide_error_codes = True 55 | main: | 56 | a = {{ val }} 57 | a.lower() # E: "bool" has no attribute "lower" 58 | 59 | - case: with_mypy_config 60 | parametrized: 61 | - allow_any: "true" 62 | - allow_any: "false" 63 | mypy_config: | 64 | disallow_any_explicit = {{ allow_any }} 65 | main: | 66 | # Anything will work, we just need to be sure that 67 | # `disallow_any_generics: Not a boolean: {{ allow_any }}` 68 | # is not raised 69 | 1 + 1 70 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_input_schema.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from typing import Sequence 3 | 4 | import jsonschema 5 | import pytest 6 | import yaml 7 | 8 | from pytest_mypy_plugins.collect import validate_schema 9 | 10 | 11 | def get_all_yaml_files(dir_path: pathlib.Path) -> Sequence[pathlib.Path]: 12 | yaml_files = [] 13 | for file in dir_path.rglob("*"): 14 | if file.suffix in (".yml", ".yaml"): 15 | yaml_files.append(file) 16 | 17 | return yaml_files 18 | 19 | 20 | files = get_all_yaml_files(pathlib.Path(__file__).parent) 21 | 22 | 23 | @pytest.mark.parametrize("yaml_file", files, ids=lambda x: x.stem) 24 | def test_yaml_files(yaml_file: pathlib.Path) -> None: 25 | validate_schema(yaml.safe_load(yaml_file.read_text())) 26 | 27 | 28 | def test_mypy_config_is_not_an_object() -> None: 29 | with pytest.raises(jsonschema.exceptions.ValidationError) as ex: 30 | validate_schema( 31 | [ 32 | { 33 | "case": "mypy_config_is_not_an_object", 34 | "main": "False", 35 | "mypy_config": [{"force_uppercase_builtins": True}, {"force_union_syntax": True}], 36 | } 37 | ] 38 | ) 39 | 40 | assert ( 41 | ex.value.message == "[{'force_uppercase_builtins': True}, {'force_union_syntax': True}] is not of type 'string'" 42 | ) 43 | 44 | 45 | def test_closed_schema() -> None: 46 | with pytest.raises(jsonschema.exceptions.ValidationError) as ex: 47 | validate_schema( 48 | [ 49 | { 50 | "case": "mypy_config_is_not_an_object", 51 | "main": "False", 52 | "extra_field": 1, 53 | } 54 | ], 55 | is_closed=True, 56 | ) 57 | 58 | assert ex.value.message == "Additional properties are not allowed ('extra_field' was unexpected)" 59 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-regex-assertions.yml: -------------------------------------------------------------------------------- 1 | - case: expected_message_regex 2 | regex: yes 3 | main: | 4 | a = 1 5 | b = 'hello' 6 | 7 | reveal_type(a) # N: Revealed type is "builtins.int" 8 | reveal_type(b) # N: .*str.* 9 | 10 | - case: expected_message_regex_with_out 11 | regex: yes 12 | main: | 13 | a = 'abc' 14 | reveal_type(a) 15 | out: | 16 | main:2: note: .*str.* 17 | 18 | - case: regex_with_out_does_not_hang 19 | expect_fail: yes 20 | regex: yes 21 | main: | 22 | 'abc'.split(4) 23 | out: | 24 | main:1: error: Argument 1 to "split" of "str" has incompatible type "int"; expected "Optional[str]" 25 | 26 | - case: regex_with_comment_does_not_hang 27 | expect_fail: yes 28 | regex: yes 29 | main: | 30 | a = 'abc'.split(4) # E: Argument 1 to "split" of "str" has incompatible type "int"; expected "Optional[str]" 31 | 32 | - case: expected_single_message_regex 33 | regex: no 34 | main: | 35 | a = 'hello' 36 | reveal_type(a) # NR: .*str.* 37 | 38 | - case: rexex_but_not_turned_on 39 | expect_fail: yes 40 | main: | 41 | a = 'hello' 42 | reveal_type(a) # N: .*str.* 43 | 44 | - case: rexex_but_turned_off 45 | expect_fail: yes 46 | regex: no 47 | main: | 48 | a = 'hello' 49 | reveal_type(a) # N: .*str.* 50 | 51 | - case: regex_does_not_match 52 | expect_fail: yes 53 | regex: no 54 | main: | 55 | a = 'hello' 56 | reveal_type(a) # NR: .*banana.* 57 | 58 | - case: regex_against_callable_comment 59 | main: | 60 | from typing import Set, Union 61 | 62 | def foo(bar: str, ham: int = 42) -> Set[Union[str, int]]: 63 | return {bar, ham} 64 | reveal_type(foo) # NR: Revealed type is "def \(bar: builtins\.str, ham: builtins\.int =\) -> .*" 65 | 66 | - case: regex_against_callable_out 67 | regex: yes 68 | main: | 69 | from typing import Set, Union 70 | 71 | def foo(bar: str, ham: int = 42) -> Set[Union[str, int]]: 72 | return {bar, ham} 73 | reveal_type(foo) 74 | out: | 75 | main:5: note: Revealed type is "def \(bar: builtins\.str, ham: builtins\.int =\) -> .*" 76 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test-simple-cases.yml: -------------------------------------------------------------------------------- 1 | - case: simplest_case 2 | main: | 3 | a = 1 4 | b = 'hello' 5 | 6 | class MyClass: 7 | pass 8 | 9 | reveal_type(a) # N: Revealed type is "builtins.int" 10 | reveal_type(b) # N: Revealed type is "builtins.str" 11 | 12 | 13 | - case: revealed_type_with_environment 14 | main: | 15 | a = 1 16 | class MyClass: 17 | def __init__(self): 18 | pass 19 | 20 | b = 'hello' 21 | 22 | reveal_type(a) # N: Revealed type is "builtins.int" 23 | reveal_type(b) # N: Revealed type is "builtins.str" 24 | env: 25 | - DJANGO_SETTINGS_MODULE=mysettings 26 | 27 | 28 | - case: revealed_type_with_disabled_cache 29 | main: | 30 | a = 1 31 | reveal_type(a) # N: Revealed type is "builtins.int" 32 | disable_cache: true 33 | 34 | 35 | - case: external_output_lines 36 | main: | 37 | a = 1 38 | reveal_type(a) 39 | out: | 40 | main:2: note: Revealed type is "builtins.int" 41 | 42 | 43 | - case: create_files 44 | main: | 45 | a = 1 46 | reveal_type(a) # N: Revealed type is "builtins.int" 47 | files: 48 | - path: myapp/__init__.py 49 | - path: myapp/models.py 50 | content: | 51 | from django.db import models 52 | class MyModel: 53 | pass 54 | - path: myapp/apps.py 55 | 56 | 57 | - case: error_case 58 | main: | 59 | a = 1 60 | a.lower() # E: "int" has no attribute "lower" [attr-defined] 61 | 62 | 63 | - case: skip_incorrect_test_case 64 | skip: yes 65 | main: | 66 | a = 1 67 | reveal_type(a) # N: boom! 68 | 69 | 70 | - case: skip_if_true 71 | skip: sys.version_info > (2, 0) 72 | main: | 73 | a = 1 74 | a.lower() # E: boom! 75 | 76 | 77 | - case: skip_if_false 78 | skip: sys.version_info < (0, 0) 79 | main: | 80 | a = 1 81 | a.lower() # E: "int" has no attribute "lower" [attr-defined] 82 | 83 | - case: fail_if_message_does_not_match 84 | expect_fail: yes 85 | main: | 86 | a = 'hello' 87 | reveal_type(a) # N: Some other message 88 | 89 | - case: fail_if_message_from_outdoes_not_match 90 | expect_fail: yes 91 | main: | 92 | a = 'abc' 93 | reveal_type(a) 94 | out: | 95 | main:2: note: Some other message 96 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/configs.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | from pathlib import Path 3 | from textwrap import dedent 4 | from typing import Final, Optional 5 | 6 | import tomlkit 7 | 8 | _TOML_TABLE_NAME: Final = "[tool.mypy]" 9 | 10 | 11 | def join_ini_configs(base_ini_fpath: Optional[str], additional_mypy_config: str, execution_path: Path) -> Optional[str]: 12 | mypy_ini_config = ConfigParser() 13 | if base_ini_fpath: 14 | mypy_ini_config.read(base_ini_fpath) 15 | if additional_mypy_config: 16 | if "[mypy]" not in additional_mypy_config: 17 | additional_mypy_config = f"[mypy]\n{additional_mypy_config}" 18 | mypy_ini_config.read_string(additional_mypy_config) 19 | 20 | if mypy_ini_config.sections(): 21 | mypy_config_file_path = execution_path / "mypy.ini" 22 | with mypy_config_file_path.open("w") as f: 23 | mypy_ini_config.write(f) 24 | return str(mypy_config_file_path) 25 | return None 26 | 27 | 28 | def join_toml_configs( 29 | base_pyproject_toml_fpath: str, additional_mypy_config: str, execution_path: Path 30 | ) -> Optional[str]: 31 | if base_pyproject_toml_fpath: 32 | with open(base_pyproject_toml_fpath) as f: 33 | toml_config = tomlkit.parse(f.read()) 34 | else: 35 | # Empty document with `[tool.mypy]` empty table, 36 | # useful for overrides further. 37 | toml_config = tomlkit.document() 38 | 39 | if "tool" not in toml_config or "mypy" not in toml_config["tool"]: # type: ignore[operator] 40 | tool = tomlkit.table(is_super_table=True) 41 | tool.append("mypy", tomlkit.table()) 42 | toml_config.append("tool", tool) 43 | 44 | if additional_mypy_config: 45 | if _TOML_TABLE_NAME not in additional_mypy_config: 46 | additional_mypy_config = f"{_TOML_TABLE_NAME}\n{dedent(additional_mypy_config)}" 47 | 48 | additional_data = tomlkit.parse(additional_mypy_config) 49 | toml_config["tool"]["mypy"].update( # type: ignore[index, union-attr] 50 | additional_data["tool"]["mypy"].value.items(), # type: ignore[index] 51 | ) 52 | 53 | mypy_config_file_path = execution_path / "pyproject.toml" 54 | with mypy_config_file_path.open("w") as f: 55 | # We don't want the whole config file, because it can contain 56 | # other sections like `[tool.isort]`, we only need `[tool.mypy]` part. 57 | tool_mypy = toml_config["tool"]["mypy"] # type: ignore[index] 58 | 59 | # construct toml output 60 | min_toml = tomlkit.document() 61 | min_tool = tomlkit.table(is_super_table=True) 62 | min_toml.append("tool", min_tool) 63 | min_tool.append("mypy", tool_mypy) 64 | 65 | f.write(min_toml.as_string()) 66 | return str(mypy_config_file_path) 67 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Version history 2 | 3 | 4 | ## 3.2.0 5 | 6 | ### Features 7 | 8 | - Drops `python3.8` support 9 | - Adds official `python3.13` support 10 | 11 | ### Bugfixes 12 | 13 | - Fixes regex for colon output `:`, #155 14 | - Fixes internal error with `TraceLastReprEntry`, #154 15 | 16 | 17 | ## 3.1.2 18 | 19 | ### Bugfixes 20 | 21 | - Fix joining `toml` configs with `[[mypy.overrides]]` 22 | 23 | 24 | ## 3.1.1 25 | 26 | ### Bugfixes 27 | 28 | - Make sure that schema is open by default: only check existing fields 29 | - Add `--mypy-schema-closed` option to check schemas with no extra fields 30 | 31 | 32 | ## 3.1.0 33 | 34 | ### Features 35 | 36 | - Add `python3.12` support 37 | - Add `mypy@1.8.0` support 38 | - Add schema definition 39 | 40 | 41 | ## 3.0.0 42 | 43 | ### Features 44 | 45 | - *Breaking*: Drop python3.7 support 46 | - Add `pyproject.toml` config file support with `--mypy-pyproject-toml-file` option 47 | 48 | ### Bugfixes 49 | 50 | - Add `tox.ini` file to `sdist` package 51 | - Add `requirements.txt` file to `sdist` package 52 | - Add `pyproject.toml` file to `sdist` package 53 | 54 | 55 | ## 2.0.0 56 | 57 | ### Features 58 | 59 | - Use `jinja2` instead of `chevron` for templating 60 | - Allow parametrizing `mypy_config` field in tests 61 | - Bump minimal `mypy` and `pytest` versions 62 | 63 | ### Bugfixes 64 | 65 | - Also include `mypy.ini` and `pytest.ini` to `sdist` package 66 | 67 | 68 | ## Version 1.11.1 69 | 70 | ### Bugfixes 71 | 72 | - Adds `tests/` subfolder to `sdist` package 73 | 74 | 75 | ## Version 1.11.0 76 | 77 | ### Features 78 | 79 | - Adds `python3.11` support and promise about `python3.12` support 80 | - Removes `pkg_resources` to use `packaging` instead 81 | 82 | 83 | ## Version 1.10.1 84 | 85 | ### Bugfixes 86 | 87 | - Removes unused depenencies for `python < 3.7` 88 | - Fixes compatibility with pytest 7.2, broken due to a private import from 89 | `py._path`. 90 | 91 | 92 | ## Version 1.10.0 93 | 94 | ### Features 95 | 96 | - Changes how `mypy>=0.970` handles `MYPYPATH` 97 | - Bumps minimal `mypy` version to `mypy>=0.970` 98 | - Drops `python3.6` support 99 | 100 | 101 | ## Version 1.9.3 102 | 103 | ### Bugfixes 104 | 105 | - Fixes `DeprecationWarning` for using `py.LocalPath` for `pytest>=7.0` #89 106 | 107 | 108 | ## Version 1.9.2 109 | 110 | ### Bugfixes 111 | 112 | - Removes usages of `distutils` #71 113 | - Fixes multiline messages #66 114 | - Fixes that empty output test cases was almost ignored #63 115 | - Fixes output formatting for expected messages #66 116 | 117 | 118 | ## Version 1.9.1 119 | 120 | ## Bugfixes 121 | 122 | - Fixes that `regex` and `dataclasses` dependencies were not listed in `setup.py` 123 | 124 | 125 | ## Version 1.9.0 126 | 127 | ## Features 128 | 129 | - Adds `regex` support in matching test output 130 | - Adds a flag for expected failures 131 | - Replaces deprecated `pystache` with `chevron` 132 | 133 | ## Misc 134 | 135 | - Updates `mypy` 136 | 137 | 138 | ## Version 1.8.0 139 | 140 | We missed this released by mistake. 141 | 142 | 143 | ## Version 1.7.0 144 | 145 | ### Features 146 | 147 | - Adds `--mypy-only-local-stub` CLI flag to ignore errors in site-packages 148 | 149 | 150 | ## Version 1.6.1 151 | 152 | ### Bugfixes 153 | 154 | - Changes how `MYPYPATH` and `PYTHONPATH` are calcualted. We now expand `$PWD` variable and also include relative paths specified in `env:` section 155 | 156 | 157 | ## Version 1.6.0 158 | 159 | ### Features 160 | 161 | - Adds `python3.9` support 162 | - Bumps required version of `pytest` to `>=6.0` 163 | - Bumps required version of `mypy` to `>=0.790` 164 | 165 | ### Misc 166 | 167 | - Moves from Travis to Github Actions 168 | 169 | 170 | ## Version 1.5.0 171 | 172 | ### Features 173 | 174 | - Adds `PYTHONPATH` and `MYPYPATH` special handling 175 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_configs/test_join_toml_configs.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from textwrap import dedent 3 | from typing import Callable, Final, Optional 4 | 5 | import pytest 6 | 7 | from pytest_mypy_plugins.configs import join_toml_configs 8 | 9 | _ADDITIONAL_CONFIG: Final = """ 10 | [tool.mypy] 11 | pretty = true 12 | show_error_codes = false 13 | show_traceback = true 14 | """ 15 | 16 | _ADDITIONAL_CONFIG_NO_TABLE: Final = """ 17 | pretty = true 18 | show_error_codes = false 19 | show_traceback = true 20 | """ 21 | 22 | _PYPROJECT1: Final = str(Path(__file__).parent / "pyproject1.toml") 23 | _PYPROJECT2: Final = str(Path(__file__).parent / "pyproject2.toml") 24 | _PYPROJECT3: Final = str(Path(__file__).parent / "pyproject3.toml") 25 | 26 | 27 | @pytest.fixture 28 | def execution_path(tmpdir_factory: pytest.TempdirFactory) -> Path: 29 | return Path(tmpdir_factory.mktemp("testproject", numbered=True)) 30 | 31 | 32 | _AssertFileContents = Callable[[Optional[str], str], None] 33 | 34 | 35 | @pytest.fixture 36 | def assert_file_contents() -> _AssertFileContents: 37 | def factory(filename: Optional[str], expected: str) -> None: 38 | assert filename 39 | 40 | expected = dedent(expected).strip() 41 | with open(filename) as f: 42 | contents = f.read().strip() 43 | assert contents == expected 44 | 45 | return factory 46 | 47 | 48 | @pytest.mark.parametrize( 49 | "additional_config", 50 | [ 51 | _ADDITIONAL_CONFIG, 52 | _ADDITIONAL_CONFIG_NO_TABLE, 53 | ], 54 | ) 55 | def test_join_existing_config( 56 | execution_path: Path, assert_file_contents: _AssertFileContents, additional_config: str 57 | ) -> None: 58 | filepath = join_toml_configs(_PYPROJECT1, additional_config, execution_path) 59 | 60 | assert_file_contents( 61 | filepath, 62 | """ 63 | [tool.mypy] 64 | warn_unused_ignores = true 65 | pretty = true 66 | show_error_codes = false 67 | show_traceback = true 68 | """, 69 | ) 70 | 71 | 72 | @pytest.mark.parametrize( 73 | "additional_config", 74 | [ 75 | _ADDITIONAL_CONFIG, 76 | _ADDITIONAL_CONFIG_NO_TABLE, 77 | ], 78 | ) 79 | def test_join_missing_config( 80 | execution_path: Path, assert_file_contents: _AssertFileContents, additional_config: str 81 | ) -> None: 82 | filepath = join_toml_configs(_PYPROJECT2, additional_config, execution_path) 83 | 84 | assert_file_contents( 85 | filepath, 86 | """ 87 | [tool.mypy] 88 | pretty = true 89 | show_error_codes = false 90 | show_traceback = true 91 | """, 92 | ) 93 | 94 | 95 | def test_join_missing_config1(execution_path: Path, assert_file_contents: _AssertFileContents) -> None: 96 | filepath = join_toml_configs(_PYPROJECT1, "", execution_path) 97 | 98 | assert_file_contents( 99 | filepath, 100 | """ 101 | [tool.mypy] 102 | warn_unused_ignores = true 103 | pretty = true 104 | show_error_codes = true 105 | """, 106 | ) 107 | 108 | 109 | def test_join_missing_config2(execution_path: Path, assert_file_contents: _AssertFileContents) -> None: 110 | filepath = join_toml_configs(_PYPROJECT2, "", execution_path) 111 | 112 | assert_file_contents( 113 | filepath, 114 | "[tool.mypy]", 115 | ) 116 | 117 | 118 | def test_join_missing_config3(execution_path: Path, assert_file_contents: _AssertFileContents) -> None: 119 | filepath = join_toml_configs(_PYPROJECT3, "", execution_path) 120 | 121 | assert_file_contents( 122 | filepath, 123 | """ 124 | [tool.mypy] 125 | warn_unused_ignores = true 126 | pretty = true 127 | show_error_codes = true 128 | 129 | [[tool.mypy.overrides]] 130 | # This section should be copied 131 | module = "mymodule" 132 | ignore_missing_imports = true 133 | """, 134 | ) 135 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | 3 | ## Mostly complete version from https://github.com/github/gitignore/blob/e5323759e387ba347a9d50f8b0ddd16502eb71d4/Python.gitignore 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | pip-wheel-metadata/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | cover/ 58 | cache/* 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/#use-with-ide 116 | .pdm.toml 117 | 118 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 119 | __pypackages__/ 120 | 121 | # Celery stuff 122 | celerybeat-schedule 123 | celerybeat.pid 124 | 125 | # SageMath parsed files 126 | *.sage.py 127 | 128 | # Environments 129 | .env 130 | .venv 131 | env/ 132 | venv/ 133 | ENV/ 134 | env.bak/ 135 | venv.bak/ 136 | 137 | # Spyder project settings 138 | .spyderproject 139 | .spyproject 140 | 141 | # Rope project settings 142 | .ropeproject 143 | 144 | # mkdocs documentation 145 | /site 146 | 147 | # mypy 148 | .mypy_cache/ 149 | .dmypy.json 150 | dmypy.json 151 | 152 | # Pyre type checker 153 | .pyre/ 154 | 155 | # pytype static type analyzer 156 | .pytype/ 157 | 158 | # Cython debug symbols 159 | cython_debug/ 160 | 161 | # PyCharm 162 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 163 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 164 | # and can be added to the global gitignore or merged into this file. For a more nuclear 165 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 166 | #.idea/ 167 | 168 | # VS code 169 | .vscode/launch.json 170 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema", 3 | "$id": "https://raw.githubusercontent.com/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json", 4 | "title": "pytest-mypy-plugins test file", 5 | "description": "JSON Schema for a pytest-mypy-plugins test file", 6 | "type": "array", 7 | "items": { 8 | "type": "object", 9 | "additionalProperties": true, 10 | "properties": { 11 | "case": { 12 | "type": "string", 13 | "pattern": "^[a-zA-Z0-9_]+$", 14 | "description": "Name of the test case, MUST comply to the `^[a-zA-Z0-9_]+$` pattern.", 15 | "examples": [ 16 | { 17 | "case": "TestCase1" 18 | }, 19 | { 20 | "case": "999" 21 | }, 22 | { 23 | "case": "test_case_1" 24 | } 25 | ] 26 | }, 27 | "main": { 28 | "type": "string", 29 | "description": "Portion of the code as if written in `.py` file. Must be valid Python code.", 30 | "examples": [ 31 | { 32 | "main": "reveal_type(1)" 33 | } 34 | ] 35 | }, 36 | "out": { 37 | "type": "string", 38 | "description": "Verbose output expected from `mypy`.", 39 | "examples": [ 40 | { 41 | "out": "main:1: note: Revealed type is \"Literal[1]?\"" 42 | } 43 | ] 44 | }, 45 | "files": { 46 | "type": "array", 47 | "items": { 48 | "$ref": "#/definitions/File" 49 | }, 50 | "description": "List of extra files to simulate imports if needed.", 51 | "examples": [ 52 | [ 53 | { 54 | "path": "myapp/__init__.py" 55 | }, 56 | { 57 | "path": "myapp/utils.py", 58 | "content": "def help(): pass" 59 | } 60 | ] 61 | ] 62 | }, 63 | "disable_cache": { 64 | "type": "boolean", 65 | "description": "Set to `true` disables `mypy` caching.", 66 | "default": false 67 | }, 68 | "mypy_config": { 69 | "type": "string", 70 | "description": "Inline `mypy` configuration, passed directly to `mypy`.", 71 | "examples": [ 72 | { 73 | "mypy_config": "force_uppercase_builtins = true\nforce_union_syntax = true\n" 74 | } 75 | ] 76 | }, 77 | "env": { 78 | "type": "array", 79 | "items": { 80 | "type": "string" 81 | }, 82 | "description": "Environmental variables to be provided inside of test run.", 83 | "examples": [ 84 | "MYPYPATH=../extras", 85 | "DJANGO_SETTINGS_MODULE=mysettings" 86 | ] 87 | }, 88 | "parametrized": { 89 | "type": "array", 90 | "items": { 91 | "$ref": "#/definitions/Parameter" 92 | }, 93 | "description": "List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html). Each entry **must** have the **exact** same set of keys.", 94 | "examples": [ 95 | [ 96 | { 97 | "val": 1, 98 | "rt": "int" 99 | }, 100 | { 101 | "val": "1", 102 | "rt": "str" 103 | } 104 | ] 105 | ] 106 | }, 107 | "skip": { 108 | "anyOf": [ 109 | { 110 | "type": "boolean" 111 | }, 112 | { 113 | "type": "string" 114 | } 115 | ], 116 | "description": "An expression set in `skip` is passed directly into [`eval`](https://docs.python.org/3/library/functions.html#eval). It is advised to take a peek and learn about how `eval` works. Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform`.", 117 | "examples": [ 118 | "yes", 119 | true, 120 | "sys.version_info > (2, 0)" 121 | ], 122 | "default": false 123 | }, 124 | "expect_fail": { 125 | "type": "boolean", 126 | "description": "Mark test case as an expected failure.", 127 | "default": false 128 | }, 129 | "regex": { 130 | "type": "boolean", 131 | "description": "Allow regular expressions in comments to be matched against actual output. _See pytest_mypy_plugins/tests/test-regex_assertions.yml for examples_", 132 | "default": false 133 | } 134 | }, 135 | "required": [ 136 | "case", 137 | "main" 138 | ] 139 | }, 140 | "definitions": { 141 | "File": { 142 | "type": "object", 143 | "properties": { 144 | "path": { 145 | "type": "string", 146 | "description": "File path.", 147 | "examples": [ 148 | "../extras/extra_module.py", 149 | "myapp/__init__.py" 150 | ] 151 | }, 152 | "content": { 153 | "type": "string", 154 | "description": "File content. Can be empty. Must be valid Python code.", 155 | "examples": [ 156 | "def help(): pass", 157 | "def help():\n pass\n" 158 | ] 159 | } 160 | }, 161 | "required": [ 162 | "path" 163 | ] 164 | }, 165 | "Parameter": { 166 | "type": "object", 167 | "additionalProperties": true, 168 | "description": "A mapping of keys to values, similar to Python's `Mapping[str, Any]`.", 169 | "examples": [ 170 | { 171 | "val": "1", 172 | "rt": "str" 173 | } 174 | ] 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/collect.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | import platform 5 | import sys 6 | import tempfile 7 | from dataclasses import dataclass 8 | from typing import ( 9 | TYPE_CHECKING, 10 | Any, 11 | Dict, 12 | Hashable, 13 | Iterator, 14 | List, 15 | Mapping, 16 | Optional, 17 | Set, 18 | ) 19 | 20 | import jsonschema 21 | import py.path 22 | import pytest 23 | import yaml 24 | from _pytest.config.argparsing import Parser 25 | from _pytest.nodes import Node 26 | 27 | from pytest_mypy_plugins import utils 28 | 29 | if TYPE_CHECKING: 30 | from pytest_mypy_plugins.item import YamlTestItem 31 | 32 | 33 | @dataclass 34 | class File: 35 | path: str 36 | content: str 37 | 38 | 39 | def validate_schema(data: Any, *, is_closed: bool = False) -> None: 40 | """Validate the schema of the file-under-test.""" 41 | # Unfortunately, yaml.safe_load() returns Any, 42 | # so we make our intention explicit here. 43 | if not isinstance(data, list): 44 | raise TypeError(f"Test file has to be YAML list, got {type(data)!r}.") 45 | 46 | schema = json.loads((pathlib.Path(__file__).parent / "schema.json").read_text("utf8")) 47 | schema["items"]["properties"]["__line__"] = { 48 | "type": "integer", 49 | "description": "Line number where the test starts (`pytest-mypy-plugins` internal)", 50 | } 51 | schema["items"]["additionalProperties"] = not is_closed 52 | 53 | jsonschema.validate(instance=data, schema=schema) 54 | 55 | 56 | def parse_test_files(test_files: List[Dict[str, Any]]) -> List[File]: 57 | files: List[File] = [] 58 | for test_file in test_files: 59 | path = test_file.get("path", "main.py") 60 | file = File(path=path, content=test_file.get("content", "")) 61 | files.append(file) 62 | return files 63 | 64 | 65 | def parse_environment_variables(env_vars: List[str]) -> Dict[str, str]: 66 | parsed_vars: Dict[str, str] = {} 67 | for env_var in env_vars: 68 | name, _, value = env_var.partition("=") 69 | parsed_vars[name] = value 70 | return parsed_vars 71 | 72 | 73 | def parse_parametrized(params: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]: 74 | if not params: 75 | return [{}] 76 | 77 | parsed_params: List[Mapping[str, Any]] = [] 78 | known_params: Optional[Set[str]] = None 79 | for idx, param in enumerate(params): 80 | param_keys = set(sorted(param.keys())) 81 | if not known_params: 82 | known_params = param_keys 83 | elif known_params.intersection(param_keys) != known_params: 84 | raise ValueError( 85 | "All parametrized entries must have same keys." 86 | f'First entry is {", ".join(known_params)} but {", ".join(param_keys)} ' 87 | "was spotted at {idx} position", 88 | ) 89 | parsed_params.append({k: v for k, v in param.items() if not k.startswith("__")}) 90 | 91 | return parsed_params 92 | 93 | 94 | class SafeLineLoader(yaml.SafeLoader): 95 | def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> Dict[Hashable, Any]: 96 | mapping = super().construct_mapping(node, deep=deep) 97 | # Add 1 so line numbering starts at 1 98 | starting_line = node.start_mark.line + 1 99 | for title_node, _contents_node in node.value: 100 | if title_node.value == "main": 101 | starting_line = title_node.start_mark.line + 1 102 | mapping["__line__"] = starting_line 103 | return mapping 104 | 105 | 106 | class YamlTestFile(pytest.File): 107 | def collect(self) -> Iterator["YamlTestItem"]: 108 | from pytest_mypy_plugins.item import YamlTestItem 109 | 110 | parsed_file = yaml.load(stream=self.path.read_text("utf8"), Loader=SafeLineLoader) 111 | if parsed_file is None: 112 | return 113 | 114 | validate_schema(parsed_file, is_closed=self.config.option.mypy_closed_schema) 115 | 116 | if not isinstance(parsed_file, list): 117 | raise ValueError(f"Test file has to be YAML list, got {type(parsed_file)!r}.") 118 | 119 | for raw_test in parsed_file: 120 | test_name_prefix = raw_test["case"] 121 | if " " in test_name_prefix: 122 | raise ValueError(f"Invalid test name {test_name_prefix!r}, only '[a-zA-Z0-9_]' is allowed.") 123 | else: 124 | parametrized = parse_parametrized(raw_test.get("parametrized", [])) 125 | 126 | for params in parametrized: 127 | if params: 128 | test_name_suffix = ",".join(f"{k}={v}" for k, v in params.items()) 129 | test_name_suffix = f"[{test_name_suffix}]" 130 | else: 131 | test_name_suffix = "" 132 | 133 | test_name = f"{test_name_prefix}{test_name_suffix}" 134 | main_content = utils.render_template(template=raw_test["main"], data=params) 135 | main_file = File(path="main.py", content=main_content) 136 | test_files = [main_file] + parse_test_files(raw_test.get("files", [])) 137 | expect_fail = raw_test.get("expect_fail", False) 138 | regex = raw_test.get("regex", False) 139 | 140 | expected_output = [] 141 | for test_file in test_files: 142 | output_lines = utils.extract_output_matchers_from_comments( 143 | test_file.path, test_file.content.split("\n"), regex=regex 144 | ) 145 | expected_output.extend(output_lines) 146 | 147 | starting_lineno = raw_test["__line__"] 148 | extra_environment_variables = parse_environment_variables(raw_test.get("env", [])) 149 | disable_cache = raw_test.get("disable_cache", False) 150 | expected_output.extend( 151 | utils.extract_output_matchers_from_out(raw_test.get("out", ""), params, regex=regex) 152 | ) 153 | additional_mypy_config = utils.render_template(template=raw_test.get("mypy_config", ""), data=params) 154 | 155 | skip = self._eval_skip(str(raw_test.get("skip", "False"))) 156 | if not skip: 157 | yield YamlTestItem.from_parent( 158 | self, 159 | name=test_name, 160 | files=test_files, 161 | starting_lineno=starting_lineno, 162 | environment_variables=extra_environment_variables, 163 | disable_cache=disable_cache, 164 | expected_output=expected_output, 165 | parsed_test_data=raw_test, 166 | mypy_config=additional_mypy_config, 167 | expect_fail=expect_fail, 168 | ) 169 | 170 | def _eval_skip(self, skip_if: str) -> bool: 171 | return bool(eval(skip_if, {"sys": sys, "os": os, "pytest": pytest, "platform": platform})) 172 | 173 | 174 | def pytest_collect_file(file_path: pathlib.Path, parent: Node) -> Optional[YamlTestFile]: 175 | if file_path.suffix in {".yaml", ".yml"} and file_path.name.startswith(("test-", "test_")): 176 | return YamlTestFile.from_parent(parent, path=file_path, fspath=None) 177 | return None 178 | 179 | 180 | def pytest_addoption(parser: Parser) -> None: 181 | group = parser.getgroup("mypy-tests") 182 | group.addoption( 183 | "--mypy-testing-base", type=str, default=tempfile.gettempdir(), help="Base directory for tests to use" 184 | ) 185 | group.addoption( 186 | "--mypy-pyproject-toml-file", 187 | type=str, 188 | help="Which `pyproject.toml` file to use as a default config for tests. Incompatible with `--mypy-ini-file`", 189 | ) 190 | group.addoption( 191 | "--mypy-ini-file", 192 | type=str, 193 | help="Which `.ini` file to use as a default config for tests. Incompatible with `--mypy-pyproject-toml-file`", 194 | ) 195 | group.addoption( 196 | "--mypy-same-process", 197 | action="store_true", 198 | help="Run in the same process. Useful for debugging, will create problems with import cache", 199 | ) 200 | group.addoption( 201 | "--mypy-extension-hook", 202 | type=str, 203 | help="Fully qualified path to the extension hook function, in case you need custom yaml keys. " 204 | "Has to be top-level.", 205 | ) 206 | group.addoption( 207 | "--mypy-only-local-stub", 208 | action="store_true", 209 | help="mypy will ignore errors from site-packages", 210 | ) 211 | group.addoption( 212 | "--mypy-closed-schema", 213 | action="store_true", 214 | help="Use closed schema to validate YAML test cases, which won't allow any extra keys (does not work well with `--mypy-extension-hook`)", 215 | ) 216 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | from typing import List, NamedTuple 3 | 4 | import pytest 5 | 6 | from pytest_mypy_plugins import utils 7 | from pytest_mypy_plugins.utils import ( 8 | OutputMatcher, 9 | TypecheckAssertionError, 10 | assert_expected_matched_actual, 11 | extract_output_matchers_from_comments, 12 | sorted_by_file_and_line, 13 | ) 14 | 15 | 16 | class ExpectMatchedActualTestData(NamedTuple): 17 | source_lines: List[str] 18 | actual_lines: List[str] 19 | expected_message_lines: List[str] 20 | 21 | 22 | def test_render_template_with_None_value() -> None: 23 | # Given 24 | template = "{{ a }} {{ b }}" 25 | data = {"a": None, "b": 99} 26 | 27 | # When 28 | actual = utils.render_template(template=template, data=data) 29 | 30 | # Then 31 | assert actual == "None 99" 32 | 33 | 34 | expect_matched_actual_data = [ 35 | ExpectMatchedActualTestData( 36 | [ 37 | '''reveal_type(42) # N: Revealed type is "Literal['foo']?"''', 38 | '''reveal_type("foo") # N: Revealed type is "Literal[42]?"''', 39 | ], 40 | ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''], 41 | [ 42 | """Invalid output: """, 43 | """Actual:""", 44 | """ main:1: note: Revealed type is "Literal[42]?" (diff)""", 45 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 46 | """Expected:""", 47 | """ main:1: note: Revealed type is "Literal['foo']?" (diff)""", 48 | """ main:2: note: Revealed type is "Literal[42]?" (diff)""", 49 | """Alignment of first line difference:""", 50 | ''' E: ...ed type is "Literal['foo']?"''', 51 | ''' A: ...ed type is "Literal[42]?"''', 52 | """ ^""", 53 | ], 54 | ), 55 | ExpectMatchedActualTestData( 56 | [ 57 | """reveal_type(42)""", 58 | '''reveal_type("foo") # N: Revealed type is "Literal['foo']?"''', 59 | ], 60 | ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''], 61 | [ 62 | """Invalid output: """, 63 | """Actual:""", 64 | """ main:1: note: Revealed type is "Literal[42]?" (diff)""", 65 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 66 | """Expected:""", 67 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 68 | """Alignment of first line difference:""", 69 | ''' E: main:2: note: Revealed type is "Literal['foo']?"''', 70 | ''' A: main:1: note: Revealed type is "Literal[42]?"''', 71 | """ ^""", 72 | ], 73 | ), 74 | ExpectMatchedActualTestData( 75 | ['''reveal_type(42) # N: Revealed type is "Literal[42]?"''', """reveal_type("foo")"""], 76 | ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''], 77 | [ 78 | """Invalid output: """, 79 | """Actual:""", 80 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 81 | """Expected:""", 82 | """ (empty)""", 83 | ], 84 | ), 85 | ExpectMatchedActualTestData( 86 | ['''42 + "foo"'''], 87 | ["""main:1: error: Unsupported operand types for + ("int" and "str")"""], 88 | [ 89 | """Output is not expected: """, 90 | """Actual:""", 91 | """ main:1: error: Unsupported operand types for + ("int" and "str") (diff)""", 92 | """Expected:""", 93 | """ (empty)""", 94 | ], 95 | ), 96 | ExpectMatchedActualTestData( 97 | [""" 1 + 1 # E: Unsupported operand types for + ("int" and "int")"""], 98 | [], 99 | [ 100 | """Invalid output: """, 101 | """Actual:""", 102 | """ (empty)""", 103 | """Expected:""", 104 | """ main:1: error: Unsupported operand types for + ("int" and "int") (diff)""", 105 | ], 106 | ), 107 | ExpectMatchedActualTestData( 108 | [ 109 | '''reveal_type(42) # N: Revealed type is "Literal[42]?"''', 110 | '''reveal_type("foo") # N: Revealed type is "builtins.int"''', 111 | ], 112 | ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''], 113 | [ 114 | """Invalid output: """, 115 | """Actual:""", 116 | """ ...""", 117 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 118 | """Expected:""", 119 | """ ...""", 120 | """ main:2: note: Revealed type is "builtins.int" (diff)""", 121 | """Alignment of first line difference:""", 122 | ''' E: ...te: Revealed type is "builtins.int"''', 123 | ''' A: ...te: Revealed type is "Literal['foo']?"''', 124 | """ ^""", 125 | ], 126 | ), 127 | ExpectMatchedActualTestData( 128 | [ 129 | '''reveal_type(42) # N: Revealed type is "Literal[42]?"''', 130 | '''reveal_type("foo") # N: Revealed type is "builtins.int"''', 131 | ], 132 | ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''], 133 | [ 134 | """Invalid output: """, 135 | """Actual:""", 136 | """ ...""", 137 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 138 | """Expected:""", 139 | """ ...""", 140 | """ main:2: note: Revealed type is "builtins.int" (diff)""", 141 | """Alignment of first line difference:""", 142 | ''' E: ...te: Revealed type is "builtins.int"''', 143 | ''' A: ...te: Revealed type is "Literal['foo']?"''', 144 | """ ^""", 145 | ], 146 | ), 147 | ExpectMatchedActualTestData( 148 | [ 149 | '''reveal_type(42.0) # N: Revealed type is "builtins.float"''', 150 | '''reveal_type("foo") # N: Revealed type is "builtins.int"''', 151 | '''reveal_type(42) # N: Revealed type is "Literal[42]?"''', 152 | ], 153 | [ 154 | '''main:1: note: Revealed type is "builtins.float"''', 155 | '''main:2: note: Revealed type is "Literal['foo']?"''', 156 | '''main:3: note: Revealed type is "Literal[42]?"''', 157 | ], 158 | [ 159 | """Invalid output: """, 160 | """Actual:""", 161 | """ ...""", 162 | """ main:2: note: Revealed type is "Literal['foo']?" (diff)""", 163 | """ ...""", 164 | """Expected:""", 165 | """ ...""", 166 | """ main:2: note: Revealed type is "builtins.int" (diff)""", 167 | """ ...""", 168 | """Alignment of first line difference:""", 169 | ''' E: ...te: Revealed type is "builtins.int"''', 170 | ''' A: ...te: Revealed type is "Literal['foo']?"''', 171 | """ ^""", 172 | ], 173 | ), 174 | ] 175 | 176 | 177 | @pytest.mark.parametrize("source_lines,actual_lines,expected_message_lines", expect_matched_actual_data) 178 | def test_assert_expected_matched_actual_failures( 179 | source_lines: List[str], actual_lines: List[str], expected_message_lines: List[str] 180 | ) -> None: 181 | expected: List[OutputMatcher] = extract_output_matchers_from_comments("main", source_lines, False) 182 | expected_error_message = "\n".join(expected_message_lines) 183 | 184 | with pytest.raises(TypecheckAssertionError) as e: 185 | assert_expected_matched_actual(expected, actual_lines) 186 | 187 | assert e.value.error_message.strip() == expected_error_message.strip() 188 | 189 | 190 | @pytest.mark.parametrize( 191 | "input_lines", 192 | [ 193 | [ 194 | '''main:12: error: No overload variant of "f" matches argument type "List[int]"''', 195 | """main:12: note: Possible overload variants:""", 196 | """main:12: note: def f(x: int) -> int""", 197 | """main:12: note: def f(x: str) -> str""", 198 | ], 199 | [ 200 | '''main_a:12: error: No overload variant of "g" matches argument type "List[int]"''', 201 | '''main_b:12: error: No overload variant of "f" matches argument type "List[int]"''', 202 | """main_b:12: note: Possible overload variants:""", 203 | """main_a:12: note: def g(b: int) -> int""", 204 | """main_a:12: note: def g(a: int) -> int""", 205 | """main_b:12: note: def f(x: int) -> int""", 206 | """main_b:12: note: def f(x: str) -> str""", 207 | ], 208 | ], 209 | ) 210 | def test_sorted_by_file_and_line_is_stable(input_lines: List[str]) -> None: 211 | def lines_for_file(lines: List[str], fname: str) -> List[str]: 212 | prefix = f"{fname}:" 213 | return [line for line in lines if line.startswith(prefix)] 214 | 215 | files = sorted({line.split(":", maxsplit=1)[0] for line in input_lines}) 216 | sorted_lines = sorted_by_file_and_line(input_lines) 217 | 218 | for f in files: 219 | assert lines_for_file(sorted_lines, f) == lines_for_file(input_lines, f) 220 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mypy logo 2 | 3 | # pytest plugin for testing mypy types, stubs, and plugins 4 | 5 | [![Tests Status](https://github.com/typeddjango/pytest-mypy-plugins/actions/workflows/test.yml/badge.svg)](https://github.com/typeddjango/pytest-mypy-plugins/actions/workflows/test.yml) 6 | [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) 7 | [![Gitter](https://badges.gitter.im/mypy-django/Lobby.svg)](https://gitter.im/mypy-django/Lobby) 8 | [![PyPI](https://img.shields.io/pypi/v/pytest-mypy-plugins?color=blue)](https://pypi.org/project/pytest-mypy-plugins/) 9 | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/pytest-mypy-plugins.svg?color=blue)](https://anaconda.org/conda-forge/pytest-mypy-plugins) 10 | 11 | ## Installation 12 | 13 | This package is available on [PyPI](https://pypi.org/project/pytest-mypy-plugins/) 14 | 15 | ```bash 16 | pip install pytest-mypy-plugins 17 | ``` 18 | 19 | and [conda-forge](https://anaconda.org/conda-forge/pytest-mypy-plugins) 20 | 21 | ```bash 22 | conda install -c conda-forge pytest-mypy-plugins 23 | ``` 24 | 25 | ## Usage 26 | 27 | ### Running 28 | 29 | Plugin, after installation, is automatically picked up by `pytest` therefore it is sufficient to 30 | just execute: 31 | 32 | ```bash 33 | pytest 34 | ``` 35 | 36 | ### Asserting types 37 | 38 | There are two ways to assert types. 39 | The custom one and regular [`typing.assert_type`](https://docs.python.org/3/library/typing.html#typing.assert_type). 40 | 41 | Our custom type assertion uses `reveal_type` helper and custom output matchers: 42 | 43 | ```yml 44 | - case: using_reveal_type 45 | main: | 46 | instance = 1 47 | reveal_type(instance) # N: Revealed type is 'builtins.int' 48 | ``` 49 | 50 | This method also allows to use `# E:` for matching exact error messages and codes. 51 | 52 | But, you can also use regular `assert_type`, examples can be [found here](https://github.com/typeddjango/pytest-mypy-plugins/blob/master/pytest_mypy_plugins/tests/test-assert-type.yml). 53 | 54 | ### Paths 55 | 56 | The `PYTHONPATH` and `MYPYPATH` environment variables, if set, are passed to `mypy` on invocation. 57 | This may be helpful if you are testing a local plugin and need to provide an import path to it. 58 | 59 | Be aware that when `mypy` is run in a subprocess (the default) the test cases are run in temporary working directories 60 | where relative paths such as `PYTHONPATH=./my_plugin` do not reference the directory which you are running `pytest` from. 61 | If you encounter this, consider invoking `pytest` with `--mypy-same-process` or make your paths absolute, 62 | e.g. `PYTHONPATH=$(pwd)/my_plugin pytest`. 63 | 64 | You can also specify `PYTHONPATH`, `MYPYPATH`, or any other environment variable in `env:` section of `yml` spec: 65 | 66 | ```yml 67 | - case: mypy_path_from_env 68 | main: | 69 | from pair import Pair 70 | 71 | instance: Pair 72 | reveal_type(instance) # N: Revealed type is 'pair.Pair' 73 | env: 74 | - MYPYPATH=../fixtures 75 | ``` 76 | 77 | 78 | ### What is a test case? 79 | 80 | In general each test case is just an element in an array written in a properly formatted `YAML` file. 81 | On top of that, each case must comply to following types: 82 | 83 | | Property | Type | Description | 84 | | --------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | 85 | | `case` | `str` | Name of the test case, complies to `[a-zA-Z0-9]` pattern | 86 | | `main` | `str` | Portion of the code as if written in `.py` file | 87 | | `files` | `Optional[List[File]]=[]`\* | List of extra files to simulate imports if needed | 88 | | `disable_cache` | `Optional[bool]=False` | Set to `true` disables `mypy` caching | 89 | | `mypy_config` | `Optional[str]` | Inline `mypy` configuration, passed directly to `mypy` as `--config-file` option, possibly joined with `--mypy-pyproject-toml-file` or `--mypy-ini-file` contents if they are passed. By default is treated as `ini`, treated as `toml` only if `--mypy-pyproject-toml-file` is passed | 90 | | `env` | `Optional[Dict[str, str]]={}` | Environmental variables to be provided inside of test run | 91 | | `parametrized` | `Optional[List[Parameter]]=[]`\* | List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html) | 92 | | `skip` | `str` | Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform` | 93 | | `expect_fail` | `bool` | Mark test case as an expected failure, like [`@pytest.mark.xfail`](https://docs.pytest.org/en/stable/skipping.html) | 94 | | `regex` | `str` | Allow regular expressions in comments to be matched against actual output. Defaults to "no", i.e. matches full text.| 95 | 96 | (*) Appendix to **pseudo** types used above: 97 | 98 | ```python 99 | class File: 100 | path: str 101 | content: Optional[str] = None 102 | Parameter = Mapping[str, Any] 103 | ``` 104 | 105 | Implementation notes: 106 | 107 | - `main` must be non-empty string that evaluates to valid **Python** code, 108 | - `content` of each of extra files must evaluate to valid **Python** code, 109 | - `parametrized` entries must all be the objects of the same _type_. It simply means that each 110 | entry must have **exact** same set of keys, 111 | - `skip` - an expression set in `skip` is passed directly into 112 | [`eval`](https://docs.python.org/3/library/functions.html#eval). It is advised to take a peek and 113 | learn about how `eval` works. 114 | 115 | Repository also offers a [JSONSchema](pytest_mypy_plugins/schema.json), with which 116 | it validates the input. It can also offer your editor auto-completions, descriptions, and validation. 117 | 118 | All you have to do, add the following line at the top of your YAML file: 119 | ```yaml 120 | # yaml-language-server: $schema=https://raw.githubusercontent.com/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json 121 | ``` 122 | 123 | ### Example 124 | 125 | #### 1. Inline type expectations 126 | 127 | ```yaml 128 | # typesafety/test_request.yml 129 | - case: request_object_has_user_of_type_auth_user_model 130 | main: | 131 | from django.http.request import HttpRequest 132 | reveal_type(HttpRequest().user) # N: Revealed type is 'myapp.models.MyUser' 133 | # check that other fields work ok 134 | reveal_type(HttpRequest().method) # N: Revealed type is 'Union[builtins.str, None]' 135 | files: 136 | - path: myapp/__init__.py 137 | - path: myapp/models.py 138 | content: | 139 | from django.db import models 140 | class MyUser(models.Model): 141 | pass 142 | ``` 143 | 144 | #### 2. `@parametrized` 145 | 146 | ```yaml 147 | - case: with_params 148 | parametrized: 149 | - val: 1 150 | rt: builtins.int 151 | - val: 1.0 152 | rt: builtins.float 153 | main: | 154 | reveal_type({{ val }}) # N: Revealed type is '{{ rt }}' 155 | ``` 156 | 157 | Properties that you can parametrize: 158 | - `main` 159 | - `mypy_config` 160 | - `out` 161 | 162 | #### 3. Longer type expectations 163 | 164 | ```yaml 165 | - case: with_out 166 | main: | 167 | reveal_type('abc') 168 | out: | 169 | main:1: note: Revealed type is 'builtins.str' 170 | ``` 171 | 172 | #### 4. Regular expressions in expectations 173 | 174 | ```yaml 175 | - case: expected_message_regex_with_out 176 | regex: yes 177 | main: | 178 | a = 'abc' 179 | reveal_type(a) 180 | out: | 181 | main:2: note: .*str.* 182 | ``` 183 | 184 | #### 5. Regular expressions specific lines of output. 185 | 186 | ```yaml 187 | - case: expected_single_message_regex 188 | main: | 189 | a = 'hello' 190 | reveal_type(a) # NR: .*str.* 191 | ``` 192 | 193 | ## Options 194 | 195 | ``` 196 | mypy-tests: 197 | --mypy-testing-base=MYPY_TESTING_BASE 198 | Base directory for tests to use 199 | --mypy-pyproject-toml-file=MYPY_PYPROJECT_TOML_FILE 200 | Which `pyproject.toml` file to use 201 | as a default config for tests. 202 | Incompatible with `--mypy-ini-file` 203 | --mypy-ini-file=MYPY_INI_FILE 204 | Which `.ini` file to use as a default config for tests. 205 | Incompatible with `--mypy-pyproject-toml-file` 206 | --mypy-same-process 207 | Run in the same process. Useful for debugging, 208 | will create problems with import cache 209 | --mypy-extension-hook=MYPY_EXTENSION_HOOK 210 | Fully qualified path to the extension hook function, 211 | in case you need custom yaml keys. Has to be top-level 212 | --mypy-only-local-stub 213 | mypy will ignore errors from site-packages 214 | --mypy-closed-schema 215 | Use closed schema to validate YAML test cases, 216 | which won't allow any extra keys 217 | (does not work well with `--mypy-extension-hook`) 218 | 219 | ``` 220 | 221 | ## Further reading 222 | 223 | - [Testing mypy stubs, plugins, and types](https://sobolevn.me/2019/08/testing-mypy-types) 224 | 225 | ## License 226 | 227 | [MIT](https://github.com/typeddjango/pytest-mypy-plugins/blob/master/LICENSE) 228 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/utils.py: -------------------------------------------------------------------------------- 1 | # Borrowed from Pew. 2 | # See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 3 | import inspect 4 | import os 5 | import re 6 | import sys 7 | from dataclasses import dataclass 8 | from itertools import zip_longest 9 | from pathlib import Path 10 | from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Tuple, Union 11 | 12 | import jinja2 13 | import regex 14 | from decorator import contextmanager 15 | 16 | _rendering_env = jinja2.Environment() 17 | 18 | 19 | @contextmanager 20 | def temp_environ() -> Iterator[None]: 21 | """Allow the ability to set os.environ temporarily""" 22 | environ = dict(os.environ) 23 | try: 24 | yield 25 | finally: 26 | os.environ.clear() 27 | os.environ.update(environ) 28 | 29 | 30 | @contextmanager 31 | def temp_path() -> Iterator[None]: 32 | """A context manager which allows the ability to set sys.path temporarily""" 33 | path = sys.path[:] 34 | try: 35 | yield 36 | finally: 37 | sys.path = path[:] 38 | 39 | 40 | @contextmanager 41 | def temp_sys_modules() -> Iterator[None]: 42 | sys_modules = sys.modules.copy() 43 | try: 44 | yield 45 | finally: 46 | sys.modules = sys_modules.copy() 47 | 48 | 49 | def fname_to_module(fpath: Path, root_path: Path) -> Optional[str]: 50 | try: 51 | relpath = fpath.relative_to(root_path).with_suffix("") 52 | return str(relpath).replace(os.sep, ".") 53 | except ValueError: 54 | return None 55 | 56 | 57 | # AssertStringArraysEqual displays special line alignment helper messages if 58 | # the first different line has at least this many characters, 59 | MIN_LINE_LENGTH_FOR_ALIGNMENT = 5 60 | 61 | 62 | @dataclass 63 | class OutputMatcher: 64 | fname: str 65 | lnum: int 66 | severity: str 67 | message: str 68 | regex: bool 69 | col: Optional[str] = None 70 | 71 | def matches(self, actual: str) -> bool: 72 | if self.regex: 73 | pattern = ( 74 | regex.escape( 75 | f"{self.fname}:{self.lnum}: {self.severity}: " 76 | if self.col is None 77 | else f"{self.fname}:{self.lnum}:{self.col}: {self.severity}: " 78 | ) 79 | + self.message 80 | ) 81 | return bool(regex.match(pattern, actual)) 82 | else: 83 | return str(self) == actual 84 | 85 | def __str__(self) -> str: 86 | if self.col is None: 87 | return f"{self.fname}:{self.lnum}: {self.severity}: {self.message}" 88 | else: 89 | return f"{self.fname}:{self.lnum}:{self.col}: {self.severity}: {self.message}" 90 | 91 | def __format__(self, format_spec: str) -> str: 92 | return format_spec.format(str(self)) 93 | 94 | def __len__(self) -> int: 95 | return len(str(self)) 96 | 97 | 98 | class TypecheckAssertionError(AssertionError): 99 | def __init__(self, error_message: Optional[str] = None, lineno: int = 0) -> None: 100 | self.error_message = error_message or "" 101 | self.lineno = lineno 102 | 103 | def first_line(self) -> str: 104 | return self.__class__.__name__ + '(message="Invalid output")' 105 | 106 | def __str__(self) -> str: 107 | return self.error_message 108 | 109 | 110 | def remove_common_prefix(lines: List[str]) -> List[str]: 111 | """Remove common directory prefix from all strings in a. 112 | 113 | This uses a naive string replace; it seems to work well enough. Also 114 | remove trailing carriage returns. 115 | """ 116 | cleaned_lines = [] 117 | for line in lines: 118 | # Ignore spaces at end of line. 119 | line = re.sub(" +$", "", line) 120 | cleaned_lines.append(re.sub("\\r$", "", line)) 121 | return cleaned_lines 122 | 123 | 124 | def _add_aligned_message(s1: str, s2: str, error_message: str) -> str: 125 | """Align s1 and s2 so that the their first difference is highlighted. 126 | 127 | For example, if s1 is 'foobar' and s2 is 'fobar', display the 128 | following lines: 129 | 130 | E: foobar 131 | A: fobar 132 | ^ 133 | 134 | If s1 and s2 are long, only display a fragment of the strings around the 135 | first difference. If s1 is very short, do nothing. 136 | """ 137 | 138 | # Seeing what went wrong is trivial even without alignment if the expected 139 | # string is very short. In this case do nothing to simplify output. 140 | if len(s1) < 4: 141 | return error_message 142 | 143 | maxw = 72 # Maximum number of characters shown 144 | 145 | error_message += "Alignment of first line difference:\n" 146 | 147 | assert s1 != s2 148 | 149 | trunc = False 150 | while s1[:30] == s2[:30]: 151 | s1 = s1[10:] 152 | s2 = s2[10:] 153 | trunc = True 154 | 155 | if trunc: 156 | s1 = "..." + s1 157 | s2 = "..." + s2 158 | 159 | max_len = max(len(s1), len(s2)) 160 | extra = "" 161 | if max_len > maxw: 162 | extra = "..." 163 | 164 | # Write a chunk of both lines, aligned. 165 | error_message += f" E: {s1[:maxw]}{extra}\n" 166 | error_message += f" A: {s2[:maxw]}{extra}\n" 167 | # Write an indicator character under the different columns. 168 | error_message += " " 169 | # sys.stderr.write(' ') 170 | for j in range(min(maxw, max(len(s1), len(s2)))): 171 | if s1[j : j + 1] != s2[j : j + 1]: 172 | error_message += "^" 173 | break 174 | else: 175 | error_message += " " 176 | error_message += "\n" 177 | return error_message 178 | 179 | 180 | def remove_empty_lines(lines: List[str]) -> List[str]: 181 | filtered_lines = [] 182 | for line in lines: 183 | if line: 184 | filtered_lines.append(line) 185 | return filtered_lines 186 | 187 | 188 | def sorted_by_file_and_line(lines: List[str]) -> List[str]: 189 | def extract_parts_as_tuple(line: str) -> Tuple[str, int]: 190 | if len(line.split(":", maxsplit=2)) < 3: 191 | return "", 0 192 | 193 | fname, line_number, _ = line.split(":", maxsplit=2) 194 | try: 195 | return fname, int(line_number) 196 | except ValueError: 197 | return "", 0 198 | 199 | return sorted(lines, key=extract_parts_as_tuple) 200 | 201 | 202 | def assert_expected_matched_actual(expected: List[OutputMatcher], actual: List[str]) -> None: 203 | """Assert that two string arrays are equal. 204 | 205 | Display any differences in a human-readable form. 206 | """ 207 | 208 | def format_mismatched_line(line: str) -> str: 209 | return f" {str(line):<45} (diff)" 210 | 211 | def format_matched_line(line: str, width: int = 100) -> str: 212 | return f" {line[:width]}..." if len(line) > width else f" {line}" 213 | 214 | def format_error_lines(lines: List[str]) -> str: 215 | return "\n".join(lines) if lines else " (empty)" 216 | 217 | expected = sorted(expected, key=lambda om: (om.fname, om.lnum)) 218 | actual = sorted_by_file_and_line(remove_empty_lines(actual)) 219 | 220 | actual = remove_common_prefix(actual) 221 | 222 | diff_lines: Dict[int, Tuple[OutputMatcher, str]] = { 223 | i: (e, a) 224 | for i, (e, a) in enumerate(zip_longest(expected, actual)) 225 | if e is None or a is None or not e.matches(a) 226 | } 227 | 228 | if diff_lines: 229 | first_diff_line = min(diff_lines.keys()) 230 | last_diff_line = max(diff_lines.keys()) 231 | 232 | expected_message_lines = [] 233 | actual_message_lines = [] 234 | 235 | for i in range(first_diff_line, last_diff_line + 1): 236 | if i in diff_lines: 237 | expected_line, actual_line = diff_lines[i] 238 | if expected_line: 239 | expected_message_lines.append(format_mismatched_line(str(expected_line))) 240 | if actual_line: 241 | actual_message_lines.append(format_mismatched_line(actual_line)) 242 | 243 | else: 244 | expected_line, actual_line = expected[i], actual[i] 245 | actual_message_lines.append(format_matched_line(actual_line)) 246 | expected_message_lines.append(format_matched_line(str(expected_line))) 247 | 248 | first_diff_expected, first_diff_actual = diff_lines[first_diff_line] 249 | 250 | failure_reason = "Output is not expected" if actual and not expected else "Invalid output" 251 | 252 | if actual_message_lines and expected_message_lines: 253 | if first_diff_line > 0: 254 | expected_message_lines.insert(0, " ...") 255 | actual_message_lines.insert(0, " ...") 256 | 257 | if last_diff_line < len(actual) - 1 and last_diff_line < len(expected) - 1: 258 | expected_message_lines.append(" ...") 259 | actual_message_lines.append(" ...") 260 | 261 | error_message = "Actual:\n{}\nExpected:\n{}\n".format( 262 | format_error_lines(actual_message_lines), format_error_lines(expected_message_lines) 263 | ) 264 | 265 | if expected_line and expected_line.regex: 266 | error_message += "The actual output does not match the expected regex." 267 | elif ( 268 | first_diff_actual is not None 269 | and first_diff_expected is not None 270 | and ( 271 | len(first_diff_actual) >= MIN_LINE_LENGTH_FOR_ALIGNMENT 272 | or len(str(first_diff_expected)) >= MIN_LINE_LENGTH_FOR_ALIGNMENT 273 | ) 274 | ): 275 | error_message = _add_aligned_message(str(first_diff_expected), first_diff_actual, error_message) 276 | 277 | raise TypecheckAssertionError( 278 | error_message=f"{failure_reason}: \n{error_message}", 279 | lineno=first_diff_expected.lnum if first_diff_expected else 0, 280 | ) 281 | 282 | 283 | def extract_output_matchers_from_comments(fname: str, input_lines: List[str], regex: bool) -> List[OutputMatcher]: 284 | """Transform comments such as '# E: message' or 285 | '# E:3: message' in input. 286 | 287 | The result is a list pf output matchers 288 | """ 289 | fname = fname.replace(".py", "") 290 | matchers = [] 291 | for index, line in enumerate(input_lines): 292 | # The first in the split things isn't a comment 293 | for possible_err_comment in line.split(" # ")[1:]: 294 | match = re.search( 295 | r"^([ENW])(?P[R])?:((?P\d+):)? (?P.*)$", possible_err_comment.strip() 296 | ) 297 | if match: 298 | if match.group(1) == "E": 299 | severity = "error" 300 | elif match.group(1) == "N": 301 | severity = "note" 302 | elif match.group(1) == "W": 303 | severity = "warning" 304 | else: 305 | severity = match.group(1) 306 | col = match.group("col") 307 | matchers.append( 308 | OutputMatcher( 309 | fname, 310 | index + 1, 311 | severity, 312 | message=match.group("message"), 313 | regex=regex or bool(match.group("regex")), 314 | col=col, 315 | ) 316 | ) 317 | return matchers 318 | 319 | 320 | def extract_output_matchers_from_out(out: str, params: Mapping[str, Any], regex: bool) -> List[OutputMatcher]: 321 | """Transform output lines such as 'function:9: E: message' 322 | 323 | The result is a list of output matchers 324 | """ 325 | matchers = [] 326 | lines = render_template(out, params).split("\n") 327 | for line in lines: 328 | match = re.search( 329 | r"^(?P.+):(?P\d+): (?P[A-Za-z]+):((?P\d+):)? (?P.*)$", line.strip() 330 | ) 331 | if match: 332 | if match.group("severity") == "E": 333 | severity = "error" 334 | elif match.group("severity") == "N": 335 | severity = "note" 336 | elif match.group("severity") == "W": 337 | severity = "warning" 338 | else: 339 | severity = match.group("severity") 340 | col = match.group("col") 341 | matchers.append( 342 | OutputMatcher( 343 | match.group("fname"), 344 | int(match.group("lnum")), 345 | severity, 346 | message=match.group("message"), 347 | regex=regex, 348 | col=col, 349 | ) 350 | ) 351 | return matchers 352 | 353 | 354 | def render_template(template: str, data: Mapping[str, Any]) -> str: 355 | if _rendering_env.variable_start_string not in template: 356 | return template 357 | 358 | t: jinja2.environment.Template = _rendering_env.from_string(template) 359 | return t.render({k: v if v is not None else "None" for k, v in data.items()}) 360 | 361 | 362 | def get_func_first_lnum(attr: Callable[..., None]) -> Optional[Tuple[int, List[str]]]: 363 | lines, _ = inspect.getsourcelines(attr) 364 | for lnum, line in enumerate(lines): 365 | no_space_line = line.strip() 366 | if f"def {attr.__name__}" in no_space_line: 367 | return lnum, lines[lnum + 1 :] 368 | raise ValueError(f'No line "def {attr.__name__}" found') 369 | 370 | 371 | @contextmanager 372 | def cd(path: Union[str, Path]) -> Iterator[None]: 373 | """Context manager to temporarily change working directories""" 374 | if not path: 375 | return 376 | prev_cwd = Path.cwd().as_posix() 377 | if isinstance(path, Path): 378 | path = path.as_posix() 379 | os.chdir(str(path)) 380 | try: 381 | yield 382 | finally: 383 | os.chdir(prev_cwd) 384 | -------------------------------------------------------------------------------- /pytest_mypy_plugins/item.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import io 3 | import os 4 | import shutil 5 | import subprocess 6 | import sys 7 | import tempfile 8 | from pathlib import Path 9 | from typing import ( 10 | TYPE_CHECKING, 11 | Any, 12 | Dict, 13 | List, 14 | Literal, 15 | Optional, 16 | TextIO, 17 | Tuple, 18 | Union, 19 | ) 20 | 21 | import py 22 | import pytest 23 | from _pytest._code import ExceptionInfo 24 | from _pytest._code.code import ReprEntry, ReprFileLocation, TerminalRepr 25 | from _pytest._io import TerminalWriter 26 | from _pytest.config import Config 27 | from mypy import build 28 | from mypy.fscache import FileSystemCache 29 | from mypy.main import process_options 30 | 31 | from pytest_mypy_plugins import configs, utils 32 | from pytest_mypy_plugins.collect import File, YamlTestFile 33 | from pytest_mypy_plugins.utils import ( 34 | OutputMatcher, 35 | TypecheckAssertionError, 36 | assert_expected_matched_actual, 37 | fname_to_module, 38 | ) 39 | 40 | if TYPE_CHECKING: 41 | # pytest 8.3.0 renamed _TracebackStyle to TracebackStyle, but there is no syntax 42 | # to assert what version you have using static conditions, so it has to be 43 | # manually re-defined here. Once minimum supported pytest version is >= 8.3.0, 44 | # the following can be replaced with `from _pytest._code.code import TracebackStyle` 45 | TracebackStyle = Literal["long", "short", "line", "no", "native", "value", "auto"] 46 | 47 | 48 | class TraceLastReprEntry(ReprEntry): 49 | def toterminal(self, tw: TerminalWriter) -> None: 50 | if not self.reprfileloc: 51 | return 52 | 53 | self.reprfileloc.toterminal(tw) 54 | for line in self.lines: 55 | red = line.startswith("E ") 56 | tw.line(line, bold=True, red=red) 57 | return 58 | 59 | 60 | def make_files(rootdir: Path, files_to_create: Dict[str, str]) -> List[str]: 61 | created_modules = [] 62 | for rel_fpath, file_contents in files_to_create.items(): 63 | fpath = rootdir / rel_fpath 64 | fpath.parent.mkdir(parents=True, exist_ok=True) 65 | fpath.write_text(file_contents) 66 | 67 | created_module = fname_to_module(fpath, root_path=rootdir) 68 | if created_module: 69 | created_modules.append(created_module) 70 | return created_modules 71 | 72 | 73 | def replace_fpath_with_module_name(line: str, rootdir: Path) -> str: 74 | if ":" not in line: 75 | return line 76 | out_fpath, res_line = line.split(":", 1) 77 | line = os.path.relpath(out_fpath, start=rootdir) + ":" + res_line 78 | return line.strip().replace(".py:", ":") 79 | 80 | 81 | def maybe_to_abspath(rel_or_abs: str, rootdir: Optional[Path]) -> str: 82 | rel_or_abs = os.path.expandvars(rel_or_abs) 83 | if rootdir is None or os.path.isabs(rel_or_abs): 84 | return rel_or_abs 85 | return str(rootdir / rel_or_abs) 86 | 87 | 88 | class ReturnCodes: 89 | SUCCESS = 0 90 | FAIL = 1 91 | FATAL_ERROR = 2 92 | 93 | 94 | def run_mypy_typechecking(cmd_options: List[str], stdout: TextIO, stderr: TextIO) -> int: 95 | fscache = FileSystemCache() 96 | sources, options = process_options(cmd_options, fscache=fscache) 97 | 98 | error_messages = [] 99 | 100 | # Different mypy versions have different arity of `flush_errors`: 2 and 3 params 101 | def flush_errors(*args: Any) -> None: 102 | new_messages: List[str] 103 | serious: bool 104 | *_, new_messages, serious = args 105 | error_messages.extend(new_messages) 106 | f = stderr if serious else stdout 107 | try: 108 | for msg in new_messages: 109 | f.write(msg + "\n") 110 | f.flush() 111 | except BrokenPipeError: 112 | sys.exit(ReturnCodes.FATAL_ERROR) 113 | 114 | try: 115 | build.build(sources, options, flush_errors=flush_errors, fscache=fscache, stdout=stdout, stderr=stderr) 116 | 117 | except SystemExit as sysexit: 118 | # The code to a SystemExit is optional 119 | # From python docs, if the code is None then the exit code is 0 120 | # Otherwise if the code is not an integer the exit code is 1 121 | code = sysexit.code 122 | if code is None: 123 | code = 0 124 | elif not isinstance(code, int): 125 | code = 1 126 | 127 | return code 128 | finally: 129 | fscache.flush() 130 | 131 | if error_messages: 132 | return ReturnCodes.FAIL 133 | 134 | return ReturnCodes.SUCCESS 135 | 136 | 137 | class MypyExecutor: 138 | def __init__( 139 | self, 140 | same_process: bool, 141 | rootdir: Union[Path, None], 142 | execution_path: Path, 143 | environment_variables: Dict[str, Any], 144 | mypy_executable: str, 145 | ) -> None: 146 | self.rootdir = rootdir 147 | self.same_process = same_process 148 | self.execution_path = execution_path 149 | self.mypy_executable = mypy_executable 150 | self.environment_variables = environment_variables 151 | 152 | def execute(self, mypy_cmd_options: List[str]) -> Tuple[int, Tuple[str, str]]: 153 | # Returns (returncode, (stdout, stderr)) 154 | if self.same_process: 155 | return self._typecheck_in_same_process(mypy_cmd_options) 156 | else: 157 | return self._typecheck_in_new_subprocess(mypy_cmd_options) 158 | 159 | def _typecheck_in_new_subprocess(self, mypy_cmd_options: List[Any]) -> Tuple[int, Tuple[str, str]]: 160 | # add current directory to path 161 | self._collect_python_path(self.rootdir) 162 | # adding proper MYPYPATH variable 163 | self._collect_mypy_path(self.rootdir) 164 | 165 | # Windows requires this to be set, otherwise the interpreter crashes 166 | if "SYSTEMROOT" in os.environ: 167 | self.environment_variables["SYSTEMROOT"] = os.environ["SYSTEMROOT"] 168 | 169 | completed = subprocess.run( 170 | [self.mypy_executable, *mypy_cmd_options], 171 | capture_output=True, 172 | cwd=os.getcwd(), 173 | env=self.environment_variables, 174 | ) 175 | captured_stdout = completed.stdout.decode() 176 | captured_stderr = completed.stderr.decode() 177 | return completed.returncode, (captured_stdout, captured_stderr) 178 | 179 | def _typecheck_in_same_process(self, mypy_cmd_options: List[Any]) -> Tuple[int, Tuple[str, str]]: 180 | return_code = -1 181 | with utils.temp_environ(), utils.temp_path(), utils.temp_sys_modules(): 182 | # add custom environment variables 183 | for key, val in self.environment_variables.items(): 184 | os.environ[key] = val 185 | 186 | # add current directory to path 187 | sys.path.insert(0, str(self.execution_path)) 188 | 189 | stdout = io.StringIO() 190 | stderr = io.StringIO() 191 | 192 | with stdout, stderr: 193 | return_code = run_mypy_typechecking(mypy_cmd_options, stdout=stdout, stderr=stderr) 194 | stdout_value = stdout.getvalue() 195 | stderr_value = stderr.getvalue() 196 | 197 | return return_code, (stdout_value, stderr_value) 198 | 199 | def _collect_python_path(self, rootdir: Optional[Path]) -> None: 200 | python_path_parts = [] 201 | 202 | existing_python_path = os.environ.get("PYTHONPATH") 203 | if existing_python_path: 204 | python_path_parts.append(existing_python_path) 205 | python_path_parts.append(str(self.execution_path)) 206 | python_path_key = self.environment_variables.get("PYTHONPATH") 207 | if python_path_key: 208 | python_path_parts.append(maybe_to_abspath(python_path_key, rootdir)) 209 | python_path_parts.append(python_path_key) 210 | 211 | self.environment_variables["PYTHONPATH"] = ":".join(python_path_parts) 212 | 213 | def _collect_mypy_path(self, rootdir: Optional[Path]) -> None: 214 | mypy_path_parts = [] 215 | 216 | existing_mypy_path = os.environ.get("MYPYPATH") 217 | if existing_mypy_path: 218 | mypy_path_parts.append(existing_mypy_path) 219 | mypy_path_key = self.environment_variables.get("MYPYPATH") 220 | if mypy_path_key: 221 | mypy_path_parts.append(maybe_to_abspath(mypy_path_key, rootdir)) 222 | mypy_path_parts.append(mypy_path_key) 223 | if rootdir: 224 | mypy_path_parts.append(str(rootdir)) 225 | 226 | self.environment_variables["MYPYPATH"] = ":".join(mypy_path_parts) 227 | 228 | 229 | class OutputChecker: 230 | def __init__(self, expect_fail: bool, execution_path: Path, expected_output: List[OutputMatcher]) -> None: 231 | self.expect_fail = expect_fail 232 | self.execution_path = execution_path 233 | self.expected_output = expected_output 234 | 235 | def check(self, ret_code: int, stdout: str, stderr: str) -> None: 236 | mypy_output = stdout + stderr 237 | if ret_code == ReturnCodes.FATAL_ERROR: 238 | print(mypy_output, file=sys.stderr) 239 | raise TypecheckAssertionError(error_message="Critical error occurred") 240 | 241 | output_lines = [] 242 | for line in mypy_output.splitlines(): 243 | output_line = replace_fpath_with_module_name(line, rootdir=self.execution_path) 244 | output_lines.append(output_line) 245 | try: 246 | assert_expected_matched_actual(expected=self.expected_output, actual=output_lines) 247 | except TypecheckAssertionError as e: 248 | if not self.expect_fail: 249 | raise e 250 | else: 251 | if self.expect_fail: 252 | raise TypecheckAssertionError("Expected failure, but test passed") 253 | 254 | 255 | class Runner: 256 | def __init__( 257 | self, 258 | *, 259 | files: List[File], 260 | config: Config, 261 | main_file: Path, 262 | config_file: Optional[str], 263 | disable_cache: bool, 264 | mypy_executor: MypyExecutor, 265 | output_checker: OutputChecker, 266 | test_only_local_stub: bool, 267 | incremental_cache_dir: str, 268 | ) -> None: 269 | self.files = files 270 | self.config = config 271 | self.main_file = main_file 272 | self.config_file = config_file 273 | self.mypy_executor = mypy_executor 274 | self.disable_cache = disable_cache 275 | self.output_checker = output_checker 276 | self.test_only_local_stub = test_only_local_stub 277 | self.incremental_cache_dir = incremental_cache_dir 278 | 279 | def run(self) -> None: 280 | # start from main.py 281 | mypy_cmd_options = self._prepare_mypy_cmd_options() 282 | mypy_cmd_options.append(str(self.main_file)) 283 | 284 | # make files 285 | for file in self.files: 286 | self._make_test_file(file) 287 | 288 | returncode, (stdout, stderr) = self.mypy_executor.execute(mypy_cmd_options) 289 | self.output_checker.check(returncode, stdout, stderr) 290 | 291 | def _make_test_file(self, file: File) -> None: 292 | current_directory = Path.cwd() 293 | fpath = current_directory / file.path 294 | fpath.parent.mkdir(parents=True, exist_ok=True) 295 | fpath.write_text(file.content) 296 | 297 | def _prepare_mypy_cmd_options(self) -> List[str]: 298 | mypy_cmd_options = [ 299 | "--show-traceback", 300 | "--no-error-summary", 301 | "--no-pretty", 302 | "--hide-error-context", 303 | ] 304 | if not self.test_only_local_stub: 305 | mypy_cmd_options.append("--no-silence-site-packages") 306 | if not self.disable_cache: 307 | mypy_cmd_options.extend(["--cache-dir", self.incremental_cache_dir]) 308 | 309 | if self.config_file: 310 | mypy_cmd_options.append(f"--config-file={self.config_file}") 311 | 312 | return mypy_cmd_options 313 | 314 | 315 | class YamlTestItem(pytest.Item): 316 | def __init__( 317 | self, 318 | name: str, 319 | parent: Optional[YamlTestFile] = None, 320 | config: Optional[Config] = None, 321 | *, 322 | files: List[File], 323 | starting_lineno: int, 324 | expected_output: List[OutputMatcher], 325 | environment_variables: Dict[str, Any], 326 | disable_cache: bool, 327 | mypy_config: str, 328 | parsed_test_data: Dict[str, Any], 329 | expect_fail: bool, 330 | ) -> None: 331 | super().__init__(name, parent, config) 332 | self.files = files 333 | self.environment_variables = environment_variables 334 | self.disable_cache = disable_cache 335 | self.expect_fail = expect_fail 336 | self.expected_output = expected_output 337 | self.starting_lineno = starting_lineno 338 | self.additional_mypy_config = mypy_config 339 | self.parsed_test_data = parsed_test_data 340 | self.same_process = self.config.option.mypy_same_process 341 | self.test_only_local_stub = self.config.option.mypy_only_local_stub 342 | 343 | # config parameters 344 | self.root_directory = self.config.option.mypy_testing_base 345 | 346 | # You cannot use both `.ini` and `pyproject.toml` files at the same time: 347 | if self.config.option.mypy_ini_file and self.config.option.mypy_pyproject_toml_file: 348 | raise ValueError("Cannot specify both `--mypy-ini-file` and `--mypy-pyproject-toml-file`") 349 | 350 | if self.config.option.mypy_ini_file: 351 | self.base_ini_fpath = os.path.abspath(self.config.option.mypy_ini_file) 352 | else: 353 | self.base_ini_fpath = None 354 | if self.config.option.mypy_pyproject_toml_file: 355 | self.base_pyproject_toml_fpath = os.path.abspath(self.config.option.mypy_pyproject_toml_file) 356 | else: 357 | self.base_pyproject_toml_fpath = None 358 | self.incremental_cache_dir = os.path.join(self.root_directory, ".mypy_cache") 359 | 360 | def remove_cache_files(self, fpath_no_suffix: Path) -> None: 361 | cache_file = Path(self.incremental_cache_dir) 362 | cache_file /= ".".join([str(part) for part in sys.version_info[:2]]) 363 | for part in fpath_no_suffix.parts: 364 | cache_file /= part 365 | 366 | data_json_file = cache_file.with_suffix(".data.json") 367 | if data_json_file.exists(): 368 | data_json_file.unlink() 369 | meta_json_file = cache_file.with_suffix(".meta.json") 370 | if meta_json_file.exists(): 371 | meta_json_file.unlink() 372 | 373 | for parent_dir in cache_file.parents: 374 | if ( 375 | parent_dir.exists() 376 | and len(list(parent_dir.iterdir())) == 0 377 | and str(self.incremental_cache_dir) in str(parent_dir) 378 | ): 379 | parent_dir.rmdir() 380 | 381 | def execute_extension_hook(self) -> None: 382 | extension_hook_fqname = self.config.option.mypy_extension_hook 383 | module_name, func_name = extension_hook_fqname.rsplit(".", maxsplit=1) 384 | module = importlib.import_module(module_name) 385 | extension_hook = getattr(module, func_name) 386 | extension_hook(self) 387 | 388 | def runtest(self) -> None: 389 | try: 390 | temp_dir = tempfile.TemporaryDirectory(prefix="pytest-mypy-", dir=self.root_directory) 391 | 392 | except (FileNotFoundError, PermissionError, NotADirectoryError) as e: 393 | raise TypecheckAssertionError( 394 | error_message=f"Testing base directory {self.root_directory} must exist and be writable" 395 | ) from e 396 | 397 | try: 398 | mypy_executable = shutil.which("mypy") 399 | assert mypy_executable is not None, "mypy executable is not found" 400 | rootdir = getattr(getattr(self.parent, "config", None), "rootdir", None) 401 | 402 | # extension point for derived packages 403 | if ( 404 | hasattr(self.config.option, "mypy_extension_hook") 405 | and self.config.option.mypy_extension_hook is not None 406 | ): 407 | self.execute_extension_hook() 408 | 409 | execution_path = Path(temp_dir.name).absolute() 410 | with utils.cd(execution_path): 411 | mypy_executor = MypyExecutor( 412 | same_process=self.same_process, 413 | execution_path=execution_path, 414 | rootdir=rootdir, 415 | environment_variables=self.environment_variables, 416 | mypy_executable=mypy_executable, 417 | ) 418 | 419 | output_checker = OutputChecker( 420 | expect_fail=self.expect_fail, execution_path=execution_path, expected_output=self.expected_output 421 | ) 422 | 423 | Runner( 424 | files=self.files, 425 | config=self.config, 426 | main_file=execution_path / "main.py", 427 | config_file=self.prepare_config_file(execution_path), 428 | disable_cache=self.disable_cache, 429 | mypy_executor=mypy_executor, 430 | output_checker=output_checker, 431 | test_only_local_stub=self.test_only_local_stub, 432 | incremental_cache_dir=self.incremental_cache_dir, 433 | ).run() 434 | finally: 435 | temp_dir.cleanup() 436 | # remove created modules 437 | if not self.disable_cache: 438 | for file in self.files: 439 | path = Path(file.path) 440 | self.remove_cache_files(path.with_suffix("")) 441 | 442 | assert not os.path.exists(temp_dir.name) 443 | 444 | def prepare_config_file(self, execution_path: Path) -> Optional[str]: 445 | # Merge (`self.base_ini_fpath` or `base_pyproject_toml_fpath`) 446 | # and `self.additional_mypy_config` 447 | # into one file and copy to the typechecking folder: 448 | if self.base_pyproject_toml_fpath: 449 | return configs.join_toml_configs( 450 | self.base_pyproject_toml_fpath, self.additional_mypy_config, execution_path 451 | ) 452 | elif self.base_ini_fpath or self.additional_mypy_config: 453 | # We might have `self.base_ini_fpath` set as well. 454 | # Or this might be a legacy case: only `mypy_config:` is set in the `yaml` test case. 455 | # This means that no real file is provided. 456 | return configs.join_ini_configs(self.base_ini_fpath, self.additional_mypy_config, execution_path) 457 | return None 458 | 459 | def repr_failure( 460 | self, excinfo: ExceptionInfo[BaseException], style: Optional["TracebackStyle"] = None 461 | ) -> Union[str, TerminalRepr]: 462 | if excinfo.errisinstance(SystemExit): 463 | # We assume that before doing exit() (which raises SystemExit) we've printed 464 | # enough context about what happened so that a stack trace is not useful. 465 | # In particular, uncaught exceptions during semantic analysis or type checking 466 | # call exit() and they already print out a stack trace. 467 | return excinfo.exconly(tryshort=True) 468 | elif excinfo.errisinstance(TypecheckAssertionError): 469 | # with traceback removed 470 | exception_repr = excinfo.getrepr(style="short") 471 | exception_repr.reprcrash.message = "" # type: ignore 472 | repr_file_location = ReprFileLocation( 473 | path=self.fspath, lineno=self.starting_lineno + excinfo.value.lineno, message="" # type: ignore 474 | ) 475 | repr_tb_entry = ReprEntry( 476 | exception_repr.reprtraceback.reprentries[-1].lines[1:], None, None, repr_file_location, "short" 477 | ) 478 | exception_repr.reprtraceback.reprentries = [repr_tb_entry] 479 | return exception_repr 480 | else: 481 | return super().repr_failure(excinfo, style="native") 482 | 483 | def reportinfo(self) -> Tuple[Union[Path, str], Optional[int], str]: 484 | return self.path, None, self.name 485 | --------------------------------------------------------------------------------