├── .github ├── dependabot.yml └── workflows │ ├── deploy.yml │ ├── format.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── plugin_test.py ├── pyproject.toml ├── pytest_github_actions_annotate_failures ├── __init__.py └── plugin.py └── tox.ini /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - main 9 | tags: 10 | - v* 11 | 12 | jobs: 13 | dist: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: hynek/build-and-inspect-python-package@v2 18 | 19 | deploy: 20 | needs: [dist] 21 | runs-on: ubuntu-latest 22 | if: startsWith(github.ref, 'refs/tags/v') 23 | permissions: 24 | id-token: write 25 | attestations: write 26 | 27 | steps: 28 | - uses: actions/download-artifact@v4 29 | with: 30 | name: Packages 31 | path: dist 32 | 33 | - name: Generate artifact attestation for sdist and wheel 34 | uses: actions/attest-build-provenance@v2 35 | with: 36 | subject-path: "dist/*" 37 | 38 | - name: Publish package 39 | uses: pypa/gh-action-pypi-publish@release/v1 40 | with: 41 | password: ${{ secrets.pypi_password }} 42 | -------------------------------------------------------------------------------- /.github/workflows/format.yml: -------------------------------------------------------------------------------- 1 | name: Format 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | pre-commit: 11 | name: Pre-commit checks 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: actions/setup-python@v5 16 | with: 17 | python-version: '3.10' 18 | - uses: pre-commit/action@v3.0.1 19 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.ref }} 12 | cancel-in-progress: true 13 | 14 | env: 15 | FORCE_COLOR: "1" 16 | 17 | jobs: 18 | test: 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | os: [ubuntu-latest, windows-latest] 23 | python-version: 24 | - '3.8' 25 | - '3.9' 26 | - '3.10' 27 | - '3.11' 28 | - '3.12' 29 | - '3.13' 30 | runs-on: ${{ matrix.os }} 31 | 32 | name: ${{ matrix.os }}, Python ${{ matrix.python-version }} 33 | steps: 34 | - uses: actions/checkout@v4 35 | 36 | - name: Set up Python ${{ matrix.python-version }} 37 | uses: actions/setup-python@v5 38 | with: 39 | python-version: ${{ matrix.python-version }} 40 | 41 | - uses: astral-sh/setup-uv@v6 42 | 43 | - name: Install tox 44 | run: uv tool install --with tox-gh-actions --with tox-uv tox 45 | 46 | - name: Run tests with PyTest 8 47 | run: tox 48 | env: 49 | PYTEST_MAJOR_VERSION: 8 50 | PYTEST_PLUGINS: pytest_github_actions_annotate_failures 51 | 52 | - name: Run tests with PyTest 7 53 | run: tox 54 | if: runner.os != 'Windows' 55 | env: 56 | PYTEST_MAJOR_VERSION: 7 57 | PYTEST_PLUGINS: pytest_github_actions_annotate_failures 58 | 59 | - name: Run tests with PyTest 6 60 | run: tox 61 | if: runner.os != 'Windows' 62 | env: 63 | PYTEST_MAJOR_VERSION: 6 64 | PYTEST_PLUGINS: pytest_github_actions_annotate_failures 65 | 66 | post-test: 67 | name: All tests passed 68 | if: always() 69 | needs: [test] 70 | runs-on: ubuntu-latest 71 | timeout-minutes: 2 72 | steps: 73 | - name: Decide whether the needed jobs succeeded or failed 74 | uses: re-actors/alls-green@release/v1 75 | with: 76 | jobs: ${{ toJSON(needs) }} 77 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: "v5.0.0" 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-case-conflict 7 | - id: check-merge-conflict 8 | - id: check-symlinks 9 | - id: check-yaml 10 | - id: debug-statements 11 | - id: end-of-file-fixer 12 | - id: mixed-line-ending 13 | - id: requirements-txt-fixer 14 | - id: trailing-whitespace 15 | 16 | - repo: https://github.com/astral-sh/ruff-pre-commit 17 | rev: "v0.9.2" 18 | hooks: 19 | - id: ruff 20 | args: ["--fix", "--show-fixes"] 21 | - id: ruff-format 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Unreleased 4 | 5 | Nothing yet. 6 | 7 | ## 0.3.0 (2025-01-17) 8 | 9 | - Test on Python 3.13 #89 10 | - Support pytest 7.4+ #97 (thanks to @edgarrmondragon) 11 | - Require Python 3.8+ #87 (thanks to @edgarrmondragon) 12 | - Require pytest 6+ #86 (thanks to @edgarrmondragon) 13 | - Speed up CI and testing #93 14 | - Use Ruff formatter #96 15 | - Use dependency-groups for tests #99 16 | - Add GitHub Attestations #100 17 | 18 | ## 0.2.0 (2023-05-04) 19 | 20 | ### Incompatible changes 21 | 22 | - Require python 3.7+ #66 (thanks to @ssbarnea) 23 | 24 | ### Other changes 25 | 26 | - Fix publish package workflow #74 27 | - Handle cases where pytest itself fails #70 (thanks to @edgarrmondragon) 28 | - Adopt PEP-621 for packaging #65 (thanks to @ssbarnea) 29 | - Bump pre-commit/action from 2.0.0 to 3.0.0 #56 30 | 31 | ## 0.1.8 (2022-12-20) 32 | 33 | No functionality change. 34 | - Change URL of PyPI project link #61 35 | - Fix CI environment #62 (thanks to @henryiii, @nicoddemus) 36 | 37 | ## 0.1.7 (2022-07-02) 38 | 39 | - add longrepr from plugin tests #50 (thanks to @helpmefindaname) 40 | - Use latest major version for actions #51 41 | 42 | ## 0.1.6 (2021-12-8) 43 | 44 | - Handle test failures without a line number #47 (thanks to @Tenzer) 45 | 46 | ## 0.1.5 (2021-10-24) 47 | 48 | - Revert changes of version 0.1.4 #42 49 | 50 | ## 0.1.4 (2021-10-24) 51 | 52 | - Ignore failures that are retried using [`pytest-rerunfailures`](https://pypi.org/project/pytest-rerunfailures/) plugin #40 (thanks to @billyvg) 53 | 54 | ## 0.1.3 (2021-07-31) 55 | 56 | - Allow specifying a run path with `PYTEST_RUN_PATH` environment variable #29 (thanks to @michamos) 57 | 58 | ## 0.1.2 (2021-03-21) 59 | 60 | - Fall back file path when ValueError on Windows #24 61 | - ci: change Python version set #25 62 | - doc: notice for Docker environments #27 63 | 64 | ## 0.1.1 (2020-10-13) 65 | 66 | - Fix #21: stdout is captured by pytest-xdist #22 (thanks to @yihuang) 67 | 68 | ## 0.1.0 (2020-08-22) 69 | 70 | - feat: better annotation structure (PR #14, thanks to @henryiii) 71 | 72 | ## 0.0.8 (2020-08-20) 73 | 74 | - Convert relative path to path from repository root (fix #8 with PR #11 and #12, thanks to @henryiii) 75 | 76 | ## 0.0.7 (2020-08-20) 77 | 78 | - Python 2.7 support 79 | 80 | ## 0.0.6 (2020-07-30) 81 | 82 | - Enable this plugin only in GitHub Actions workflow 83 | 84 | ## 0.0.5 (2020-05-10) 85 | 86 | - Remove unnecessary environment (PYTEST_PLUGINS) variable in README (thanks to @AlphaMycelium) 87 | 88 | ## 0.0.4 (2020-05-09) 89 | 90 | - Always enabled whether or not in GitHub Actions workflow 91 | 92 | ## 0.0.3 (2020-05-09) 93 | 94 | - Requires pytest >= 4.0.0 95 | 96 | ## 0.0.2 (2020-05-09) 97 | 98 | - Add short description 99 | 100 | ## 0.0.1 (2020-05-09) 101 | 102 | - First release 103 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 utagawa kiki 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.py 2 | include *.txt 3 | include *.yaml 4 | include .pre-commit-config.yaml 5 | include CHANGELOG.md 6 | include LICENSE 7 | include tox.ini 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pytest-github-actions-annotate-failures 2 | [Pytest](https://pypi.org/project/pytest/) plugin to annotate failed tests with a [workflow command for GitHub Actions](https://help.github.com/en/actions/reference/workflow-commands-for-github-actions) 3 | 4 | ## Usage 5 | Just install and run pytest with this plugin in your workflow. For example, 6 | 7 | ```yaml 8 | name: test 9 | 10 | on: 11 | push: 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - uses: actions/setup-python@v4 21 | with: 22 | python-version: 3.8 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install -r requirements.txt 28 | 29 | - name: Install plugin 30 | run: pip install pytest-github-actions-annotate-failures 31 | 32 | - run: pytest 33 | ``` 34 | 35 | If your test is running in a Docker container, you have to install this plugin and manually set `GITHUB_ACTIONS` environment variable to `true` inside of Docker container. (For example, `docker-compose run --rm -e GITHUB_ACTIONS=true app -- pytest`) 36 | 37 | If your tests are run from a subdirectory of the git repository, you have to set the `PYTEST_RUN_PATH` environment variable to the path of that directory relative to the repository root in order for GitHub to identify the files with errors correctly. 38 | 39 | ### Warning annotations 40 | 41 | This plugin also supports warning annotations when used with Pytest 6.0+. To disable warning annotations, pass `--exclude-warning-annotations` to pytest. 42 | 43 | ## Screenshot 44 | [![Image from Gyazo](https://i.gyazo.com/b578304465dd1b755ceb0e04692a57d9.png)](https://gyazo.com/b578304465dd1b755ceb0e04692a57d9) 45 | -------------------------------------------------------------------------------- /plugin_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | 5 | import pytest 6 | from packaging import version 7 | 8 | PYTEST_VERSION = version.parse(pytest.__version__) 9 | pytest_plugins = "pytester" 10 | 11 | 12 | # result.stderr.no_fnmatch_line() was added to testdir on pytest 5.3.0 13 | # https://docs.pytest.org/en/stable/changelog.html#pytest-5-3-0-2019-11-19 14 | def no_fnmatch_line(result: pytest.RunResult, pattern: str): 15 | result.stderr.no_fnmatch_line(pattern + "*") 16 | 17 | 18 | def test_annotation_succeed_no_output(testdir: pytest.Testdir): 19 | testdir.makepyfile( 20 | """ 21 | import pytest 22 | pytest_plugins = 'pytest_github_actions_annotate_failures' 23 | 24 | def test_success(): 25 | assert 1 26 | """ 27 | ) 28 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 29 | result = testdir.runpytest_subprocess() 30 | 31 | no_fnmatch_line(result, "::error file=test_annotation_succeed_no_output.py") 32 | 33 | 34 | def test_annotation_pytest_error(testdir: pytest.Testdir): 35 | testdir.makepyfile( 36 | """ 37 | import pytest 38 | pytest_plugins = 'pytest_github_actions_annotate_failures' 39 | 40 | @pytest.fixture 41 | def fixture(): 42 | return 1 43 | 44 | def test_error(): 45 | assert fixture() == 1 46 | """ 47 | ) 48 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 49 | result = testdir.runpytest_subprocess() 50 | 51 | result.stderr.re_match_lines( 52 | [ 53 | r"::error file=test_annotation_pytest_error\.py,line=8::test_error.*", 54 | ] 55 | ) 56 | 57 | 58 | def test_annotation_fail(testdir: pytest.Testdir): 59 | testdir.makepyfile( 60 | """ 61 | import pytest 62 | pytest_plugins = 'pytest_github_actions_annotate_failures' 63 | 64 | def test_fail(): 65 | assert 0 66 | """ 67 | ) 68 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 69 | result = testdir.runpytest_subprocess() 70 | result.stderr.fnmatch_lines( 71 | [ 72 | "::error file=test_annotation_fail.py,line=5::test_fail*assert 0*", 73 | ] 74 | ) 75 | 76 | 77 | def test_annotation_exception(testdir: pytest.Testdir): 78 | testdir.makepyfile( 79 | """ 80 | import pytest 81 | pytest_plugins = 'pytest_github_actions_annotate_failures' 82 | 83 | def test_fail(): 84 | raise Exception('oops') 85 | assert 1 86 | """ 87 | ) 88 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 89 | result = testdir.runpytest_subprocess() 90 | result.stderr.fnmatch_lines( 91 | [ 92 | "::error file=test_annotation_exception.py,line=5::test_fail*oops*", 93 | ] 94 | ) 95 | 96 | 97 | def test_annotation_warning(testdir: pytest.Testdir): 98 | testdir.makepyfile( 99 | """ 100 | import warnings 101 | import pytest 102 | pytest_plugins = 'pytest_github_actions_annotate_failures' 103 | 104 | def test_warning(): 105 | warnings.warn('beware', Warning) 106 | assert 1 107 | """ 108 | ) 109 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 110 | result = testdir.runpytest_subprocess() 111 | result.stderr.fnmatch_lines( 112 | [ 113 | "::warning file=test_annotation_warning.py,line=6::beware", 114 | ] 115 | ) 116 | 117 | 118 | def test_annotation_exclude_warnings(testdir: pytest.Testdir): 119 | testdir.makepyfile( 120 | """ 121 | import warnings 122 | import pytest 123 | pytest_plugins = 'pytest_github_actions_annotate_failures' 124 | 125 | def test_warning(): 126 | warnings.warn('beware', Warning) 127 | assert 1 128 | """ 129 | ) 130 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 131 | result = testdir.runpytest_subprocess("--exclude-warning-annotations") 132 | assert not result.stderr.lines 133 | 134 | 135 | def test_annotation_third_party_exception(testdir: pytest.Testdir): 136 | testdir.makepyfile( 137 | my_module=""" 138 | def fn(): 139 | raise Exception('oops') 140 | """ 141 | ) 142 | 143 | testdir.makepyfile( 144 | """ 145 | import pytest 146 | from my_module import fn 147 | pytest_plugins = 'pytest_github_actions_annotate_failures' 148 | 149 | def test_fail(): 150 | fn() 151 | """ 152 | ) 153 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 154 | result = testdir.runpytest_subprocess() 155 | result.stderr.fnmatch_lines( 156 | [ 157 | "::error file=test_annotation_third_party_exception.py,line=6::test_fail*oops*", 158 | ] 159 | ) 160 | 161 | 162 | def test_annotation_third_party_warning(testdir: pytest.Testdir): 163 | testdir.makepyfile( 164 | my_module=""" 165 | import warnings 166 | 167 | def fn(): 168 | warnings.warn('beware', Warning) 169 | """ 170 | ) 171 | 172 | testdir.makepyfile( 173 | """ 174 | import pytest 175 | from my_module import fn 176 | pytest_plugins = 'pytest_github_actions_annotate_failures' 177 | 178 | def test_warning(): 179 | fn() 180 | """ 181 | ) 182 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 183 | result = testdir.runpytest_subprocess() 184 | result.stderr.fnmatch_lines( 185 | # ["::warning file=test_annotation_third_party_warning.py,line=6::beware",] 186 | [ 187 | "::warning file=my_module.py,line=4::beware", 188 | ] 189 | ) 190 | 191 | 192 | def test_annotation_fail_disabled_outside_workflow(testdir: pytest.Testdir): 193 | testdir.makepyfile( 194 | """ 195 | import pytest 196 | pytest_plugins = 'pytest_github_actions_annotate_failures' 197 | 198 | def test_fail(): 199 | assert 0 200 | """ 201 | ) 202 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "") 203 | result = testdir.runpytest_subprocess() 204 | no_fnmatch_line( 205 | result, "::error file=test_annotation_fail_disabled_outside_workflow.py*" 206 | ) 207 | 208 | 209 | def test_annotation_fail_cwd(testdir: pytest.Testdir): 210 | testdir.makepyfile( 211 | """ 212 | import pytest 213 | pytest_plugins = 'pytest_github_actions_annotate_failures' 214 | 215 | def test_fail(): 216 | assert 0 217 | """ 218 | ) 219 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 220 | testdir.monkeypatch.setenv("GITHUB_WORKSPACE", os.path.dirname(str(testdir.tmpdir))) 221 | testdir.mkdir("foo") 222 | testdir.makefile(".ini", pytest="[pytest]\ntestpaths=..") 223 | result = testdir.runpytest_subprocess("--rootdir=foo") 224 | result.stderr.fnmatch_lines( 225 | [ 226 | "::error file=test_annotation_fail_cwd0/test_annotation_fail_cwd.py,line=5::test_fail*assert 0*", 227 | ] 228 | ) 229 | 230 | 231 | def test_annotation_fail_runpath(testdir: pytest.Testdir): 232 | testdir.makepyfile( 233 | """ 234 | import pytest 235 | pytest_plugins = 'pytest_github_actions_annotate_failures' 236 | 237 | def test_fail(): 238 | assert 0 239 | """ 240 | ) 241 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 242 | testdir.monkeypatch.setenv("PYTEST_RUN_PATH", "some_path") 243 | result = testdir.runpytest_subprocess() 244 | result.stderr.fnmatch_lines( 245 | [ 246 | "::error file=some_path/test_annotation_fail_runpath.py,line=5::test_fail*assert 0*", 247 | ] 248 | ) 249 | 250 | 251 | def test_annotation_long(testdir: pytest.Testdir): 252 | testdir.makepyfile( 253 | """ 254 | import pytest 255 | pytest_plugins = 'pytest_github_actions_annotate_failures' 256 | 257 | def f(x): 258 | return x 259 | 260 | def test_fail(): 261 | x = 1 262 | x += 1 263 | x += 1 264 | x += 1 265 | x += 1 266 | x += 1 267 | x += 1 268 | x += 1 269 | 270 | assert f(x) == 3 271 | """ 272 | ) 273 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 274 | result = testdir.runpytest_subprocess() 275 | result.stderr.fnmatch_lines( 276 | [ 277 | "::error file=test_annotation_long.py,line=17::test_fail*assert 8 == 3*where 8 = f(8)*", 278 | ] 279 | ) 280 | no_fnmatch_line(result, "::*assert x += 1*") 281 | 282 | 283 | def test_class_method(testdir: pytest.Testdir): 284 | testdir.makepyfile( 285 | """ 286 | import pytest 287 | pytest_plugins = 'pytest_github_actions_annotate_failures' 288 | 289 | class TestClass(object): 290 | def test_method(self): 291 | x = 1 292 | assert x == 2 293 | """ 294 | ) 295 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 296 | result = testdir.runpytest_subprocess() 297 | result.stderr.fnmatch_lines( 298 | [ 299 | "::error file=test_class_method.py,line=7::TestClass.test_method*assert 1 == 2*", 300 | ] 301 | ) 302 | no_fnmatch_line(result, "::*x = 1*") 303 | 304 | 305 | def test_annotation_param(testdir: pytest.Testdir): 306 | testdir.makepyfile( 307 | """ 308 | import pytest 309 | pytest_plugins = 'pytest_github_actions_annotate_failures' 310 | 311 | @pytest.mark.parametrize("a", [1]) 312 | @pytest.mark.parametrize("b", [2], ids=["other"]) 313 | def test_param(a, b): 314 | 315 | a += 1 316 | b += 1 317 | 318 | assert a == b 319 | """ 320 | ) 321 | testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true") 322 | result = testdir.runpytest_subprocess() 323 | result.stderr.fnmatch_lines( 324 | [ 325 | "::error file=test_annotation_param.py,line=11::test_param?other?1*assert 2 == 3*", 326 | ] 327 | ) 328 | 329 | 330 | # Debugging / development tip: 331 | # Add a breakpoint() to the place you are going to check, 332 | # uncomment this example, and run it with: 333 | # GITHUB_ACTIONS=true pytest -k test_example 334 | # def test_example(): 335 | # x = 3 336 | # y = 4 337 | # assert x == y 338 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 64.0.0", # required by pyproject+setuptools_scm integration 4 | "setuptools_scm[toml] >= 7.0.5", # required for "no-local-version" scheme 5 | 6 | ] 7 | build-backend = "setuptools.build_meta" 8 | 9 | [project] 10 | # https://peps.python.org/pep-0621/#readme 11 | requires-python = ">=3.8" 12 | version = "0.3.0" 13 | name = "pytest-github-actions-annotate-failures" 14 | description = "pytest plugin to annotate failed tests with a workflow command for GitHub Actions" 15 | readme = "README.md" 16 | authors = [{ "name" = "utgwkk", "email" = "utagawakiki@gmail.com" }] 17 | maintainers = [{ "name" = "utgwkk", "email" = "utagawakiki@gmail.com" }] 18 | license = { text = "MIT" } 19 | classifiers = [ 20 | "Development Status :: 5 - Production/Stable", 21 | "Environment :: Console", 22 | "Intended Audience :: Developers", 23 | "Intended Audience :: Information Technology", 24 | "Intended Audience :: System Administrators", 25 | "License :: OSI Approved :: MIT License", 26 | "Framework :: Pytest", 27 | "Programming Language :: Python :: 3", 28 | "Programming Language :: Python :: 3.8", 29 | "Programming Language :: Python :: 3.9", 30 | "Programming Language :: Python :: 3.10", 31 | "Programming Language :: Python :: 3.11", 32 | "Programming Language :: Python :: 3.12", 33 | "Programming Language :: Python :: 3.13", 34 | "Programming Language :: Python :: 3 :: Only", 35 | "Programming Language :: Python", 36 | "Topic :: System :: Systems Administration", 37 | "Topic :: Software Development :: Quality Assurance", 38 | "Topic :: Software Development :: Testing", 39 | "Topic :: Utilities", 40 | ] 41 | keywords = ["ansible", "testing", "molecule", "plugin"] 42 | dependencies = [ 43 | "pytest>=6.0.0" 44 | ] 45 | 46 | [project.urls] 47 | homepage = "https://github.com/pytest-dev/pytest-github-actions-annotate-failures" 48 | repository = "https://github.com/pytest-dev/pytest-github-actions-annotate-failures" 49 | changelog = "https://github.com/pytest-dev/pytest-github-actions-annotate-failures/releases" 50 | 51 | [project.entry-points.pytest11] 52 | pytest_github_actions_annotate_failures = "pytest_github_actions_annotate_failures.plugin" 53 | 54 | [dependency-groups] 55 | dev = [{ include-group = "test"}] 56 | test = ["packaging"] 57 | 58 | 59 | [tool.ruff.lint] 60 | extend-select = [ 61 | "B", # flake8-bugbear 62 | "I", # isort 63 | "ARG", # flake8-unused-arguments 64 | "C4", # flake8-comprehensions 65 | "EM", # flake8-errmsg 66 | "ICN", # flake8-import-conventions 67 | "ISC", # flake8-implicit-str-concat 68 | "G", # flake8-logging-format 69 | "PGH", # pygrep-hooks 70 | "PIE", # flake8-pie 71 | "PL", # pylint 72 | "PT", # flake8-pytest-style 73 | "RET", # flake8-return 74 | "RUF", # Ruff-specific 75 | "SIM", # flake8-simplify 76 | "UP", # pyupgrade 77 | "YTT", # flake8-2020 78 | "EXE", # flake8-executable 79 | ] 80 | ignore = [ 81 | "PLR", # Design related pylint codes 82 | ] 83 | isort.required-imports = ["from __future__ import annotations"] 84 | 85 | [tool.ruff.lint.per-file-ignores] 86 | "tests/**" = ["T20"] 87 | -------------------------------------------------------------------------------- /pytest_github_actions_annotate_failures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pytest-dev/pytest-github-actions-annotate-failures/bdfbf81a8c54803669537d72d3219886dc5ac654/pytest_github_actions_annotate_failures/__init__.py -------------------------------------------------------------------------------- /pytest_github_actions_annotate_failures/plugin.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import contextlib 4 | import os 5 | import sys 6 | from typing import TYPE_CHECKING 7 | 8 | import pytest 9 | from _pytest._code.code import ExceptionRepr, ReprEntry 10 | from packaging import version 11 | 12 | if TYPE_CHECKING: 13 | from _pytest.nodes import Item 14 | from _pytest.reports import CollectReport 15 | 16 | 17 | # Reference: 18 | # https://docs.pytest.org/en/latest/writing_plugins.html#hookwrapper-executing-around-other-hooks 19 | # https://docs.pytest.org/en/latest/writing_plugins.html#hook-function-ordering-call-example 20 | # https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_runtest_makereport 21 | # 22 | # Inspired by: 23 | # https://github.com/pytest-dev/pytest/blob/master/src/_pytest/terminal.py 24 | 25 | 26 | PYTEST_VERSION = version.parse(pytest.__version__) 27 | 28 | 29 | @pytest.hookimpl(tryfirst=True, hookwrapper=True) 30 | def pytest_runtest_makereport(item: Item, call): # noqa: ARG001 31 | # execute all other hooks to obtain the report object 32 | outcome = yield 33 | report: CollectReport = outcome.get_result() 34 | 35 | # enable only in a workflow of GitHub Actions 36 | # ref: https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables 37 | if os.environ.get("GITHUB_ACTIONS") != "true": 38 | return 39 | 40 | if report.when == "call" and report.failed: 41 | filesystempath, lineno, _ = report.location 42 | 43 | if lineno is not None: 44 | # 0-index to 1-index 45 | lineno += 1 46 | 47 | longrepr = report.head_line or item.name 48 | 49 | # get the error message and line number from the actual error 50 | if isinstance(report.longrepr, ExceptionRepr): 51 | if report.longrepr.reprcrash is not None: 52 | longrepr += "\n\n" + report.longrepr.reprcrash.message 53 | tb_entries = report.longrepr.reprtraceback.reprentries 54 | if tb_entries: 55 | entry = tb_entries[0] 56 | # Handle third-party exceptions 57 | if isinstance(entry, ReprEntry) and entry.reprfileloc is not None: 58 | lineno = entry.reprfileloc.lineno 59 | filesystempath = entry.reprfileloc.path 60 | 61 | elif report.longrepr.reprcrash is not None: 62 | lineno = report.longrepr.reprcrash.lineno 63 | elif isinstance(report.longrepr, tuple): 64 | filesystempath, lineno, message = report.longrepr 65 | longrepr += "\n\n" + message 66 | elif isinstance(report.longrepr, str): 67 | longrepr += "\n\n" + report.longrepr 68 | 69 | workflow_command = _build_workflow_command( 70 | "error", 71 | compute_path(filesystempath), 72 | lineno, 73 | message=longrepr, 74 | ) 75 | print(workflow_command, file=sys.stderr) 76 | 77 | 78 | def compute_path(filesystempath: str) -> str: 79 | """Extract and process location information from the report.""" 80 | runpath = os.environ.get("PYTEST_RUN_PATH") 81 | if runpath: 82 | filesystempath = os.path.join(runpath, filesystempath) 83 | 84 | # try to convert to absolute path in GitHub Actions 85 | workspace = os.environ.get("GITHUB_WORKSPACE") 86 | if workspace: 87 | full_path = os.path.abspath(filesystempath) 88 | try: 89 | rel_path = os.path.relpath(full_path, workspace) 90 | except ValueError: 91 | # os.path.relpath() will raise ValueError on Windows 92 | # when full_path and workspace have different mount points. 93 | rel_path = filesystempath 94 | if not rel_path.startswith(".."): 95 | filesystempath = rel_path 96 | 97 | return filesystempath 98 | 99 | 100 | class _AnnotateWarnings: 101 | def pytest_warning_recorded(self, warning_message, when, nodeid, location): # noqa: ARG002 102 | # enable only in a workflow of GitHub Actions 103 | # ref: https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables 104 | if os.environ.get("GITHUB_ACTIONS") != "true": 105 | return 106 | 107 | filesystempath = warning_message.filename 108 | workspace = os.environ.get("GITHUB_WORKFLOW") 109 | 110 | if workspace: 111 | try: 112 | rel_path = os.path.relpath(filesystempath, workspace) 113 | except ValueError: 114 | # os.path.relpath() will raise ValueError on Windows 115 | # when full_path and workspace have different mount points. 116 | rel_path = filesystempath 117 | if not rel_path.startswith(".."): 118 | filesystempath = rel_path 119 | else: 120 | with contextlib.suppress(ValueError): 121 | filesystempath = os.path.relpath(filesystempath) 122 | 123 | workflow_command = _build_workflow_command( 124 | "warning", 125 | filesystempath, 126 | warning_message.lineno, 127 | message=warning_message.message.args[0], 128 | ) 129 | print(workflow_command, file=sys.stderr) 130 | 131 | 132 | def pytest_addoption(parser): 133 | group = parser.getgroup("pytest_github_actions_annotate_failures") 134 | group.addoption( 135 | "--exclude-warning-annotations", 136 | action="store_true", 137 | default=False, 138 | help="Annotate failures in GitHub Actions.", 139 | ) 140 | 141 | 142 | def pytest_configure(config): 143 | if not config.option.exclude_warning_annotations: 144 | config.pluginmanager.register(_AnnotateWarnings(), "annotate_warnings") 145 | 146 | 147 | def _build_workflow_command( 148 | command_name: str, 149 | file: str, 150 | line: int, 151 | end_line: int | None = None, 152 | column: int | None = None, 153 | end_column: int | None = None, 154 | title: str | None = None, 155 | message: str | None = None, 156 | ): 157 | """Build a command to annotate a workflow.""" 158 | result = f"::{command_name} " 159 | 160 | entries = [ 161 | ("file", file), 162 | ("line", line), 163 | ("endLine", end_line), 164 | ("col", column), 165 | ("endColumn", end_column), 166 | ("title", title), 167 | ] 168 | 169 | result = result + ",".join(f"{k}={v}" for k, v in entries if v is not None) 170 | 171 | if message is not None: 172 | result = result + "::" + _escape(message) 173 | 174 | return result 175 | 176 | 177 | def _escape(s: str) -> str: 178 | return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") 179 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{38,39,310,311,312,313}-pytest{6,7,8} 4 | 5 | [gh-actions] 6 | python = 7 | 3.8: py38 8 | 3.9: py39 9 | 3.10: py310 10 | 3.11: py311 11 | 3.12: py312 12 | 3.13: py313 13 | 14 | [gh-actions:env] 15 | PYTEST_MAJOR_VERSION = 16 | 6: pytest6 17 | 7: pytest7 18 | 8: pytest8 19 | 20 | [testenv] 21 | min_version = 4.22.0 22 | groups = test 23 | deps = 24 | pytest6: pytest>=6.0.0,<7.0.0 25 | pytest7: pytest>=7.0.0,<8.0.0 26 | pytest8: pytest>=8.0.0,<9.0.0 27 | 28 | commands = {envpython} -m pytest {posargs} 29 | --------------------------------------------------------------------------------