├── .craft.yml ├── .flake8 ├── .github └── workflows │ ├── main.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── assets └── pytest-sentry-screenshot.png ├── dev-requirements.txt ├── pytest_sentry ├── __init__.py ├── client.py ├── consts.py ├── fixtures.py ├── helpers.py ├── hooks.py └── integration.py ├── scripts └── bump-version.sh ├── setup.cfg ├── setup.py └── tests ├── test_envvars.py ├── test_fixture.py ├── test_retries.py ├── test_scope.py ├── test_skip.py └── test_tracing.py /.craft.yml: -------------------------------------------------------------------------------- 1 | minVersion: 0.34.0 2 | targets: 3 | - name: pypi 4 | - name: github 5 | - name: sentry-pypi 6 | internalPypiRepo: getsentry/pypi 7 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, E402, E731 3 | max-line-length = 80 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude=checkouts,lol*,.tox 7 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | push: 4 | branches: [main, release/**, test-me-*] 5 | 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: actions/setup-python@v4 15 | with: 16 | python-version: ${{ matrix.python }} 17 | - run: pip install -r dev-requirements.txt 18 | - run: PYTEST_SENTRY_ALWAYS_REPORT=1 pytest tests 19 | dist: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v3 23 | - uses: actions/setup-python@v4 24 | with: 25 | python-version: '3.x' 26 | - run: | 27 | pip install build 28 | python -m build 29 | - uses: actions/upload-artifact@v4 30 | with: 31 | name: ${{ github.sha }} 32 | path: dist/* 33 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version: 7 | description: Version to release 8 | required: true 9 | force: 10 | description: Force a release even when there are release-blockers (optional) 11 | required: false 12 | 13 | jobs: 14 | release: 15 | runs-on: ubuntu-latest 16 | name: "Release a new version" 17 | steps: 18 | - name: Get auth token 19 | id: token 20 | uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 21 | with: 22 | app-id: ${{ vars.SENTRY_RELEASE_BOT_CLIENT_ID }} 23 | private-key: ${{ secrets.SENTRY_RELEASE_BOT_PRIVATE_KEY }} 24 | - uses: actions/checkout@v3 25 | with: 26 | token: ${{ steps.token.outputs.token }} 27 | fetch-depth: 0 28 | - name: Prepare release 29 | uses: getsentry/action-prepare-release@v1 30 | env: 31 | GITHUB_TOKEN: ${{ steps.token.outputs.token }} 32 | with: 33 | version: ${{ github.event.inputs.version }} 34 | force: ${{ github.event.inputs.force }} 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.log 3 | *.egg 4 | *.db 5 | *.pid 6 | .python-version 7 | .coverage* 8 | .DS_Store 9 | .tox 10 | pip-log.txt 11 | *.egg-info 12 | /build 13 | /dist 14 | .cache 15 | .idea 16 | .eggs 17 | venv 18 | .venv 19 | .vscode/tags 20 | .pytest_cache 21 | .hypothesis 22 | semaphore 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 Markus Unterwaditzer and individual contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | release: 2 | rm -rf dist/ 3 | python setup.py sdist bdist_wheel 4 | twine upload dist/* 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pytest-sentry 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/pytest-sentry)](https://pypi.org/project/pytest-sentry/) 4 | [![License](https://img.shields.io/pypi/l/pytest-sentry)](https://pypi.org/project/pytest-sentry/) 5 | 6 | `pytest-sentry` is a [pytest](https://pytest.org>) plugin that uses [Sentry](https://sentry.io/) to store and aggregate information about your testruns. 7 | 8 | > [!IMPORTANT] 9 | > **This is not an official Sentry product.** 10 | 11 | ![Screenshot of tracing data of a pytest test suite](assets/pytest-sentry-screenshot.png) 12 | 13 | This is what a test run will look like in Sentry. 14 | 15 | ## Quickstart 16 | ### Prerequisites 17 | 18 | * You are using [pytest](https://pytest.org) to run your tests. 19 | * You are using [pytest-rerunfailures](https://github.com/pytest-dev/pytest-rerunfailures) to rerun flaky tests. 20 | 21 | ### Configuration 22 | 23 | You can configure `pytest-sentry` with environment variables: 24 | 25 | * `PYTEST_SENTRY_DSN`: The Sentry DSN to send the data to. 26 | 27 | * `PYTEST_SENTRY_ALWAYS_REPORT`: If not set, only flaky tests are reported as errors. If set to `1` all test failures are reported. If set to `0` no test failures are reported at all. 28 | 29 | * `PYTEST_SENTRY_TRACES_SAMPLE_RATE`: The sample rate for tracing data. See https://docs.sentry.io/platforms/python/configuration/options/#traces_sample_rate 30 | 31 | * `PYTEST_SENTRY_PROFILES_SAMPLE_RATE`: The sample rate for profiling data. 32 | 33 | * `PYTEST_SENTRY_DEBUG`: Set to `1` to display Sentry debug output. See https://docs.sentry.io/platforms/python/configuration/options/#debug 34 | 35 | ### Running 36 | 37 | If you have `pytest-rerunfailures` plugin enabled you set the environment variables and run `pytest` as usual: 38 | ```bash 39 | export PYTEST_SENTRY_DSN="https://xx@xxx.ingest.sentry.io/xxx" # your DSN 40 | export PYTEST_SENTRY_TRACES_SAMPLE_RATE=1 41 | export PYTEST_SENTRY_PROFILES_SAMPLE_RATE=1 42 | export SENTRY_ENVIRONMENT="test-suite" 43 | 44 | pytest --reruns=5 45 | ``` 46 | 47 | Now all flaky tests will report to the configured DSN in Sentry.io including trace information and profiles of your tests and test fixtures in an Sentry environment calld `test-suite`. 48 | 49 | # Tracking flaky tests as errors 50 | 51 | Let's say you have a testsuite with some flaky tests that randomly break your 52 | CI build due to network issues, race conditions or other stuff that you don't 53 | want to fix immediately. The known workaround is to retry those tests 54 | automatically, for example using [pytest-rerunfailures](https://github.com/pytest-dev/pytest-rerunfailures). 55 | 56 | One concern against plugins like this is that they just hide the bugs in your 57 | testsuite or even other code. After all your CI build is green and your code 58 | probably works most of the time. 59 | 60 | `pytest-sentry` tries to make that choice a bit easier by tracking flaky test 61 | failures in a place separate from your build status. Sentry is already a 62 | good choice for keeping tabs on all kinds of errors, important or not, in 63 | production, so let's try to use it in testsuites too. 64 | 65 | The prerequisite is that you already make use of `pytest` and 66 | `pytest-rerunfailures` in CI. Now install `pytest-sentry` and set the 67 | `PYTEST_SENTRY_DSN` environment variable to the DSN of a new Sentry project. 68 | 69 | Now every test failure that is "fixed" by retrying the test is reported to 70 | Sentry, but still does not break CI. Tests that consistently fail will not be 71 | reported. 72 | 73 | # Tracking the performance of your testsuite 74 | 75 | By default `pytest-sentry` will send [Performance](https://sentry.io/for/performance/) data to Sentry: 76 | 77 | * Fixture setup is reported as "transaction" to Sentry, such that you can 78 | answer questions like "what is my slowest test fixture" and "what is my most 79 | used test fixture". 80 | 81 | * Calls to the test function itself are reported as separate transaction such 82 | that you can find large, slow tests as well. 83 | 84 | * Fixture setup related to a particular test item will be in the same trace, 85 | i.e. will have same trace ID. There is no common parent transaction though. 86 | It is purposefully dropped to spare quota as it does not contain interesting 87 | information:: 88 | 89 | pytest.runtest.protocol [one time, not sent] 90 | pytest.fixture.setup [multiple times, sent] 91 | pytest.runtest.call [one time, sent] 92 | 93 | The trace is per-test-item. For correlating transactions across an entire 94 | test run, use the automatically attached CI tags or attach some tag on your 95 | own. 96 | 97 | To measure performance data, install `pytest-sentry` and set 98 | `PYTEST_SENTRY_DSN`, like with errors. By default, the extension will send all 99 | performance data to Sentry. If you want to limit the amount of data sent, you 100 | can set the `PYTEST_SENTRY_TRACES_SAMPLE_RATE` environment variable to a float 101 | between `0` and `1`. This will cause only a random sample of transactions to 102 | be sent to Sentry. 103 | 104 | Transactions can have noticeable runtime overhead over just reporting errors. 105 | To disable, use a marker:: 106 | 107 | import pytest 108 | import pytest_sentry 109 | 110 | pytestmarker = pytest.mark.sentry_client({"traces_sample_rate": 0.0}) 111 | 112 | # Advanced Options 113 | 114 | `pytest-sentry` supports marking your tests to use a different DSN, client or 115 | scope per-test. You can use this to provide custom options to the `Client` 116 | object from the [Sentry SDK for Python](https://github.com/getsentry/sentry-python): 117 | 118 | ```python 119 | import random 120 | import pytest 121 | 122 | from sentry_sdk import Scope 123 | from pytest_sentry import Client 124 | 125 | @pytest.mark.sentry_client(None) 126 | def test_no_sentry(): 127 | # Even though flaky, this test never gets reported to sentry 128 | assert random.random() > 0.5 129 | 130 | @pytest.mark.sentry_client("MY NEW DSN") 131 | def test_custom_dsn(): 132 | # Use a different DSN to report errors for this one 133 | assert random.random() > 0.5 134 | 135 | # Other invocations: 136 | 137 | @pytest.mark.sentry_client(Client("CUSTOM DSN")) 138 | @pytest.mark.sentry_client(lambda: Client("CUSTOM DSN")) 139 | @pytest.mark.sentry_client(Scope(Client("CUSTOM DSN"))) 140 | @pytest.mark.sentry_client({"dsn": ..., "debug": True}) 141 | ``` 142 | 143 | The `Client` class exposed by `pytest-sentry` only has different default 144 | integrations. It disables some of the error-capturing integrations to avoid 145 | sending random expected errors into your project. 146 | 147 | # Accessing the used Sentry client 148 | 149 | You will notice that the global functions such as 150 | `sentry_sdk.capture_message` will not actually send events into the same DSN 151 | you configured this plugin with. That's because `pytest-sentry` goes to 152 | extreme lenghts to keep its own SDK setup separate from the SDK setup of the 153 | tested code. 154 | 155 | `pytest-sentry` exposes the `sentry_test_scope` fixture whose return value is 156 | the `Scope` being used to send events to Sentry. Use `with use_scope(entry_test_scope):` 157 | to temporarily switch context. You can use this to set custom tags like so:: 158 | 159 | ```python 160 | def test_foo(sentry_test_scope): 161 | with use_scope(sentry_test_scope): 162 | sentry_sdk.set_tag("pull_request", os.environ['EXAMPLE_CI_PULL_REQUEST']) 163 | ``` 164 | 165 | Why all the hassle with the context manager? Just imagine if your tested 166 | application would start to log some (expected) errors on its own. You would 167 | immediately exceed your quota! 168 | 169 | # Always reporting test failures 170 | 171 | You can always report all test failures to Sentry by setting the environment 172 | variable `PYTEST_SENTRY_ALWAYS_REPORT=1`. 173 | 174 | This can be enabled for builds on the `main` or release branch, to catch 175 | certain kinds of tests that are flaky across builds, but consistently fail or 176 | pass within one testrun. 177 | 178 | # License 179 | 180 | Licensed under 2-clause BSD, see [LICENSE](LICENSE). 181 | -------------------------------------------------------------------------------- /assets/pytest-sentry-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getsentry/pytest-sentry/9871391848f2eeab0c137cf43f7a1971666722be/assets/pytest-sentry-screenshot.png -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | pytest-rerunfailures 3 | pytest<4; python_version < '3.0' 4 | -------------------------------------------------------------------------------- /pytest_sentry/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import Client # noqa: F401 2 | from .fixtures import sentry_test_scope # noqa: F401 3 | from .hooks import ( # noqa: F401 4 | pytest_fixture_setup, 5 | pytest_load_initial_conftests, 6 | pytest_runtest_call, 7 | pytest_runtest_makereport, 8 | pytest_runtest_protocol, 9 | ) 10 | -------------------------------------------------------------------------------- /pytest_sentry/client.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sentry_sdk 3 | 4 | from .integration import PytestIntegration 5 | 6 | 7 | class Client(sentry_sdk.Client): 8 | """ 9 | A client that is used to report errors from tests. 10 | Configured via `PYTEST_SENTRY_*` environment variables. 11 | """ 12 | 13 | def __init__(self, *args, **kwargs): 14 | kwargs.setdefault( 15 | "dsn", 16 | os.environ.get("PYTEST_SENTRY_DSN", None), 17 | ) 18 | kwargs.setdefault( 19 | "traces_sample_rate", 20 | float(os.environ.get("PYTEST_SENTRY_TRACES_SAMPLE_RATE", 1.0)), 21 | ) 22 | kwargs.setdefault( 23 | "profiles_sample_rate", 24 | float(os.environ.get("PYTEST_SENTRY_PROFILES_SAMPLE_RATE", 0.0)), 25 | ) 26 | kwargs.setdefault( 27 | "_experiments", 28 | {}, 29 | ).setdefault( 30 | "auto_enabling_integrations", 31 | True, 32 | ) 33 | kwargs.setdefault( 34 | "environment", 35 | os.environ.get("SENTRY_ENVIRONMENT", "test"), 36 | ) 37 | kwargs.setdefault("integrations", []).append(PytestIntegration()) 38 | 39 | debug = os.environ.get("PYTEST_SENTRY_DEBUG", "").lower() in ("1", "true", "yes") 40 | kwargs.setdefault("debug", debug) 41 | 42 | sentry_sdk.Client.__init__(self, *args, **kwargs) 43 | -------------------------------------------------------------------------------- /pytest_sentry/consts.py: -------------------------------------------------------------------------------- 1 | _ENVVARS_AS_TAGS = frozenset( 2 | [ 3 | "GITHUB_WORKFLOW", # The name of the workflow. 4 | "GITHUB_RUN_ID", # A unique number for each run within a repository. This number does not change if you re-run the workflow run. 5 | "GITHUB_RUN_NUMBER", # A unique number for each run of a particular workflow in a repository. This number begins at 1 for the workflow's first run, and increments with each new run. This number does not change if you re-run the workflow run. 6 | "GITHUB_ACTION", # The unique identifier (id) of the action. 7 | "GITHUB_ACTOR", # The name of the person or app that initiated the workflow. For example, octocat. 8 | "GITHUB_REPOSITORY", # The owner and repository name. For example, octocat/Hello-World. 9 | "GITHUB_EVENT_NAME", # The name of the webhook event that triggered the workflow. 10 | "GITHUB_EVENT_PATH", # The path of the file with the complete webhook event payload. For example, /github/workflow/event.json. 11 | "GITHUB_WORKSPACE", # The GitHub workspace directory path. The workspace directory is a copy of your repository if your workflow uses the actions/checkout action. If you don't use the actions/checkout action, the directory will be empty. For example, /home/runner/work/my-repo-name/my-repo-name. 12 | "GITHUB_SHA", # The commit SHA that triggered the workflow. For example, ffac537e6cbbf934b08745a378932722df287a53. 13 | "GITHUB_REF", # The branch or tag ref that triggered the workflow. For example, refs/heads/feature-branch-1. If neither a branch or tag is available for the event type, the variable will not exist. 14 | "GITHUB_HEAD_REF", # Only set for pull request events. The name of the head branch. 15 | "GITHUB_BASE_REF", # Only set for pull request events. The name of the base branch. 16 | "GITHUB_SERVER_URL", # Returns the URL of the GitHub server. For example: https://github.com. 17 | "GITHUB_API_URL", # Returns the API URL. For example: https://api.github.com. 18 | # Gitlab CI variables, as defined here https://docs.gitlab.com/ee/ci/variables/predefined_variables.html 19 | "CI_COMMIT_REF_NAME", # Branch or tag name 20 | "CI_JOB_ID", # Unique job ID 21 | "CI_JOB_URL", # Job details URL 22 | "CI_PIPELINE_ID", # Unique pipeline ID 23 | "CI_PROJECT_NAME", 24 | "CI_PROJECT_PATH", 25 | "CI_SERVER_URL", 26 | "GITLAB_USER_NAME", # The name of the user who started the job. 27 | # CircleCI variables, as defined here https://circleci.com/docs/variables/#built-in-environment-variables 28 | "CIRCLE_BRANCH", # The name of the Git branch currently being built. 29 | "CIRCLE_BUILD_NUM", # The number of the current job. Job numbers are unique for each job. 30 | "CIRCLE_BUILD_URL", # The URL for the current job on CircleCI. 31 | "CIRCLE_JOB", # The name of the current job. 32 | "CIRCLE_NODE_INDEX", # For jobs that run with parallelism enabled, this is the index of the current parallel run. 33 | "CIRCLE_PR_NUMBER", # The number of the associated GitHub or Bitbucket pull request. 34 | "CIRCLE_PR_REPONAME", # The name of the GitHub or Bitbucket repository where the pull request was created. 35 | "CIRCLE_PR_USERNAME", # The GitHub or Bitbucket username of the user who created the pull request. 36 | "CIRCLE_PROJECT_REPONAME", # The name of the repository of the current project. 37 | "CIRCLE_PROJECT_USERNAME", # The GitHub or Bitbucket username of the current project. 38 | "CIRCLE_PULL_REQUEST", # The URL of the associated pull request. 39 | "CIRCLE_REPOSITORY_URL", # The URL of your GitHub or Bitbucket repository. 40 | "CIRCLE_SHA1", # The SHA1 hash of the last commit of the current build. 41 | "CIRCLE_TAG", # The name of the git tag, if the current build is tagged. 42 | "CIRCLE_USERNAME", # The GitHub or Bitbucket username of the user who triggered the pipeline. 43 | "CIRCLE_WORKFLOW_ID", # A unique identifier for the workflow instance of the current job. 44 | "CIRCLE_WORKFLOW_JOB_ID", # A unique identifier for the current job. 45 | "CIRCLE_WORKFLOW_WORKSPACE_ID", # An identifier for the workspace of the current job. 46 | ] 47 | ) 48 | -------------------------------------------------------------------------------- /pytest_sentry/fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from .helpers import _resolve_scope_marker_value 4 | 5 | 6 | @pytest.fixture 7 | def sentry_test_scope(request): 8 | """ 9 | Gives back the current scope. 10 | """ 11 | 12 | item = request.node 13 | return _resolve_scope_marker_value(item.get_closest_marker("sentry_client")) 14 | -------------------------------------------------------------------------------- /pytest_sentry/helpers.py: -------------------------------------------------------------------------------- 1 | import sentry_sdk 2 | 3 | from pytest_sentry.client import Client 4 | 5 | 6 | DEFAULT_SCOPE = sentry_sdk.Scope(client=Client()) 7 | 8 | _scope_cache = {} 9 | 10 | 11 | def _resolve_scope_marker_value(marker_value): 12 | if id(marker_value) not in _scope_cache: 13 | _scope_cache[id(marker_value)] = rv = _resolve_scope_marker_value_uncached( 14 | marker_value 15 | ) 16 | return rv 17 | 18 | return _scope_cache[id(marker_value)] 19 | 20 | 21 | def _resolve_scope_marker_value_uncached(marker_value): 22 | if marker_value is None: 23 | # If no special configuration is provided 24 | # (like pytestmark or @pytest.mark.sentry_client() decorator) 25 | # use the default scope 26 | marker_value = DEFAULT_SCOPE 27 | else: 28 | marker_value = marker_value.args[0] 29 | 30 | if callable(marker_value): 31 | # If a callable is provided, call it to get the real marker value 32 | marker_value = marker_value() 33 | 34 | if marker_value is None: 35 | # The user explicitly disabled reporting 36 | return sentry_sdk.Scope() 37 | 38 | if isinstance(marker_value, str): 39 | # If a DSN string is provided, create a new client and use that 40 | scope = sentry_sdk.get_current_scope() 41 | scope.set_client(Client(marker_value)) 42 | return scope 43 | 44 | if isinstance(marker_value, dict): 45 | # If a dict is provided, create a new client using the dict as Client options 46 | scope = sentry_sdk.get_current_scope() 47 | scope.set_client(Client(**marker_value)) 48 | return scope 49 | 50 | if isinstance(marker_value, Client): 51 | # If a Client instance is provided, use that 52 | scope = sentry_sdk.get_current_scope() 53 | scope.set_client(marker_value) 54 | return scope 55 | 56 | if isinstance(marker_value, sentry_sdk.Scope): 57 | # If a Scope instance is provided, use the client from it 58 | scope = sentry_sdk.get_current_scope() 59 | scope.set_client(marker_value.client) 60 | return marker_value 61 | 62 | raise RuntimeError( 63 | "The `sentry_client` value must be a client, scope or string, not {}".format( 64 | repr(type(marker_value)) 65 | ) 66 | ) 67 | -------------------------------------------------------------------------------- /pytest_sentry/hooks.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import wrapt 3 | 4 | import sentry_sdk 5 | 6 | from .helpers import _resolve_scope_marker_value 7 | from .integration import PytestIntegration 8 | 9 | 10 | def pytest_load_initial_conftests(early_config, parser, args): 11 | """ 12 | Pytest hook that is called when pytest starts. 13 | See https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_load_initial_conftests 14 | """ 15 | early_config.addinivalue_line( 16 | "markers", 17 | "sentry_client(client=None): Use this client instance for reporting tests. You can also pass a DSN string directly, or a `Scope` if you need it.", 18 | ) 19 | 20 | 21 | def hookwrapper(itemgetter, **kwargs): 22 | """ 23 | A version of pytest.hookimpl that sets the current scope to the correct one 24 | and skips the hook if the integration is disabled. 25 | 26 | Assumes the function is a hookwrapper, ie yields once 27 | """ 28 | 29 | @wrapt.decorator 30 | def _with_scope(wrapped, instance, args, kwargs): 31 | item = itemgetter(*args, **kwargs) 32 | scope = _resolve_scope_marker_value(item.get_closest_marker("sentry_client")) 33 | 34 | if scope.client.get_integration(PytestIntegration) is None: 35 | yield 36 | else: 37 | with sentry_sdk.use_scope(scope): 38 | gen = wrapped(*args, **kwargs) 39 | 40 | while True: 41 | try: 42 | with sentry_sdk.use_scope(scope): 43 | chunk = next(gen) 44 | 45 | y = yield chunk 46 | 47 | with sentry_sdk.use_scope(scope): 48 | gen.send(y) 49 | 50 | except StopIteration: 51 | break 52 | 53 | def inner(f): 54 | return pytest.hookimpl(hookwrapper=True, **kwargs)(_with_scope(f)) 55 | 56 | return inner 57 | 58 | 59 | @hookwrapper(itemgetter=lambda item: item) 60 | def pytest_runtest_protocol(item): 61 | """ 62 | Pytest hook that is called when one test is run. 63 | The runtest protocol includes setup phase, call phase and teardown phase. 64 | See https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_runtest_protocol 65 | """ 66 | op = "pytest.runtest.protocol" 67 | name = "{} {}".format(op, item.nodeid) 68 | 69 | # We use the full name including parameters because then we can identify 70 | # how often a single test has run as part of the same GITHUB_RUN_ID. 71 | # Purposefully drop transaction to spare quota. We only created it to 72 | # have a trace_id to correlate by. 73 | with sentry_sdk.start_span(op=op, name=name, sampled=False): 74 | yield 75 | 76 | 77 | @hookwrapper(itemgetter=lambda item: item) 78 | def pytest_runtest_call(item): 79 | """ 80 | Pytest hook that is called when the call phase of a test is run. 81 | See https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_runtest_call 82 | """ 83 | op = "pytest.runtest.call" 84 | is_rerun = hasattr(item, "execution_count") and item.execution_count is not None and item.execution_count > 1 85 | if is_rerun: 86 | name = "{} (rerun {}) {}".format(op, item.execution_count - 1, item.nodeid) 87 | else: 88 | name = "{} {}".format(op, item.nodeid) 89 | 90 | # We use the full name including parameters because then we can identify 91 | # how often a single test has run as part of the same GITHUB_RUN_ID. 92 | with sentry_sdk.continue_trace(dict(sentry_sdk.get_current_scope().iter_trace_propagation_headers())): 93 | with sentry_sdk.start_span(op=op, name=name) as span: 94 | span.set_attribute("pytest-sentry.rerun", is_rerun) 95 | if is_rerun: 96 | span.set_attribute("pytest-sentry.execution_count", item.execution_count) 97 | 98 | yield 99 | 100 | 101 | @hookwrapper(itemgetter=lambda fixturedef, request: request._pyfuncitem) 102 | def pytest_fixture_setup(fixturedef, request): 103 | """ 104 | Pytest hook that is called when the fixtures are initially set up. 105 | See: https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_fixture_setup 106 | """ 107 | op = "pytest.fixture.setup" 108 | name = "{} {}".format(op, fixturedef.argname) 109 | 110 | with sentry_sdk.continue_trace(dict(sentry_sdk.get_current_scope().iter_trace_propagation_headers())): 111 | with sentry_sdk.start_span(op=op, name=name) as root_span: 112 | root_span.set_tag("pytest.fixture.scope", fixturedef.scope) 113 | yield 114 | 115 | 116 | @hookwrapper(tryfirst=True, itemgetter=lambda item, call: item) 117 | def pytest_runtest_makereport(item, call): 118 | """ 119 | Pytest hook that is called when the report is made for a test. 120 | This is executed for the setup, call and teardown phases. 121 | See: https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_runtest_makereport 122 | """ 123 | sentry_sdk.set_tag("pytest.result", "pending") 124 | 125 | report = yield 126 | outcome = report.get_result().outcome 127 | 128 | sentry_sdk.set_tag("pytest.result", outcome) 129 | 130 | if call.when == "call" and outcome != "skipped": 131 | cur_exc_chain = getattr(item, "pytest_sentry_exc_chain", []) 132 | 133 | if call.excinfo is not None: 134 | item.pytest_sentry_exc_chain = cur_exc_chain = cur_exc_chain + [ 135 | call.excinfo 136 | ] 137 | 138 | scope = _resolve_scope_marker_value(item.get_closest_marker("sentry_client")) 139 | integration = scope.client.get_integration(PytestIntegration) 140 | 141 | if (cur_exc_chain and call.excinfo is None) or (integration is not None and integration.always_report): 142 | for exc_info in cur_exc_chain: 143 | sentry_sdk.capture_exception((exc_info.type, exc_info.value, exc_info.tb)) 144 | -------------------------------------------------------------------------------- /pytest_sentry/integration.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import sentry_sdk 4 | from sentry_sdk.integrations import Integration 5 | from sentry_sdk.scope import add_global_event_processor 6 | 7 | from .consts import _ENVVARS_AS_TAGS 8 | 9 | 10 | def _process_stacktrace(stacktrace): 11 | for frame in stacktrace["frames"]: 12 | frame["in_app"] = not frame["module"].startswith( 13 | ("_pytest.", "pytest.", "pluggy.") 14 | ) 15 | 16 | 17 | class PytestIntegration(Integration): 18 | # Right now this integration type is only a carrier for options, and to 19 | # disable the pytest plugin. `setup_once` is unused. 20 | 21 | identifier = "pytest" 22 | 23 | def __init__(self, always_report=None): 24 | if always_report is None: 25 | always_report = os.environ.get( 26 | "PYTEST_SENTRY_ALWAYS_REPORT", "" 27 | ).lower() in ("1", "true", "yes") 28 | 29 | self.always_report = always_report 30 | 31 | @staticmethod 32 | def setup_once(): 33 | @add_global_event_processor 34 | def processor(event, hint): 35 | if sentry_sdk.get_client().get_integration(PytestIntegration) is None: 36 | return event 37 | 38 | for key in _ENVVARS_AS_TAGS: 39 | value = os.environ.get(key) 40 | if not value: 41 | continue 42 | event.setdefault("tags", {})["pytest_environ.{}".format(key)] = value 43 | 44 | if "exception" in event: 45 | for exception in event["exception"]["values"]: 46 | if "stacktrace" in exception: 47 | _process_stacktrace(exception["stacktrace"]) 48 | 49 | if "stacktrace" in event: 50 | _process_stacktrace(event["stacktrace"]) 51 | 52 | return event 53 | -------------------------------------------------------------------------------- /scripts/bump-version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euxo pipefail 3 | 4 | sed -i "s/^version =.*/version = $2/" setup.cfg 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = pytest_sentry 3 | version = 0.4.1 4 | description = A pytest plugin to send testrun information to Sentry.io 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | url = https://github.com/untitaker/pytest-sentry 8 | author = Markus Unterwaditzer 9 | author_email = markus@unterwaditzer.net 10 | license = BSD-2-Clause 11 | license_file = LICENSE 12 | classifiers = 13 | License :: OSI Approved :: BSD License 14 | 15 | [options] 16 | packages = pytest_sentry 17 | install_requires = 18 | pytest 19 | sentry-sdk>=3.0.0a1 20 | wrapt 21 | 22 | [options.entry_points] 23 | pytest11 = 24 | sentry = pytest_sentry 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | setup() 3 | -------------------------------------------------------------------------------- /tests/test_envvars.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_sentry 3 | 4 | import sentry_sdk 5 | 6 | 7 | events = [] 8 | envelopes = [] 9 | 10 | 11 | @pytest.fixture(autouse=True) 12 | def clear_events(monkeypatch): 13 | monkeypatch.setenv("GITHUB_RUN_ID", "123abc") 14 | events.clear() 15 | envelopes.clear() 16 | 17 | 18 | class MyTransport(sentry_sdk.Transport): 19 | def capture_event(self, event): 20 | events.append(event) 21 | 22 | def capture_envelope(self, envelope): 23 | envelopes.append(envelope) 24 | if envelope.get_event() is not None: 25 | events.append(envelope.get_event()) 26 | 27 | 28 | GLOBAL_TRANSPORT = MyTransport() 29 | GLOBAL_CLIENT = pytest_sentry.Client(transport=GLOBAL_TRANSPORT) 30 | 31 | pytestmark = pytest.mark.sentry_client(GLOBAL_CLIENT) 32 | 33 | 34 | def test_basic(sentry_test_scope): 35 | with sentry_sdk.use_scope(sentry_test_scope): 36 | sentry_test_scope.capture_message("hi") 37 | 38 | (event,) = events 39 | assert event["tags"]["pytest_environ.GITHUB_RUN_ID"] == "123abc" 40 | 41 | 42 | def test_transaction(request): 43 | @request.addfinalizer 44 | def _(): 45 | for transaction in envelopes: 46 | assert ( 47 | transaction.get_transaction_event()["tags"][ 48 | "pytest_environ.GITHUB_RUN_ID" 49 | ] 50 | == "123abc" 51 | ) 52 | -------------------------------------------------------------------------------- /tests/test_fixture.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_sentry 3 | 4 | 5 | GLOBAL_CLIENT = pytest_sentry.Client() 6 | 7 | pytestmark = pytest.mark.sentry_client(GLOBAL_CLIENT) 8 | 9 | 10 | def test_basic(sentry_test_scope): 11 | assert sentry_test_scope.client is GLOBAL_CLIENT 12 | 13 | 14 | @pytest.mark.sentry_client(None) 15 | def test_func(sentry_test_scope): 16 | assert not sentry_test_scope.client.is_active() 17 | -------------------------------------------------------------------------------- /tests/test_retries.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_sentry 3 | 4 | import sentry_sdk 5 | 6 | 7 | i = 0 8 | events = [] 9 | 10 | 11 | class MyTransport(sentry_sdk.Transport): 12 | def capture_event(self, event): 13 | events.append(event) 14 | 15 | def capture_envelope(self, envelope): 16 | if envelope.get_event() is not None: 17 | events.append(envelope.get_event()) 18 | 19 | 20 | @pytest.mark.flaky(reruns=2) 21 | @pytest.mark.sentry_client(pytest_sentry.Client(transport=MyTransport(), traces_sample_rate=0.0)) 22 | def test_basic(request): 23 | global i 24 | i += 1 25 | if i < 2: 26 | 1 / 0 27 | 28 | 29 | @pytest.fixture(scope="module", autouse=True) 30 | def assert_report(): 31 | yield 32 | (event,) = events 33 | (exception,) = event["exception"]["values"] 34 | assert exception["type"] == "ZeroDivisionError" 35 | -------------------------------------------------------------------------------- /tests/test_scope.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_sentry 3 | import unittest 4 | 5 | import sentry_sdk 6 | 7 | 8 | GLOBAL_CLIENT = pytest_sentry.Client() 9 | 10 | pytestmark = pytest.mark.sentry_client(GLOBAL_CLIENT) 11 | 12 | _DEFAULT_GLOBAL_SCOPE = sentry_sdk.Scope.get_global_scope() 13 | _DEFAULT_ISOLATION_SCOPE = sentry_sdk.Scope.get_isolation_scope() 14 | 15 | 16 | def _assert_right_scopes(): 17 | global_scope = sentry_sdk.Scope.get_global_scope() 18 | assert global_scope is _DEFAULT_GLOBAL_SCOPE 19 | 20 | isolation_scope = sentry_sdk.Scope.get_isolation_scope() 21 | assert isolation_scope is _DEFAULT_ISOLATION_SCOPE 22 | 23 | 24 | def test_basic(): 25 | _assert_right_scopes() 26 | 27 | 28 | def test_sentry_test_scope(sentry_test_scope): 29 | # Ensure that we are within a root span (started by the pytest_runtest_call hook) 30 | assert sentry_test_scope.span is not None 31 | 32 | 33 | class TestSimpleClass(object): 34 | def setup_method(self): 35 | _assert_right_scopes() 36 | 37 | def test_basic(self): 38 | _assert_right_scopes() 39 | 40 | def teardown_method(self): 41 | _assert_right_scopes() 42 | 43 | 44 | class TestUnittestClass(unittest.TestCase): 45 | def setUp(self): 46 | _assert_right_scopes() 47 | 48 | def test_basic(self): 49 | _assert_right_scopes() 50 | 51 | def tearDown(self): 52 | _assert_right_scopes() 53 | -------------------------------------------------------------------------------- /tests/test_skip.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_sentry 3 | 4 | 5 | events = [] 6 | 7 | 8 | @pytest.mark.flaky(reruns=2) 9 | @pytest.mark.sentry_client(pytest_sentry.Client(transport=events.append, traces_sample_rate=0.0)) 10 | def test_skip(): 11 | pytest.skip("bye") 12 | 13 | 14 | @pytest.mark.flaky(reruns=2) 15 | @pytest.mark.skipif(True, reason="bye") 16 | @pytest.mark.sentry_client(pytest_sentry.Client(transport=events.append, traces_sample_rate=0.0)) 17 | def test_skipif(): 18 | pass 19 | 20 | 21 | @pytest.mark.flaky(reruns=2) 22 | @pytest.mark.xfail(True, reason="bye") 23 | @pytest.mark.sentry_client(pytest_sentry.Client(transport=events.append, traces_sample_rate=0.0)) 24 | def test_mark_xfail(): 25 | pass 26 | 27 | 28 | @pytest.mark.flaky(reruns=2) 29 | @pytest.mark.sentry_client(pytest_sentry.Client(transport=events.append, traces_sample_rate=0.0)) 30 | def test_xfail(): 31 | pytest.xfail("bye") 32 | 33 | 34 | @pytest.fixture(scope="session", autouse=True) 35 | def assert_report(): 36 | yield 37 | assert not events 38 | -------------------------------------------------------------------------------- /tests/test_tracing.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytest_sentry 3 | 4 | import sentry_sdk 5 | 6 | 7 | transactions = [] 8 | 9 | 10 | class MyTransport(sentry_sdk.Transport): 11 | def __init__(self): 12 | pass 13 | 14 | def capture_envelope(self, envelope): 15 | transactions.append(envelope.get_transaction_event()) 16 | 17 | 18 | GLOBAL_TRANSPORT = MyTransport() 19 | GLOBAL_CLIENT = pytest_sentry.Client(transport=GLOBAL_TRANSPORT) 20 | 21 | pytestmark = pytest.mark.sentry_client(GLOBAL_CLIENT) 22 | 23 | 24 | @pytest.fixture 25 | def foo_fixture(): 26 | return 42 27 | 28 | 29 | def test_basic(foo_fixture): 30 | assert foo_fixture == 42 31 | 32 | 33 | @pytest.fixture(scope="module", autouse=True) 34 | def assert_report(): 35 | yield 36 | 37 | self_transaction, fixture_transaction, test_transaction = transactions 38 | 39 | assert self_transaction["type"] == "transaction" 40 | assert self_transaction["transaction"] == "pytest.fixture.setup assert_report" 41 | 42 | assert fixture_transaction["type"] == "transaction" 43 | assert fixture_transaction["transaction"] == "pytest.fixture.setup foo_fixture" 44 | 45 | assert test_transaction["type"] == "transaction" 46 | assert ( 47 | test_transaction["transaction"] 48 | == "pytest.runtest.call tests/test_tracing.py::test_basic" 49 | ) 50 | 51 | assert ( 52 | fixture_transaction["contexts"]["trace"]["trace_id"] 53 | == test_transaction["contexts"]["trace"]["trace_id"] 54 | ) 55 | assert ( 56 | self_transaction["contexts"]["trace"]["trace_id"] 57 | == fixture_transaction["contexts"]["trace"]["trace_id"] 58 | ) 59 | --------------------------------------------------------------------------------