├── .coveragerc ├── .github └── workflows │ ├── main.yml │ └── pypi.yml ├── .gitignore ├── LICENSE ├── README.md ├── pytest_tinybird ├── __init__.py ├── plugin.py └── tinybird.py ├── requirements-dev.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py └── test_tinybird.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = pytest_tinybird 4 | 5 | [paths] 6 | source = 7 | pytest_tinybird 8 | .tox/*/lib/python*/site-packages/pytest_tinybird 9 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: ['3.8', '3.9', '3.10', '3.11'] 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Set up tox 23 | run: | 24 | pip install --upgrade pip tox 25 | - name: Run tox 26 | env: 27 | TINYBIRD_URL: https://api.tinybird.co 28 | TINYBIRD_DATASOURCE: ci_tests 29 | TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }} 30 | CI_COMMIT_BRANCH: ${{ github.ref_name }} 31 | CI_COMMIT_SHA: ${{ github.sha }} 32 | CI_JOB_ID: ${{ github.job }}-${{ matrix.python-version }} 33 | CI_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} 34 | CI_JOB_NAME: ${{ github.job }}-${{ matrix.python-version }} 35 | TINYBIRD_TIMEOUT: 10 36 | TINYBIRD_WAIT: false 37 | run: | 38 | tox -e py -- --report-to-tinybird 39 | lint: 40 | runs-on: ubuntu-latest 41 | steps: 42 | - uses: actions/checkout@v2 43 | - name: Set up Python 44 | uses: actions/setup-python@v2 45 | - name: Set up tox 46 | run: | 47 | pip install --upgrade pip tox 48 | - name: Run tox 49 | run: | 50 | tox -e lint 51 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI.org 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | pypi: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v3 11 | with: 12 | fetch-depth: 0 13 | - run: python3 -m pip install --upgrade build && python3 -m build 14 | - name: Publish package 15 | uses: pypa/gh-action-pypi-publish@release/v1 16 | with: 17 | password: ${{ secrets.PYPI_API_TOKEN }} 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | *$py.class 4 | *.egg-info/ 5 | .coverage 6 | .cache/ 7 | .report.json 8 | TODO 9 | .tox 10 | build/ 11 | dist/ 12 | .pytest_cache/ 13 | .coverage.* 14 | .env 15 | .idea/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Tinybird 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pytest-tinybird 2 | =================================== 3 | 4 | A pytest plugin to report test results to Tinybird. At the end of every run, this plugin posts results using the [Tinybird Events API](https://www.tinybird.co/docs/ingest/events-api.html). 5 | 6 | [![PyPI version](https://badge.fury.io/py/pytest-tinybird.svg)](https://badge.fury.io/py/pytest-tinybird) 7 | ![Passed](https://github.com/jlmadurga/pytest-tinybird/actions/workflows/main.yml/badge.svg) 8 | ![Top test passed](https://img.shields.io/endpoint?url=https://api.tinybird.co/v0/pipes/top_test_passed.ndjson?token=p.eyJ1IjogIjNhZjhlMTBhLTM2MjEtNDQ3OC04MWJmLTE5MDQ5N2UwNjBjYiIsICJpZCI6ICJkNDNmZGQ2Ni03NzY1LTQzZGYtYjEyNS0wYzNjYWJiMDgxZjUifQ.yWypEczMfJlgkjNt29pCf45XaxE1dMOr-oznll5tjpY) 9 | 10 | 11 | 12 | Requirements 13 | ------------ 14 | 15 | - Python >=3.8 16 | - pytest 3.8 or newer (previous versions might be compatible) 17 | - [Tinybird account](https://www.tinybird.co/) and a [token with append permissions](https://www.tinybird.co/docs/concepts/auth-tokens.html?highlight=token#auth-token-scopes) 18 | 19 | 20 | Installation 21 | ------------ 22 | 23 | pypi install package: https://pypi.org/project/pytest-tinybird/ 24 | 25 | You can install the plugin with the following bash command: 26 | 27 | ```bash 28 | $ pip install pytest-tinybird 29 | ``` 30 | 31 | 32 | Usage 33 | ------------ 34 | 35 | You just need your [Tinybird](https://www.tinybird.co/) account and a Token with append permissions 36 | 37 | Set these env variables: 38 | 39 | ```bash 40 | export TINYBIRD_URL= # depends on your region 41 | export TINYBIRD_DATASOURCE= # will be created with first results posted 42 | export TINYBIRD_TOKEN= 43 | ``` 44 | 45 | Then run pytest with `--report-to-tinybird`. 46 | 47 | 48 | ```bash 49 | $ pytest tests --report-to-tinybird 50 | ``` 51 | 52 | CI execution info can also be set using some env variables. These are from GitLab: 53 | 54 | ```bash 55 | CI_COMMIT_SHA 56 | CI_COMMIT_BRANCH 57 | CI_JOB_ID 58 | CI_JOB_NAME 59 | CI_JOB_URL 60 | ``` 61 | 62 | If you are not using GitLab, you will need to set them manually. For instance, for GitHub actions you can check our current [GitHub actions workflow](https://github.com/tinybirdco/pytest-tinybird/blob/master/.github/workflows/main.yml). 63 | 64 | Data Source details 65 | -------------------- 66 | 67 | The `pytest-tinybird` plugin creates and sends `report` objects via the [Events API](https://www.tinybird.co/docs/ingest/events-api.html) with this structure: 68 | 69 | ``` 70 | { 71 | 'date': now, 72 | 'commit': self.commit, 73 | 'branch': self.branch, 74 | 'job_id': self.job_id, 75 | 'job_url': self.job_url, 76 | 'job_name': self.job_name, 77 | 'test_nodeid': test.nodeid, 78 | 'test_name': test.head_line, 79 | 'test_part': test.when, 80 | 'duration': test.duration, 81 | 'outcome': test.outcome 82 | } 83 | ``` 84 | 85 | There are also additional optional values that can be set for multi-repository and multi-workflow setups (e.g., in GitHub Actions): 86 | 87 | ``` 88 | { 89 | 'repository': self.repository 90 | 'workflow': self.workflow 91 | } 92 | ``` 93 | 94 | To use these, you will need to set the `CI_REPOSITORY_NAME` and `CI_WORKFLOW_NAME` environment variables respectively. 95 | 96 | 97 | When a `report` object is first sent to Tinybird, a Data Source with the following definition and schema is created: 98 | 99 | ```sql 100 | TOKEN "pytest-executor-write" APPEND 101 | 102 | SCHEMA > 103 | `commit` String `json:$.commit`, 104 | `branch` String `json:$.branch`, 105 | `date` DateTime `json:$.date`, 106 | `duration` Float32 `json:$.duration`, 107 | `job_id` String `json:$.job_id`, 108 | `job_name` String `json:$.job_name`, 109 | `job_url` String `json:$.job_url`, 110 | `outcome` LowCardinality(String) `json:$.outcome`, 111 | `test_name` String `json:$.test_name`, 112 | `test_nodeid` String `json:$.test_nodeid`, 113 | `test_part` LowCardinality(String) `json:$.test_part` 114 | 115 | ENGINE MergeTree 116 | ENGINE_PARTITION_KEY toYYYYMM(date) 117 | ``` 118 | 119 | You can also see the Data Source schema with this [data sample](https://api.tinybird.co/v0/pipes/ci_tests_sample.json?token=p.eyJ1IjogIjNhZjhlMTBhLTM2MjEtNDQ3OC04MWJmLTE5MDQ5N2UwNjBjYiIsICJpZCI6ICIwNzMwZTJjYy1mYzA4LTQxMDMtOTMwNy1jMThjYWY5OGI4OGUifQ.kpCQfin0KFC8olEju1qVqDH14nlSzGgqjAWpl1k7RUI) 120 | from an API Endpoint created from the Data Source the `pytest-tinybird` plugin populates. 121 | -------------------------------------------------------------------------------- /pytest_tinybird/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinybirdco/pytest-tinybird/13dd7eacffecaad0e9bddc72715afd2f757a43e4/pytest_tinybird/__init__.py -------------------------------------------------------------------------------- /pytest_tinybird/plugin.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from . import tinybird 4 | 5 | 6 | def pytest_addoption(parser): 7 | group = parser.getgroup('tinybird', 'reporting test results to tinybird') 8 | group.addoption( 9 | '--report-to-tinybird', default=False, action='store_true', 10 | help='send report to tinybird') 11 | 12 | 13 | def pytest_configure(config): 14 | if not config.getoption("--report-to-tinybird"): 15 | return 16 | plugin = tinybird.TinybirdReport(config) 17 | config._tinybird = plugin 18 | config.pluginmanager.register(plugin) 19 | 20 | 21 | def pytest_unconfigure(config): 22 | plugin = getattr(config, '_tinybird', None) 23 | if plugin is not None: 24 | del config._tinybird 25 | config.pluginmanager.unregister(plugin) 26 | -------------------------------------------------------------------------------- /pytest_tinybird/tinybird.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import time 5 | from datetime import datetime 6 | from typing import Union 7 | 8 | import pytest 9 | import requests 10 | from _pytest.config import Config, ExitCode 11 | from _pytest.main import Session 12 | from _pytest.terminal import TerminalReporter 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | REQUEST_TIMEOUT = 2 17 | 18 | 19 | class TinybirdReport: 20 | def __init__(self, config: Config): 21 | self.config = config 22 | self.base_url = os.environ.get("TINYBIRD_URL") 23 | self.timeout = int(os.environ.get("TINYBIRD_TIMEOUT", REQUEST_TIMEOUT)) 24 | self.retries = int(os.environ.get('TINYBIRD_RETRIES', 0)) 25 | self.wait = os.environ.get("TINYBIRD_WAIT", "false") 26 | self.datasource_name = os.environ.get("TINYBIRD_DATASOURCE") 27 | self.token = os.environ.get("TINYBIRD_TOKEN") 28 | self.commit = os.environ.get('CI_COMMIT_SHA', 'ci_commit_sha_unknown') 29 | self.job_id = os.environ.get('CI_JOB_ID', 'ci_job_id_unknown') 30 | self.job_url = os.environ.get('CI_JOB_URL', 'job_url_unknown') 31 | self.job_name = os.environ.get('CI_JOB_NAME', 'job_name_unknown') 32 | self.branch = os.environ.get( 33 | 'CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', 34 | os.environ.get('CI_COMMIT_BRANCH', 'ci_commit_branch_unknown') 35 | ) 36 | self.url = f"{self.base_url}/v0/events?name={self.datasource_name}" \ 37 | f"&token={self.token}" \ 38 | f"&wait={self.wait}" 39 | 40 | # optional values for multi-repository and multi-workflow usage 41 | self.repository = os.environ.get('CI_REPOSITORY_NAME', None) 42 | self.workflow = os.environ.get('CI_WORKFLOW_NAME', None) 43 | 44 | def report(self, session: Session): 45 | if None in [self.base_url, self.datasource_name, self.token]: 46 | log.error("Required values for environment variables") 47 | return 48 | now = str(datetime.now()) 49 | terminalreporter: TerminalReporter = session.config.pluginmanager.get_plugin( 50 | "terminalreporter" 51 | ) 52 | # special check for pytest-xdist plugin, we do not want to send report for each worker. 53 | if hasattr(terminalreporter.config, 'workerinput'): 54 | return 55 | report = [] 56 | for k in terminalreporter.stats: 57 | for test in terminalreporter.stats[k]: 58 | try: 59 | report_entry = { 60 | 'date': now, 61 | 'commit': self.commit, 62 | 'branch': self.branch, 63 | 'job_id': self.job_id, 64 | 'job_url': self.job_url, 65 | 'job_name': self.job_name, 66 | 'test_nodeid': test.nodeid, 67 | 'test_name': test.head_line, 68 | 'test_part': test.when, 69 | 'duration': test.duration, 70 | 'outcome': test.outcome 71 | } 72 | report_optionals = { 73 | 'repository': self.repository, 74 | 'workflow': self.workflow 75 | } 76 | # only add optional values if they're not None 77 | report_entry.update({k: v for k, v 78 | in report_optionals.items() 79 | if v is not None}) 80 | report.append(report_entry) 81 | except AttributeError: 82 | pass 83 | data = '\n'.join(json.dumps(x) for x in report) 84 | for attempt in range(self.retries + 1): 85 | try: 86 | # This goes to the Internal workspace in EU 87 | response = requests.post( 88 | self.url, 89 | data=data, 90 | timeout=self.timeout 91 | ) 92 | if response.status_code in [200, 202]: 93 | break 94 | log.error("Error while uploading to tinybird %s (Attempt %s/%s)", 95 | response.status_code, attempt + 1, self.retries + 1) 96 | except requests.exceptions.RequestException as e: 97 | log.error("Request failed: %s (Attempt %s/%s)", 98 | e, attempt + 1, self.retries + 1) 99 | if attempt < self.retries: 100 | time.sleep(2 ** attempt) 101 | else: 102 | log.error("All %s attempts failed to upload to tinybird", self.retries + 1) 103 | 104 | @pytest.hookimpl(trylast=True) 105 | def pytest_sessionfinish(self, session: Session, exitstatus: Union[int, ExitCode]): 106 | try: 107 | self.report(session) 108 | except Exception as e: 109 | log.error("Tinybird report error: %s - %s", self.url, e) 110 | raise e 111 | 112 | @pytest.hookimpl(trylast=True) 113 | def pytest_terminal_summary( 114 | self, 115 | terminalreporter: TerminalReporter, 116 | exitstatus: Union[int, ExitCode], 117 | config: Config, 118 | ): 119 | terminalreporter.write_sep("-", "send report to Tinybird") 120 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | flake8==6.0.0 2 | pylint==2.16.2 3 | pytest==7.2.1 4 | requests>=2.28.1 5 | pytest-xdist==2.5.0 6 | tox==4.2.8 7 | 8 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | 4 | [pylint] 5 | disable = 6 | missing-docstring, 7 | invalid-name, 8 | unused-argument, 9 | too-few-public-methods, 10 | too-many-public-methods, 11 | protected-access, 12 | no-self-use, 13 | too-many-instance-attributes, 14 | fixme, 15 | consider-using-f-string, -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | import sys 3 | 4 | from setuptools import setup 5 | 6 | # Open encoding isn't available for Python 2.7 (sigh) 7 | if sys.version_info < (3, 0): 8 | from io import open 9 | 10 | 11 | this_directory = path.abspath(path.dirname(__file__)) 12 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 13 | long_description = f.read() 14 | 15 | 16 | setup( 17 | name='pytest-tinybird', 18 | description='A pytest plugin to report test results to tinybird', 19 | long_description=long_description, 20 | long_description_content_type='text/markdown', 21 | packages=['pytest_tinybird'], 22 | author='jlmadurga', 23 | author_email='jlmadurga@gmail.com', 24 | version='0.5.0', 25 | url='https://github.com/jlmadurga/pytest-tinybird', 26 | license='MIT', 27 | install_requires=[ 28 | 'pytest>=3.8.0', 29 | 'requests>=2.28.1' 30 | ], 31 | entry_points={ 32 | 'pytest11': [ 33 | 'pytest_tinybird = pytest_tinybird.plugin', 34 | ] 35 | }, 36 | classifiers=[ 37 | 'Development Status :: 4 - Beta', 38 | 'Programming Language :: Python', 39 | 'Programming Language :: Python :: 3', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: 3.9', 42 | 'Programming Language :: Python :: 3.10', 43 | 'Programming Language :: Python :: 3.11', 44 | 'Framework :: Pytest', 45 | ], 46 | ) 47 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinybirdco/pytest-tinybird/13dd7eacffecaad0e9bddc72715afd2f757a43e4/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | pytest_plugins = 'pytester' 2 | -------------------------------------------------------------------------------- /tests/test_tinybird.py: -------------------------------------------------------------------------------- 1 | import random 2 | from unittest import mock 3 | import os 4 | 5 | 6 | def test_report_to_tinybird(testdir): 7 | 8 | # create a temporary pytest test module 9 | testdir.makepyfile(""" 10 | import pytest 11 | from random import randint 12 | def test_passed(): 13 | assert True 14 | def test_flaky(): 15 | assert randint(1,10) >=5 16 | """) 17 | 18 | tinybird_url = os.environ["TINYBIRD_URL"] = 'https://fake-api.tinybird.co' 19 | datasource_name = os.environ["TINYBIRD_DATASOURCE"] = "test_datasource" 20 | tinybird_token = os.environ["TINYBIRD_TOKEN"] = 'test_token' 21 | timeout = os.environ["TINYBIRD_TIMEOUT"] = "10" 22 | wait = os.environ["TINYBIRD_WAIT"] = random.choice(['true', 'false']) 23 | 24 | with mock.patch('requests.post') as mock_post: 25 | mock_post.return_value.status_code = 200 if wait == 'true' else 202 26 | testdir.runpytest( 27 | "-n 2", 28 | '--report-to-tinybird', 29 | '-vvv' 30 | ) 31 | mock_post.assert_called_once_with(f"{tinybird_url}/v0/events?name={datasource_name}" 32 | f"&token={tinybird_token}" 33 | f"&wait={wait}", 34 | data=mock.ANY, 35 | timeout=int(timeout)) 36 | 37 | 38 | def test_retry_to_tinybird(testdir): 39 | # create a temporary pytest test module 40 | testdir.makepyfile(""" 41 | import pytest 42 | def test_passed(): 43 | assert True 44 | """) 45 | 46 | os.environ["TINYBIRD_URL"] = 'https://fake-api.tinybird.co' 47 | os.environ["TINYBIRD_DATASOURCE"] = "test_datasource" 48 | os.environ["TINYBIRD_TOKEN"] = 'test_token' 49 | os.environ["TINYBIRD_TIMEOUT"] = "1" 50 | os.environ["TINYBIRD_RETRIES"] = "1" 51 | 52 | with mock.patch('requests.post') as mock_post: 53 | mock_post.return_value.status_code = 503 54 | testdir.runpytest( 55 | "-n 1", 56 | '--report-to-tinybird', 57 | '-vvv' 58 | ) 59 | assert mock_post.call_count == 2 60 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{38},py{39},py{310},py{311},lint,coverage-report 3 | 4 | 5 | [testenv] 6 | passenv = * 7 | deps = 8 | coverage 9 | pytest 10 | pytest-xdist 11 | flaky 12 | commands = 13 | coverage run --parallel -m pytest -v {posargs} 14 | 15 | 16 | [testenv:coverage-report] 17 | basepython = python3.10 18 | skip_install = true 19 | deps = coverage 20 | commands = 21 | coverage combine 22 | coverage report 23 | 24 | [testenv:lint] 25 | deps = 26 | flake8 27 | pylint 28 | commands = 29 | flake8 --max-line-length 100 30 | pylint --rcfile setup.cfg pytest_tinybird/ 31 | 32 | [testenv:release] 33 | deps = 34 | wheel 35 | twine 36 | commands = 37 | rm -rf *.egg-info build/ dist/ 38 | python setup.py bdist_wheel sdist 39 | twine upload -r pypi dist/* 40 | rm -rf *.egg-info build/ dist/ 41 | 42 | 43 | --------------------------------------------------------------------------------