├── .coveragerc ├── .editorconfig ├── .gitignore ├── .travis.yml ├── CHANGELOG.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── pytest_pipeline ├── __init__.py ├── core.py ├── mark.py ├── plugin.py └── utils.py ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests └── test_pipelines.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = pytest_pipeline 4 | 5 | [report] 6 | exclude_lines = 7 | if self.debug: 8 | pragma: no cover 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | omit = 12 | tests/* 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | htmlcov 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | # Complexity 39 | output/*.html 40 | output/*/index.html 41 | 42 | # Sphinx 43 | docs/_build -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | sudo: false 6 | 7 | python: 8 | - "3.6" 9 | - "3.7" 10 | - "3.8" 11 | 12 | before_install: 13 | - pip install -r requirements-dev.txt 14 | 15 | install: 16 | - pip install -e . 17 | - pip install codecov==2.0.15 18 | 19 | script: 20 | - flake8 --statistics pytest_pipeline tests 21 | - coverage run --source=pytest_pipeline $(which py.test) tests pytest_pipeline 22 | - isort --check-only --recursive pytest_pipeline tests setup.py 23 | - radon cc --total-average --show-closures --show-complexity --min C pytest_pipeline 24 | 25 | after_success: 26 | - coverage report -m 27 | - codecov 28 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | Changelog 4 | ========= 5 | 6 | Version 0.4 7 | ----------- 8 | 9 | Release 0.4.0 10 | ^^^^^^^^^^^^^ 11 | 12 | Release date: TBD 13 | 14 | * Allow for the passed in command to access the run directory name 15 | via ``{run_dir}``. 16 | * Drop official support for Python 2.7, 3.3, 3.4, and 3.5. 17 | * Drop official support for pytest <5.0.0. 18 | 19 | 20 | Version 0.3 21 | ----------- 22 | 23 | Release 0.3.0 24 | ^^^^^^^^^^^^^ 25 | 26 | Release date: 24 January 2017 27 | 28 | * Allow stdout and/or stderr capture in-memory. This can be done by 29 | setting their respective keyword arguments to ``True`` when creating 30 | the run fixture. 31 | 32 | 33 | Version 0.2 34 | ----------- 35 | 36 | Release 0.2.0 37 | ^^^^^^^^^^^^^ 38 | 39 | Release date: 31 March 2015 40 | 41 | * Pipeline runs are now modelled differently. Instead of a class attribute, 42 | they are now created as pytest fixtures. This allows the pipeline runs 43 | to be used in non-`unittest.TestCase` tests. 44 | 45 | * The `after_run` decorator is deprecated. 46 | 47 | * The command line flags `--xfail-pipeline` and `--skip-run` are deprecated. 48 | 49 | 50 | Version 0.1 51 | ----------- 52 | 53 | Release 0.1.0 54 | ^^^^^^^^^^^^^ 55 | 56 | Release date: 25 August 2014 57 | 58 | * First release on PyPI. 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2020 Wibowo Arindrarto 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | * Neither the name of pytest-pipeline nor the names of its contributors may be 16 | used to endorse or promote products derived from this software without 17 | specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CHANGELOG.rst 3 | include LICENSE 4 | include README.rst 5 | include requirements.txt 6 | include requirements-dev.txt 7 | 8 | recursive-include tests * 9 | recursive-exclude * __pycache__ 10 | recursive-exclude * *.py[co] 11 | 12 | recursive-include docs *.rst conf.py Makefile make.bat 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs clean 2 | 3 | help: 4 | @echo "clean-build - remove build artifacts" 5 | @echo "clean-pyc - remove Python file artifacts" 6 | @echo "lint - check style with flake8" 7 | @echo "test - run tests quickly with the default Python" 8 | @echo "test-all - run tests on every Python version with tox" 9 | @echo "coverage - check code coverage quickly with the default Python" 10 | @echo "docs - generate Sphinx HTML documentation, including API docs" 11 | @echo "release - package and upload a release" 12 | @echo "dist - package" 13 | 14 | clean: clean-build clean-pyc 15 | rm -fr htmlcov/ 16 | 17 | clean-build: 18 | rm -fr build/ 19 | rm -fr dist/ 20 | rm -fr *.egg-info 21 | 22 | clean-pyc: 23 | find . -name '*.pyc' -exec rm -f {} + 24 | find . -name '*.pyo' -exec rm -f {} + 25 | find . -name '*~' -exec rm -f {} + 26 | 27 | lint: 28 | flake8 pytest_pipeline tests 29 | 30 | test: 31 | python setup.py test 32 | 33 | test-all: 34 | tox 35 | 36 | coverage: 37 | coverage run --source pytest_pipeline setup.py test 38 | coverage report -m 39 | coverage html 40 | open htmlcov/index.html 41 | 42 | docs: 43 | rm -f docs/pytest_pipeline.rst 44 | rm -f docs/modules.rst 45 | sphinx-apidoc -o docs/ pytest_pipeline 46 | $(MAKE) -C docs clean 47 | $(MAKE) -C docs html 48 | open docs/_build/html/index.html 49 | 50 | release: clean 51 | python setup.py sdist upload 52 | python setup.py bdist_wheel upload 53 | 54 | dist: clean 55 | python setup.py sdist 56 | python setup.py bdist_wheel 57 | ls -l dist -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | :warning: archived 3 | ================== 4 | 5 | I am no longer working on this repository and have therefore decided to archive it. 6 | 7 | 8 | =============== 9 | pytest-pipeline 10 | =============== 11 | 12 | |ci| |coverage| |pypi| 13 | 14 | .. |ci| image:: https://travis-ci.org/bow/pytest-pipeline.png?branch=master 15 | :target: https://travis-ci.org/bow/pytest-pipeline 16 | 17 | .. |coverage| image:: https://codecov.io/gh/bow/pytest-pipeline/branch/master/graph/badge.svg 18 | :target: https://codecov.io/gh/bow/pytest-pipeline 19 | 20 | .. |pypi| image:: https://badge.fury.io/py/pytest-pipeline.svg 21 | :target: http://badge.fury.io/py/pytest-pipeline 22 | 23 | 24 | pytest-pipeline is a pytest plugin for functional testing of data analysis 25 | pipelines. They are usually long-running scripts or executables with multiple 26 | input and/or output files + directories. The plugin is meant for end-to-end 27 | testing where you test for conditions before the pipeline run and after the 28 | pipeline runs (output files, checksums, etc.). 29 | 30 | pytest-pipeline is tested against Python versions 3.6, 3.7. and 3.8. 31 | 32 | 33 | Installation 34 | ============ 35 | 36 | :: 37 | 38 | pip install pytest-pipeline 39 | 40 | 41 | Walkthrough 42 | =========== 43 | 44 | For our example, we will use a super simple pipeline (an executable, really) 45 | that writes a file and prints to stdout: 46 | 47 | .. code-block:: python 48 | 49 | #!/usr/bin/env python 50 | 51 | from __future__ import print_function 52 | 53 | if __name__ == "__main__": 54 | 55 | with open("result.txt", "w") as result: 56 | result.write("42\n") 57 | print("Result computed") 58 | 59 | At this point it's just a simple script, but it is enough to represent a single, 60 | long running task to be tested. If you want to follow along, save the above file as 61 | ``run_pipeline``. 62 | 63 | With the pipeline above, here's how your test would look like with 64 | ``pytest_pipeline``: 65 | 66 | .. code-block:: python 67 | 68 | import os 69 | import shutil 70 | import unittest 71 | from pytest_pipeline import PipelineRun, mark, utils 72 | 73 | # we can subclass `PipelineRun` to add custom methods 74 | # using `PipelineRun` as-is is also possible 75 | class MyRun(PipelineRun): 76 | 77 | # before_run-marked functions will be run before the pipeline is executed 78 | @mark.before_run 79 | def test_prep_executable(self): 80 | # copy the executable to the run directory 81 | shutil.copy2("/path/to/run_pipeline", "run_pipeline") 82 | # ensure that the file is executable 83 | assert os.access("run_pipeline", os.X_OK) 84 | 85 | # a pipeline run is treated as a test fixture 86 | run = MyRun.class_fixture(cmd="./run_pipeline", stdout="run.stdout") 87 | 88 | # the fixture is bound to a unittest.TestCase using the usefixtures mark 89 | @pytest.mark.usefixtures("run") 90 | # tests per-pipeline run are grouped in one unittest.TestCase instance 91 | class TestMyPipeline(unittest.TestCase): 92 | 93 | def test_result_md5(self): 94 | assert utils.file_md5sum("result.txt") == "50a2fabfdd276f573ff97ace8b11c5f4" 95 | 96 | def test_exit_code(self): 97 | # the run fixture is stored as the `run_fixture` attribute 98 | assert self.run_fixture.exit_code == 0 99 | 100 | # we can check the stdout that we capture as well 101 | def test_stdout(self): 102 | assert open("run.stdout", "r").read().strip() == "Result computed" 103 | 104 | If the test above is saved as ``test_demo.py``, you can then run the test by 105 | executing ``py.test -v test_demo.py``. You should see that four passing tests: 106 | one that is done prior to the script run and three that are done afterwards. 107 | 108 | What just happened? 109 | ------------------- 110 | 111 | You just executed your first pipeline test. The plugin itself gives you: 112 | 113 | - Test directory creation (one class gets one directory). 114 | By default, testdirectories are all created in the ``/tmp/pipeline_test`` 115 | directory. You can tweak this location by supplying the 116 | ``--base-pipeline-dir`` command line flag. Need access to the directory 117 | name for the command invocation? Use the ``{run_dir}`` placeholder 118 | in the string command. 119 | 120 | - Automatic execution of the pipeline. 121 | No need to ``import subprocess``, just define the command via the 122 | ``PipelineRun`` object. We optionally captured the standard output to a file 123 | called ``run.stdout``. Not a fan of doing disk IO? You can also set ``stdout`` 124 | and/or ``stderr`` to ``True`` and have their values captured in-memory. 125 | 126 | - Timeout control. 127 | For long running pipelines, you can also supply a ``timeout`` argument which 128 | limits how long the pipeline process can run. 129 | 130 | - Some simple utility methods for working with files. 131 | 132 | And since this is a py.test plugin, test discovery and execution is done via 133 | py.test. 134 | 135 | 136 | Getting + giving help 137 | ===================== 138 | 139 | Please use the `issue tracker `_ 140 | to report bugs or feature requests. You can always fork and submit a pull 141 | request as well. 142 | 143 | Local Development 144 | ----------------- 145 | 146 | Setting up a local development requires any of the supported Python version. It is ideal if you have support Python 2.x 147 | and 3.x versions installed, as that will allow you to run the full tests suite against all versions using ``tox``. 148 | 149 | In any case, the following steps can be your guide for setting up your local development environment: 150 | 151 | .. code-block:: bash 152 | 153 | # Clone the repository and cd into it 154 | $ git clone {repo-url} 155 | $ cd pytest-pipeline 156 | 157 | # Create your virtualenv, using pyenv for example (recommended, https://github.com/pyenv/pyenv) 158 | $ pyenv virtualenv 3.7.0 pytest-pipeline-dev 159 | # or using virtualenvwrapper (https://virtualenvwrapper.readthedocs.io/en/latest/) 160 | $ mkvirtualenv -p /usr/bin/python3.7 pytest-pipeline-dev 161 | 162 | # From within the root directory and with an active virtualenv, install the dependencies and package itself 163 | $ pip install -e .[dev] 164 | 165 | 166 | License 167 | ======= 168 | 169 | See LICENSE. 170 | -------------------------------------------------------------------------------- /pytest_pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | pytest_pipeline 4 | ~~~~~~~~~~~~~~~ 5 | 6 | Pytest plugin for functional testing of data analysis pipelines. 7 | 8 | :license: BSD 9 | 10 | """ 11 | # (c) 2014-2020 Wibowo Arindrarto 12 | 13 | RELEASE = False 14 | 15 | __version_info__ = ("0", "4", "0") 16 | __version__ = ".".join(__version_info__) 17 | __version__ += "-dev" if not RELEASE else "" 18 | 19 | __author__ = "Wibowo Arindrarto" 20 | __contact__ = "bow@bow.web.id" 21 | __homepage__ = "https://github.com/bow/pytest-pipeline" 22 | 23 | # so we can keep the info above for setup.py 24 | try: 25 | from .core import PipelineRun # noqa 26 | except ImportError: 27 | pass 28 | -------------------------------------------------------------------------------- /pytest_pipeline/core.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | pytest_pipeline.core 4 | ~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Core pipeline test classes. 7 | 8 | """ 9 | # (c) 2014-2020 Wibowo Arindrarto 10 | 11 | import inspect 12 | import os 13 | import shlex 14 | import shutil 15 | import subprocess 16 | import tempfile 17 | import threading 18 | import time 19 | from uuid import uuid4 20 | 21 | import pytest 22 | 23 | 24 | # TODO: allow multiple runs to be executed in test pipelines 25 | class PipelineRun: 26 | 27 | def __init__( 28 | self, 29 | cmd, 30 | stdout=None, 31 | stderr=None, 32 | poll_time=0.01, 33 | timeout=None, 34 | ): 35 | self.cmd = cmd 36 | self.stdout = stdout 37 | self.stderr = stderr 38 | self.poll_time = poll_time 39 | self.timeout = float(timeout) if timeout is not None else timeout 40 | self._process = None 41 | # set by the make_fixture functions at runtime 42 | self.run_dir = None 43 | 44 | def __repr__(self): 45 | return f"{self.__class__.__name__}(run_id={self.run_id}, ...)" 46 | 47 | def __launch_main_process(self): 48 | 49 | if isinstance(self.stdout, str): 50 | self.stdout = open(self.stdout, "w") 51 | elif self.stdout is None: 52 | self.stdout = open(os.devnull, "w") 53 | elif self.stdout is True: 54 | self.stdout = subprocess.PIPE 55 | 56 | if isinstance(self.stderr, str): 57 | self.stderr = open(self.stderr, "w") 58 | elif self.stderr is None: 59 | self.stderr = open(os.devnull, "w") 60 | elif self.stderr is True: 61 | self.stderr = subprocess.PIPE 62 | 63 | def target(): 64 | toks = shlex.split( 65 | self.cmd.format(run_dir="'" + self.run_dir + "'") 66 | ) 67 | self._process = subprocess.Popen( 68 | toks, 69 | stdout=self.stdout, 70 | stderr=self.stderr, 71 | ) 72 | while self._process.poll() is None: 73 | time.sleep(self.poll_time) 74 | 75 | thread = threading.Thread(target=target) 76 | thread.start() 77 | 78 | thread.join(self.timeout) 79 | if thread.is_alive(): 80 | self._process.terminate() 81 | pytest.fail(f"Process is taking longer than {self.timeout} seconds") 82 | 83 | if self.stdout == subprocess.PIPE: 84 | self.stdout = self._process.stdout.read() 85 | if self.stderr == subprocess.PIPE: 86 | self.stderr = self._process.stderr.read() 87 | 88 | return None 89 | 90 | @classmethod 91 | def _get_before_run_funcs(cls): 92 | funcs = [] 93 | 94 | def pred(obj): 95 | return hasattr(obj, "_before_run_order") 96 | 97 | for _, func in inspect.getmembers(cls, predicate=pred): 98 | funcs.append(func) 99 | 100 | return sorted(funcs, key=lambda f: getattr(f, "_before_run_order")) 101 | 102 | @classmethod 103 | def class_fixture(cls, *args, **kwargs): 104 | return cls.make_fixture("class", *args, **kwargs) 105 | 106 | @classmethod 107 | def make_fixture(cls, scope, *args, **kwargs): 108 | 109 | @pytest.fixture(scope=scope) 110 | def fixture(request): 111 | run = cls(*args, **kwargs) 112 | 113 | init_dir = os.getcwd() 114 | # create base pipeline dir if it does not exist 115 | root_test_dir = request.config.option.base_pipeline_dir 116 | if root_test_dir is None: 117 | root_test_dir = os.path.join( 118 | tempfile.gettempdir(), 119 | "pipeline_tests", 120 | ) 121 | if not os.path.exists(root_test_dir): 122 | os.makedirs(root_test_dir) 123 | test_dir = os.path.join(root_test_dir, run.run_id) 124 | run.run_dir = test_dir 125 | # warn if we are removing existing directory? 126 | if os.path.exists(test_dir): 127 | shutil.rmtree 128 | os.makedirs(test_dir) 129 | 130 | def done(): 131 | os.chdir(init_dir) 132 | 133 | request.addfinalizer(done) 134 | 135 | os.chdir(test_dir) 136 | for func in cls._get_before_run_funcs(): 137 | func(run) 138 | 139 | run.__launch_main_process() 140 | 141 | if scope != "class": 142 | return run 143 | 144 | request.cls.run_fixture = run 145 | 146 | return fixture 147 | 148 | @property 149 | def run_id(self): 150 | if not hasattr(self, "_run_id"): 151 | self._run_id = f"{self.__class__.__name__}_{uuid4()}" 152 | 153 | return self._run_id 154 | 155 | @property 156 | def exit_code(self): 157 | if self._process is not None: 158 | return self._process.returncode 159 | 160 | return None 161 | -------------------------------------------------------------------------------- /pytest_pipeline/mark.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | pytest_pipeline.mark 4 | ~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Marks for pipeline run tests. 7 | 8 | """ 9 | # (c) 2014-2020 Wibowo Arindrarto 10 | 11 | import sys 12 | from functools import wraps 13 | 14 | 15 | # trying to emulate Python's builtin argument handling here 16 | # so we can have decorators with optional arguments 17 | def before_run(__firstarg=None, order=sys.maxsize, **kwargz): 18 | # first case: when decorator has no args 19 | # TODO: can we have less duplication here? 20 | if callable(__firstarg) and len(kwargz) == 0: 21 | 22 | func = __firstarg 23 | assert not hasattr(func, "_before_run_order"), \ 24 | "Can not set existing '_before_run_order' attribute" 25 | 26 | func._before_run_order = order 27 | 28 | @wraps(func) 29 | def wrapped(self, *args, **kwargs): 30 | return func(self, *args, **kwargs) 31 | 32 | return wrapped 33 | # other cases: when decorator has args 34 | elif __firstarg is None: 35 | def onion(func): # layers, right? 36 | assert not hasattr(func, "_before_run_order"), ( 37 | "Can not set existing '_before_run_order' attribute" 38 | ) 39 | func._before_run_order = order 40 | 41 | @wraps(func) 42 | def wrapped(self, *args, **kwargs): 43 | return func(self, *args, **kwargs) 44 | 45 | return wrapped 46 | 47 | return onion 48 | 49 | # fall through other cases 50 | raise ValueError("Decorator 'before_run' set incorrectly") 51 | -------------------------------------------------------------------------------- /pytest_pipeline/plugin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | pytest_pipeline.plugin 4 | ~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | pytest plugin entry point. 7 | 8 | """ 9 | # (c) 2014-2020 Wibowo Arindrarto 10 | 11 | 12 | def pytest_addoption(parser): 13 | group = parser.getgroup("general") 14 | group.addoption( 15 | "--base-pipeline-dir", 16 | dest="base_pipeline_dir", 17 | default=None, 18 | metavar="dir", 19 | help="Base directory to put all pipeline test directories", 20 | ) 21 | -------------------------------------------------------------------------------- /pytest_pipeline/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | pytest_pipeline.utils 4 | ~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | General utilities. 7 | 8 | """ 9 | # (c) 2014-2020 Wibowo Arindrarto 10 | 11 | import gzip 12 | import hashlib 13 | import os 14 | 15 | 16 | def file_md5sum(fname, unzip=False, blocksize=65536, encoding="utf-8"): 17 | if unzip: 18 | opener = gzip.open 19 | else: 20 | opener = open 21 | 22 | hasher = hashlib.md5() 23 | with opener(fname, "rb") as src: 24 | buf = src.read(blocksize) 25 | while len(buf) > 0: 26 | hasher.update(buf) 27 | buf = src.read(blocksize) 28 | 29 | return hasher.hexdigest() 30 | 31 | 32 | def isexecfile(fname): 33 | return os.path.isfile(fname) and os.access(fname, os.X_OK) 34 | 35 | 36 | def which(program): 37 | # can not do anything meaningful without PATH 38 | if "PATH" not in os.environ: 39 | return None 40 | 41 | for possible in os.environ["PATH"].split(":"): 42 | qualname = os.path.join(possible, program) 43 | if isexecfile(qualname): 44 | return qualname 45 | 46 | return None 47 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | apipkg==1.5 2 | attrs==19.3.0 3 | colorama==0.4.3 4 | coverage==5.0.1 5 | entrypoints==0.3 6 | execnet==1.7.1 7 | filelock==3.0.12 8 | flake8==3.7.9 9 | flake8-polyfill==1.0.2 10 | future==0.18.2 11 | isort==4.3.21 12 | mando==0.6.4 13 | mccabe==0.6.1 14 | more-itertools==8.0.2 15 | mypy==0.761 16 | mypy-extensions==0.4.3 17 | packaging==19.2 18 | pluggy==0.13.1 19 | py==1.8.1 20 | pycodestyle==2.5.0 21 | pyflakes==2.1.1 22 | pyparsing==2.4.6 23 | pytest>=5.0.0 24 | pytest-forked==1.1.3 25 | pytest-sugar==0.9.2 26 | pytest-xdist==1.31.0 27 | radon==4.0.0 28 | six==1.13.0 29 | termcolor==1.1.0 30 | toml==0.10.0 31 | tox==3.14.3 32 | typed-ast==1.4.0 33 | typing-extensions==3.7.4.1 34 | virtualenv==16.7.9 35 | wcwidth==0.1.8 36 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest>=5.0.0 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E402,F403,F405,E731,W503 3 | max-line-length = 80 4 | 5 | [isort] 6 | default_section = FIRSTPARTY 7 | indent = 4 8 | known_future_library = future 9 | length_sort = false 10 | line_length = 80 11 | lines_between_types = 0 12 | multi_line_output = 3 13 | no_lines_before = LOCALFOLDER 14 | use_parentheses = true 15 | 16 | [bdist_wheel] 17 | universal = 1 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # (c) 2014-2020 Wibowo Arindrarto 4 | 5 | from setuptools import find_packages, setup 6 | 7 | from pytest_pipeline import __author__, __contact__, __homepage__, __version__ 8 | 9 | with open("README.rst") as src: 10 | readme = src.read() 11 | with open("CHANGELOG.rst") as src: 12 | changelog = src.read().replace(".. :changelog:", "") 13 | 14 | with open("requirements.txt") as src: 15 | requirements = [line.strip() for line in src] 16 | with open("requirements-dev.txt") as src: 17 | test_requirements = [line.strip() for line in src] 18 | 19 | 20 | setup( 21 | name="pytest-pipeline", 22 | version=__version__, 23 | description="Pytest plugin for functional testing of data analysis" 24 | "pipelines", 25 | long_description=readme + "\n\n" + changelog, 26 | author=__author__, 27 | author_email=__contact__, 28 | url=__homepage__, 29 | packages=find_packages(), 30 | include_package_data=True, 31 | install_requires=requirements, 32 | extras_require={"dev": test_requirements}, 33 | license="BSD", 34 | zip_safe=False, 35 | keywords="pytest pipeline plugin testing", 36 | entry_points={ 37 | "pytest11": [ 38 | "pytest-pipeline = pytest_pipeline.plugin", 39 | ], 40 | }, 41 | classifiers=[ 42 | "Development Status :: 3 - Alpha", 43 | "Environment :: Console", 44 | "Intended Audience :: Developers", 45 | "License :: OSI Approved :: BSD License", 46 | "Operating System :: POSIX", 47 | "Programming Language :: Python :: 3.6", 48 | "Programming Language :: Python :: 3.7", 49 | "Programming Language :: Python :: 3.8", 50 | "Topic :: Utilities", 51 | "Topic :: Software Development :: Testing", 52 | "Topic :: Software Development :: Libraries", 53 | ], 54 | ) 55 | -------------------------------------------------------------------------------- /tests/test_pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | plugin tests 4 | ~~~~~~~~~~~~ 5 | 6 | """ 7 | # (c) 2014-2020 Wibowo Arindrarto 8 | 9 | import glob 10 | import os 11 | import sys 12 | 13 | import pytest 14 | 15 | pytest_plugins = "pytester" 16 | 17 | 18 | MOCK_PIPELINE = """ 19 | #!/usr/bin/env python 20 | 21 | if __name__ == "__main__": 22 | 23 | import os 24 | import sys 25 | 26 | OUT_DIR = "output_dir" 27 | 28 | if len(sys.argv) > 1: 29 | sys.exit(1) 30 | 31 | sys.stdout.write("stdout stream") 32 | sys.stderr.write("stderr stream") 33 | 34 | with open("log.txt", "w") as log: 35 | log.write("not really\\n") 36 | 37 | if not os.path.exists(OUT_DIR): 38 | os.makedirs(OUT_DIR) 39 | 40 | with open(os.path.join(OUT_DIR, "results.txt"), "w") as result: 41 | result.write("42\\n") 42 | """ 43 | 44 | 45 | @pytest.fixture(scope="function") 46 | def mockpipe(request, testdir): 47 | """Mock pipeline script""" 48 | mp = testdir.makefile("", pipeline=MOCK_PIPELINE) 49 | return mp 50 | 51 | 52 | TEST_OK = f""" 53 | import os, shutil, unittest 54 | 55 | import pytest 56 | 57 | from pytest_pipeline import PipelineRun, mark 58 | 59 | 60 | class MyRun(PipelineRun): 61 | 62 | @mark.before_run 63 | def prep_executable(self): 64 | shutil.copy2("../pipeline", "pipeline") 65 | assert os.path.exists("pipeline") 66 | 67 | 68 | run = MyRun.make_fixture("class", "{sys.executable} pipeline") 69 | 70 | 71 | @pytest.mark.usefixtures("run") 72 | class TestMyPipeline(unittest.TestCase): 73 | 74 | def test_exit_code(self): 75 | assert self.run_fixture.exit_code == 0 76 | """ 77 | 78 | 79 | def test_pipeline_basic(mockpipe, testdir): 80 | """Test for basic run""" 81 | test = testdir.makepyfile(TEST_OK) 82 | result = testdir.inline_run( 83 | "-v", 84 | f"--base-pipeline-dir={test.dirname}", 85 | test 86 | ) 87 | passed, skipped, failed = result.listoutcomes() 88 | 89 | assert len(passed) == 1 90 | assert len(skipped) == 0 91 | assert len(failed) == 0 92 | 93 | 94 | TEST_OK_CLASS_FIXTURE = f""" 95 | import os, shutil, unittest 96 | 97 | import pytest 98 | 99 | from pytest_pipeline import PipelineRun, mark 100 | 101 | 102 | class MyRun(PipelineRun): 103 | 104 | @mark.before_run 105 | def prep_executable(self): 106 | shutil.copy2("../pipeline", "pipeline") 107 | assert os.path.exists("pipeline") 108 | 109 | 110 | run = MyRun.class_fixture("{sys.executable} pipeline") 111 | 112 | 113 | @pytest.mark.usefixtures("run") 114 | class TestMyPipelineAgain(unittest.TestCase): 115 | 116 | def test_exit_code(self): 117 | assert self.run_fixture.exit_code == 0 118 | """ 119 | 120 | 121 | def test_pipeline_class_fixture(mockpipe, testdir): 122 | """Test for basic run""" 123 | test = testdir.makepyfile(TEST_OK_CLASS_FIXTURE) 124 | result = testdir.inline_run( 125 | "-v", 126 | f"--base-pipeline-dir={test.dirname}", 127 | test 128 | ) 129 | passed, skipped, failed = result.listoutcomes() 130 | 131 | assert len(passed) == 1 132 | assert len(skipped) == 0 133 | assert len(failed) == 0 134 | 135 | 136 | TEST_REDIRECTION = f""" 137 | import os, shutil, unittest 138 | 139 | import pytest 140 | 141 | from pytest_pipeline import PipelineRun, mark 142 | 143 | 144 | class MyRun(PipelineRun): 145 | 146 | @mark.before_run 147 | def prep_executable(self): 148 | shutil.copy2("../pipeline", "pipeline") 149 | assert os.path.exists("pipeline") 150 | 151 | 152 | run = MyRun.make_fixture( 153 | "class", 154 | cmd="{sys.executable} pipeline", 155 | stdout="stream.out", 156 | stderr="stream.err", 157 | ) 158 | 159 | 160 | @pytest.mark.usefixtures("run") 161 | class TestMyPipeline(unittest.TestCase): 162 | 163 | def test_exit_code(self): 164 | assert self.run_fixture.exit_code == 0 165 | """ 166 | 167 | 168 | def test_pipeline_redirection(mockpipe, testdir): 169 | test = testdir.makepyfile(TEST_REDIRECTION) 170 | result = testdir.inline_run( 171 | "-v", 172 | f"--base-pipeline-dir={test.dirname}", 173 | test 174 | ) 175 | passed, skipped, failed = result.listoutcomes() 176 | 177 | assert len(passed) == 1 178 | assert len(skipped) == 0 179 | assert len(failed) == 0 180 | 181 | testdir_matches = glob.glob(os.path.join(test.dirname, "MyRun*")) 182 | 183 | assert len(testdir_matches) == 1 184 | 185 | testdir_pipeline = testdir_matches[0] 186 | stdout = os.path.join(testdir_pipeline, "stream.out") 187 | 188 | assert os.path.exists(stdout) 189 | assert open(stdout).read() == "stdout stream" 190 | 191 | stderr = os.path.join(testdir_pipeline, "stream.err") 192 | 193 | assert os.path.exists(stderr) 194 | assert open(stderr).read() == "stderr stream" 195 | 196 | 197 | TEST_REDIRECTION_MEM = f""" 198 | import os, shutil, unittest 199 | 200 | import pytest 201 | 202 | from pytest_pipeline import PipelineRun, mark 203 | 204 | 205 | class MyRun(PipelineRun): 206 | 207 | @mark.before_run 208 | def prep_executable(self): 209 | shutil.copy2("../pipeline", "pipeline") 210 | assert os.path.exists("pipeline") 211 | 212 | 213 | run = MyRun.make_fixture( 214 | "class", 215 | cmd="{sys.executable} pipeline", 216 | stdout=True, 217 | stderr=True, 218 | ) 219 | 220 | 221 | @pytest.mark.usefixtures("run") 222 | class TestMyPipeline(unittest.TestCase): 223 | 224 | def test_exit_code(self): 225 | assert self.run_fixture.exit_code == 0 226 | 227 | def test_stdout(self): 228 | assert self.run_fixture.stdout == b"stdout stream" 229 | 230 | def test_stderr(self): 231 | assert self.run_fixture.stderr == b"stderr stream" 232 | """ 233 | 234 | 235 | def test_pipeline_redirection_mem(mockpipe, testdir): 236 | test = testdir.makepyfile(TEST_REDIRECTION_MEM) 237 | result = testdir.inline_run( 238 | "-v", 239 | f"--base-pipeline-dir={test.dirname}", 240 | test 241 | ) 242 | passed, skipped, failed = result.listoutcomes() 243 | 244 | assert len(passed) == 3 245 | assert len(skipped) == 0 246 | assert len(failed) == 0 247 | 248 | testdir_matches = glob.glob(os.path.join(test.dirname, "MyRun*")) 249 | 250 | assert len(testdir_matches) == 1 251 | 252 | 253 | TEST_AS_NONCLASS_FIXTURE = f""" 254 | import os, shutil, unittest 255 | 256 | import pytest 257 | 258 | from pytest_pipeline import PipelineRun, mark 259 | 260 | 261 | class MyRun(PipelineRun): 262 | 263 | @mark.before_run 264 | def prep_executable(self): 265 | shutil.copy2("../pipeline", "pipeline") 266 | assert os.path.exists("pipeline") 267 | 268 | 269 | run = MyRun.make_fixture("module", "{sys.executable} pipeline") 270 | 271 | 272 | def test_exit_code(run): 273 | assert run.exit_code == 0 274 | """ 275 | 276 | 277 | def test_pipeline_as_nonclass_fixture(mockpipe, testdir): 278 | """Test for PipelineTest classes without run attribute""" 279 | test = testdir.makepyfile(TEST_AS_NONCLASS_FIXTURE) 280 | result = testdir.inline_run( 281 | "-v", 282 | f"--base-pipeline-dir={test.dirname}", 283 | test 284 | ) 285 | passed, skipped, failed = result.listoutcomes() 286 | 287 | assert len(passed) == 1 288 | assert len(skipped) == 0 289 | assert len(failed) == 0 290 | 291 | 292 | TEST_OK_GRANULAR = f""" 293 | import os, shutil, unittest 294 | 295 | import pytest 296 | 297 | from pytest_pipeline import PipelineRun, mark 298 | 299 | 300 | class MyRun(PipelineRun): 301 | 302 | @mark.before_run(order=2) 303 | def prep_executable(self): 304 | shutil.copy2("../pipeline", "pipeline") 305 | assert os.path.exists("pipeline") 306 | 307 | @mark.before_run(order=1) 308 | def check_init_condition(self): 309 | assert not os.path.exists("pipeline") 310 | 311 | 312 | run = MyRun.make_fixture("class", cmd="{sys.executable} pipeline") 313 | 314 | 315 | @pytest.mark.usefixtures("run") 316 | class TestMyPipeline(unittest.TestCase): 317 | 318 | def test_exit_code(self): 319 | assert self.run_fixture.exit_code == 0 320 | 321 | def test_output_file(self): 322 | assert os.path.exists(os.path.join("output_dir", "results.txt")) 323 | """ 324 | 325 | 326 | def test_pipeline_granular(mockpipe, testdir): 327 | """Test for execution with 'order' specified in before_run and after_run""" 328 | test = testdir.makepyfile(TEST_OK_GRANULAR) 329 | result = testdir.inline_run( 330 | "-v", 331 | f"--base-pipeline-dir={test.dirname}", 332 | test 333 | ) 334 | passed, skipped, failed = result.listoutcomes() 335 | 336 | assert len(passed) == 2 337 | assert len(skipped) == 0 338 | assert len(failed) == 0 339 | 340 | 341 | MOCK_PIPELINE_TIMEOUT = """ 342 | #!/usr/bin/env python 343 | 344 | if __name__ == "__main__": 345 | 346 | import time 347 | time.sleep(10) 348 | """ 349 | 350 | 351 | TEST_TIMEOUT = f""" 352 | import os, shutil, unittest 353 | 354 | import pytest 355 | 356 | from pytest_pipeline import PipelineRun, mark 357 | 358 | 359 | class MyRun(PipelineRun): 360 | 361 | @mark.before_run 362 | def test_and_prep_executable(self): 363 | shutil.copy2("../pipeline", "pipeline") 364 | assert os.path.exists("pipeline") 365 | 366 | 367 | run = PipelineRun.make_fixture( 368 | "class", 369 | cmd="{sys.executable} pipeline", 370 | timeout=0.01, 371 | ) 372 | 373 | 374 | @pytest.mark.usefixtures("run") 375 | class TestMyPipeline(unittest.TestCase): 376 | 377 | def test_exit_code(self): 378 | assert self.run_fixture.exit_code != 0 379 | """ 380 | 381 | 382 | @pytest.fixture(scope="function") 383 | def mockpipe_timeout(request, testdir): 384 | """Mock pipeline script with timeout""" 385 | mp = testdir.makefile("", pipeline=MOCK_PIPELINE_TIMEOUT) 386 | return mp 387 | 388 | 389 | def test_pipeline_timeout(mockpipe_timeout, testdir): 390 | """Test for execution with timeout""" 391 | test = testdir.makepyfile(TEST_TIMEOUT) 392 | result = testdir.inline_run( 393 | "-v", 394 | f"--base-pipeline-dir={test.dirname}", 395 | test 396 | ) 397 | passed, skipped, failed = result.listoutcomes() 398 | 399 | assert len(passed) == 0 400 | assert len(skipped) == 0 401 | assert len(failed) == 1 402 | 403 | 404 | MOCK_PIPELINE_FMT = """ 405 | #!/usr/bin/env python 406 | 407 | import sys 408 | 409 | if __name__ == "__main__": 410 | 411 | print(sys.argv[1]) 412 | """ 413 | 414 | 415 | TEST_FMT = f""" 416 | import os, shutil, unittest 417 | 418 | import pytest 419 | 420 | from pytest_pipeline import PipelineRun, mark 421 | 422 | 423 | class MyRun(PipelineRun): 424 | 425 | @mark.before_run 426 | def prep_executable(self): 427 | shutil.copy2("../pipeline", "pipeline") 428 | assert os.path.exists("pipeline") 429 | 430 | 431 | run = MyRun.make_fixture( 432 | "class", 433 | "{sys.executable} pipeline {{run_dir}}", 434 | stdout=True, 435 | ) 436 | 437 | 438 | @pytest.mark.usefixtures("run") 439 | class TestMyPipeline(unittest.TestCase): 440 | 441 | def test_exit_code(self): 442 | assert self.run_fixture.exit_code == 0 443 | 444 | def test_stdout(self): 445 | stdout = self.run_fixture.stdout.decode("utf-8").strip() 446 | assert self.run_fixture.run_dir == stdout 447 | """ 448 | 449 | 450 | @pytest.fixture(scope="function") 451 | def mockpipe_fmt(request, testdir): 452 | """Mock pipeline script with timeout""" 453 | mp = testdir.makefile("", pipeline=MOCK_PIPELINE_FMT) 454 | return mp 455 | 456 | 457 | def test_pipeline_fmt(mockpipe_fmt, testdir): 458 | """Test for run with templated command""" 459 | test = testdir.makepyfile(TEST_FMT) 460 | result = testdir.inline_run( 461 | "-v", 462 | f"--base-pipeline-dir={test.dirname}", 463 | test 464 | ) 465 | passed, skipped, failed = result.listoutcomes() 466 | 467 | assert len(passed) == 2 468 | assert len(skipped) == 0 469 | assert len(failed) == 0 470 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36, py37, py38 3 | 4 | [testenv] 5 | usedevelop = True 6 | commands = 7 | flake8 --statistics pytest_pipeline tests 8 | coverage run --source=pytest_pipeline {envbindir}/py.test tests pytest_pipeline 9 | isort --check-only --recursive pytest_pipeline tests setup.py 10 | radon cc --total-average --show-closures --show-complexity --min C pytest_pipeline 11 | deps = 12 | -rrequirements.txt 13 | -rrequirements-dev.txt 14 | --------------------------------------------------------------------------------