├── .github ├── dependabot.yml └── workflows │ └── ci_workflows.yml ├── .gitignore ├── CHANGES.md ├── LICENSE ├── MANIFEST.in ├── README.rst ├── pyproject.toml ├── pytest_arraydiff ├── __init__.py └── plugin.py ├── setup.cfg ├── setup.py ├── tests ├── baseline │ ├── test_absolute_tolerance.fits │ ├── test_relative_tolerance.fits │ ├── test_single_reference.fits │ ├── test_succeeds_class.fits │ ├── test_succeeds_func_default.txt │ ├── test_succeeds_func_fits.fits │ ├── test_succeeds_func_fits_hdu.fits │ ├── test_succeeds_func_pdhdf.h5 │ └── test_succeeds_func_text.txt └── test_pytest_arraydiff.py └── tox.ini /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: ".github/workflows" 5 | schedule: 6 | interval: "monthly" 7 | groups: 8 | actions: 9 | patterns: 10 | - "*" 11 | -------------------------------------------------------------------------------- /.github/workflows/ci_workflows.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | push: 6 | schedule: 7 | # Run every Sunday at 06:53 UTC 8 | - cron: 53 6 * * 0 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | tests: 17 | uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@8c0fde6f7e926df6ed7057255d29afa9c1ad5320 # v1.16.0 18 | with: 19 | envs: | 20 | - linux: codestyle 21 | - windows: py38-test-pytestoldest 22 | - linux: py38-test-pytest53 23 | - macos: py39-test-pytest60 24 | - windows: py39-test-pytest61 25 | - linux: py310-test-pytest62 26 | - macos: py310-test-pytest70 27 | - windows: py310-test-pytest71 28 | - linux: py311-test-pytest72 29 | - macos: py311-test-pytest73 30 | - windows: py312-test-pytest74 31 | - linux: py312-test-devdeps 32 | publish: 33 | needs: tests 34 | uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@8c0fde6f7e926df6ed7057255d29afa9c1ad5320 # v1.16.0 35 | with: 36 | test_extras: test 37 | test_command: pytest $GITHUB_WORKSPACE/tests; pytest --arraydiff $GITHUB_WORKSPACE/tests 38 | secrets: 39 | pypi_token: ${{ secrets.pypi_password }} 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled files 2 | *.py[cod] 3 | *.a 4 | *.o 5 | *.so 6 | *.pyd 7 | __pycache__ 8 | 9 | # Ignore .c files by default to avoid including generated code. If you want to 10 | # add a non-generated .c extension, use `git add -f filename.c`. 11 | *.c 12 | 13 | # Other generated files 14 | MANIFEST 15 | 16 | # Sphinx 17 | _build 18 | _generated 19 | docs/api 20 | docs/generated 21 | 22 | # Packages/installer info 23 | *.egg 24 | *.egg-info 25 | dist 26 | build 27 | eggs 28 | .eggs 29 | parts 30 | bin 31 | var 32 | sdist 33 | develop-eggs 34 | .installed.cfg 35 | distribute-*.tar.gz 36 | 37 | # Other 38 | .cache 39 | .tox 40 | .*.swp 41 | .*.swo 42 | *~ 43 | .project 44 | .pydevproject 45 | .settings 46 | .coverage 47 | cover 48 | htmlcov 49 | .pytest_cache 50 | 51 | # Env 52 | .venv 53 | venv 54 | .env 55 | 56 | # Mac OSX 57 | .DS_Store 58 | 59 | # PyCharm 60 | .idea 61 | 62 | */version.py 63 | pip-wheel-metadata/ 64 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 0.7 (unreleased) 2 | ---------------- 3 | 4 | - Minimum Python version is now 3.8. [#49] 5 | 6 | 0.6.1 (2023-11-27) 7 | ------------------ 8 | 9 | - Fix broken ``single_reference=True`` usage. [#43] 10 | 11 | 0.6 (2023-11-15) 12 | ---------------- 13 | 14 | - Add ability to compare to Pandas DataFrames and store them as HDF5 files [#23] 15 | 16 | - Fix ``array_compare`` so that the ``atol`` parameter is correctly used with 17 | FITS files. [#33] 18 | 19 | - Test inside ``pytest_runtest_call`` hook. [#36] 20 | 21 | 0.5 (2022-01-12) 22 | ---------------- 23 | 24 | - Removed `astropy` as required dependency. [#31] 25 | 26 | - Formally register `array_compare` as marker. 27 | 28 | 0.4 (2021-12-31) 29 | ---------------- 30 | 31 | - Minimum Python version is now 3.7. [#30] 32 | 33 | - Various infrastructure updates. 34 | 35 | 0.3 (2018-12-05) 36 | ---------------- 37 | 38 | - Fixed compatibility with pytest 4+. [#15] 39 | 40 | 0.2 (2018-01-29) 41 | ---------------- 42 | 43 | - Fix compatibility with recent versions of Astropy and Numpy. [#8, #10] 44 | 45 | - Add back support for returning HDUs from tests. [#5] 46 | 47 | 0.1 (2016-11-26) 48 | ---------------- 49 | 50 | - Initial version 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Thomas P. Robitaille 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | This package was adapted from pytest-mpl, which is released under a BSD 26 | license and can be found here: 27 | 28 | https://github.com/astrofrog/pytest-mpl 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include CHANGES.md 4 | include tox.ini 5 | include pyproject.toml 6 | include setup.cfg 7 | include setup.py 8 | 9 | recursive-include tests *.py *.fits *.txt 10 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5811772.svg 2 | :target: https://doi.org/10.5281/zenodo.5811772 3 | :alt: 10.5281/zenodo.5811772 4 | 5 | .. image:: https://github.com/astropy/pytest-arraydiff/workflows/CI/badge.svg 6 | :target: https://github.com/astropy/pytest-arraydiff/actions 7 | :alt: CI Status 8 | 9 | .. image:: https://img.shields.io/pypi/v/pytest-arraydiff.svg 10 | :target: https://pypi.org/project/pytest-arraydiff 11 | :alt: PyPI Status 12 | 13 | About 14 | ----- 15 | 16 | This is a `py.test `__ plugin to facilitate the 17 | generation and comparison of data arrays produced during tests, in particular 18 | in cases where the arrays are too large to conveniently hard-code them 19 | in the tests (e.g. ``np.testing.assert_allclose(x, [1, 2, 3])``). 20 | 21 | The basic idea is that you can write a test that generates a Numpy array (or 22 | other related objects depending on the format, e.g. pandas DataFrame). 23 | You can then either run the 24 | tests in a mode to **generate** reference files from the arrays, or you can run 25 | the tests in **comparison** mode, which will compare the results of the tests to 26 | the reference ones within some tolerance. 27 | 28 | At the moment, the supported file formats for the reference files are: 29 | 30 | - A plain text-based format (based on Numpy ``loadtxt`` output) 31 | - The FITS format (requires `astropy `__). With this 32 | format, tests can return either a Numpy array for a FITS HDU object. 33 | - A pandas HDF5 format using the pandas HDFStore 34 | 35 | For more information on how to write tests to do this, see the **Using** 36 | section below. 37 | 38 | Installing 39 | ---------- 40 | 41 | This plugin is compatible with Python 2.7, and 3.5 and later, and 42 | requires `pytest `__ and 43 | `numpy `__ to be installed. 44 | 45 | To install, you can do:: 46 | 47 | pip install pytest-arraydiff 48 | 49 | You can check that the plugin is registered with pytest by doing:: 50 | 51 | py.test --version 52 | 53 | which will show a list of plugins:: 54 | 55 | This is pytest version 2.7.1, imported from ... 56 | setuptools registered plugins: 57 | pytest-arraydiff-0.1 at ... 58 | 59 | Using 60 | ----- 61 | 62 | To use, you simply need to mark the function where you want to compare 63 | arrays using ``@pytest.mark.array_compare``, and make sure that the 64 | function returns a plain Numpy array:: 65 | 66 | python 67 | import pytest 68 | import numpy as np 69 | 70 | @pytest.mark.array_compare 71 | def test_succeeds(): 72 | return np.arange(3 * 5 * 4).reshape((3, 5, 4)) 73 | 74 | To generate the reference data files, run the tests with the 75 | ``--arraydiff-generate-path`` option with the name of the directory 76 | where the generated files should be placed:: 77 | 78 | py.test --arraydiff-generate-path=reference 79 | 80 | If the directory does not exist, it will be created. The directory will 81 | be interpreted as being relative to where you are running ``py.test``. 82 | Make sure you manually check the reference arrays to ensure they are 83 | correct. 84 | 85 | Once you are happy with the generated data files, you should move them 86 | to a sub-directory called ``reference`` relative to the test files (this 87 | name is configurable, see below). You can also generate the baseline 88 | arrays directly in the right directory. 89 | 90 | You can then run the tests simply with:: 91 | 92 | py.test --arraydiff 93 | 94 | and the tests will pass if the arrays are the same. If you omit the 95 | ``--arraydiff`` option, the tests will run but will only check that the 96 | code runs without checking the output arrays. 97 | 98 | Options 99 | ------- 100 | 101 | The ``@pytest.mark.array_compare`` marker take an argument to specify 102 | the format to use for the reference files: 103 | 104 | .. code:: python 105 | 106 | @pytest.mark.array_compare(file_format='text') 107 | def test_array(): 108 | ... 109 | 110 | The default file format can also be specified using the 111 | ``--arraydiff-default-format=`` flag when running ``py.test``, 112 | and ```` should be either ``fits`` or ``text``. 113 | 114 | The supported formats at this time are ``text`` and ``fits``, and 115 | contributions for other formats are welcome. The default format is 116 | ``text``. 117 | 118 | Additional arguments are the relative and absolute tolerances for floating 119 | point values (which default to 1e-7 and 0, respectively): 120 | 121 | .. code:: python 122 | 123 | @pytest.mark.array_compare(rtol=20, atol=0.1) 124 | def test_array(): 125 | ... 126 | 127 | You can also pass keyword arguments to the writers using the 128 | ``write_kwargs``. For the ``text`` format, these arguments are passed to 129 | ``savetxt`` while for the ``fits`` format they are passed to Astropy's 130 | ``fits.writeto`` function. 131 | 132 | .. code:: python 133 | 134 | @pytest.mark.array_compare(file_format='fits', write_kwargs={'output_verify': 'silentfix'}) 135 | def test_array(): 136 | ... 137 | 138 | Other options include the name of the reference directory (which 139 | defaults to ``reference`` ) and the filename for the reference file 140 | (which defaults to the name of the test with a format-dependent 141 | extension). 142 | 143 | .. code:: python 144 | 145 | @pytest.mark.array_compare(reference_dir='baseline_arrays', 146 | filename='other_name.fits') 147 | def test_array(): 148 | ... 149 | 150 | The reference directory in the decorator above will be interpreted as 151 | being relative to the test file. Note that the baseline directory can 152 | also be a URL (which should start with ``http://`` or ``https://`` and 153 | end in a slash). 154 | 155 | Finally, you can also set a custom baseline directory globally when 156 | running tests by running ``py.test`` with:: 157 | 158 | py.test --arraydiff --arraydiff-reference-path=baseline_arrays 159 | 160 | This directory will be interpreted as being relative to where the tests 161 | are run. In addition, if both this option and the ``reference_dir`` 162 | option in the ``array_compare`` decorator are used, the one in the 163 | decorator takes precedence. 164 | 165 | Test failure example 166 | -------------------- 167 | 168 | If the arrays produced by the tests are correct, then the test will 169 | pass, but if they are not, the test will fail with a message similar to 170 | the following:: 171 | 172 | E AssertionError: 173 | E 174 | E a: /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmpbvjkzt_q/test_to_mask_rect-mode_subpixels-subpixels_18.txt 175 | E b: /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmpbvjkzt_q/reference-test_to_mask_rect-mode_subpixels-subpixels_18.txt 176 | E 177 | E Not equal to tolerance rtol=1e-07, atol=0 178 | E 179 | E (mismatch 47.22222222222222%) 180 | E x: array([[ 0. , 0. , 0. , 0. , 0.404012, 0.55 , 181 | E 0.023765, 0. , 0. ], 182 | E [ 0. , 0. , 0. , 0.112037, 1.028704, 1.1 ,... 183 | E y: array([[ 0. , 0. , 0. , 0. , 0.367284, 0.5 , 184 | E 0.021605, 0. , 0. ], 185 | E [ 0. , 0. , 0. , 0.101852, 0.935185, 1. ,... 186 | 187 | The file paths included in the exception are then available for 188 | inspection. 189 | 190 | Running the tests for pytest-arraydiff 191 | -------------------------------------- 192 | 193 | If you are contributing some changes and want to run the tests, first 194 | install the latest version of the plugin then do:: 195 | 196 | cd tests 197 | py.test --arraydiff 198 | 199 | The reason for having to install the plugin first is to ensure that the 200 | plugin is correctly loaded as part of the test suite. 201 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=30.3.0", 3 | "setuptools_scm", 4 | "wheel"] 5 | build-backend = 'setuptools.build_meta' 6 | -------------------------------------------------------------------------------- /pytest_arraydiff/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | 3 | from .version import version as __version__ # noqa 4 | -------------------------------------------------------------------------------- /pytest_arraydiff/plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016, Thomas P. Robitaille 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions and the following disclaimer. 9 | # 10 | # 2. Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 18 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | # POSSIBILITY OF SUCH DAMAGE. 25 | # 26 | # This package was derived from pytest-mpl, which is released under a BSD 27 | # license and can be found here: 28 | # 29 | # https://github.com/astrofrog/pytest-mpl 30 | 31 | import os 32 | import abc 33 | import shutil 34 | import tempfile 35 | import warnings 36 | from urllib.request import urlopen 37 | 38 | import pytest 39 | import numpy as np 40 | 41 | 42 | abstractstaticmethod = abc.abstractstaticmethod 43 | abstractclassmethod = abc.abstractclassmethod 44 | 45 | 46 | class BaseDiff(metaclass=abc.ABCMeta): 47 | 48 | @abstractstaticmethod 49 | def read(filename): 50 | """ 51 | Given a filename, return a data object. 52 | """ 53 | raise NotImplementedError() 54 | 55 | @abstractstaticmethod 56 | def write(filename, data, **kwargs): 57 | """ 58 | Given a filename and a data object (and optional keyword arguments), 59 | write the data to a file. 60 | """ 61 | raise NotImplementedError() 62 | 63 | @abstractclassmethod 64 | def compare(self, reference_file, test_file, atol=None, rtol=None): 65 | """ 66 | Given a reference and test filename, compare the data to the specified 67 | absolute (``atol``) and relative (``rtol``) tolerances. 68 | 69 | Should return two arguments: a boolean indicating whether the data are 70 | identical, and a string giving the full error message if not. 71 | """ 72 | raise NotImplementedError() 73 | 74 | 75 | class SimpleArrayDiff(BaseDiff): 76 | 77 | @classmethod 78 | def compare(cls, reference_file, test_file, atol=None, rtol=None): 79 | 80 | array_ref = cls.read(reference_file) 81 | array_new = cls.read(test_file) 82 | 83 | try: 84 | np.testing.assert_allclose(array_ref, array_new, atol=atol, rtol=rtol) 85 | except AssertionError as exc: 86 | message = f"\n\na: {test_file}" + '\n' 87 | message += f"b: {reference_file}" + '\n' 88 | message += exc.args[0] 89 | return False, message 90 | else: 91 | return True, "" 92 | 93 | 94 | class FITSDiff(BaseDiff): 95 | 96 | extension = 'fits' 97 | 98 | @staticmethod 99 | def read(filename): 100 | from astropy.io import fits 101 | return fits.getdata(filename) 102 | 103 | @staticmethod 104 | def write(filename, data, **kwargs): 105 | from astropy.io import fits 106 | if isinstance(data, np.ndarray): 107 | data = fits.PrimaryHDU(data) 108 | return data.writeto(filename, **kwargs) 109 | 110 | @classmethod 111 | def compare(cls, reference_file, test_file, atol=None, rtol=None): 112 | import astropy 113 | from astropy.io.fits.diff import FITSDiff 114 | from astropy.utils.introspection import minversion 115 | if minversion(astropy, '2.0'): 116 | diff = FITSDiff(reference_file, test_file, rtol=rtol, atol=atol) 117 | else: 118 | # `atol` is not supported prior to Astropy 2.0 119 | diff = FITSDiff(reference_file, test_file, tolerance=rtol) 120 | return diff.identical, diff.report() 121 | 122 | 123 | class TextDiff(SimpleArrayDiff): 124 | 125 | extension = 'txt' 126 | 127 | @staticmethod 128 | def read(filename): 129 | return np.loadtxt(filename) 130 | 131 | @staticmethod 132 | def write(filename, data, **kwargs): 133 | fmt = kwargs.get('fmt', '%g') 134 | kwargs['fmt'] = fmt 135 | return np.savetxt(filename, data, **kwargs) 136 | 137 | 138 | class PDHDFDiff(BaseDiff): 139 | 140 | extension = 'h5' 141 | 142 | @staticmethod 143 | def read(filename): 144 | import pandas as pd 145 | return pd.read_hdf(filename) 146 | 147 | @staticmethod 148 | def write(filename, data, **kwargs): 149 | import pandas as pd # noqa: F401 150 | key = os.path.basename(filename).replace('.h5', '') 151 | return data.to_hdf(filename, key=key, **kwargs) 152 | 153 | @classmethod 154 | def compare(cls, reference_file, test_file, atol=None, rtol=None): 155 | import pandas.testing as pdt 156 | import pandas as pd 157 | 158 | ref_data = pd.read_hdf(reference_file) 159 | test_data = pd.read_hdf(test_file) 160 | try: 161 | pdt.assert_frame_equal(ref_data, test_data) 162 | except AssertionError as exc: 163 | message = f"\n\na: {test_file}" + '\n' 164 | message += f"b: {reference_file}" + '\n' 165 | message += exc.args[0] 166 | return False, message 167 | else: 168 | return True, "" 169 | 170 | 171 | FORMATS = {} 172 | FORMATS['fits'] = FITSDiff 173 | FORMATS['text'] = TextDiff 174 | FORMATS['pd_hdf'] = PDHDFDiff 175 | 176 | 177 | def _download_file(url): 178 | u = urlopen(url) 179 | result_dir = tempfile.mkdtemp() 180 | filename = os.path.join(result_dir, 'downloaded') 181 | with open(filename, 'wb') as tmpfile: 182 | tmpfile.write(u.read()) 183 | return filename 184 | 185 | 186 | def pytest_addoption(parser): 187 | group = parser.getgroup("general") 188 | group.addoption('--arraydiff', action='store_true', 189 | help="Enable comparison of arrays to reference arrays stored in files") 190 | group.addoption('--arraydiff-generate-path', 191 | help="directory to generate reference files in, relative to location where py.test is run", action='store') 192 | group.addoption('--arraydiff-reference-path', 193 | help="directory containing reference files, relative to location where py.test is run", action='store') 194 | group.addoption('--arraydiff-default-format', 195 | help="Default format for the reference arrays (can be 'fits' or 'text' currently)") 196 | 197 | 198 | def pytest_configure(config): 199 | config.getini('markers').append( 200 | 'array_compare: for functions using array comparison') 201 | 202 | if config.getoption("--arraydiff") or config.getoption("--arraydiff-generate-path") is not None: 203 | 204 | reference_dir = config.getoption("--arraydiff-reference-path") 205 | generate_dir = config.getoption("--arraydiff-generate-path") 206 | 207 | if reference_dir is not None and generate_dir is not None: 208 | warnings.warn("Ignoring --arraydiff-reference-path since --arraydiff-generate-path is set") 209 | 210 | if reference_dir is not None: 211 | reference_dir = os.path.abspath(reference_dir) 212 | if generate_dir is not None: 213 | reference_dir = os.path.abspath(generate_dir) 214 | 215 | default_format = config.getoption("--arraydiff-default-format") or 'text' 216 | 217 | config.pluginmanager.register(ArrayComparison(config, 218 | reference_dir=reference_dir, 219 | generate_dir=generate_dir, 220 | default_format=default_format)) 221 | else: 222 | config.pluginmanager.register(ArrayInterceptor(config)) 223 | 224 | 225 | def generate_test_name(item): 226 | """ 227 | Generate a unique name for this test. 228 | """ 229 | if item.cls is not None: 230 | name = f"{item.module.__name__}.{item.cls.__name__}.{item.name}" 231 | else: 232 | name = f"{item.module.__name__}.{item.name}" 233 | return name 234 | 235 | 236 | def wrap_array_interceptor(plugin, item): 237 | """ 238 | Intercept and store arrays returned by test functions. 239 | """ 240 | # Only intercept array on marked array tests 241 | if item.get_closest_marker('array_compare') is not None: 242 | 243 | # Use the full test name as a key to ensure correct array is being retrieved 244 | test_name = generate_test_name(item) 245 | 246 | def array_interceptor(store, obj): 247 | def wrapper(*args, **kwargs): 248 | store.return_value[test_name] = obj(*args, **kwargs) 249 | return wrapper 250 | 251 | item.obj = array_interceptor(plugin, item.obj) 252 | 253 | 254 | class ArrayComparison: 255 | 256 | def __init__(self, config, reference_dir=None, generate_dir=None, default_format='text'): 257 | self.config = config 258 | self.reference_dir = reference_dir 259 | self.generate_dir = generate_dir 260 | self.default_format = default_format 261 | self.return_value = {} 262 | 263 | @pytest.hookimpl(hookwrapper=True) 264 | def pytest_runtest_call(self, item): 265 | 266 | compare = item.get_closest_marker('array_compare') 267 | 268 | if compare is None: 269 | yield 270 | return 271 | 272 | file_format = compare.kwargs.get('file_format', self.default_format) 273 | 274 | if file_format not in FORMATS: 275 | raise ValueError(f"Unknown format: {file_format}") 276 | 277 | if 'extension' in compare.kwargs: 278 | extension = compare.kwargs['extension'] 279 | else: 280 | extension = FORMATS[file_format].extension 281 | 282 | atol = compare.kwargs.get('atol', 0.) 283 | rtol = compare.kwargs.get('rtol', 1e-7) 284 | 285 | single_reference = compare.kwargs.get('single_reference', False) 286 | 287 | write_kwargs = compare.kwargs.get('write_kwargs', {}) 288 | 289 | reference_dir = compare.kwargs.get('reference_dir', None) 290 | if reference_dir is None: 291 | if self.reference_dir is None: 292 | reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), 'reference') 293 | else: 294 | reference_dir = self.reference_dir 295 | else: 296 | if not reference_dir.startswith(('http://', 'https://')): 297 | reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), reference_dir) 298 | 299 | baseline_remote = reference_dir.startswith('http') 300 | 301 | # Run test and get array object 302 | wrap_array_interceptor(self, item) 303 | yield 304 | test_name = generate_test_name(item) 305 | if test_name not in self.return_value: 306 | # Test function did not complete successfully 307 | return 308 | array = self.return_value[test_name] 309 | 310 | # Find test name to use as plot name 311 | filename = compare.kwargs.get('filename', None) 312 | if filename is None: 313 | if single_reference: 314 | filename = item.originalname + '.' + extension 315 | else: 316 | filename = item.name + '.' + extension 317 | filename = filename.replace('[', '_').replace(']', '_') 318 | filename = filename.replace('_.' + extension, '.' + extension) 319 | 320 | # What we do now depends on whether we are generating the reference 321 | # files or simply running the test. 322 | if self.generate_dir is None: 323 | 324 | # Save the figure 325 | result_dir = tempfile.mkdtemp() 326 | test_array = os.path.abspath(os.path.join(result_dir, filename)) 327 | 328 | FORMATS[file_format].write(test_array, array, **write_kwargs) 329 | 330 | # Find path to baseline array 331 | if baseline_remote: 332 | baseline_file_ref = _download_file(reference_dir + filename) 333 | else: 334 | baseline_file_ref = os.path.abspath(os.path.join(os.path.dirname(item.fspath.strpath), reference_dir, filename)) 335 | 336 | if not os.path.exists(baseline_file_ref): 337 | raise Exception("""File not found for comparison test 338 | Generated file: 339 | \t{test} 340 | This is expected for new tests.""".format( 341 | test=test_array)) 342 | 343 | # setuptools may put the baseline arrays in non-accessible places, 344 | # copy to our tmpdir to be sure to keep them in case of failure 345 | baseline_file = os.path.abspath(os.path.join(result_dir, 'reference-' + filename)) 346 | shutil.copyfile(baseline_file_ref, baseline_file) 347 | 348 | identical, msg = FORMATS[file_format].compare(baseline_file, test_array, atol=atol, rtol=rtol) 349 | 350 | if identical: 351 | shutil.rmtree(result_dir) 352 | else: 353 | raise Exception(msg) 354 | 355 | else: 356 | 357 | if not os.path.exists(self.generate_dir): 358 | os.makedirs(self.generate_dir) 359 | 360 | FORMATS[file_format].write(os.path.abspath(os.path.join(self.generate_dir, filename)), array, **write_kwargs) 361 | 362 | pytest.skip("Skipping test, since generating data") 363 | 364 | 365 | class ArrayInterceptor: 366 | """ 367 | This is used in place of ArrayComparison when the array comparison option is not used, 368 | to make sure that we still intercept arrays returned by tests. 369 | """ 370 | 371 | def __init__(self, config): 372 | self.config = config 373 | self.return_value = {} 374 | 375 | @pytest.hookimpl(hookwrapper=True) 376 | def pytest_runtest_call(self, item): 377 | 378 | if item.get_closest_marker('array_compare') is not None: 379 | wrap_array_interceptor(self, item) 380 | 381 | yield 382 | return 383 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = pytest-arraydiff 3 | url = https://github.com/astropy/pytest-arraydiff 4 | author = The Astropy Developers 5 | author_email = astropy.team@gmail.com 6 | classifiers = 7 | Development Status :: 4 - Beta 8 | Framework :: Pytest 9 | Intended Audience :: Developers 10 | License :: OSI Approved :: BSD License 11 | Operating System :: OS Independent 12 | Programming Language :: Python 13 | Programming Language :: Python :: 3 14 | Programming Language :: Python :: 3 :: Only 15 | Programming Language :: Python :: 3.8 16 | Programming Language :: Python :: 3.9 17 | Programming Language :: Python :: 3.10 18 | Programming Language :: Python :: 3.11 19 | Programming Language :: Python :: 3.12 20 | Programming Language :: Python :: Implementation :: CPython 21 | Topic :: Software Development :: Testing 22 | Topic :: Utilities 23 | license = BSD 24 | description = pytest plugin to help with comparing array output from tests 25 | long_description = file: README.rst 26 | long_description_content_type = text/x-rst 27 | 28 | [options] 29 | zip_safe = False 30 | packages = find: 31 | python_requires = >=3.8 32 | setup_requires = 33 | setuptools_scm 34 | install_requires = 35 | pytest>=5.0 36 | numpy 37 | 38 | # tables limitation is until 3.9.3 is out as that supports ARM OSX. 39 | [options.extras_require] 40 | test = 41 | astropy 42 | pandas 43 | tables;platform_machine!='arm64' 44 | 45 | [options.entry_points] 46 | pytest11 = 47 | pytest_arraydiff = pytest_arraydiff.plugin 48 | 49 | [tool:pytest] 50 | minversion = 5.0 51 | testpaths = tests 52 | xfail_strict = true 53 | markers = 54 | array_compare: for functions using array comparison 55 | filterwarnings = 56 | error 57 | # Can be removed when min Python is >=3.8 58 | ignore:distutils Version classes are deprecated 59 | 60 | [flake8] 61 | max-line-length = 150 62 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from setuptools import setup 5 | 6 | setup(use_scm_version={'write_to': os.path.join('pytest_arraydiff', 'version.py')}) 7 | -------------------------------------------------------------------------------- /tests/baseline/test_absolute_tolerance.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/astropy/pytest-arraydiff/069039b469f102b8819f18bacad7c9dff5cc73f1/tests/baseline/test_absolute_tolerance.fits -------------------------------------------------------------------------------- /tests/baseline/test_relative_tolerance.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/astropy/pytest-arraydiff/069039b469f102b8819f18bacad7c9dff5cc73f1/tests/baseline/test_relative_tolerance.fits -------------------------------------------------------------------------------- /tests/baseline/test_single_reference.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/astropy/pytest-arraydiff/069039b469f102b8819f18bacad7c9dff5cc73f1/tests/baseline/test_single_reference.fits -------------------------------------------------------------------------------- /tests/baseline/test_succeeds_class.fits: -------------------------------------------------------------------------------- 1 | SIMPLE = T / conforms to FITS standard BITPIX = 64 / array data type NAXIS = 3 / number of array dimensions NAXIS1 = 3 NAXIS2 = 4 NAXIS3 = 2 EXTEND = T END  2 |  -------------------------------------------------------------------------------- /tests/baseline/test_succeeds_func_default.txt: -------------------------------------------------------------------------------- 1 | 0 1 2 3 4 2 | 5 6 7 8 9 3 | 10 11 12 13 14 4 | -------------------------------------------------------------------------------- /tests/baseline/test_succeeds_func_fits.fits: -------------------------------------------------------------------------------- 1 | SIMPLE = T / conforms to FITS standard BITPIX = 64 / array data type NAXIS = 2 / number of array dimensions NAXIS1 = 5 NAXIS2 = 3 EXTEND = T END  2 |  -------------------------------------------------------------------------------- /tests/baseline/test_succeeds_func_fits_hdu.fits: -------------------------------------------------------------------------------- 1 | SIMPLE = T / conforms to FITS standard BITPIX = 64 / array data type NAXIS = 2 / number of array dimensions NAXIS1 = 5 NAXIS2 = 3 EXTEND = T END  2 |  -------------------------------------------------------------------------------- /tests/baseline/test_succeeds_func_pdhdf.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/astropy/pytest-arraydiff/069039b469f102b8819f18bacad7c9dff5cc73f1/tests/baseline/test_succeeds_func_pdhdf.h5 -------------------------------------------------------------------------------- /tests/baseline/test_succeeds_func_text.txt: -------------------------------------------------------------------------------- 1 | 0 1 2 3 4 2 | 5 6 7 8 9 3 | 10 11 12 13 14 4 | -------------------------------------------------------------------------------- /tests/test_pytest_arraydiff.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import tempfile 4 | 5 | import pytest 6 | import numpy as np 7 | from packaging.version import Version 8 | 9 | NUMPY_LT_2_0 = Version(np.__version__) < Version("2.0.dev") 10 | 11 | reference_dir = 'baseline' 12 | 13 | 14 | @pytest.mark.array_compare(reference_dir=reference_dir) 15 | def test_succeeds_func_default(): 16 | return np.arange(3 * 5).reshape((3, 5)) 17 | 18 | 19 | @pytest.mark.array_compare(file_format='text', reference_dir=reference_dir) 20 | def test_succeeds_func_text(): 21 | return np.arange(3 * 5).reshape((3, 5)) 22 | 23 | 24 | @pytest.mark.skipif(not NUMPY_LT_2_0, reason="AttributeError: `np.unicode_` was removed in the NumPy 2.0 release. Use `np.str_` instead.") 25 | @pytest.mark.array_compare(file_format='pd_hdf', reference_dir=reference_dir) 26 | def test_succeeds_func_pdhdf(): 27 | pd = pytest.importorskip('pandas') 28 | return pd.DataFrame(data=np.arange(20, dtype='int64'), 29 | columns=['test_data']) 30 | 31 | 32 | @pytest.mark.array_compare(file_format='fits', reference_dir=reference_dir) 33 | def test_succeeds_func_fits(): 34 | return np.arange(3 * 5).reshape((3, 5)).astype(np.int64) 35 | 36 | 37 | @pytest.mark.array_compare(file_format='fits', reference_dir=reference_dir) 38 | def test_succeeds_func_fits_hdu(): 39 | from astropy.io import fits 40 | return fits.PrimaryHDU(np.arange(3 * 5).reshape((3, 5)).astype(np.int64)) 41 | 42 | 43 | class TestClass: 44 | 45 | @pytest.mark.array_compare(file_format='fits', reference_dir=reference_dir) 46 | def test_succeeds_class(self): 47 | return np.arange(2 * 4 * 3).reshape((2, 4, 3)).astype(np.int64) 48 | 49 | 50 | TEST_FAILING = """ 51 | import pytest 52 | import numpy as np 53 | from astropy.io import fits 54 | @pytest.mark.array_compare 55 | def test_fail(): 56 | return np.ones((3, 4)) 57 | """ 58 | 59 | 60 | def test_fails(): 61 | 62 | tmpdir = tempfile.mkdtemp() 63 | 64 | test_file = os.path.join(tmpdir, 'test.py') 65 | with open(test_file, 'w') as f: 66 | f.write(TEST_FAILING) 67 | 68 | # If we use --arraydiff, it should detect that the file is missing 69 | code = subprocess.call(f'pytest --arraydiff {test_file}', shell=True) 70 | assert code != 0 71 | 72 | # If we don't use --arraydiff option, the test should succeed 73 | code = subprocess.call(f'pytest {test_file}', shell=True) 74 | assert code == 0 75 | 76 | 77 | TEST_GENERATE = """ 78 | import pytest 79 | import numpy as np 80 | from astropy.io import fits 81 | @pytest.mark.array_compare(file_format='{file_format}') 82 | def test_gen(): 83 | return np.arange(6 * 5).reshape((6, 5)) 84 | """ 85 | 86 | 87 | @pytest.mark.parametrize('file_format', ('fits', 'text')) 88 | def test_generate(file_format): 89 | 90 | tmpdir = tempfile.mkdtemp() 91 | 92 | test_file = os.path.join(tmpdir, 'test.py') 93 | with open(test_file, 'w') as f: 94 | f.write(TEST_GENERATE.format(file_format=file_format)) 95 | 96 | gen_dir = os.path.join(tmpdir, 'spam', 'egg') 97 | 98 | # If we don't generate, the test will fail 99 | try: 100 | subprocess.check_output(['pytest', '--arraydiff', test_file], timeout=10) 101 | except subprocess.CalledProcessError as grepexc: 102 | assert b'File not found for comparison test' in grepexc.output 103 | 104 | # If we do generate, the test should succeed and a new file will appear 105 | code = subprocess.call(['pytest', f'--arraydiff-generate-path={gen_dir}', test_file], 106 | timeout=10) 107 | assert code == 0 108 | assert os.path.exists(os.path.join(gen_dir, 'test_gen.' + ('fits' if file_format == 'fits' else 'txt'))) 109 | 110 | 111 | TEST_DEFAULT = """ 112 | import pytest 113 | import numpy as np 114 | from astropy.io import fits 115 | @pytest.mark.array_compare 116 | def test_default(): 117 | return np.arange(6 * 5).reshape((6, 5)) 118 | """ 119 | 120 | 121 | @pytest.mark.parametrize('file_format', ('fits', 'text')) 122 | def test_default_format(file_format): 123 | 124 | tmpdir = tempfile.mkdtemp() 125 | 126 | test_file = os.path.join(tmpdir, 'test.py') 127 | with open(test_file, 'w') as f: 128 | f.write(TEST_DEFAULT) 129 | 130 | gen_dir = os.path.join(tmpdir, 'spam', 'egg') 131 | 132 | # If we do generate, the test should succeed and a new file will appear 133 | code = subprocess.call('pytest -s --arraydiff-default-format={}' 134 | ' --arraydiff-generate-path={} {}'.format(file_format, gen_dir, test_file), shell=True) 135 | assert code == 0 136 | assert os.path.exists(os.path.join(gen_dir, 'test_default.' + ('fits' if file_format == 'fits' else 'txt'))) 137 | 138 | 139 | @pytest.mark.array_compare(reference_dir=reference_dir, rtol=0.5, 140 | file_format='fits') 141 | def test_relative_tolerance(): 142 | # Scale up the output values by 1.5 to ensure the large `rtol` value is 143 | # needed. (The comparison file contains all 1.6.) 144 | return np.ones((3, 4)) * 1.6 * 1.5 145 | 146 | 147 | @pytest.mark.array_compare(reference_dir=reference_dir, atol=1.5, 148 | file_format='fits') 149 | def test_absolute_tolerance(): 150 | # Increase the output values by 1.4 to ensure the large `atol` value is 151 | # needed. (The comparison file contains all 1.6.) 152 | return np.ones((3, 4)) * 1.6 + 1.4 153 | 154 | 155 | @pytest.mark.array_compare( 156 | reference_dir=reference_dir, 157 | atol=1.5, 158 | file_format='fits', 159 | single_reference=True) 160 | @pytest.mark.parametrize('spam', ('egg', 'bacon')) 161 | def test_single_reference(spam): 162 | return np.ones((3, 4)) * 1.6 + 1.4 163 | 164 | 165 | class TestSingleReferenceClass: 166 | 167 | @pytest.mark.array_compare( 168 | reference_dir=reference_dir, 169 | atol=1.5, 170 | file_format='fits', 171 | single_reference=True) 172 | @pytest.mark.parametrize('spam', ('egg', 'bacon')) 173 | def test_single_reference(self, spam): 174 | return np.ones((3, 4)) * 1.6 + 1.4 175 | 176 | 177 | def test_nofile(): 178 | pass 179 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{38,39,310,311,312}-test{,-pytestoldest,-pytest52,-pytest53,-pytest60,-pytest61,-pytest62,-pytest70,-pytest71,-pytest72,-pytest73,-pytest74,-devdeps} 4 | codestyle 5 | requires = 6 | setuptools >= 30.3.0 7 | pip >= 19.3.1 8 | isolated_build = true 9 | 10 | [testenv] 11 | changedir = .tmp/{envname} 12 | setenv = 13 | devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple 14 | description = run tests 15 | deps = 16 | pytestoldest: pytest==5.0.0 17 | pytest52: pytest==5.2.* 18 | pytest53: pytest==5.3.* 19 | pytest60: pytest==6.0.* 20 | pytest61: pytest==6.1.* 21 | pytest62: pytest==6.2.* 22 | pytest70: pytest==7.0.* 23 | pytest71: pytest==7.1.* 24 | pytest72: pytest==7.2.* 25 | pytest73: pytest==7.3.* 26 | pytest74: pytest==7.4.* 27 | devdeps: git+https://github.com/pytest-dev/pytest#egg=pytest 28 | devdeps: numpy>=0.0.dev0 29 | devdeps: pandas>=0.0.dev0 30 | devdeps: pyerfa>=0.0.dev0 31 | devdeps: astropy>=0.0.dev0 32 | extras = 33 | test 34 | commands = 35 | # Force numpy-dev after something in the stack downgrades it 36 | devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy 37 | pip freeze 38 | pytest {toxinidir}/tests {posargs} 39 | pytest {toxinidir}/tests --arraydiff {posargs} 40 | 41 | [testenv:codestyle] 42 | skip_install = true 43 | changedir = {toxinidir} 44 | description = check code style, e.g. with flake8 45 | deps = flake8 46 | commands = flake8 pytest_arraydiff --count 47 | --------------------------------------------------------------------------------