├── .coveragerc ├── .flake8 ├── .gitattributes ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .mypy.ini ├── .pre-commit-config.yaml ├── CHANGELOG.rst ├── LICENSE.txt ├── LICENSES └── headers │ └── apache.txt ├── NOTICE ├── README.rst ├── doc ├── api.rst ├── changelog.rst ├── conf.py ├── fixtures.rst ├── how-to-use.rst ├── index.rst ├── post-processing.rst ├── requirements.txt ├── test-case.rst ├── test_executable.rst └── usage.rst ├── mpi_runner.sh ├── pylintrc ├── pyproject.toml ├── pytest.ini ├── report-conf ├── conf.py ├── generate_report.py └── index_template.rst ├── src └── pytest_executable │ ├── __init__.py │ ├── file_tools.py │ ├── plugin.py │ ├── report-db-schema.yaml │ ├── report.py │ ├── script_runner.py │ ├── settings.py │ ├── test-settings-schema.yaml │ ├── test-settings.yaml │ ├── test_executable.py │ └── yaml_helper.py ├── tests ├── __init__.py ├── conftest.py ├── data │ ├── collect_order │ │ ├── b │ │ │ ├── a │ │ │ │ └── test_aaa.py │ │ │ └── test-settings.yaml │ │ ├── test_a.py │ │ └── z │ │ │ ├── test-settings.yaml │ │ │ └── test_aa.py │ ├── find_references │ │ ├── ref-dir │ │ │ └── 0 │ │ │ │ ├── 1.prf │ │ │ │ └── dir │ │ │ │ └── 2.prf │ │ └── test-dir │ │ │ └── 0 │ │ │ └── dummy-file-for-git-to-store-the-directory-tree │ ├── report │ │ └── report_db.yaml │ ├── runners │ │ ├── error.sh │ │ ├── nproc.sh │ │ └── timeout.sh │ ├── shallow_dir_copy │ │ ├── dst_dir │ │ │ ├── dir │ │ │ │ └── file │ │ │ └── file │ │ └── src_dir │ │ │ ├── dir │ │ │ └── file │ │ │ ├── file │ │ │ └── file-to-ignore │ ├── test___init__ │ │ └── tests-inputs │ │ │ ├── case1 │ │ │ ├── test-settings.yaml │ │ │ └── test_dummy.py │ │ │ └── case2 │ │ │ ├── test-settings.yaml │ │ │ └── test_dummy.py │ ├── test_cli_check │ │ └── dummy-file-for-git-to-store-the-directory-tree │ ├── test_marks_from_yaml │ │ └── tests-inputs │ │ │ ├── a │ │ │ ├── __init__.py │ │ │ └── test_dummy.py │ │ │ ├── test-settings.yaml │ │ │ └── test_dummy.py │ ├── test_output_dir_fixture │ │ ├── tests-inputs │ │ │ └── case │ │ │ │ └── test-settings.yaml │ │ └── tests-output │ │ │ └── case │ │ │ └── dummy-file-for-git-to-store-the-directory-tree │ ├── test_regression_file_path_fixture │ │ ├── references │ │ │ └── case │ │ │ │ ├── 0.xmf │ │ │ │ └── 1.xmf │ │ └── tests-inputs │ │ │ ├── case-no-references │ │ │ └── test-settings.yaml │ │ │ └── case │ │ │ ├── test-settings.yaml │ │ │ └── test_fixture.py │ ├── test_regression_path_fixture │ │ ├── references │ │ │ └── case │ │ │ │ └── dummy-file-for-git-to-store-the-directory-tree │ │ └── tests-inputs │ │ │ └── case │ │ │ ├── test-settings.yaml │ │ │ └── test_fixture.py │ ├── test_report │ │ ├── report │ │ │ ├── generator-ko.sh │ │ │ └── generator.sh │ │ └── tests-inputs │ │ │ ├── case │ │ │ ├── description.rst │ │ │ └── test-settings.yaml │ │ │ └── empty-case │ │ │ └── dummy-file-for-git-to-store-the-directory-tree │ ├── test_runner_fixture │ │ ├── runner.sh │ │ ├── settings.yaml │ │ └── tests-inputs │ │ │ ├── case-global-settings │ │ │ └── test-settings.yaml │ │ │ └── case-local-settings │ │ │ └── test-settings.yaml │ └── test_tolerances_fixture │ │ └── tests-inputs │ │ └── case │ │ ├── test-settings.yaml │ │ └── test_fixture.py ├── test_file_tools.py ├── test_plugin │ ├── __init__.py │ ├── test_fixtures.py │ ├── test_misc.py │ └── test_report.py ├── test_report.py ├── test_script_runner.py └── test_settings.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | plugins = covdefaults 3 | source = pytest_executable 4 | 5 | [report] 6 | # Override covdefaults. 7 | fail_under = 90 8 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # See http://www.pydocstyle.org/en/latest/error_codes.html for more details. 3 | ignore = 4 | # D105 Missing docstring in magic method 5 | D105, 6 | # D107 Missing docstring in __init__: because we use google style docstring in class. 7 | D107, 8 | # D413 Missing blank line after last section: see above. 9 | D413, 10 | # E501 line too long, use bugbear warning instead, see https://github.com/psf/black#line-length 11 | E501, 12 | exclude = tests/data 13 | max-line-length = 88 14 | select = B,C,D,E,F,G,N,T,W,B950 15 | docstring-convention = google 16 | per-file-ignores = 17 | conf.py:D 18 | tests/**/*.py:D 19 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | pytest_executable/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - "*.rst" 7 | - "LICENSE**" 8 | - ".gitignore" 9 | pull_request: 10 | paths-ignore: 11 | - "*.rst" 12 | - "LICENSE**" 13 | - ".gitignore" 14 | 15 | jobs: 16 | test: 17 | runs-on: ${{ matrix.os }} 18 | strategy: 19 | matrix: 20 | os: [ubuntu-latest, macos-latest] 21 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Setup Python 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install tox 29 | run: pip install tox 30 | - name: Run tox 31 | # Run tox using the version of Python in `PATH`. 32 | run: tox -e py-coverage 33 | - name: Upload coverage to Codecov 34 | uses: codecov/codecov-action@v3 35 | with: 36 | files: ./coverage.xml 37 | verbose: true 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /src/*.egg-info 3 | /.tox 4 | -------------------------------------------------------------------------------- /.mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | strict = True 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v4.4.0 5 | hooks: 6 | - id: trailing-whitespace 7 | - id: end-of-file-fixer 8 | - id: check-yaml 9 | - id: check-added-large-files 10 | - id: check-json 11 | - id: check-toml 12 | - id: destroyed-symlinks 13 | - id: check-symlinks 14 | 15 | - repo: https://github.com/pre-commit/pygrep-hooks 16 | rev: v1.10.0 17 | hooks: 18 | - id: rst-backticks 19 | - id: rst-directive-colons 20 | - id: rst-inline-touching-normal 21 | 22 | - repo: https://github.com/PyCQA/autoflake 23 | rev: v2.2.1 24 | hooks: 25 | - id: autoflake 26 | args: [ 27 | --in-place, 28 | --remove-all-unused-imports, 29 | ] 30 | 31 | - repo: https://github.com/asottile/reorder_python_imports 32 | rev: v3.12.0 33 | hooks: 34 | - id: reorder-python-imports 35 | name: reorder python imports inside src 36 | args: [ 37 | --application-directories, 38 | src, 39 | --py38-plus, 40 | --add-import, 41 | "from __future__ import annotations", 42 | ] 43 | files: ^src 44 | - id: reorder-python-imports 45 | name: reorder python imports outside src 46 | exclude: ^src 47 | args: [ 48 | --py38-plus, 49 | --add-import, 50 | "from __future__ import annotations", 51 | ] 52 | 53 | - repo: https://github.com/myint/docformatter 54 | rev: v1.7.5 55 | hooks: 56 | - id: docformatter 57 | exclude: ^tests/.*$ 58 | args: [ 59 | --in-place, 60 | --black, 61 | ] 62 | 63 | - repo: https://github.com/asottile/pyupgrade 64 | rev: v3.14.0 65 | hooks: 66 | - id: pyupgrade 67 | args: [--py38-plus] 68 | 69 | - repo: https://github.com/psf/black 70 | rev: 23.9.1 71 | hooks: 72 | - id: black 73 | 74 | - repo: https://github.com/PyCQA/flake8 75 | rev: 6.1.0 76 | hooks: 77 | - id: flake8 78 | additional_dependencies: 79 | - flake8-bugbear==23.7.10 80 | - flake8-docstrings==1.7.0 81 | - flake8-logging-format==0.9.0 82 | - flake8-print==5.0.0 83 | - pep8-naming==0.13.3 84 | 85 | - repo: https://github.com/pre-commit/mirrors-mypy 86 | rev: v1.5.1 87 | hooks: 88 | - id: mypy 89 | exclude: ^(report-conf|tests|doc) 90 | additional_dependencies: 91 | - types-dataclasses 92 | - types-PyYAML 93 | - types-jsonschema 94 | - pytest 95 | - types-pytest-lazy-fixture 96 | 97 | - repo: https://github.com/Lucas-C/pre-commit-hooks 98 | rev: v1.5.4 99 | hooks: 100 | - id: insert-license 101 | name: insert apache license 102 | files: \.py$ 103 | args: 104 | - --license-filepath 105 | - LICENSES/headers/apache.txt 106 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | .. _`changelog`: 2 | 3 | Changelog 4 | ========= 5 | 6 | All notable changes to this project will be documented here. 7 | 8 | The format is based on `Keep a Changelog 9 | `_, and this project adheres to `Semantic 10 | Versioning `_. 11 | 12 | 0.5.5 - 2023-03-25 13 | ------------------ 14 | 15 | Fixed 16 | ~~~~~ 17 | - #24: removed deprecation warning ``PytestRemovedIn8Warning``. 18 | 19 | 20 | 0.5.4 - 2022-11-05 21 | ------------------ 22 | 23 | Fixed 24 | ~~~~~ 25 | - #22: support for pytest 7. 26 | 27 | Added 28 | ~~~~~ 29 | - Support for Python 3.11. 30 | 31 | Removed 32 | ~~~~~~~ 33 | - Support for Python 3.6. 34 | 35 | 0.5.3 - 2021-11-10 36 | ------------------ 37 | 38 | Added 39 | ~~~~~ 40 | - Support for Python 3.10. 41 | 42 | 0.5.2 - 2020-08-09 43 | ------------------ 44 | 45 | Fixed 46 | ~~~~~ 47 | - Typing issues. 48 | - #6: pytest 6 support. 49 | 50 | 0.5.1 - 2020-06-08 51 | ------------------ 52 | 53 | Fixed 54 | ~~~~~ 55 | - Bad version constraint on a dependency. 56 | 57 | 0.5.0 - 2020-06-05 58 | ------------------ 59 | 60 | Changed 61 | ~~~~~~~ 62 | - The name of the runner shell script in the output directories is the one 63 | passed to the CLI instead of the hardcoded one. 64 | - All the names of the CLI options have been prefixed with :option:`--exe-` to 65 | prevent name clashes with other plugins options. 66 | - It is easier to define the settings to execute the runner shell script for a 67 | test case thanks to a dedicated section in test-settings.yaml. 68 | - Rename *test_case.yaml* to *test-settings.yaml*. 69 | 70 | Added 71 | ~~~~~ 72 | - Testing on MacOS. 73 | - :option:`--exe-test-module` CLI option for setting the default test module 74 | - Add timeout setting for the runner execution. 75 | 76 | Removed 77 | ~~~~~~~ 78 | - The log files testing in the builtin test module. 79 | 80 | Fixed 81 | ~~~~~ 82 | - Tests execution order when a test module is in sub-directory of the yaml 83 | settings. 84 | - Marks of a test case not propagated to all test modules. 85 | 86 | 0.4.0 - 2020-05-03 87 | ------------------ 88 | 89 | Removed 90 | ~~~~~~~ 91 | - equal_nan option is too specific and can easily be added with a custom 92 | fixture. 93 | 94 | 0.3.1 - 2020-03-30 95 | ------------------ 96 | 97 | Added 98 | ~~~~~ 99 | - Report generation can handle a sphinx _static directory. 100 | 101 | 0.3.0 - 2020-03-19 102 | ------------------ 103 | 104 | Added 105 | ~~~~~ 106 | - How to use skip and xfail marks in the docs. 107 | - How to use a proxy with anaconda in the docs. 108 | - Better error message when :option:`--runner` do not get a script. 109 | 110 | Changed 111 | ~~~~~~~ 112 | - Placeholder in the runner script are compliant with bash (use {{}} instead of 113 | {}). 114 | - Report generation is done for all the tests at once and only requires a 115 | report generator script. 116 | 117 | Fixed 118 | ~~~~~ 119 | - #8393: check that :option:`--clean-output` and :option:`--overwrite-output` 120 | are not used both. 121 | - Output directory creation no longer fails when the input directory tree has 122 | one level. 123 | 124 | Removed 125 | ~~~~~~~ 126 | - Useless :option:`--nproc` command line argument, because this can be done 127 | with a custom default :file:`test_case.yaml` passed to the command line 128 | argument :option:`--default-settings`. 129 | 130 | 0.2.1 - 2020-01-14 131 | ------------------ 132 | 133 | Fixed 134 | ~~~~~ 135 | - #7043: skip regression tests when reference files are missing, no longer 136 | raise error. 137 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /LICENSES/headers/apache.txt: -------------------------------------------------------------------------------- 1 | Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | 3 | This file is part of pytest-executable 4 | https://www.github.com/CS-SI/pytest-executable 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | pytest-executable Copyright 2020 CS Systèmes d'Information 2 | 3 | This software is distributed under the Apache Software License (ASL) v2.0, see 4 | LICENSE file or http://www.apache.org/licenses/LICENSE-2.0 for details. 5 | 6 | 7 | This software includes code from many other open source projects. See below for 8 | details about the license of each project. 9 | 10 | ================================================================ 11 | The MIT License 12 | ================================================================ 13 | 14 | The following components are provided under the MIT License 15 | (http://www.opensource.org/licenses/mit-license.php). 16 | 17 | https://pyyaml.org 18 | https://pytest.org 19 | https://python-jsonschema.readthedocs.io 20 | https://github.com/rlgomes/delta 21 | 22 | ================================================================ 23 | The BSD License 24 | ================================================================ 25 | 26 | The following components are provided under the BSD License 27 | (https://opensource.org/licenses/bsd-license.php) 28 | 29 | https://palletsprojects.com/p/jinja 30 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Pytest-executable 2 | ================= 3 | 4 | .. inclusion-marker-do-not-remove 5 | 6 | .. image:: https://img.shields.io/pypi/l/pytest-executable.svg 7 | :target: `quick summary`_ 8 | 9 | .. image:: https://img.shields.io/pypi/v/pytest-executable.svg 10 | :target: https://pypi.org/project/pytest-executable 11 | 12 | .. image:: https://img.shields.io/conda/vn/conda-forge/pytest-executable 13 | :target: https://anaconda.org/conda-forge/pytest-executable 14 | 15 | .. image:: https://img.shields.io/pypi/pyversions/pytest-executable.svg 16 | 17 | .. image:: https://img.shields.io/badge/platform-linux%20%7C%20macos-lightgrey 18 | 19 | .. image:: https://img.shields.io/readthedocs/pytest-executable/stable 20 | :target: https://pytest-executable.readthedocs.io/en/stable/?badge=stable 21 | :alt: Read The Docs Status 22 | 23 | .. image:: https://img.shields.io/travis/CS-SI/pytest-executable/master 24 | :target: https://travis-ci.org/CS-SI/pytest-executable 25 | :alt: Travis-CI Build Status 26 | 27 | .. image:: https://img.shields.io/codecov/c/gh/CS-SI/pytest-executable/develop 28 | :target: https://codecov.io/gh/CS-SI/pytest-executable 29 | :alt: Codecov coverage report 30 | 31 | .. image:: https://img.shields.io/badge/code%20style-black-000000.svg 32 | :target: https://github.com/psf/black 33 | 34 | *pytest-executable* is a `pytest`_ plugin for simplifying the black-box 35 | testing of an executable, be it written in python or not. It helps to avoid 36 | writing the boilerplate test code to: 37 | 38 | - define the settings of a test case in a yaml file, 39 | - spawn a subprocess for running an executable, 40 | - reorder the tests properly either for a single test case or across several test cases, 41 | - handle the outputs and references directory trees, 42 | - provide convenient fixtures to customize the checking of the outcome of an executable. 43 | 44 | It integrates naturally with standard test scripts written for pytest. 45 | 46 | This plugin is originally intended for testing executables that create 47 | scientific data but it may hopefully be helpful for other kinds of executables. 48 | This project is still young, but already used in a professional environment. 49 | 50 | 51 | Documentation 52 | ------------- 53 | 54 | The project documentation and installation instructions are available `online`_. 55 | 56 | 57 | Contributing 58 | ------------ 59 | 60 | A contributing guide will be soon available (just a matter of free time). 61 | 62 | Please fill an issue on the `Github issue tracker`_ for any bug report, feature 63 | request or question. 64 | 65 | 66 | Authors 67 | ------- 68 | 69 | - `Antoine Dechaume`_ - *Project creator and maintainer* 70 | 71 | 72 | Copyright and License 73 | --------------------- 74 | 75 | Copyright 2020, `CS GROUP`_ 76 | 77 | *pytest-executable* is a free and open source software, distributed under the 78 | Apache License 2.0. See the `LICENSE.txt`_ file for more information, or the 79 | `quick summary`_ of this license on `tl;drLegal`_ website. 80 | 81 | 82 | .. _conda: https://docs.conda.io 83 | .. _pip: https://pip-installer.org 84 | .. _pytest: https://docs.pytest.org 85 | .. _online: https://pytest-executable.readthedocs.io 86 | .. _Github issue tracker: https://github.com/CS-SI/pytest-executable/issues 87 | .. _Antoine Dechaume: https://github.com/AntoineD 88 | .. _CS GROUP: http://www.csgroup.eu 89 | .. _`LICENSE.txt`: LICENSE.txt 90 | .. _quick summary: https://tldrlegal.com/license/apache-license-2.0-(apache-2.0) 91 | .. _tl;drLegal: https://tldrlegal.com 92 | -------------------------------------------------------------------------------- /doc/api.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | API documentation 19 | ================= 20 | 21 | Below are some of the classes API used by |ptx|. 22 | 23 | .. autoclass:: pytest_executable.script_runner.ScriptRunner 24 | :members: 25 | 26 | .. autoclass:: pytest_executable.settings.Tolerances 27 | -------------------------------------------------------------------------------- /doc/changelog.rst: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.rst -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # Configuration file for the Sphinx documentation builder. 18 | # 19 | # This file only contains a selection of the most common options. For a full 20 | # list see the documentation: 21 | # http://www.sphinx-doc.org/en/master/config 22 | # -- Path setup -------------------------------------------------------------- 23 | # If extensions (or modules to document with autodoc) are in another directory, 24 | # add these directories to sys.path here. If the directory is relative to the 25 | # documentation root, use os.path.abspath to make it absolute, like shown here. 26 | # 27 | from __future__ import annotations 28 | 29 | from sphinx.ext import autodoc 30 | 31 | # import os 32 | # import sys 33 | # sys.path.insert(0, os.path.abspath('.')) 34 | # -- Project information ----------------------------------------------------- 35 | 36 | project = "pytest-executable" 37 | copyright = "2019, C-S" 38 | author = "C-S" 39 | 40 | 41 | # -- General configuration --------------------------------------------------- 42 | 43 | # Add any Sphinx extension module names here, as strings. They can be 44 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 45 | # ones. 46 | extensions = [ 47 | "sphinx.ext.napoleon", 48 | "sphinxcontrib.spelling", 49 | ] 50 | 51 | # Add any paths that contain templates here, relative to this directory. 52 | templates_path = ["_templates"] 53 | 54 | # List of patterns, relative to source directory, that match files and 55 | # directories to ignore when looking for source files. 56 | # This pattern also affects html_static_path and html_extra_path. 57 | exclude_patterns = [] 58 | 59 | 60 | # -- Options for HTML output ------------------------------------------------- 61 | 62 | # The theme to use for HTML and HTML Help pages. See the documentation for 63 | # a list of builtin themes. 64 | # 65 | html_theme = "sphinx_rtd_theme" 66 | 67 | # Add any paths that contain custom static files (such as style sheets) here, 68 | # relative to this directory. They are copied after the builtin static files, 69 | # so a file named "default.css" will overwrite the builtin "default.css". 70 | html_static_path = ["_static"] 71 | 72 | 73 | # -- Extension configuration ------------------------------------------------- 74 | rst_prolog = """ 75 | .. _pytest: https://docs.pytest.org 76 | .. |ptx| replace:: *pytest-executable* 77 | .. |exe| replace:: :program:`executable` 78 | .. |pytest| replace:: `pytest`_ 79 | .. |yaml| replace:: :file:`test-settings.yaml` 80 | .. |runner| replace:: *runner shell script* 81 | """ 82 | 83 | 84 | # class DocsonlyMethodDocumenter(autodoc.MethodDocumenter): 85 | # def format_args(self): 86 | # return None 87 | # 88 | # autodoc.add_documenter(DocsonlyMethodDocumenter) 89 | 90 | 91 | class SimpleDocumenter(autodoc.MethodDocumenter): 92 | objtype = "simple" 93 | 94 | # do not indent the content 95 | content_indent = "" 96 | 97 | # do not add a header to the docstring 98 | def add_directive_header(self, sig): 99 | pass 100 | 101 | 102 | def setup(app): 103 | app.add_autodocumenter(SimpleDocumenter) 104 | -------------------------------------------------------------------------------- /doc/fixtures.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | .. _Path: https://docs.python.org/3/library/pathlib.html#basic-use 19 | .. _Sphinx: https://www.sphinx-doc.org 20 | 21 | .. _fixtures: 22 | 23 | Fixtures 24 | ======== 25 | 26 | The purpose of the test fixtures is to ease the writing of test functions by 27 | providing information and data automatically. You may find more documentation 28 | on |pytest| fixture in its `official documentation 29 | `_. We describe here the 30 | fixtures defined in |ptx|. Some of them are used in the default test module, 31 | see :ref:`builtin-test-module`. 32 | 33 | .. _fixture-runner: 34 | 35 | Runner fixture 36 | -------------- 37 | 38 | The :py:data:`runner` fixture is used to execute the |runner| passed with 39 | :option:`--exe-runner`. This fixture is an :py:class:`object 40 | ` which can execute the script 41 | with the :py:meth:`run` method. This method returns the exit status of the 42 | script execution. The value of the exit status shall be **0** when the 43 | execution is successful. 44 | 45 | When :option:`--exe-runner` is not set, a function that uses this fixture will 46 | be skipped. 47 | 48 | .. _fixture-output_path: 49 | 50 | Output path fixture 51 | ------------------- 52 | 53 | The :py:data:`output_path` fixture provides the absolute path to the output 54 | directory of a test case as a `Path`_ object. 55 | 56 | .. _regression-path-fixtures: 57 | 58 | Regression path fixture 59 | ----------------------- 60 | 61 | The :py:data:`regression_file_path` fixture provides the paths to the reference 62 | data of a test case. A test function that use this fixture is called once per 63 | reference item (file or directory) declared in the :ref:`yaml-ref` of a |yaml| 64 | (thanks to the `parametrize 65 | `_ feature). The 66 | :py:data:`regression_file_path` object has the attributes: 67 | 68 | - :py:attr:`relative`: a `Path`_ object that contains the path to a reference 69 | item relatively to the output directory of the test case. 70 | - :py:attr:`absolute`: a `Path`_ object that contains the absolute path to a 71 | reference item. 72 | 73 | If :option:`--exe-regression-root` is not set then a test function that uses 74 | the fixture is skipped. 75 | 76 | You may use this fixture with the :ref:`fixture-output_path` to get the path to 77 | an output file that shall be compared to a reference file. 78 | 79 | For instance, if a |yaml| under :file:`inputs/case` contains: 80 | 81 | .. code-block:: yaml 82 | 83 | references: 84 | - output/file 85 | - '**/*.txt' 86 | 87 | and if :option:`--exe-regression-root` is set to a directory :file:`references` 88 | that contains: 89 | 90 | .. code-block:: text 91 | 92 | references 93 | └── case 94 | ├── 0.txt 95 | └── output 96 | ├── a.txt 97 | └── file 98 | 99 | then a test function that uses the fixture will be called once per item of the 100 | following list: 101 | 102 | .. code-block:: py 103 | 104 | [ 105 | "references/case/output/file", 106 | "references/case/0.txt", 107 | "references/case/output/a.txt", 108 | ] 109 | 110 | and for each these items, the :py:data:`regression_file_path` is set as 111 | described above with the relative and absolute paths. 112 | 113 | .. _tolerances-fixtures: 114 | 115 | Tolerances fixture 116 | ------------------ 117 | 118 | The :py:data:`tolerances` fixture provides the contents of the :ref:`yaml-tol` 119 | of a |yaml| as a dictionary that maps names to :py:class:`Tolerances 120 | ` objects. 121 | 122 | For instance, if a |yaml| contains: 123 | 124 | .. code-block:: yaml 125 | 126 | tolerances: 127 | data-name1: 128 | abs: 1. 129 | data-name2: 130 | rel: 0. 131 | abs: 0. 132 | 133 | then the fixture object is such that: 134 | 135 | .. code-block:: py 136 | 137 | tolerances["data-name1"].abs = 1. 138 | tolerances["data-name1"].rel = 0. 139 | tolerances["data-name2"].abs = 0. 140 | tolerances["data-name2"].rel = 0. 141 | -------------------------------------------------------------------------------- /doc/how-to-use.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | Overview 19 | ======== 20 | 21 | Directory trees 22 | --------------- 23 | 24 | The |ptx| plugin deals with multiple directory trees: 25 | 26 | - the inputs 27 | - the outputs 28 | - the regression references 29 | 30 | The inputs tree contains the files required to run an |exe| and to check its 31 | outcomes for different settings. It is composed of test cases as directories at 32 | the leaves of the tree. To create a test case, see :ref:`add-test-case-label`. 33 | 34 | All the directory trees have the same hierarchy, this convention allows |ptx| 35 | to work out what to test and what to check. The outputs tree is automatically 36 | created by |ptx|, inside it, a test case directory typically contains: 37 | 38 | - symbolic links to the |exe| input files for the corresponding test case in 39 | the inputs tree 40 | - a |runner| to execute |exe| 41 | - the files produced by the execution of |exe| 42 | - eventually, the files produced by the additional test modules 43 | 44 | At the beginning, a regression reference tree is generally created from an 45 | existing outputs tree. In a regression references tree, a test case directory 46 | shall contain all the result files required for performing the comparisons for 47 | the regression testing. There can be more than one regression references trees 48 | for storing different sets of references, for instance for comparing the 49 | results against more than one version of |exe|. 50 | 51 | Execution order 52 | --------------- 53 | 54 | The |ptx| plugin will reorder the execution such that the |pytest| tests are 55 | executed in the following order: 56 | 57 | 1. in a test case, the tests defined in the default test module (see 58 | :option:`--exe-test-module`), 59 | 2. any other tests defined in a test case directory, with |pytest| natural 60 | order, 61 | 3. any other tests defined in the parent directories of a test case. 62 | 63 | The purposes of this order is to make sure that the |runner| and the other 64 | default tests are executed first before the tests in other modules can be 65 | performed on the outcome of the |exe|. It also allows to create test modules in 66 | the parent directory of several test cases to gather their outcomes. 67 | 68 | How to use 69 | ========== 70 | 71 | Run the |exe| only 72 | ------------------ 73 | 74 | :command:`pytest --exe-runner -k runner` 75 | 76 | This command will execute the |exe| for all the test cases that are found in 77 | the input tree under :file:`path/to/tests/inputs`. A test case is identified by 78 | a directory that contains a |yaml| file. For each of the test cases found, 79 | |ptx| will create an output directory with the same directory hierarchy and run 80 | the cases in that output directory. By default, the root directory of the 81 | output tree is :file:`tests-output`, this can be changed with the option 82 | :option:`--exe-output-root`. Finally, the :option:`-k runner` option instructs 83 | |pytest| to only execute the |runner| and nothing more, see :ref:`filter` for 84 | more information on doing only some of the processing. 85 | 86 | For instance, if the tests input tree contains:: 87 | 88 | path/to/tests/inputs 89 | ├── case-1 90 | │ ├── input 91 | │ └── test-settings.yaml 92 | └── case-2 93 | ├── input 94 | └── test-settings.yaml 95 | 96 | Then the tests output tree is:: 97 | 98 | tests-output 99 | ├── case-1 100 | │ ├── input -> path/to/tests/inputs/case-1/input 101 | │ ├── output 102 | │ ├── executable.stderr 103 | │ ├── executable.stdout 104 | │ ├── runner.sh 105 | │ ├── runner.sh.stderr 106 | │ └── runner.sh.stdout 107 | └── case-2 108 | ├── input -> path/to/tests/inputs/case-2/input 109 | ├── output 110 | ├── executable.stderr 111 | ├── executable.stdout 112 | ├── runner.sh 113 | ├── runner.sh.stderr 114 | └── runner.sh.stdout 115 | 116 | For a given test case, for instance :file:`tests-output/case-1`, 117 | the output directory contains: 118 | 119 | output 120 | the output file produced by the execution of the |exe|, in practice there 121 | can be any number of output files and directories produced. 122 | 123 | input 124 | a symbolic link to the file in the test input directory, in practice 125 | there can be any number of input files. 126 | 127 | executable.stderr 128 | contains the error messages from the |exe| execution 129 | 130 | executable.stdout 131 | contains the log messages from the |exe| execution 132 | 133 | runner.sh 134 | a copy of the |runner| defined with :option:`--exe-runner`, eventually 135 | modified by |ptx| for replacing the placeholders. Executing this script 136 | directly from a console shall produce the same results as when it is 137 | executed by |ptx|. This script is intended to be as much as possible 138 | independent of the execution context such that it can be executed 139 | independently of |ptx| in a reproducible way, i.e. it is self contained 140 | and does not depend on the shell context. 141 | 142 | runner.sh.stderr 143 | contains the error messages from the |runner| execution 144 | 145 | runner.sh.stdout 146 | contains the log messages from the |runner| execution 147 | 148 | If you need to manually run the |exe| for a test case, for debugging 149 | purposes for instance, just go to its output directory, for instance 150 | :command:`cd tests-output/case-1`, and execute the |runner|. 151 | 152 | 153 | Check regressions without running the |exe| 154 | ------------------------------------------- 155 | 156 | :command:`pytest --exe-regression-root --exe-overwrite-output` 157 | 158 | We assume that the |exe| results have already been produced for the test cases 159 | considered. This is not enough though because the output directory already 160 | exists and |ptx| will by default prevent the user from silently modifying any 161 | existing test output directories. In that case, the option 162 | :option:`--exe-overwrite-output` shall be used. The above command line will 163 | compare the results in the default output tree with the references, if the 164 | existing |exe| results are in a different directory then you need to add the 165 | path to it with :option:`--exe-output-root`. 166 | 167 | The option :option:`--exe-regression-root` points to the root directory with 168 | the regression references tree . This tree shall have the same hierarchy as the 169 | output tree but it only contains the results files that are used for doing the 170 | regression checks. 171 | 172 | 173 | Run the |exe| and do default regression checks 174 | ---------------------------------------------- 175 | 176 | :command:`pytest --exe-runner --exe-regression-root ` 177 | 178 | .. note:: 179 | 180 | Currently this can only be used when the |exe| execution is done on the same 181 | machine as the one that execute the regression checks, i.e. this will not 182 | work when the |exe| is executed on another machine. 183 | 184 | Finally, checks are done on the |exe| log files to verify that the file 185 | :file:`executable.stdout` exists and is not empty, and that the file 186 | :file:`executable.stderr` exists and is empty. 187 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | Welcome to |ptx| documentation! 19 | =============================== 20 | 21 | .. include:: ../README.rst 22 | :start-after: inclusion-marker-do-not-remove 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | :caption: Contents: 27 | 28 | usage 29 | how-to-use 30 | test-case 31 | post-processing 32 | fixtures 33 | test_executable 34 | api 35 | changelog 36 | 37 | 38 | Indices and tables 39 | ------------------ 40 | 41 | * :ref:`genindex` 42 | * :ref:`modindex` 43 | * :ref:`search` 44 | -------------------------------------------------------------------------------- /doc/post-processing.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | Add a post-processing 19 | ===================== 20 | 21 | This section show how to a add post-processing that will be run by |ptx|. 22 | 23 | 24 | Pytest functions 25 | ---------------- 26 | 27 | In a test case input directory, create a python module with a name starting 28 | by :file:`test_`. Then in that module, create |pytest| functions with a name 29 | starting by :data:`test_`. Those functions will be executed and |pytest| will 30 | catch the :py:data:`assert` statements to determine if the processing done by a 31 | function is considered as **passed** or **failed**. The outcome of a function 32 | could also be **skipped** if for some reason no assertion could be evaluated. 33 | If an exception is raised in a function, the function execution will be 34 | considered as **failed**. 35 | 36 | The functions are executed is a defined order: first by the test directory 37 | name, then by the module name and finally by the function name. The sorting is 38 | done by alphabetical order. There are 2 exceptions to this behavior: 39 | 40 | - the |yaml| file is always processes before all other modules in a given 41 | directory 42 | - a module in a parent directory is always run after the modules in the 43 | children directories, this allows for gathering the results from the 44 | children directories 45 | 46 | The |pytest| functions shall take advantages of the fixtures for automatically 47 | retrieved data from the execution context, such as the information stored in 48 | the |yaml| or the path to the current output directory. 49 | 50 | See :ref:`fixtures` for more information on fixtures. 51 | 52 | See :ref:`builtin-test-module` for |pytest| function examples. 53 | 54 | 55 | Best practices 56 | -------------- 57 | 58 | Script naming 59 | ~~~~~~~~~~~~~ 60 | 61 | If a post-processing script has the same name in different test case 62 | directories then each of those directories shall have a :file:`__init__.py` 63 | file so |pytest| can use them. 64 | 65 | 66 | External python module 67 | ~~~~~~~~~~~~~~~~~~~~~~ 68 | 69 | If you import an external python module in a |pytest| function, you shall use 70 | the following code snippet to prevent |pytest| from failing if the module is 71 | not available. 72 | 73 | .. code-block:: python 74 | 75 | pytest.importorskip('external_module', 76 | reason='skip test because external_module cannot be imported') 77 | from external_module import a_function, a_class 78 | 79 | If the external module is installed in an environment not compatible with the 80 | anaconda environment of |ptx|, then execute the module through a `subprocess 81 | `_ 82 | call. For instance: 83 | 84 | .. code-block:: python 85 | 86 | import subprocess 87 | command = 'python external_module.py' 88 | subprocess.run(command.split()) 89 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=2 2 | sphinx-autodoc-typehints 3 | sphinx_rtd_theme 4 | sphinxcontrib.spelling 5 | delta 6 | -------------------------------------------------------------------------------- /doc/test-case.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | .. _add-test-case-label: 19 | 20 | Add a test case 21 | =============== 22 | 23 | A test case is composed of an input directory with: 24 | 25 | - the input files required by the |runner|, 26 | - a |yaml| file with the |ptx| settings, 27 | - any optional |pytest| python modules for performing additional tests. 28 | 29 | .. warning:: 30 | 31 | The input directory of a test case shall not contain any of the files created by 32 | the execution of the |exe| or of the additional python modules, otherwise 33 | they may badly interfere with the executions done by |ptx|. In other words: 34 | do not run anything in the input directory of a test case, this directory 35 | shall only contain input data. 36 | 37 | The |yaml| file is used by |ptx| for several things. When this file is 38 | found, |ptx| will: 39 | 40 | 1. create the output directory of the test case and, if needed, its parents, 41 | 2. execute the tests defined in the default test module, 42 | 3. execute the tests defined in the additional test modules. 43 | 4. execute the tests defined in the parent directories. 44 | 45 | The parents of an output directory are created such that the path from the 46 | directory where |pytest| is executed to the input directory of the test case is 47 | the same but for the first parent. This way, the directories hierarchy below 48 | the first parent of both the inputs and the outputs trees are the same. 49 | 50 | If |yaml| is empty, then the default settings are used. If 51 | :option:`--exe-default-settings` is not set, the default settings are the 52 | builtin ones: 53 | 54 | .. literalinclude:: ../src/pytest_executable/test-settings.yaml 55 | 56 | The following gives a description of the contents of |yaml|. 57 | 58 | .. note:: 59 | 60 | If other settings not described below exist in |yaml|, they will be ignored 61 | by |ptx|. This means that you can use |yaml| to store settings for other 62 | purposes than |ptx|. 63 | 64 | .. _yaml-runner: 65 | 66 | Runner section 67 | -------------- 68 | 69 | The purpose of this section is to be able to precisely define how to run the 70 | |exe| for each test case. The *runner* section contains key-value pairs of 71 | settings to be used for replacing placeholders in the |runner| passed to 72 | :option:`--exe-runner`. For a key to be replaced, the |runner| shall contain 73 | the key between double curly braces. 74 | 75 | For instance, if |yaml| of a test case contains: 76 | 77 | .. code-block:: yaml 78 | 79 | runner: 80 | nproc: 10 81 | 82 | and the |runner| passed to :option:`--exe-runner` contains: 83 | 84 | .. code-block:: console 85 | 86 | mpirun -np {{nproc}} executable 87 | 88 | then this line in the actual |runner| used to run the test case will be: 89 | 90 | .. code-block:: console 91 | 92 | mpirun -np 10 executable 93 | 94 | The runner section may also contain the *timeout* key to set the maximum 95 | duration of the |runner| execution. When this duration is reached and if the 96 | execution is not finished then the execution is failed and likely the other 97 | tests that rely on the outcome of the |exe|. If *timeout* is not set then there 98 | is no duration limit. The duration can be expressed with one or more numbers 99 | followed by its unit and separated by a space, for instance: 100 | 101 | .. code-block:: yaml 102 | 103 | runner: 104 | timeout: 1h 2m 3s 105 | 106 | The available units are: 107 | 108 | - y, year, years 109 | - m, month, months 110 | - w, week, weeks 111 | - d, day, days 112 | - h, hour, hours 113 | - min, minute, minutes 114 | - s, second, seconds 115 | - ms, millis, millisecond, milliseconds 116 | 117 | .. _yaml-ref: 118 | 119 | Reference section 120 | ----------------- 121 | 122 | The reference files are used to check for regressions on the files created by 123 | the |exe|. Those checks can be done by comparing the files with a tolerance 124 | , see :ref:`yaml-tol`. The *references* section shall contain a list of paths 125 | to the files to be compared. A path shall be defined relatively to the test 126 | case output directory, it may use any shell pattern like :file:`**`, 127 | :file:`*`, :file:`?`, for instance: 128 | 129 | .. code-block:: yaml 130 | 131 | references: 132 | - output/file 133 | - '**/*.txt' 134 | 135 | Note that |ptx| does not know how to check for regression on files, you have to 136 | implement the |pytest| tests by yourself. To get the path to the references 137 | files in a test function, use the fixture :ref:`regression-path-fixtures`. 138 | 139 | .. _yaml-tol: 140 | 141 | Tolerances section 142 | ------------------ 143 | 144 | A tolerance is used to define how close shall be 2 data to be considered as 145 | equal. It can be used when checking for regression by comparing files, see 146 | :ref:`yaml-ref`. To set the tolerances for the data named *data-name1* and 147 | *data-name2*: 148 | 149 | .. code-block:: yaml 150 | 151 | tolerances: 152 | data-name1: 153 | abs: 1. 154 | data-name2: 155 | rel: 0. 156 | abs: 0. 157 | 158 | For a given name, if one of the tolerance value is not defined, like the 159 | **rel** one for the **data-name1**, then its value will be set to **0.**. 160 | 161 | Note that |ptx| does not know how to use a tolerance, you have to implement it 162 | by yourself in a |pytest| tests. To get the tolerance in a test function, use 163 | the :ref:`tolerances-fixtures`. 164 | 165 | .. _yaml-marks: 166 | 167 | Marks section 168 | ------------- 169 | 170 | A mark is a |pytest| feature that allows to select some of the tests to be 171 | executed, see :ref:`mark_usage`. This is how to add marks to a test case, for 172 | instance the **slow** and **big** marks: 173 | 174 | .. code-block:: yaml 175 | 176 | marks: 177 | - slow 178 | - big 179 | 180 | Such a declared mark will be set to all the test functions in the directory of 181 | a test case, either from the default test module or from an additional |pytest| 182 | module. 183 | 184 | You can also use the marks that already existing. In particular, the ``skip`` and 185 | ``xfail`` marks provided by |pytest| can be used. The ``skip`` mark tells pytest to 186 | record but not execute the built-in test events of a test case. The ``xfail`` 187 | mark tells pytest to expect that at least one of the built-in test events will 188 | fail. 189 | 190 | Marks declaration 191 | ----------------- 192 | 193 | The marks defined in all test cases shall be declared to |pytest| in order to 194 | be used. This is done in the file :file:`pytest.ini` that shall be created in 195 | the parent folder of the test inputs directory tree, where the |pytest| command 196 | is executed. This file shall have the format: 197 | 198 | .. code-block:: ini 199 | 200 | [pytest] 201 | markers = 202 | slow: one line explanation of what slow means 203 | big: one line explanation of what big means 204 | -------------------------------------------------------------------------------- /doc/test_executable.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | .. _builtin-test-module: 19 | 20 | Default test module 21 | =================== 22 | 23 | This is the default python module executed when the testing tool finds a 24 | |yaml|, this module can be used as an example for writing new test modules. 25 | 26 | .. literalinclude:: ../src/pytest_executable/test_executable.py 27 | -------------------------------------------------------------------------------- /doc/usage.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | .. _conda: https://docs.conda.io 19 | .. _pip: https://pip.pypa.io/en/stable/installing 20 | .. _report-conf: https://github.com/CS-SI/pytest-executable/tree/master/report-conf 21 | 22 | 23 | Installation 24 | ============ 25 | 26 | Install using `pip`_: 27 | 28 | .. code-block:: console 29 | 30 | pip install pytest-executable 31 | 32 | Install using `conda`_: 33 | 34 | .. code-block:: console 35 | 36 | conda install pytest-executable -c conda-forge 37 | 38 | 39 | Command line interface 40 | ====================== 41 | 42 | The |pytest| command line shall be executed from the directory that contains 43 | the inputs root directory. 44 | 45 | 46 | Plugin options 47 | -------------- 48 | 49 | .. option:: --exe-runner PATH 50 | 51 | use the shell script at PATH to run the |exe|. 52 | 53 | This shell script may contain placeholders, such as *{{output_path}}* or 54 | others defined in the :ref:`yaml-runner` of a |yaml|. A final |runner|, 55 | with replaced placeholders, is written in the output directory of a test 56 | case (*{{output_path}}* is set to this path). This final script is then 57 | executed before any other test functions of a test case. See 58 | :ref:`fixture-runner` for further information. 59 | 60 | If this option is not defined then the |runner| will not be executed, but 61 | all the other test functions will. 62 | 63 | A typical |runner| for running the |exe| with MPI could be: 64 | 65 | .. literalinclude:: ../mpi_runner.sh 66 | :language: bash 67 | 68 | .. option:: --exe-output-root PATH 69 | 70 | use PATH as the root for the output directory tree, default: tests-output 71 | 72 | .. option:: --exe-overwrite-output 73 | 74 | overwrite existing files in the output directories 75 | 76 | .. option:: --exe-clean-output 77 | 78 | clean the output directories before executing the tests 79 | 80 | .. option:: --exe-regression-root PATH 81 | 82 | use PATH as the root directory with the references for the regression 83 | testing, if omitted then the tests using the regression_path fixture will be 84 | skipped 85 | 86 | .. option:: --exe-default-settings PATH 87 | 88 | use PATH as the yaml file with the default test settings instead of the 89 | built-in ones 90 | 91 | .. option:: --exe-test-module PATH 92 | 93 | use PATH as the default test module instead of the built-in one 94 | 95 | .. option:: --exe-report-generator PATH 96 | 97 | use PATH as the script to generate the test report 98 | 99 | See :file:`generate_report.py` in the `report-conf`_ directory for an 100 | example of such a script. 101 | 102 | .. note:: 103 | 104 | The report generator script may require to install additional 105 | dependencies, such as sphinx, which are not install by the |ptx| plugin. 106 | 107 | 108 | .. _filter: 109 | 110 | Standard pytest options 111 | ----------------------- 112 | 113 | You can get all the standard command line options of |pytest| by executing 114 | :command:`pytest -h`. In particular, to run only some of the test cases in the 115 | inputs tree, or to execute only some of the test functions, you may use one of 116 | the following ways: 117 | 118 | Use multiple path patterns 119 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 120 | 121 | Instead of providing the path to the root of the inputs tree, you may 122 | provide the path to one or more of its sub-directories, for instance: 123 | 124 | :command:`pytest --exe-runner ` 125 | 126 | You may also use shell patterns (with ``*`` and ``?`` characters) in the paths 127 | like: 128 | 129 | :command:`pytest --exe-runner ` 130 | 131 | .. _mark_usage: 132 | 133 | Use marks 134 | ~~~~~~~~~ 135 | 136 | A test case could be assigned one or more marks in the |yaml| file, see 137 | :ref:`yaml-marks`. Use the :option:`-m` to execute only the test cases that 138 | match a given mark expression. A mark expression is a logical expression that 139 | combines marks and yields a truth value. For example, to run only the tests 140 | that have the mark1 mark but not the mark2 mark, use :option:`-m "mark1 and not 141 | mark2"`. The logical operator ``or`` could be used as well. 142 | 143 | Use sub-string expression 144 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 145 | 146 | Like the marks, any part (sub-string) of the name of a test case or of a test 147 | function can be used to filter what will be executed. For instance to only 148 | execute the tests that have the string ``transition`` anywhere in their name, use 149 | :option:`-k "transition"`. Or, to execute only the functions that have ``runner`` 150 | in their names, use :option:`-k "runner"`. Logical expressions could be used to 151 | combine more sub-strings as well. 152 | 153 | Process last failed tests only 154 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 155 | 156 | To only execute the tests that previously failed, use :option:`--last-failed`. 157 | 158 | Show the markers 159 | ~~~~~~~~~~~~~~~~ 160 | 161 | Use :option:`--markers` to show the available markers without executing the 162 | tests. 163 | 164 | Show the tests to be executed 165 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 166 | 167 | Use :option:`--collect-only` to show the test cases and the test events 168 | (functions) selected without executing them. You may combine this option with 169 | other options, like the one above to filter the test cases. 170 | -------------------------------------------------------------------------------- /mpi_runner.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | env=/path/to/env/settings 4 | exe=/path/to/executable 5 | 6 | source $env 7 | 8 | mpirun -np {{nproc}} \ 9 | $exe \ 10 | --options \ 11 | 1> executable.stdout \ 12 | 2> executable.stderr 13 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS,_version.py 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 21 | # number of processors available to use. 22 | jobs=0 23 | 24 | # Control the amount of potential inferred values when inferring a single 25 | # object. This can help the performance when dealing with large functions or 26 | # complex, nested conditions. 27 | limit-inference-results=100 28 | 29 | # List of plugins (as comma separated values of python modules names) to load, 30 | # usually to register additional checkers. 31 | load-plugins= 32 | 33 | # Pickle collected data for later comparisons. 34 | persistent=yes 35 | 36 | # Specify a configuration file. 37 | #rcfile= 38 | 39 | # When enabled, pylint would attempt to guess common misconfiguration and emit 40 | # user-friendly hints instead of false-positive error messages. 41 | suggestion-mode=yes 42 | 43 | # Allow loading of arbitrary C extensions. Extensions are imported into the 44 | # active Python interpreter and may run arbitrary code. 45 | unsafe-load-any-extension=no 46 | 47 | 48 | [MESSAGES CONTROL] 49 | 50 | # Only show warnings with the listed confidence levels. Leave empty to show 51 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 52 | confidence= 53 | 54 | # Disable the message, report, category or checker with the given id(s). You 55 | # can either give multiple identifiers separated by comma (,) or put this 56 | # option multiple times (only on the command line, not in the configuration 57 | # file where it should appear only once). You can also use "--disable=all" to 58 | # disable everything first and then reenable specific checks. For example, if 59 | # you want to run only the similarities checker, you can use "--disable=all 60 | # --enable=similarities". If you want to run only the classes checker, but have 61 | # no Warning level messages displayed, use "--disable=all --enable=classes 62 | # --disable=W". 63 | disable=print-statement, 64 | parameter-unpacking, 65 | unpacking-in-except, 66 | old-raise-syntax, 67 | backtick, 68 | long-suffix, 69 | old-ne-operator, 70 | old-octal-literal, 71 | import-star-module-level, 72 | non-ascii-bytes-literal, 73 | raw-checker-failed, 74 | bad-inline-option, 75 | locally-disabled, 76 | file-ignored, 77 | suppressed-message, 78 | useless-suppression, 79 | deprecated-pragma, 80 | use-symbolic-message-instead, 81 | apply-builtin, 82 | basestring-builtin, 83 | buffer-builtin, 84 | cmp-builtin, 85 | coerce-builtin, 86 | execfile-builtin, 87 | file-builtin, 88 | long-builtin, 89 | raw_input-builtin, 90 | reduce-builtin, 91 | standarderror-builtin, 92 | unicode-builtin, 93 | xrange-builtin, 94 | coerce-method, 95 | delslice-method, 96 | getslice-method, 97 | setslice-method, 98 | no-absolute-import, 99 | old-division, 100 | dict-iter-method, 101 | dict-view-method, 102 | next-method-called, 103 | metaclass-assignment, 104 | indexing-exception, 105 | raising-string, 106 | reload-builtin, 107 | oct-method, 108 | hex-method, 109 | nonzero-method, 110 | cmp-method, 111 | input-builtin, 112 | round-builtin, 113 | intern-builtin, 114 | unichr-builtin, 115 | map-builtin-not-iterating, 116 | zip-builtin-not-iterating, 117 | range-builtin-not-iterating, 118 | filter-builtin-not-iterating, 119 | using-cmp-argument, 120 | eq-without-hash, 121 | div-method, 122 | idiv-method, 123 | rdiv-method, 124 | exception-message-attribute, 125 | invalid-str-codec, 126 | sys-max-int, 127 | bad-python3-import, 128 | deprecated-string-function, 129 | deprecated-str-translate-call, 130 | deprecated-itertools-function, 131 | deprecated-types-field, 132 | next-method-defined, 133 | dict-items-not-iterating, 134 | dict-keys-not-iterating, 135 | dict-values-not-iterating, 136 | deprecated-operator-function, 137 | deprecated-urllib-function, 138 | xreadlines-attribute, 139 | deprecated-sys-function, 140 | exception-escape, 141 | comprehension-escape, 142 | too-few-public-methods, 143 | bad-continuation, 144 | line-too-long 145 | 146 | # Enable the message, report, category or checker with the given id(s). You can 147 | # either give multiple identifier separated by comma (,) or put this option 148 | # multiple time (only on the command line, not in the configuration file where 149 | # it should appear only once). See also the "--disable" option for examples. 150 | enable=c-extension-no-member 151 | 152 | 153 | [REPORTS] 154 | 155 | # Python expression which should return a note less than 10 (10 is the highest 156 | # note). You have access to the variables errors warning, statement which 157 | # respectively contain the number of errors / warnings messages and the total 158 | # number of statements analyzed. This is used by the global evaluation report 159 | # (RP0004). 160 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 161 | 162 | # Template used to display messages. This is a python new-style format string 163 | # used to format the message information. See doc for all details. 164 | #msg-template= 165 | 166 | # Set the output format. Available formats are text, parseable, colorized, json 167 | # and msvs (visual studio). You can also give a reporter class, e.g. 168 | # mypackage.mymodule.MyReporterClass. 169 | output-format=text 170 | 171 | # Tells whether to display a full report or only the messages. 172 | reports=no 173 | 174 | # Activate the evaluation score. 175 | score=yes 176 | 177 | 178 | [REFACTORING] 179 | 180 | # Maximum number of nested blocks for function / method body 181 | max-nested-blocks=5 182 | 183 | # Complete name of functions that never returns. When checking for 184 | # inconsistent-return-statements if a never returning function is called then 185 | # it will be considered as an explicit return statement and no message will be 186 | # printed. 187 | never-returning-functions=sys.exit 188 | 189 | 190 | [VARIABLES] 191 | 192 | # List of additional names supposed to be defined in builtins. Remember that 193 | # you should avoid defining new builtins when possible. 194 | additional-builtins= 195 | 196 | # Tells whether unused global variables should be treated as a violation. 197 | allow-global-unused-variables=yes 198 | 199 | # List of strings which can identify a callback function by name. A callback 200 | # name must start or end with one of those strings. 201 | callbacks=cb_, 202 | _cb 203 | 204 | # A regular expression matching the name of dummy variables (i.e. expected to 205 | # not be used). 206 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 207 | 208 | # Argument names that match this expression will be ignored. Default to name 209 | # with leading underscore. 210 | ignored-argument-names=_.*|^ignored_|^unused_ 211 | 212 | # Tells whether we should check for unused import in __init__ files. 213 | init-import=no 214 | 215 | # List of qualified module names which can have objects that can redefine 216 | # builtins. 217 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 218 | 219 | 220 | [MISCELLANEOUS] 221 | 222 | # List of note tags to take in consideration, separated by a comma. 223 | notes=FIXME, 224 | XXX 225 | # TODO 226 | 227 | 228 | [BASIC] 229 | 230 | # Naming style matching correct argument names. 231 | argument-naming-style=snake_case 232 | 233 | # Regular expression matching correct argument names. Overrides argument- 234 | # naming-style. 235 | #argument-rgx= 236 | 237 | # Naming style matching correct attribute names. 238 | attr-naming-style=snake_case 239 | 240 | # Regular expression matching correct attribute names. Overrides attr-naming- 241 | # style. 242 | #attr-rgx= 243 | 244 | # Bad variable names which should always be refused, separated by a comma. 245 | bad-names=foo, 246 | bar, 247 | baz, 248 | toto, 249 | tutu, 250 | tata 251 | 252 | # Naming style matching correct class attribute names. 253 | class-attribute-naming-style=any 254 | 255 | # Regular expression matching correct class attribute names. Overrides class- 256 | # attribute-naming-style. 257 | #class-attribute-rgx= 258 | 259 | # Naming style matching correct class names. 260 | class-naming-style=PascalCase 261 | 262 | # Regular expression matching correct class names. Overrides class-naming- 263 | # style. 264 | #class-rgx= 265 | 266 | # Naming style matching correct constant names. 267 | const-naming-style=UPPER_CASE 268 | 269 | # Regular expression matching correct constant names. Overrides const-naming- 270 | # style. 271 | #const-rgx= 272 | 273 | # Minimum line length for functions/classes that require docstrings, shorter 274 | # ones are exempt. 275 | docstring-min-length=-1 276 | 277 | # Naming style matching correct function names. 278 | function-naming-style=snake_case 279 | 280 | # Regular expression matching correct function names. Overrides function- 281 | # naming-style. 282 | #function-rgx= 283 | 284 | # Good variable names which should always be accepted, separated by a comma. 285 | good-names=i, 286 | j, 287 | k, 288 | ex, 289 | Run, 290 | _ 291 | 292 | # Include a hint for the correct naming format with invalid-name. 293 | include-naming-hint=no 294 | 295 | # Naming style matching correct inline iteration names. 296 | inlinevar-naming-style=any 297 | 298 | # Regular expression matching correct inline iteration names. Overrides 299 | # inlinevar-naming-style. 300 | #inlinevar-rgx= 301 | 302 | # Naming style matching correct method names. 303 | method-naming-style=snake_case 304 | 305 | # Regular expression matching correct method names. Overrides method-naming- 306 | # style. 307 | #method-rgx= 308 | 309 | # Naming style matching correct module names. 310 | module-naming-style=snake_case 311 | 312 | # Regular expression matching correct module names. Overrides module-naming- 313 | # style. 314 | #module-rgx= 315 | 316 | # Colon-delimited sets of names that determine each other's naming style when 317 | # the name regexes allow several styles. 318 | name-group= 319 | 320 | # Regular expression which should only match function or class names that do 321 | # not require a docstring. 322 | no-docstring-rgx=^_ 323 | 324 | # List of decorators that produce properties, such as abc.abstractproperty. Add 325 | # to this list to register other decorators that produce valid properties. 326 | # These decorators are taken in consideration only for invalid-name. 327 | property-classes=abc.abstractproperty 328 | 329 | # Naming style matching correct variable names. 330 | variable-naming-style=snake_case 331 | 332 | # Regular expression matching correct variable names. Overrides variable- 333 | # naming-style. 334 | #variable-rgx= 335 | 336 | 337 | [TYPECHECK] 338 | 339 | # List of decorators that produce context managers, such as 340 | # contextlib.contextmanager. Add to this list to register other decorators that 341 | # produce valid context managers. 342 | contextmanager-decorators=contextlib.contextmanager 343 | 344 | # List of members which are set dynamically and missed by pylint inference 345 | # system, and so shouldn't trigger E1101 when accessed. Python regular 346 | # expressions are accepted. 347 | generated-members= 348 | 349 | # Tells whether missing members accessed in mixin class should be ignored. A 350 | # mixin class is detected if its name ends with "mixin" (case insensitive). 351 | ignore-mixin-members=yes 352 | 353 | # Tells whether to warn about missing members when the owner of the attribute 354 | # is inferred to be None. 355 | ignore-none=yes 356 | 357 | # This flag controls whether pylint should warn about no-member and similar 358 | # checks whenever an opaque object is returned when inferring. The inference 359 | # can return multiple potential results while evaluating a Python object, but 360 | # some branches might not be evaluated, which results in partial inference. In 361 | # that case, it might be useful to still emit no-member and other checks for 362 | # the rest of the inferred objects. 363 | ignore-on-opaque-inference=yes 364 | 365 | # List of class names for which member attributes should not be checked (useful 366 | # for classes with dynamically set attributes). This supports the use of 367 | # qualified names. 368 | ignored-classes=optparse.Values,thread._local,_thread._local 369 | 370 | # List of module names for which member attributes should not be checked 371 | # (useful for modules/projects where namespaces are manipulated during runtime 372 | # and thus existing member attributes cannot be deduced by static analysis. It 373 | # supports qualified module names, as well as Unix pattern matching. 374 | ignored-modules= 375 | 376 | # Show a hint with possible names when a member name was not found. The aspect 377 | # of finding the hint is based on edit distance. 378 | missing-member-hint=yes 379 | 380 | # The minimum edit distance a name should have in order to be considered a 381 | # similar match for a missing member name. 382 | missing-member-hint-distance=1 383 | 384 | # The total number of similar names that should be taken in consideration when 385 | # showing a hint for a missing member. 386 | missing-member-max-choices=1 387 | 388 | 389 | [FORMAT] 390 | 391 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 392 | expected-line-ending-format= 393 | 394 | # Regexp for a line that is allowed to be longer than the limit. 395 | ignore-long-lines=^\s*(# )??$ 396 | 397 | # Number of spaces of indent required inside a hanging or continued line. 398 | indent-after-paren=4 399 | 400 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 401 | # tab). 402 | indent-string=' ' 403 | 404 | # Maximum number of characters on a single line. 405 | max-line-length=100 406 | 407 | # Maximum number of lines in a module. 408 | max-module-lines=1000 409 | 410 | # List of optional constructs for which whitespace checking is disabled. `dict- 411 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 412 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 413 | # `empty-line` allows space-only lines. 414 | no-space-check=trailing-comma, 415 | dict-separator 416 | 417 | # Allow the body of a class to be on the same line as the declaration if body 418 | # contains single statement. 419 | single-line-class-stmt=no 420 | 421 | # Allow the body of an if to be on the same line as the test if there is no 422 | # else. 423 | single-line-if-stmt=no 424 | 425 | 426 | [SPELLING] 427 | 428 | # Limits count of emitted suggestions for spelling mistakes. 429 | max-spelling-suggestions=4 430 | 431 | # Spelling dictionary name. Available dictionaries: none. To make it working 432 | # install python-enchant package.. 433 | spelling-dict= 434 | 435 | # List of comma separated words that should not be checked. 436 | spelling-ignore-words= 437 | 438 | # A path to a file that contains private dictionary; one word per line. 439 | spelling-private-dict-file= 440 | 441 | # Tells whether to store unknown words to indicated private dictionary in 442 | # --spelling-private-dict-file option instead of raising a message. 443 | spelling-store-unknown-words=no 444 | 445 | 446 | [LOGGING] 447 | 448 | # Format style used to check logging format string. `old` means using % 449 | # formatting, while `new` is for `{}` formatting. 450 | logging-format-style=old 451 | 452 | # Logging modules to check that the string format arguments are in logging 453 | # function parameter format. 454 | logging-modules=logging 455 | 456 | 457 | [SIMILARITIES] 458 | 459 | # Ignore comments when computing similarities. 460 | ignore-comments=yes 461 | 462 | # Ignore docstrings when computing similarities. 463 | ignore-docstrings=yes 464 | 465 | # Ignore imports when computing similarities. 466 | ignore-imports=no 467 | 468 | # Minimum lines number of a similarity. 469 | min-similarity-lines=4 470 | 471 | 472 | [CLASSES] 473 | 474 | # List of method names used to declare (i.e. assign) instance attributes. 475 | defining-attr-methods=__init__, 476 | __new__, 477 | setUp 478 | 479 | # List of member names, which should be excluded from the protected access 480 | # warning. 481 | exclude-protected=_asdict, 482 | _fields, 483 | _replace, 484 | _source, 485 | _make 486 | 487 | # List of valid names for the first argument in a class method. 488 | valid-classmethod-first-arg=cls 489 | 490 | # List of valid names for the first argument in a metaclass class method. 491 | valid-metaclass-classmethod-first-arg=cls 492 | 493 | 494 | [DESIGN] 495 | 496 | # Maximum number of arguments for function / method. 497 | max-args=5 498 | 499 | # Maximum number of attributes for a class (see R0902). 500 | max-attributes=7 501 | 502 | # Maximum number of boolean expressions in an if statement. 503 | max-bool-expr=5 504 | 505 | # Maximum number of branch for function / method body. 506 | max-branches=12 507 | 508 | # Maximum number of locals for function / method body. 509 | max-locals=15 510 | 511 | # Maximum number of parents for a class (see R0901). 512 | max-parents=7 513 | 514 | # Maximum number of public methods for a class (see R0904). 515 | max-public-methods=20 516 | 517 | # Maximum number of return / yield for function / method body. 518 | max-returns=6 519 | 520 | # Maximum number of statements in function / method body. 521 | max-statements=50 522 | 523 | # Minimum number of public methods for a class (see R0903). 524 | min-public-methods=2 525 | 526 | 527 | [IMPORTS] 528 | 529 | # Allow wildcard imports from modules that define __all__. 530 | allow-wildcard-with-all=no 531 | 532 | # Analyse import fallback blocks. This can be used to support both Python 2 and 533 | # 3 compatible code, which means that the block might have code that exists 534 | # only in one or another interpreter, leading to false positives when analysed. 535 | analyse-fallback-blocks=no 536 | 537 | # Deprecated modules which should not be used, separated by a comma. 538 | deprecated-modules=optparse,tkinter.tix 539 | 540 | # Create a graph of external dependencies in the given file (report RP0402 must 541 | # not be disabled). 542 | ext-import-graph= 543 | 544 | # Create a graph of every (i.e. internal and external) dependencies in the 545 | # given file (report RP0402 must not be disabled). 546 | import-graph= 547 | 548 | # Create a graph of internal dependencies in the given file (report RP0402 must 549 | # not be disabled). 550 | int-import-graph= 551 | 552 | # Force import order to recognize a module as part of the standard 553 | # compatibility libraries. 554 | known-standard-library= 555 | 556 | # Force import order to recognize a module as part of a third party library. 557 | known-third-party=enchant 558 | 559 | 560 | [EXCEPTIONS] 561 | 562 | # Exceptions that will emit a warning when being caught. Defaults to 563 | # "Exception". 564 | overgeneral-exceptions=Exception 565 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=45", 4 | "setuptools_scm[toml]>=6.2", 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | 8 | [project] 9 | name = "pytest-executable" 10 | description = "pytest plugin for testing executables" 11 | readme = "README.rst" 12 | authors = [ 13 | {name = "Antoine Dechaume"}, 14 | ] 15 | classifiers = [ 16 | "Environment :: Console", 17 | "Framework :: Pytest", 18 | "License :: OSI Approved :: Apache Software License", 19 | "Operating System :: MacOS", 20 | "Operating System :: POSIX :: Linux", 21 | "Programming Language :: Python", 22 | "Programming Language :: Python :: 3", 23 | "Programming Language :: Python :: 3 :: Only", 24 | "Programming Language :: Python :: 3.8", 25 | "Programming Language :: Python :: 3.9", 26 | "Programming Language :: Python :: 3.10", 27 | "Programming Language :: Python :: 3.11", 28 | "Programming Language :: Python :: 3.12", 29 | "Topic :: Software Development :: Libraries", 30 | "Topic :: Software Development :: Testing", 31 | "Topic :: Utilities", 32 | ] 33 | dynamic = ["version"] 34 | requires-python = ">=3.8,<3.13" 35 | dependencies = [ 36 | "delta >=0.4,<0.5", 37 | "jinja2 >=2.7,<3.2", 38 | "jsonschema >=2,<5", 39 | "pytest >=5,<8", 40 | "pyyaml >=3,<6.1", 41 | ] 42 | license = {text = "Apache Software License 2.0"} 43 | 44 | [project.urls] 45 | Homepage = "https://www.github.com/CS-SI/pytest-executable" 46 | 47 | [project.optional-dependencies] 48 | test = [ 49 | "covdefaults", 50 | "pytest-cov", 51 | ] 52 | 53 | [project.entry-points.pytest11] 54 | pytest_executable = "pytest_executable.plugin" 55 | 56 | [tool.setuptools_scm] 57 | 58 | [tool.setuptools] 59 | license-files = [ 60 | "LICENSE.txt", 61 | "CREDITS.md", 62 | ] 63 | 64 | [tool.black] 65 | target-version = ['py38'] 66 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | addopts = --ignore tests/data 4 | filterwarnings = 5 | ignore::pytest.PytestExperimentalApiWarning 6 | -------------------------------------------------------------------------------- /report-conf/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # Configuration file for the Sphinx documentation builder. 19 | # 20 | # This file does only contain a selection of the most common options. For a 21 | # full list see the documentation: 22 | # http://www.sphinx-doc.org/en/master/config 23 | # -- Path setup -------------------------------------------------------------- 24 | # If extensions (or modules to document with autodoc) are in another directory, 25 | # add these directories to sys.path here. If the directory is relative to the 26 | # documentation root, use os.path.abspath to make it absolute, like shown here. 27 | # 28 | # import os 29 | # import sys 30 | # sys.path.insert(0, os.path.abspath('.')) 31 | from __future__ import annotations 32 | 33 | 34 | def setup(app): 35 | app.add_css_file("html_width.css") 36 | 37 | 38 | # -- Project information ----------------------------------------------------- 39 | 40 | project = "Validation Case" 41 | copyright = "" 42 | author = "" 43 | 44 | # The short X.Y version 45 | version = "" 46 | # The full version, including alpha/beta/rc tags 47 | release = "V1.0" 48 | 49 | # -- General configuration --------------------------------------------------- 50 | 51 | # If your documentation needs a minimal Sphinx version, state it here. 52 | # 53 | # needs_sphinx = '1.0' 54 | 55 | # Add any Sphinx extension module names here, as strings. They can be 56 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 57 | # ones. 58 | extensions = [ 59 | "sphinx.ext.autodoc", 60 | "sphinx.ext.mathjax", 61 | ] 62 | mathjax_path = ( 63 | "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" 64 | "?config=TeX-AMS_CHTML-full" 65 | ) 66 | 67 | # extensions = ['sphinxcontrib.katex'] 68 | 69 | 70 | # Add any paths that contain templates here, relative to this directory. 71 | templates_path = ["_templates"] 72 | 73 | # The suffix(es) of source filenames. 74 | # You can specify multiple suffix as a list of string: 75 | # 76 | # source_suffix = ['.rst', '.md'] 77 | source_suffix = ".rst" 78 | 79 | # The master toctree document. 80 | master_doc = "index" 81 | 82 | # The language for content autogenerated by Sphinx. Refer to documentation 83 | # for a list of supported languages. 84 | # 85 | # This is also used if you do content translation via gettext catalogs. 86 | # Usually you set "language" from the command line for these cases. 87 | language = None 88 | 89 | # List of patterns, relative to source directory, that match files and 90 | # directories to ignore when looking for source files. 91 | # This pattern also affects html_static_path and html_extra_path. 92 | exclude_patterns = [] 93 | 94 | # The name of the Pygments (syntax highlighting) style to use. 95 | pygments_style = None 96 | 97 | # -- Options for HTML output ------------------------------------------------- 98 | 99 | # The theme to use for HTML and HTML Help pages. See the documentation for 100 | # a list of builtin themes. 101 | # 102 | 103 | html_theme = "sphinx_rtd_theme" 104 | html_theme_path = [ 105 | "_themes", 106 | ] 107 | html_theme_options = { 108 | "canonical_url": "", 109 | "analytics_id": "UA-XXXXXXX-1", # Provided by Google in your dashboard 110 | "display_version": True, 111 | "prev_next_buttons_location": "bottom", 112 | "style_external_links": False, 113 | "style_nav_header_background": "white", 114 | # Toc options 115 | "collapse_navigation": True, 116 | "sticky_navigation": True, 117 | "navigation_depth": 4, 118 | "includehidden": True, 119 | "titles_only": False, 120 | } 121 | 122 | # Theme options are theme-specific and customize the look and feel of a theme 123 | # further. For a list of options available for each theme, see the 124 | # documentation. 125 | # 126 | # Add any paths that contain custom static files (such as style sheets) here, 127 | # relative to this directory. They are copied after the builtin static files, 128 | # so a file named "default.css" will overwrite the builtin "default.css". 129 | html_static_path = ["_static"] 130 | 131 | # Custom sidebar templates, must be a dictionary that maps document names 132 | # to template names. 133 | # 134 | # The default sidebars (for documents that don't match any pattern) are 135 | # defined by theme itself. Builtin themes are using these templates by 136 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 137 | # 'searchbox.html']``. 138 | # 139 | # html_sidebars = {} 140 | 141 | numfig = True 142 | 143 | # -- Options for HTMLHelp output --------------------------------------------- 144 | 145 | # Output file base name for HTML help builder. 146 | htmlhelp_basename = "Validation Case" 147 | 148 | # -- Options for LaTeX output ------------------------------------------------ 149 | 150 | latex_elements = { 151 | # The paper size ('letterpaper' or 'a4paper'). 152 | # 153 | # 'papersize': 'letterpaper', 154 | # The font size ('10pt', '11pt' or '12pt'). 155 | # 156 | # 'pointsize': '10pt', 157 | # Additional stuff for the LaTeX preamble. 158 | # 159 | # 'preamble': '', 160 | # Latex figure (float) alignment 161 | # 162 | # 'figure_align': 'htbp', 163 | } 164 | 165 | # Grouping the document tree into LaTeX files. List of tuples 166 | # (source start file, target name, title, 167 | # author, documentclass [howto, manual, or own class]). 168 | latex_documents = [ 169 | (master_doc, "Validation_case.tex", "Validation case", " ", "manual"), 170 | ] 171 | 172 | # -- Options for manual page output ------------------------------------------ 173 | 174 | # One entry per manual page. List of tuples 175 | # (source start file, name, description, authors, manual section). 176 | man_pages = [(master_doc, "validation_case", "Validation case", [author], 1)] 177 | 178 | # -- Options for Texinfo output ---------------------------------------------- 179 | 180 | # Grouping the document tree into Texinfo files. List of tuples 181 | # (source start file, target name, title, author, 182 | # dir menu entry, description, category) 183 | texinfo_documents = [ 184 | ( 185 | master_doc, 186 | "Validation_case", 187 | "Validation case", 188 | author, 189 | "Validation_case", 190 | "One line description of project.", 191 | "Miscellaneous", 192 | ), 193 | ] 194 | 195 | # -- Options for Epub output ------------------------------------------------- 196 | 197 | # Bibliographic Dublin Core info. 198 | epub_title = project 199 | 200 | # The unique identifier of the text. This can be a ISBN. 201 | # or the project homepage. 202 | # 203 | # epub_identifier = '' 204 | 205 | # A unique identification for the text. 206 | # 207 | # epub_uid = '' 208 | 209 | # A list of files that should not be packed into the epub file. 210 | epub_exclude_files = ["search.html"] 211 | 212 | # -- Extension configuration ------------------------------------------------- 213 | -------------------------------------------------------------------------------- /report-conf/generate_report.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 3 | # 4 | # This file is part of pytest-executable 5 | # https://www.github.com/CS-SI/pytest-executable 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | """This module generates the test report. 19 | 20 | It shall be called from the test output root directory where the tests report 21 | database file is (tests_report_db.yaml). Under the output root directory, it 22 | will create a: 23 | - the index.rst file from the index_template.rst, 24 | - the directory named report that contains the generated report. 25 | 26 | In the template index, the summary table and the table of content are inserted. 27 | A line of the summary table contains a case: 28 | - path relative to the output root directory, 29 | - test status (passed, failed, error) 30 | - test messages when the status is not passed 31 | The table of content contains the path to the description.rst files relative to 32 | the output root directory. 33 | 34 | This module requires the packages tabulate and sphinx, these could be installed 35 | with the command: conda install tabluate sphinx 36 | """ 37 | from __future__ import annotations 38 | 39 | import shutil 40 | import subprocess 41 | import textwrap 42 | from pathlib import Path 43 | 44 | import yaml 45 | from tabulate import tabulate 46 | 47 | # indentation used in the toc tree of the index template 48 | TOCTREE_INDENTATION = " " 49 | # name of the directory under the test output directory that will contain the 50 | # generated report 51 | REPORT_OUTPUT_DIRNAME = "report" 52 | # description file name in each test case output directory 53 | DESCRIPTION_FILENAME = "description.rst" 54 | # the directory that contains the current module 55 | DOC_GENERATOR_DIRPATH = Path(__file__).parent 56 | # sphinx documentation builder type 57 | DOC_BUILDER = "html" 58 | # name of the test report database 59 | REPORT_DB_FILENAME = "tests_report_db.yaml" 60 | # template file for creating the index.rst 61 | INDEX_TEMPLATE_RST = "index_template.rst" 62 | 63 | 64 | def create_summary_table(output_root: Path) -> str: 65 | """Create the summary table in rst. 66 | 67 | The summary table is sorted alphabetically. 68 | 69 | Args: 70 | output_root: Path to the test results output directory. 71 | 72 | Returns: 73 | The summary table string in rst format. 74 | """ 75 | with (output_root / REPORT_DB_FILENAME).open() as stream: 76 | report_db = yaml.safe_load(stream) 77 | 78 | report_data: list[tuple[str]] = [] 79 | for case, data in report_db.items(): 80 | messages = "\n".join(data.get("messages", [])) 81 | report_data += [(case, data["status"], messages)] 82 | 83 | return tabulate( 84 | sorted(report_data), headers=["Case", "Status", "Messages"], tablefmt="rst" 85 | ) 86 | 87 | 88 | def create_index_rst(output_root: Path) -> None: 89 | """Create the index.rst. 90 | 91 | Args: 92 | output_root: Path to the test results output directory. 93 | """ 94 | # check that the output directory exists 95 | output_root = Path(output_root).resolve(True) 96 | 97 | # find the paths to the description rst files relatively to the output_root 98 | description_paths: list[Path] = [] 99 | for path in output_root.glob(f"**/{DESCRIPTION_FILENAME}"): 100 | description_paths += [path.relative_to(output_root)] 101 | 102 | summary_table = create_summary_table(output_root) 103 | 104 | # the toc tree is sorted alphabetically like the summary table 105 | toctree_cases = textwrap.indent( 106 | "\n".join(map(str, sorted(description_paths))), TOCTREE_INDENTATION 107 | ) 108 | 109 | # read the index template 110 | index_template_path = DOC_GENERATOR_DIRPATH / INDEX_TEMPLATE_RST 111 | with index_template_path.open() as stream: 112 | index_rst_template = stream.read() 113 | 114 | # replace the placeholders in the template 115 | index_rst = index_rst_template.format( 116 | summary_table=summary_table, toctree_cases=toctree_cases 117 | ) 118 | 119 | # write the final index.rst 120 | index_path = output_root / "index.rst" 121 | with index_path.open("w") as stream: 122 | stream.write(index_rst) 123 | 124 | 125 | if __name__ == "__main__": 126 | # common working directory shall be the test output root directory 127 | output_root = Path.cwd() 128 | 129 | # create the report index.rst 130 | create_index_rst(output_root) 131 | # copy the _static directory if it exists 132 | static_path = DOC_GENERATOR_DIRPATH / "_static" 133 | if static_path.exists(): 134 | shutil.copytree(static_path, output_root / "_static") 135 | 136 | # command line to build the report 137 | cmd = ( 138 | f"sphinx-build -b {DOC_BUILDER} -c {DOC_GENERATOR_DIRPATH} " 139 | f"{output_root} {output_root}/{REPORT_OUTPUT_DIRNAME}" 140 | ) 141 | 142 | # build the report 143 | subprocess.run(cmd.split(), check=True) 144 | -------------------------------------------------------------------------------- /report-conf/index_template.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2020 CS Systemes d'Information, http://www.c-s.fr 2 | .. 3 | .. This file is part of pytest-executable 4 | .. https://www.github.com/CS-SI/pytest-executable 5 | .. 6 | .. Licensed under the Apache License, Version 2.0 (the "License"); 7 | .. you may not use this file except in compliance with the License. 8 | .. You may obtain a copy of the License at 9 | .. 10 | .. http://www.apache.org/licenses/LICENSE-2.0 11 | .. 12 | .. Unless required by applicable law or agreed to in writing, software 13 | .. distributed under the License is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | 18 | Welcome to the test report 19 | ========================== 20 | 21 | {summary_table} 22 | 23 | .. toctree:: 24 | :maxdepth: 2 25 | :hidden: 26 | :caption: Contents: 27 | 28 | {toctree_cases} 29 | -------------------------------------------------------------------------------- /src/pytest_executable/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Entry point into pytest_executable tools. 18 | 19 | This module contains function that do not depend on pytest to allow easier testing. 20 | """ 21 | from __future__ import annotations 22 | -------------------------------------------------------------------------------- /src/pytest_executable/file_tools.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Output directory tree creation functions.""" 18 | from __future__ import annotations 19 | 20 | import logging 21 | import shutil 22 | from dataclasses import dataclass 23 | from pathlib import Path 24 | from typing import Iterable 25 | 26 | LOG = logging.getLogger(__name__) 27 | 28 | 29 | @dataclass 30 | class FilePath: 31 | """Relative and absolute file path. 32 | 33 | Attributes: 34 | relative: Relative path to a file. 35 | absolute: Absolute path to a file. 36 | """ 37 | 38 | absolute: Path 39 | relative: Path 40 | 41 | 42 | def get_mirror_path(path_from: Path, path_to: Path) -> Path: 43 | """Return the mirror path from a path to another one. 44 | 45 | The mirrored path is determined from the current working directory (cwd) 46 | and the path_from. The path_from shall be under the cwd. The mirrored 47 | relative path is the part of path_from that has no common ancestor with 48 | path_to, without its first parent. For example, given cwd, if path_from is 49 | c/d/e and path_to is /x/y, then it returns /x/y/d/e (no common ancestor 50 | between path_from and path_to). If path_to is c/x/y, then cwd/c/x/y/e is 51 | returned. 52 | 53 | Args: 54 | path_from: Path to be mirrored from. 55 | path_to: Root path to be mirrored to. 56 | 57 | Returns: 58 | Mirror path. 59 | 60 | Raises: 61 | ValueError: If path_from is not under the cwd. 62 | """ 63 | cwd = str(Path.cwd().resolve()) 64 | path_from = path_from.resolve(True) 65 | 66 | try: 67 | relative_from = path_from.relative_to(cwd) 68 | except ValueError: 69 | # re-raise with better message 70 | msg = ( 71 | f"the current working directory {cwd} shall be a parent directory of " 72 | f"the inputs directory {path_from}" 73 | ) 74 | raise ValueError(msg) 75 | 76 | # number of directory levels to skip for determining the mirrored relative 77 | # path 78 | offset = 1 79 | 80 | try: 81 | relative_to = path_to.relative_to(cwd) 82 | except ValueError: 83 | pass 84 | else: 85 | # find the common path part between from and to 86 | for part_from, part_to in zip(relative_from.parts, relative_to.parts): 87 | if part_from == part_to: 88 | offset += 1 89 | 90 | return path_to.joinpath(*relative_from.parts[offset:]) 91 | 92 | 93 | def find_references(ref_dir: Path, ref_files: Iterable[str]) -> list[FilePath]: 94 | """Return the paths to the references files. 95 | 96 | Args: 97 | ref_dir: Path to a case directory under the references tree. 98 | ref_files: Path patterns to the references files. 99 | 100 | Returns: 101 | Absolute and relative paths from a reference case directory to the 102 | reference files. 103 | """ 104 | abs_paths: list[Path] = [] 105 | for ref in ref_files: 106 | abs_paths += ref_dir.glob(ref) 107 | 108 | if not abs_paths: 109 | return [] 110 | 111 | file_paths: list[FilePath] = [] 112 | for abs_path in abs_paths: 113 | rel_path = abs_path.relative_to(ref_dir) 114 | file_paths += [FilePath(abs_path, rel_path)] 115 | 116 | return file_paths 117 | 118 | 119 | def create_output_directory( 120 | src_dir: Path, 121 | dst_dir: Path, 122 | check_dst: bool, 123 | clean_dst: bool, 124 | ignored_files: Iterable[str], 125 | ) -> None: 126 | """Create a directory copy with symbolic links. 127 | 128 | The destination directory is created if it does not exist or if clean_dst 129 | is true. Only the specified input files will be symlinked, the other files 130 | will be ignored. 131 | 132 | Args: 133 | src_dir: Path to the source directory. 134 | dst_dir: Path to the destination directory. 135 | check_dst: Whether to check that destination exists. 136 | clean_dst: Whether to remove an existing destination. 137 | ignored_files: Files to be ignored when creating the destination 138 | directory. 139 | 140 | Raises: 141 | FileExistsError: If the destination directory exists when check_dst is 142 | true and clean_dst is false. 143 | """ 144 | # destination checking: force erasing or fail 145 | if check_dst and dst_dir.is_dir(): 146 | if clean_dst: 147 | LOG.debug("removing output directory %s", dst_dir) 148 | shutil.rmtree(dst_dir) 149 | else: 150 | raise FileExistsError 151 | 152 | LOG.debug("creating a shallow copy from %s to %s", src_dir, dst_dir) 153 | _shallow_dir_copy(src_dir, dst_dir, ignored_files) 154 | 155 | 156 | def _shallow_dir_copy( 157 | src_dir: Path, dst_dir: Path, ignored_files: Iterable[str] 158 | ) -> None: 159 | """Shallow copy a directory tree. 160 | 161 | Directories are duplicated, files are symlinked. The destination directory 162 | shall not exist and will be created. 163 | 164 | Args: 165 | src_dir: Path to the source directory. 166 | dst_dir: Path to the destination directory. 167 | ignored_files: Files to be ignored. 168 | """ 169 | dst_dir.mkdir(parents=True, exist_ok=True) 170 | for src_entry in src_dir.iterdir(): 171 | dst_entry = dst_dir / src_entry.name 172 | if src_entry.name in ignored_files: 173 | pass 174 | elif src_entry.is_dir(): 175 | # directories are not symlinked but created such that we can not modify a 176 | # child file by accident 177 | _shallow_dir_copy(src_entry, dst_entry, ignored_files) 178 | else: 179 | # symlink files, first removing an already existing symlink 180 | if dst_entry.exists(): 181 | dst_entry.unlink() 182 | dst_entry.symlink_to(src_entry) 183 | -------------------------------------------------------------------------------- /src/pytest_executable/plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Entry point into the pytest executable plugin.""" 18 | from __future__ import annotations 19 | 20 | import logging 21 | import sys 22 | from functools import cmp_to_key 23 | from pathlib import Path 24 | from types import ModuleType 25 | from typing import Any 26 | from typing import Callable 27 | from typing import TYPE_CHECKING 28 | 29 | import _pytest 30 | import py 31 | import pytest 32 | from _pytest._code.code import ExceptionChainRepr 33 | 34 | from . import report 35 | from .file_tools import create_output_directory 36 | from .file_tools import find_references 37 | from .file_tools import get_mirror_path 38 | from .script_runner import ScriptRunner 39 | from .settings import Settings 40 | from .settings import Tolerances 41 | 42 | if TYPE_CHECKING: 43 | from _pytest.compat import LEGACY_PATH 44 | from _pytest.config.argparsing import Parser 45 | from _pytest.fixtures import SubRequest 46 | from _pytest.main import Session 47 | from _pytest.nodes import Collector 48 | from _pytest.nodes import Item 49 | from _pytest.python import Metafunc 50 | from _pytest.reports import CollectReport 51 | from _pytest.reports import TestReport 52 | from _pytest.runner import CallInfo 53 | 54 | LOGGER = logging.getLogger(__name__) 55 | 56 | # files to be ignored when creating the output directories symlinks 57 | OUTPUT_IGNORED_FILES = ("__pycache__", "conftest.py", "test-settings.yaml") 58 | 59 | # file with the test default settings 60 | SETTINGS_PATH = Path(__file__).parent / "test-settings.yaml" 61 | TEST_MODULE_PATH = Path(__file__).parent / "test_executable.py" 62 | 63 | # caches the test case directory path to marks to propagate them to all the 64 | # test modules of a test case 65 | _marks_cache: dict[Path, set[str]] = {} 66 | 67 | PYTEST_USE_FSPATH = pytest.__version__ < "7.0.0" 68 | 69 | 70 | def _get_path(obj: Any) -> Path | LEGACY_PATH: 71 | if PYTEST_USE_FSPATH: # pragma: no cover 72 | return obj.fspath 73 | else: 74 | return obj.path 75 | 76 | 77 | def pytest_addoption(parser: Parser) -> None: 78 | """CLI options for the plugin.""" 79 | group = parser.getgroup("executable", "executable testing") 80 | 81 | group.addoption( 82 | "--exe-runner", 83 | metavar="PATH", 84 | help="use the shell script at PATH to run an executable", 85 | ) 86 | 87 | group.addoption( 88 | "--exe-output-root", 89 | default="tests-output", 90 | metavar="PATH", 91 | help="use PATH as the root directory of the tests output, default: %(default)s", 92 | ) 93 | 94 | group.addoption( 95 | "--exe-overwrite-output", 96 | action="store_true", 97 | help="overwrite existing files in the tests output directories", 98 | ) 99 | 100 | group.addoption( 101 | "--exe-clean-output", 102 | action="store_true", 103 | help="clean the tests output directories before executing the tests", 104 | ) 105 | 106 | group.addoption( 107 | "--exe-regression-root", 108 | metavar="PATH", 109 | help="use PATH as the root directory with the references for the " 110 | "regression testing", 111 | ) 112 | 113 | group.addoption( 114 | "--exe-default-settings", 115 | default=SETTINGS_PATH, 116 | metavar="PATH", 117 | help="use PATH as the yaml file with the global default test settings instead " 118 | "of the built-in ones", 119 | ) 120 | 121 | group.addoption( 122 | "--exe-test-module", 123 | default=TEST_MODULE_PATH, 124 | metavar="PATH", 125 | help="use PATH as the default test module", 126 | ) 127 | 128 | group.addoption( 129 | "--exe-report-generator", 130 | metavar="PATH", 131 | help="use PATH as the script to generate the test report", 132 | ) 133 | 134 | # change default traceback settings to get only the message without the 135 | # traceback 136 | term_rep_options = parser.getgroup("terminal reporting").options 137 | tb_option = next( 138 | option for option in term_rep_options if option.names() == ["--tb"] 139 | ) 140 | tb_option.default = "line" 141 | 142 | 143 | def pytest_sessionstart(session: Session) -> None: 144 | """Check the CLI arguments and resolve their paths.""" 145 | option = session.config.option 146 | 147 | # check options clash 148 | if option.exe_clean_output and option.exe_overwrite_output: 149 | msg = "options --exe-clean-output and --exe-overwrite-output are not compatible" 150 | raise pytest.UsageError(msg) 151 | 152 | # check paths are valid 153 | for option_name in ( 154 | "exe_runner", 155 | "exe_test_module", 156 | "exe_default_settings", 157 | "exe_regression_root", 158 | "exe_report_generator", 159 | ): 160 | path = getattr(option, option_name) 161 | try: 162 | path = Path(path).resolve(True) 163 | except FileNotFoundError: 164 | msg = ( 165 | f"argument --{option_name.replace('_', '-')}: " 166 | f"no such file or directory: {path}" 167 | ) 168 | raise pytest.UsageError(msg) 169 | except TypeError: 170 | # path is None, i.e. no option is defined 171 | pass 172 | else: 173 | # overwrite the option with the resolved path 174 | setattr(option, option_name, path) 175 | 176 | # convert remaining option with pat 177 | option.exe_output_root = Path(option.exe_output_root).resolve() 178 | 179 | 180 | def _get_parent_path(path: Path) -> Path: 181 | """Return the resolved path to a parent directory. 182 | 183 | Args: 184 | path: Path object from pytest. 185 | 186 | Returns: 187 | Resolved path to the parent directory of the given pat. 188 | """ 189 | return Path(path).parent.resolve(True) 190 | 191 | 192 | @pytest.fixture(scope="module") 193 | def create_output_tree(request: SubRequest) -> None: 194 | """Fixture to create and return the path to the output directory tree.""" 195 | option = request.config.option 196 | parent_path = _get_parent_path(_get_path(request.node)) 197 | output_path = get_mirror_path(parent_path, option.exe_output_root) 198 | 199 | try: 200 | create_output_directory( 201 | parent_path, 202 | output_path, 203 | not option.exe_overwrite_output, 204 | option.exe_clean_output, 205 | OUTPUT_IGNORED_FILES, 206 | ) 207 | except FileExistsError: 208 | msg = ( 209 | f'the output directory "{output_path}" already exists: either ' 210 | "remove it manually or use the --exe-clean-output option to remove " 211 | "it or use the --exe-overwrite-output to overwrite it" 212 | ) 213 | raise FileExistsError(msg) 214 | 215 | 216 | @pytest.fixture(scope="module") 217 | def output_path(request: SubRequest) -> Path: 218 | """Fixture to return the path to the output directory.""" 219 | return get_mirror_path( 220 | _get_parent_path(_get_path(request.node)), 221 | request.config.option.exe_output_root, 222 | ) 223 | 224 | 225 | def _get_settings(config: _pytest.config.Config, path: Path) -> Settings: 226 | """Return the settings from global and local test-settings.yaml. 227 | 228 | Args: 229 | config: Config from pytest. 230 | path: Path to a test case directory. 231 | 232 | Returns: 233 | The settings from the test case yaml. 234 | """ 235 | return Settings.from_local_file( 236 | Path(config.option.exe_default_settings), 237 | _get_parent_path(path) / SETTINGS_PATH.name, 238 | ) 239 | 240 | 241 | @pytest.fixture(scope="module") 242 | def tolerances(request: SubRequest) -> dict[str, Tolerances]: 243 | """Fixture that provides the tolerances from the settings.""" 244 | return _get_settings(request.config, _get_path(request.node)).tolerances 245 | 246 | 247 | @pytest.fixture(scope="module") 248 | def runner( 249 | request: SubRequest, 250 | create_output_tree: Callable[[ScriptRunner], None], 251 | output_path: Path, 252 | ) -> ScriptRunner: 253 | """Fixture to execute the runner script.""" 254 | runner_path = request.config.option.exe_runner 255 | if runner_path is None: 256 | pytest.skip("no runner provided with --exe-runner") 257 | 258 | settings = _get_settings(request.config, _get_path(request.node)).runner 259 | settings["output_path"] = str(output_path) 260 | 261 | return ScriptRunner(runner_path, settings, output_path) 262 | 263 | 264 | def _get_regression_path(config: _pytest.config.Config, path: Path) -> Path | None: 265 | """Return the path to the reference directory of a test case. 266 | 267 | None is returned if --exe-regression-root is not passed to the CLI. 268 | 269 | Args: 270 | config: Config from pytest. 271 | path: Path to a test case directory. 272 | 273 | Returns: 274 | The path to the reference directory of the test case or None. 275 | """ 276 | regression_path = config.option.exe_regression_root 277 | if regression_path is None: 278 | return None 279 | return get_mirror_path(_get_parent_path(path), regression_path) 280 | 281 | 282 | @pytest.fixture(scope="module") 283 | def regression_path(request: SubRequest) -> Path | None: 284 | """Fixture to return the path of a test case under the references tree.""" 285 | regression_path = _get_regression_path(request.config, _get_path(request.node)) 286 | if regression_path is None: 287 | pytest.skip( 288 | "no tests references root directory provided to --exe-regression-root" 289 | ) 290 | return regression_path 291 | 292 | 293 | def pytest_generate_tests(metafunc: Metafunc) -> None: 294 | """Create the regression_file_path parametrized fixture. 295 | 296 | Used for accessing the references files. 297 | 298 | If --exe-regression-root is not set then no reference files will be provided. 299 | """ 300 | if "regression_file_path" not in metafunc.fixturenames: 301 | return 302 | 303 | # result absolute and relative file paths to be provided by the fixture parameter 304 | # empty means skip the test function that use the fixture 305 | file_paths = [] 306 | 307 | regression_path = _get_regression_path( 308 | metafunc.config, _get_path(metafunc.definition) 309 | ) 310 | 311 | if regression_path is not None: 312 | settings_path = _get_path(metafunc.definition) 313 | settings = _get_settings(metafunc.config, settings_path) 314 | 315 | if settings.references: 316 | file_paths = find_references(regression_path, settings.references) 317 | 318 | metafunc.parametrize( 319 | "regression_file_path", 320 | file_paths, 321 | scope="function", 322 | ids=list(map(str, [f.relative for f in file_paths])), 323 | ) 324 | 325 | 326 | if PYTEST_USE_FSPATH: # pragma: no cover 327 | 328 | def pytest_collect_file( 329 | parent: Collector, 330 | path: LEGACY_PATH, 331 | ) -> Collector | None: 332 | """Collect test cases defined with a yaml file.""" 333 | if path.basename != SETTINGS_PATH.name: 334 | return None 335 | if hasattr(TestExecutableModule, "from_parent"): 336 | return TestExecutableModule.from_parent(parent, fspath=path) # type:ignore 337 | else: 338 | return TestExecutableModule(path, parent) 339 | 340 | else: 341 | 342 | def pytest_collect_file( # type:ignore 343 | parent: Collector, 344 | file_path: Path, 345 | ) -> Collector | None: 346 | """Collect test cases defined with a yaml file.""" 347 | if file_path.name != SETTINGS_PATH.name: 348 | return None 349 | if hasattr(TestExecutableModule, "from_parent"): 350 | return TestExecutableModule.from_parent( # type:ignore 351 | parent, path=file_path 352 | ) 353 | else: 354 | return TestExecutableModule(file_path, parent) 355 | 356 | 357 | def pytest_configure(config: _pytest.config.Config) -> None: 358 | """Register the possible markers and change default error display. 359 | 360 | Display only the last error line without the traceback. 361 | """ 362 | config.addinivalue_line( 363 | "markers", 'slow: marks tests as slow (deselect with -m "not slow")' 364 | ) 365 | 366 | # show only the last line with the error message when displaying a 367 | # traceback 368 | if config.option.tbstyle == "auto": 369 | config.option.tbstyle = "line" 370 | 371 | 372 | class TestExecutableModule(pytest.Module): 373 | """Collector for tests defined with a yaml file.""" 374 | 375 | def _getobj(self) -> ModuleType: 376 | """Override the base class method. 377 | 378 | To swap the yaml file with the test module. 379 | """ 380 | test_module_path = Path(self.config.option.exe_test_module) 381 | 382 | # prevent python from using the module cache, otherwise the module 383 | # object will be the same for all the tests 384 | try: 385 | del sys.modules[test_module_path.stem] 386 | except KeyError: 387 | pass 388 | 389 | # backup the attribute before a temporary override of it 390 | path = _get_path(self) 391 | if PYTEST_USE_FSPATH: # pragma: no cover 392 | self.fspath = py.path.local(test_module_path) 393 | else: 394 | self.path = test_module_path 395 | module: ModuleType = self._importtestmodule() # type: ignore 396 | 397 | # restore the backed up attribute 398 | if PYTEST_USE_FSPATH: # pragma: no cover 399 | self.fspath = path 400 | else: 401 | self.path = path 402 | 403 | # set the test case marks from test-settings.yaml 404 | settings = _get_settings(self.config, path) 405 | 406 | # store the marks for applying them later 407 | if settings.marks: 408 | _marks_cache[Path(path).parent] = settings.marks 409 | 410 | return module 411 | 412 | 413 | def pytest_exception_interact( 414 | node: Item | Collector, 415 | call: CallInfo[Any], 416 | report: CollectReport | TestReport, 417 | ) -> None: 418 | """Change exception display to only show the test path and error message. 419 | 420 | Avoid displaying the test file path and the Exception type. 421 | """ 422 | excinfo = call.excinfo 423 | if excinfo is None: 424 | return 425 | if excinfo.typename == "CollectError" and str(excinfo.value).startswith( 426 | "import file mismatch:\n" 427 | ): 428 | # handle when a custom test script is used in more than one test case with 429 | # the same name 430 | path = Path(_get_path(node)) 431 | dirname = path.parent 432 | filename = path.name 433 | report.longrepr = ( 434 | f"{dirname}\nshall have a __init__.py because {filename} " 435 | "exists in other directories" 436 | ) 437 | if isinstance(report.longrepr, ExceptionChainRepr): 438 | report.longrepr.reprcrash = f"{report.nodeid}: {excinfo.value}" # type:ignore 439 | 440 | 441 | def pytest_collection_modifyitems(items: list[_pytest.nodes.Item]) -> None: 442 | """Change the tests execution order. 443 | 444 | Such that: 445 | - the tests in parent directories are executed after the tests in children 446 | directories 447 | - in a test case directory, the yaml defined tests are executed before the 448 | others 449 | """ 450 | items.sort(key=cmp_to_key(_sort_parent_last)) 451 | items.sort(key=cmp_to_key(_sort_yaml_first)) 452 | _set_marks(items) 453 | 454 | 455 | def _sort_yaml_first(item_1: _pytest.nodes.Item, item_2: _pytest.nodes.Item) -> int: 456 | """Sort yaml item first vs module at or below yaml parent directory.""" 457 | path_1 = Path(_get_path(item_1)) 458 | path_2 = Path(_get_path(item_2)) 459 | if path_1 == path_2 or path_1.suffix == path_2.suffix: 460 | return 0 461 | if path_2.suffix == ".yaml" and (path_2.parent in path_1.parents): 462 | return 1 463 | if path_1.suffix == ".yaml" and (path_1.parent in path_2.parents): 464 | return -1 465 | return 0 466 | 467 | 468 | def _sort_parent_last(item_1: _pytest.nodes.Item, item_2: _pytest.nodes.Item) -> int: 469 | """Sort item in parent directory last.""" 470 | dir_1 = Path(_get_path(item_1)).parent 471 | dir_2 = Path(_get_path(item_2)).parent 472 | if dir_1 == dir_2: 473 | return 0 474 | if dir_2 in dir_1.parents: 475 | return -1 476 | return 1 477 | 478 | 479 | def _set_marks(items: list[_pytest.nodes.Item]) -> None: 480 | """Set the marks to all the test functions of a test case.""" 481 | for dir_path, marks in _marks_cache.items(): 482 | for item in items: 483 | if dir_path in Path(_get_path(item)).parents: 484 | for mark in marks: 485 | item.add_marker(mark) 486 | 487 | 488 | def pytest_terminal_summary( 489 | terminalreporter: _pytest.terminal.TerminalReporter, 490 | config: _pytest.config.Config, 491 | ) -> None: 492 | """Create the custom report. 493 | 494 | In the directory that contains the report generator, the report database is created 495 | and the report generator is called. 496 | """ 497 | # path to the report generator 498 | reporter_path = config.option.exe_report_generator 499 | if reporter_path is None: 500 | return 501 | 502 | if not terminalreporter.stats: 503 | # no test have been run thus no report to create or update 504 | return 505 | 506 | terminalreporter.write_sep("=", "starting report generation") 507 | 508 | try: 509 | report.generate(reporter_path, config.option.exe_output_root, terminalreporter) 510 | except Exception as e: 511 | terminalreporter.write_line(str(e), red=True) 512 | terminalreporter.write_sep("=", "report generation failed", red=True) 513 | else: 514 | terminalreporter.write_sep("=", "report generation done") 515 | -------------------------------------------------------------------------------- /src/pytest_executable/report-db-schema.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This is the schema for validating the report database yaml file. 19 | # 20 | # For more information on schema and validation, see 21 | # https://json-schema.org/understanding-json-schema/index.html 22 | # json-schema.org/latest/json-schema-validation.html 23 | 24 | $schema: "http://json-schema.org/draft-07/schema#" 25 | 26 | type: object 27 | propertyNames: 28 | type: string 29 | additionalProperties: 30 | type: object 31 | status: 32 | enum: [error, failed, passed, skipped] 33 | messages: 34 | type: array 35 | items: 36 | type: string 37 | required: [status] 38 | -------------------------------------------------------------------------------- /src/pytest_executable/report.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Provide the report database dumper and report generation.""" 18 | from __future__ import annotations 19 | 20 | import subprocess 21 | from collections import defaultdict 22 | from pathlib import Path 23 | from typing import Any 24 | from typing import Dict 25 | 26 | from _pytest.terminal import TerminalReporter 27 | 28 | from .yaml_helper import YamlHelper 29 | 30 | YAML_HELPER = YamlHelper(Path(__file__).parent / "report-db-schema.yaml") 31 | 32 | REPORT_DB_FILENAME = "tests_report_db.yaml" 33 | REPORT_DIRNAME = "report" 34 | 35 | ReportDBType = Dict[str, Dict[str, Any]] 36 | 37 | 38 | def create(terminalreporter: TerminalReporter) -> ReportDBType: 39 | """Create the report database. 40 | 41 | A test case is errored if at least one test function is error. A test case 42 | is failed if at least one test function is failed. A test case is skipped 43 | if all the test functions are skipped. Otherwise, it is passed. 44 | 45 | All the error and failure messages are listed for a given path. 46 | 47 | Args: 48 | terminalreporter: Pytest terminal reporter. 49 | 50 | Returns: 51 | The report database. 52 | """ 53 | report_db: ReportDBType = defaultdict(dict) 54 | seen_dirs: set[str] = set() 55 | 56 | for status in ("error", "failed", "passed", "skipped"): 57 | stats = terminalreporter.stats.get(status, []) 58 | for test_report in stats: 59 | try: 60 | path = test_report.path 61 | except AttributeError: 62 | path = test_report.fspath 63 | path_from_root = Path(*Path(path).parts[1:]) 64 | dir_path = str(path_from_root.parent) 65 | db_entry = report_db[dir_path] 66 | messages = db_entry.setdefault("messages", []) 67 | if status in ("error", "failed"): 68 | messages += [test_report.longreprtext] 69 | if dir_path in seen_dirs: 70 | continue 71 | seen_dirs.add(dir_path) 72 | db_entry["status"] = status 73 | 74 | return dict(report_db) 75 | 76 | 77 | def merge(db_path: Path, new_db: ReportDBType) -> ReportDBType: 78 | """Merge a report database into an existing database on disk. 79 | 80 | The entries in db_path but not in new_db are left untouched. The entries in 81 | both are overwritten with the ones in new_db. The entries only in new_db 82 | are added. 83 | 84 | Args: 85 | db_path: Path to a database file. 86 | new_db: Database to be merged from. 87 | 88 | Returns: 89 | The merged database. 90 | """ 91 | merged_db = YAML_HELPER.load(db_path) 92 | merged_db.update(new_db) 93 | return merged_db 94 | 95 | 96 | def dump(report_db_path: Path, terminalreporter: TerminalReporter) -> None: 97 | """Dump the report database. 98 | 99 | If the database file already exists then new test results are merged with 100 | the ones in the file. 101 | 102 | The database yaml file have the following format: 103 | 104 | path/to/a/test/case: 105 | status: the test case status among error, failure, passed and skipped 106 | messages: list of error and failure messages for all the test functions 107 | of the test case 108 | 109 | Args: 110 | report_db_path: Path to the report db file. 111 | terminalreporter: Pytest terminal reporter. 112 | """ 113 | report_db = create(terminalreporter) 114 | 115 | if report_db_path.is_file(): 116 | report_db = merge(report_db_path, report_db) 117 | 118 | YAML_HELPER.dump(report_db, report_db_path.open("w")) 119 | 120 | 121 | def generate( 122 | script_path: str, output_root: Path, terminalreporter: TerminalReporter 123 | ) -> None: 124 | """Generate the report in the output root. 125 | 126 | The directory that contains script_path is shallow copied in output_root, 127 | the report is created there. 128 | 129 | Args: 130 | script_path: Path to the reporter generator script. 131 | output_root: Path to the test results root directory. 132 | terminalreporter: Pytest terminal reporter. 133 | """ 134 | # check that the report generator script is there 135 | reporter_path = Path(script_path).resolve(True) 136 | 137 | # write the report database 138 | dump(output_root / REPORT_DB_FILENAME, terminalreporter) 139 | 140 | # generate the report 141 | subprocess.run(str(reporter_path), cwd=output_root, check=True, shell=True) 142 | -------------------------------------------------------------------------------- /src/pytest_executable/script_runner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Provides the shell script creation and execution routines.""" 18 | from __future__ import annotations 19 | 20 | import logging 21 | import stat 22 | import subprocess 23 | from pathlib import Path 24 | from typing import cast 25 | 26 | import delta 27 | import jinja2 28 | 29 | LOG = logging.getLogger(__name__) 30 | 31 | 32 | class ScriptExecutionError(Exception): 33 | """Error for script execution.""" 34 | 35 | 36 | class ScriptRunner: 37 | """Class for creating and running a runner script. 38 | 39 | It can create and execute a shell script from the path to a script with 40 | placeholders and the settings. 41 | 42 | Attributes: 43 | path: Path to the script. 44 | settings: Runner settings from the yaml file. 45 | workdir: Path to the script working directory. 46 | STDOUT_EXT: Suffix for the file with the script standard output (class 47 | attribute). 48 | STDERR_EXT: Suffix for the file with the script standard error (class 49 | attribute). 50 | SHELL: Shell used to execute the script (class attribute). 51 | """ 52 | 53 | STDOUT_EXT = "stdout" 54 | STDERR_EXT = "stderr" 55 | SHELL = "/usr/bin/env bash" 56 | 57 | def __init__(self, path: Path, settings: dict[str, str], workdir: Path): 58 | """Docstring just to prevent the arguments to appear in the autodoc. 59 | 60 | Args: 61 | path: Path to the script. 62 | settings: Runner settings from the yaml file. 63 | workdir: Path to the script working directory. 64 | """ 65 | self.path = path 66 | self.workdir = workdir 67 | self.settings = settings 68 | self._content = self._substitute() 69 | 70 | def _substitute(self) -> str: 71 | """Return the script contents with replaced placeholders. 72 | 73 | Returns: 74 | The final script contents. 75 | 76 | Raises: 77 | TypeError: if the script cannot be processed. 78 | ValueError: if a placeholder is undefined. 79 | """ 80 | try: 81 | template = jinja2.Template( 82 | self.path.read_text(), undefined=jinja2.StrictUndefined 83 | ) 84 | except UnicodeDecodeError: 85 | raise TypeError(f"cannot read the script {self.path}") from None 86 | try: 87 | return cast(str, template.render(**self.settings)) 88 | except jinja2.exceptions.UndefinedError as error: 89 | raise ValueError(f"in {self.path}: {error}") from None 90 | 91 | def run(self) -> int: 92 | """Execute the script. 93 | 94 | The script is created and executed in the working directory. The stdout 95 | and stderr of the script are each redirected to files named after the 96 | script and suffixed with :py:data:`STDOUT_EXT` and :py:data:`STDERR_EXT`. 97 | 98 | Returns: 99 | The return code of the executed subprocess. 100 | 101 | Raises: 102 | ScriptExecutionError: If the execution fails. 103 | """ 104 | filename = self.path.name 105 | script_path = self.workdir / filename 106 | 107 | # write the script 108 | with script_path.open("w") as script_file: 109 | LOG.debug("writing the shell script %s", script_path) 110 | script_file.write(self._content) 111 | 112 | # make it executable for the user and the group 113 | permission = stat.S_IMODE(script_path.stat().st_mode) 114 | script_path.chmod(permission | stat.S_IXUSR | stat.S_IXGRP) 115 | 116 | # redirect the stdout and stderr to files 117 | stdout = open(self.workdir / f"{filename}.{self.STDOUT_EXT}", "w") 118 | stderr = open(self.workdir / f"{filename}.{self.STDERR_EXT}", "w") 119 | 120 | LOG.debug("executing the shell script %s", script_path) 121 | cmd = self.SHELL.split() + [filename] 122 | 123 | timeout = self.settings.get("timeout") 124 | 125 | if timeout is not None: 126 | # convert to seconds 127 | timeout = delta.parse(timeout).seconds 128 | 129 | try: 130 | process = subprocess.run( 131 | cmd, 132 | cwd=self.workdir, 133 | stdout=stdout, 134 | stderr=stderr, 135 | check=True, 136 | timeout=timeout, # type: ignore 137 | ) 138 | except subprocess.CalledProcessError: 139 | # inform about the log files 140 | raise ScriptExecutionError( 141 | "execution failure, see the stdout and stderr files in " 142 | f"{self.workdir}" 143 | ) 144 | finally: 145 | stdout.close() 146 | stderr.close() 147 | 148 | return process.returncode 149 | -------------------------------------------------------------------------------- /src/pytest_executable/settings.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Provides the container for the settings of a test case. 18 | 19 | We use a dataclass because a dictionary does not offer easy checking and code 20 | completion. 21 | """ 22 | from __future__ import annotations 23 | 24 | from dataclasses import dataclass 25 | from dataclasses import fields 26 | from pathlib import Path 27 | 28 | from .yaml_helper import YamlHelper 29 | 30 | # the yaml file with the default settings is in the same directory as the 31 | # current module, the yaml schema too 32 | SETTINGS_SCHEMA_FILE = Path(__file__).parent / "test-settings-schema.yaml" 33 | 34 | 35 | @dataclass 36 | class Tolerances: 37 | """Comparison tolerances. 38 | 39 | Attributes: 40 | rel: The relative tolerance. 41 | abs: The absolute tolerance. 42 | """ 43 | 44 | rel: float = 0.0 45 | abs: float = 0.0 46 | 47 | 48 | @dataclass 49 | class Settings: 50 | """Test settings container. 51 | 52 | This contains the test settings read from a yaml file. 53 | 54 | Attributes: 55 | runner: The settings for the script runner. 56 | marks: The pytest marks. 57 | references: The reference files path patterns. 58 | tolerances: The comparison tolerances. 59 | """ 60 | 61 | runner: dict[str, str] 62 | marks: set[str] 63 | references: set[str] 64 | tolerances: dict[str, Tolerances] 65 | 66 | def __post_init__(self) -> None: 67 | """Coerce the attributes types.""" 68 | self.marks = set(self.marks) 69 | self.references = set(self.references) 70 | for key, value in self.tolerances.copy().items(): 71 | self.tolerances[key] = Tolerances(**value) # type:ignore 72 | 73 | @classmethod 74 | def from_local_file(cls, path_global: Path, path_local: Path) -> Settings: 75 | """Create a :class:`Settings` object from 2 yaml files. 76 | 77 | The contents of the local file overrides or extends the contents of the 78 | global one. The items that have no corresponding attributes in the 79 | current class are ignored. 80 | 81 | Args: 82 | path_global: Path to a yaml file with global settings. 83 | path_local: Path to a yaml file with local settings. 84 | 85 | Returns: 86 | Settings object. 87 | """ 88 | loader = YamlHelper(SETTINGS_SCHEMA_FILE) 89 | # contains the settings and eventually additional items 90 | loaded_settings = loader.load_merge(path_global, path_local) 91 | # keep the used settings 92 | settings = {} 93 | for field in fields(cls): 94 | name = field.name 95 | settings[name] = loaded_settings[name] 96 | return cls(**settings) 97 | -------------------------------------------------------------------------------- /src/pytest_executable/test-settings-schema.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This is the schema for validating the settings from a yaml file. 19 | # 20 | # For more information on schema and validation, see 21 | # https://json-schema.org/understanding-json-schema/index.html 22 | # json-schema.org/latest/json-schema-validation.html 23 | 24 | $schema: "http://json-schema.org/draft-07/schema#" 25 | 26 | definitions: 27 | stringArray: 28 | type: array 29 | items: 30 | type: string 31 | uniqueItems: true 32 | positiveNumber: 33 | type: number 34 | minimum: 0. 35 | 36 | type: object 37 | properties: 38 | runner: 39 | type: object 40 | properties: 41 | timeout: 42 | type: string 43 | marks: 44 | $ref: "#/definitions/stringArray" 45 | references: 46 | $ref: "#/definitions/stringArray" 47 | tolerances: 48 | type: object 49 | propertyNames: 50 | pattern: "^[A-Za-z_][A-Za-z0-9_]*$" 51 | additionalProperties: 52 | type: object 53 | additionalProperties: false 54 | properties: 55 | rel: 56 | $ref: "#/definitions/positiveNumber" 57 | abs: 58 | $ref: "#/definitions/positiveNumber" 59 | -------------------------------------------------------------------------------- /src/pytest_executable/test-settings.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # Settings for the runner script 19 | runner: {} 20 | 21 | # List of keywords used to mark a test case 22 | marks: [] 23 | 24 | # File path patterns relative to a test case to identify the regression 25 | # reference files used for the regression assertions. 26 | references: [] 27 | 28 | # Tolerances used for the assertions when comparing the fields, all default 29 | # tolerances are 0. 30 | tolerances: {} 31 | -------------------------------------------------------------------------------- /src/pytest_executable/test_executable.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Builtin test module. 18 | 19 | This module is automatically executed when a test-settings.yaml file is found. 20 | """ 21 | from __future__ import annotations 22 | 23 | from pytest_executable.script_runner import ScriptRunner 24 | 25 | 26 | def test_runner(runner: ScriptRunner) -> None: 27 | """Check the runner execution. 28 | 29 | An OK process execution shall return the code 0. 30 | 31 | Args: 32 | runner: Runner object to be run. 33 | """ 34 | assert runner.run() == 0 35 | -------------------------------------------------------------------------------- /src/pytest_executable/yaml_helper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Provide a yaml file loader.""" 18 | from __future__ import annotations 19 | 20 | from pathlib import Path 21 | from typing import Any 22 | from typing import Dict 23 | from typing import TextIO 24 | 25 | import jsonschema 26 | import yaml 27 | 28 | # basic type for the content of a yaml with a mapping at the root level 29 | DataType = Dict[str, Any] 30 | 31 | 32 | class YamlHelper: 33 | """Yaml file helper class. 34 | 35 | Can load and validate a yaml file, can also merge tha data 2 yaml files. 36 | 37 | Args: 38 | schema_path: Path to a schema file used for validating a yaml. 39 | """ 40 | 41 | def __init__(self, schema_path: Path): 42 | with schema_path.open() as file_: 43 | self.__schema = yaml.safe_load(file_) 44 | 45 | def load(self, path: Path) -> DataType: 46 | """Return the validated data from a yaml file. 47 | 48 | Args: 49 | path: Path to a yaml file. 50 | 51 | Returns: 52 | The validated data. 53 | """ 54 | data: DataType 55 | with path.open() as file_: 56 | data = yaml.safe_load(file_) 57 | 58 | if data is None: 59 | data = {} 60 | 61 | jsonschema.validate(data, self.__schema) 62 | 63 | return data 64 | 65 | def dump(self, data: DataType, stream: TextIO) -> None: 66 | """Validate and dump data to a yaml file. 67 | 68 | Args: 69 | data: Data to be dumped. 70 | stream: IO stream to be written to. 71 | """ 72 | jsonschema.validate(data, self.__schema) 73 | yaml.safe_dump(data, stream) 74 | 75 | def load_merge(self, ref: Path, new: Path) -> DataType: 76 | """Merge the data of 2 yaml files. 77 | 78 | Args: 79 | ref: Path to the file to be updated with new. 80 | new: Path to the file to be merged into ref. 81 | 82 | Return: 83 | The merged dictionary. 84 | """ 85 | return self.__recursive_update(self.load(ref), self.load(new)) 86 | 87 | @classmethod 88 | def __recursive_update(cls, ref: DataType, new: DataType) -> DataType: 89 | """Merge recursively 2 dictionaries. 90 | 91 | The lists are concatenated. 92 | """ 93 | for key, value in new.items(): 94 | if isinstance(value, dict): 95 | ref[key] = cls.__recursive_update(ref.get(key, {}), value) 96 | elif isinstance(value, list): 97 | # append the items that were not already in the list 98 | for val in value: 99 | if val not in ref[key]: 100 | ref[key] += [val] 101 | else: 102 | ref[key] = value 103 | return ref 104 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Entry point to pytest_executable tests.""" 18 | from __future__ import annotations 19 | 20 | from pathlib import Path 21 | 22 | # path to the data directory 23 | ROOT_DATA_DIR = Path(__file__).parent / "data" 24 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Configuration for testing the plugin.""" 18 | from __future__ import annotations 19 | 20 | 21 | pytest_plugins = "pytester" 22 | -------------------------------------------------------------------------------- /tests/data/collect_order/b/a/test_aaa.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | 20 | def test_dummy(): 21 | pass 22 | -------------------------------------------------------------------------------- /tests/data/collect_order/b/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/collect_order/b/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/collect_order/test_a.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | 20 | def test_dummy(): 21 | pass 22 | -------------------------------------------------------------------------------- /tests/data/collect_order/z/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/collect_order/z/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/collect_order/z/test_aa.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | 20 | def test_dummy(): 21 | pass 22 | -------------------------------------------------------------------------------- /tests/data/find_references/ref-dir/0/1.prf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/find_references/ref-dir/0/1.prf -------------------------------------------------------------------------------- /tests/data/find_references/ref-dir/0/dir/2.prf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/find_references/ref-dir/0/dir/2.prf -------------------------------------------------------------------------------- /tests/data/find_references/test-dir/0/dummy-file-for-git-to-store-the-directory-tree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/find_references/test-dir/0/dummy-file-for-git-to-store-the-directory-tree -------------------------------------------------------------------------------- /tests/data/report/report_db.yaml: -------------------------------------------------------------------------------- 1 | .: 2 | status: error 3 | messages: 4 | - error message 0 5 | - failure message 0 6 | dir1: 7 | status: failed 8 | messages: 9 | - failure message 1 10 | dir2: 11 | status: passed 12 | messages: [] 13 | dir3: 14 | status: skipped 15 | messages: [] 16 | -------------------------------------------------------------------------------- /tests/data/runners/error.sh: -------------------------------------------------------------------------------- 1 | ls non-existing-file 2 | -------------------------------------------------------------------------------- /tests/data/runners/nproc.sh: -------------------------------------------------------------------------------- 1 | echo {{nproc}} 2 | -------------------------------------------------------------------------------- /tests/data/runners/timeout.sh: -------------------------------------------------------------------------------- 1 | sleep 1 2 | -------------------------------------------------------------------------------- /tests/data/shallow_dir_copy/dst_dir/dir/file: -------------------------------------------------------------------------------- 1 | ../../src_dir/dir/file -------------------------------------------------------------------------------- /tests/data/shallow_dir_copy/dst_dir/file: -------------------------------------------------------------------------------- 1 | ../src_dir/file -------------------------------------------------------------------------------- /tests/data/shallow_dir_copy/src_dir/dir/file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/shallow_dir_copy/src_dir/dir/file -------------------------------------------------------------------------------- /tests/data/shallow_dir_copy/src_dir/file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/shallow_dir_copy/src_dir/file -------------------------------------------------------------------------------- /tests/data/shallow_dir_copy/src_dir/file-to-ignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/shallow_dir_copy/src_dir/file-to-ignore -------------------------------------------------------------------------------- /tests/data/test___init__/tests-inputs/case1/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test___init__/tests-inputs/case1/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test___init__/tests-inputs/case1/test_dummy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | -------------------------------------------------------------------------------- /tests/data/test___init__/tests-inputs/case2/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test___init__/tests-inputs/case2/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test___init__/tests-inputs/case2/test_dummy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | -------------------------------------------------------------------------------- /tests/data/test_cli_check/dummy-file-for-git-to-store-the-directory-tree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_cli_check/dummy-file-for-git-to-store-the-directory-tree -------------------------------------------------------------------------------- /tests/data/test_marks_from_yaml/tests-inputs/a/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | -------------------------------------------------------------------------------- /tests/data/test_marks_from_yaml/tests-inputs/a/test_dummy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | 20 | def test_marks(): 21 | pass 22 | -------------------------------------------------------------------------------- /tests/data/test_marks_from_yaml/tests-inputs/test-settings.yaml: -------------------------------------------------------------------------------- 1 | marks: 2 | - mark1 3 | -------------------------------------------------------------------------------- /tests/data/test_marks_from_yaml/tests-inputs/test_dummy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | 20 | def test_marks(): 21 | pass 22 | -------------------------------------------------------------------------------- /tests/data/test_output_dir_fixture/tests-inputs/case/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_output_dir_fixture/tests-inputs/case/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test_output_dir_fixture/tests-output/case/dummy-file-for-git-to-store-the-directory-tree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_output_dir_fixture/tests-output/case/dummy-file-for-git-to-store-the-directory-tree -------------------------------------------------------------------------------- /tests/data/test_regression_file_path_fixture/references/case/0.xmf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_regression_file_path_fixture/references/case/0.xmf -------------------------------------------------------------------------------- /tests/data/test_regression_file_path_fixture/references/case/1.xmf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_regression_file_path_fixture/references/case/1.xmf -------------------------------------------------------------------------------- /tests/data/test_regression_file_path_fixture/tests-inputs/case-no-references/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_regression_file_path_fixture/tests-inputs/case-no-references/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test_regression_file_path_fixture/tests-inputs/case/test-settings.yaml: -------------------------------------------------------------------------------- 1 | references: 2 | - '*.xmf' 3 | -------------------------------------------------------------------------------- /tests/data/test_regression_file_path_fixture/tests-inputs/case/test_fixture.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | from fnmatch import fnmatch 20 | 21 | 22 | def test_fixture(regression_file_path): 23 | assert fnmatch(str(regression_file_path.relative), "[01].xmf") 24 | assert fnmatch( 25 | str(regression_file_path.absolute), 26 | "*/test_regression_file_path_fixture0/references/case/[01].xmf", 27 | ) 28 | -------------------------------------------------------------------------------- /tests/data/test_regression_path_fixture/references/case/dummy-file-for-git-to-store-the-directory-tree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_regression_path_fixture/references/case/dummy-file-for-git-to-store-the-directory-tree -------------------------------------------------------------------------------- /tests/data/test_regression_path_fixture/tests-inputs/case/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_regression_path_fixture/tests-inputs/case/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test_regression_path_fixture/tests-inputs/case/test_fixture.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | from fnmatch import fnmatch 20 | 21 | 22 | def test_fixture(regression_path): 23 | assert fnmatch( 24 | str(regression_path), "*/test_regression_path_fixture0/references/case" 25 | ) 26 | -------------------------------------------------------------------------------- /tests/data/test_report/report/generator-ko.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | echo REPORT GENERATION FAILED 4 | exit 1 5 | -------------------------------------------------------------------------------- /tests/data/test_report/report/generator.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | echo "I SUCCEEDED" 4 | -------------------------------------------------------------------------------- /tests/data/test_report/tests-inputs/case/description.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_report/tests-inputs/case/description.rst -------------------------------------------------------------------------------- /tests/data/test_report/tests-inputs/case/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_report/tests-inputs/case/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test_report/tests-inputs/empty-case/dummy-file-for-git-to-store-the-directory-tree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_report/tests-inputs/empty-case/dummy-file-for-git-to-store-the-directory-tree -------------------------------------------------------------------------------- /tests/data/test_runner_fixture/runner.sh: -------------------------------------------------------------------------------- 1 | echo {{nproc}} 2 | -------------------------------------------------------------------------------- /tests/data/test_runner_fixture/settings.yaml: -------------------------------------------------------------------------------- 1 | runner: 2 | nproc: 100 3 | marks: [] 4 | references: [] 5 | tolerances: {} 6 | -------------------------------------------------------------------------------- /tests/data/test_runner_fixture/tests-inputs/case-global-settings/test-settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CS-SI/pytest-executable/ceff006f3a8e5dd46ecc9109469584987957c5dc/tests/data/test_runner_fixture/tests-inputs/case-global-settings/test-settings.yaml -------------------------------------------------------------------------------- /tests/data/test_runner_fixture/tests-inputs/case-local-settings/test-settings.yaml: -------------------------------------------------------------------------------- 1 | runner: 2 | nproc: 100 3 | -------------------------------------------------------------------------------- /tests/data/test_tolerances_fixture/tests-inputs/case/test-settings.yaml: -------------------------------------------------------------------------------- 1 | tolerances: 2 | field_name: 3 | abs: 123. 4 | -------------------------------------------------------------------------------- /tests/data/test_tolerances_fixture/tests-inputs/case/test_fixture.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | from __future__ import annotations 18 | 19 | 20 | def test_fixture(tolerances): 21 | assert tolerances["field_name"].abs == 123.0 22 | assert tolerances["field_name"].rel == 0.0 23 | -------------------------------------------------------------------------------- /tests/test_file_tools.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tests for the file tools.""" 18 | from __future__ import annotations 19 | 20 | import os 21 | from pathlib import Path 22 | 23 | import pytest 24 | from pytest_executable.file_tools import create_output_directory 25 | from pytest_executable.file_tools import find_references 26 | from pytest_executable.file_tools import get_mirror_path 27 | 28 | from . import ROOT_DATA_DIR 29 | 30 | DATA_DIR = ROOT_DATA_DIR / "shallow_dir_copy" 31 | 32 | 33 | @pytest.fixture(scope="module") 34 | def shared_tmp_path(tmp_path_factory): 35 | """Shared tmp_path fixture for the current module.""" 36 | return tmp_path_factory.mktemp("dummy") 37 | 38 | 39 | def test_non_existing_destination(shared_tmp_path): 40 | """Test shallow directory copy with non-existing destination.""" 41 | _helper(shared_tmp_path, True, False) 42 | 43 | 44 | def test_existing_destination_clean(shared_tmp_path): 45 | """Test clean shallow directory copy with existing destination.""" 46 | # this test shall be executed after test_non_existing_destination 47 | _helper(shared_tmp_path, True, True) 48 | 49 | 50 | def test_existing_destination_ko(shared_tmp_path): 51 | """Test errors.""" 52 | # force creation for using with xdist 53 | _helper(shared_tmp_path, True, True) 54 | with pytest.raises(FileExistsError): 55 | _helper(shared_tmp_path, True, False) 56 | 57 | 58 | def test_existing_destination_overwrite(shared_tmp_path): 59 | """Test overwrite shallow directory copy with existing destination.""" 60 | _helper(shared_tmp_path, False, False) 61 | 62 | 63 | def _helper(shared_tmp_path, check, overwrite): 64 | create_output_directory( 65 | DATA_DIR / "src_dir", 66 | shared_tmp_path / "dst_dir", 67 | check, 68 | overwrite, 69 | ["file-to-ignore"], 70 | ) 71 | _compare_directory_trees( 72 | DATA_DIR / "dst_dir", shared_tmp_path / "dst_dir/data/shallow_dir_copy/src_dir" 73 | ) 74 | 75 | 76 | def _compare_directory_trees(exp_dir, dst_dir): 77 | for exp_entries, dst_entries in zip(os.walk(exp_dir), os.walk(dst_dir)): 78 | # check the names of the subdirectories and the files 79 | assert dst_entries[1:] == exp_entries[1:] 80 | 81 | # check the directories type 82 | assert [Path(dst_entries[0], e).is_dir() for e in dst_entries[1]] == [ 83 | Path(exp_entries[0], e).is_dir() for e in exp_entries[1] 84 | ] 85 | 86 | # check the symlinks type 87 | assert [Path(dst_entries[0], e).is_symlink() for e in dst_entries[2]] == [ 88 | Path(exp_entries[0], e).is_symlink() for e in exp_entries[2] 89 | ] 90 | 91 | # check symlinks pointed path 92 | assert [Path(dst_entries[0], e).resolve() for e in dst_entries[2]] == [ 93 | Path(exp_entries[0], e).resolve() for e in exp_entries[2] 94 | ] 95 | 96 | 97 | @pytest.fixture 98 | def _tmp_path(tmp_path): 99 | cwd = Path.cwd() 100 | os.chdir(tmp_path) 101 | yield tmp_path 102 | os.chdir(cwd) 103 | 104 | 105 | def test_get_mirror_path_ok(_tmp_path): 106 | """Test get_mirror_path.""" 107 | path_from = _tmp_path / "c/d/e" 108 | path_from.mkdir(parents=True) 109 | assert get_mirror_path(path_from, _tmp_path / "x/y") == _tmp_path / "x/y/d/e" 110 | assert get_mirror_path(path_from, Path("/x/y")) == Path("/x/y/d/e") 111 | 112 | # with common parents 113 | path_from = _tmp_path / "a/b/c/d/e" 114 | path_from.mkdir(parents=True) 115 | assert ( 116 | get_mirror_path(path_from, _tmp_path / "a/b/x/y") == _tmp_path / "a/b/x/y/d/e" 117 | ) 118 | 119 | 120 | def test_get_mirror_path_ko(tmp_path): 121 | """Test get_mirror_path failures.""" 122 | msg = ( 123 | f"the current working directory {Path.cwd()} shall be a parent directory of " 124 | f"the inputs directory {tmp_path}" 125 | ) 126 | with pytest.raises(ValueError, match=msg): 127 | get_mirror_path(tmp_path, "") 128 | 129 | 130 | def test_find_references(): 131 | """Test find_references.""" 132 | data_dir = ROOT_DATA_DIR / "find_references" 133 | file_paths = find_references(data_dir / "ref-dir", ["**/*.prf"]) 134 | abs_paths = [f.absolute for f in file_paths] 135 | assert abs_paths == [data_dir / "ref-dir/0/1.prf", data_dir / "ref-dir/0/dir/2.prf"] 136 | rel_paths = [f.relative for f in file_paths] 137 | assert rel_paths == [Path("0/1.prf"), Path("0/dir/2.prf")] 138 | # empty case 139 | assert not find_references(data_dir / "ref-dir", ["**/*.dummy"]) 140 | -------------------------------------------------------------------------------- /tests/test_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Test for the plugin itself.""" 18 | from __future__ import annotations 19 | 20 | import inspect 21 | 22 | 23 | def assert_outcomes(result, **kwargs): 24 | """Wrap result.assert_outcomes different API vs pytest versions.""" 25 | signature_params = inspect.signature(result.assert_outcomes).parameters 26 | if "errors" not in signature_params and "errors" in kwargs: 27 | kwargs["error"] = kwargs.pop("errors") 28 | result.assert_outcomes(**kwargs) 29 | -------------------------------------------------------------------------------- /tests/test_plugin/test_fixtures.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tests for the plugin fixtures.""" 18 | from __future__ import annotations 19 | 20 | from . import assert_outcomes 21 | 22 | 23 | def test_tolerances_fixture(testdir): 24 | """Test tolerances fixture from test-settings.yaml.""" 25 | directory = testdir.copy_example("tests/data/test_tolerances_fixture") 26 | result = testdir.runpytest(directory / "tests-inputs") 27 | # skip runner because no --exe-runner 28 | # pass fixture 29 | assert_outcomes(result, passed=1, skipped=1) 30 | 31 | 32 | def test_regression_path_fixture(testdir): 33 | """Test regression_path fixture.""" 34 | directory = testdir.copy_example("tests/data/test_regression_path_fixture") 35 | result = testdir.runpytest( 36 | directory / "tests-inputs", "--exe-regression-root", directory / "references" 37 | ) 38 | # skip runner because no --exe-runner 39 | # pass fixture test 40 | assert_outcomes(result, skipped=1, passed=1) 41 | 42 | 43 | def test_regression_path_fixture_no_regression_root(testdir): 44 | """Test skipping regression_path fixture without --exe-regression-root option.""" 45 | directory = testdir.copy_example("tests/data/test_regression_path_fixture") 46 | result = testdir.runpytest(directory / "tests-inputs") 47 | # skip runner because no --exe-runner 48 | assert_outcomes(result, skipped=2) 49 | 50 | 51 | def test_regression_file_path_fixture_no_regression_root(testdir): 52 | """Test skipping regression_file_path fixture without --exe-regression-root.""" 53 | directory = testdir.copy_example("tests/data/test_regression_file_path_fixture") 54 | result = testdir.runpytest(directory / "tests-inputs/case-no-references") 55 | assert_outcomes(result, skipped=1) 56 | 57 | 58 | def test_regression_file_path_fixture_no_references(testdir): 59 | """Test skipping regression_file_path fixture whitout references.""" 60 | directory = testdir.copy_example("tests/data/test_regression_file_path_fixture") 61 | result = testdir.runpytest( 62 | directory / "tests-inputs/case-no-references", 63 | "--exe-regression-root", 64 | directory / "references", 65 | ) 66 | assert_outcomes(result, skipped=1) 67 | 68 | 69 | RUNNER_DATA_DIR = "tests/data/test_runner_fixture" 70 | 71 | 72 | def test_runner_fixture_no_runner(testdir): 73 | """Test skipping runner fixture without runner.""" 74 | directory = testdir.copy_example(RUNNER_DATA_DIR) 75 | result = testdir.runpytest(directory / "tests-inputs/case-local-settings") 76 | assert_outcomes(result, skipped=1) 77 | 78 | 79 | def test_runner_fixture_with_local_settings(testdir): 80 | """Test runner fixture with placeholder from local test settings.""" 81 | directory = testdir.copy_example(RUNNER_DATA_DIR) 82 | result = testdir.runpytest( 83 | directory / "tests-inputs/case-local-settings", 84 | "--exe-runner", 85 | directory / "runner.sh", 86 | ) 87 | assert_outcomes(result, passed=1) 88 | stdout = ( 89 | (directory / "tests-output/case-local-settings/runner.sh.stdout") 90 | .open() 91 | .read() 92 | .strip() 93 | ) 94 | assert stdout == "100" 95 | 96 | 97 | def test_runner_not_script(testdir): 98 | """Test error when the runner is not a text script.""" 99 | directory = testdir.copy_example(RUNNER_DATA_DIR) 100 | result = testdir.runpytest( 101 | directory / "tests-inputs/case-local-settings", 102 | "--exe-runner", 103 | "/bin/bash", 104 | ) 105 | assert_outcomes(result, errors=1) 106 | result.stdout.fnmatch_lines(["E TypeError: cannot read the script */bin/bash"]) 107 | 108 | 109 | def test_runner_fixture_with_global_settings(testdir): 110 | """Test runner fixture with nproc from default settings.""" 111 | directory = testdir.copy_example(RUNNER_DATA_DIR) 112 | result = testdir.runpytest( 113 | directory / "tests-inputs/case-global-settings", 114 | "--exe-runner", 115 | directory / "runner.sh", 116 | "--exe-default-settings", 117 | directory / "settings.yaml", 118 | ) 119 | assert_outcomes(result, passed=1) 120 | stdout = ( 121 | (directory / "tests-output/case-global-settings/runner.sh.stdout") 122 | .open() 123 | .read() 124 | .strip() 125 | ) 126 | assert stdout == "100" 127 | 128 | 129 | def test_runner_error_with_undefined_placeholder(testdir): 130 | """Test error with runner fixture when a placeholder is not replaced.""" 131 | directory = testdir.copy_example(RUNNER_DATA_DIR) 132 | result = testdir.runpytest( 133 | directory / "tests-inputs/case-global-settings", 134 | "--exe-runner", 135 | directory / "runner.sh", 136 | ) 137 | assert_outcomes(result, errors=1) 138 | result.stdout.fnmatch_lines( 139 | ["E ValueError: in */runner.sh: 'nproc' is undefined"] 140 | ) 141 | -------------------------------------------------------------------------------- /tests/test_plugin/test_misc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tests for the plugin itself.""" 18 | from __future__ import annotations 19 | 20 | import pytest 21 | 22 | from . import assert_outcomes 23 | 24 | 25 | def test_collect_order(testdir): 26 | """Check the tests order. 27 | 28 | The default test module functions shall be first. Then the additional test 29 | modules, and finally the test modules from the parent directories. 30 | """ 31 | directory = testdir.copy_example("tests/data/collect_order") 32 | result = testdir.runpytest(directory, "--collect-only") 33 | result.stdout.re_match_lines( 34 | [ 35 | "collected 5 items", 36 | "", 37 | " ", 38 | "", 39 | " ", 40 | "", 41 | " ", 42 | "", 43 | " ", 44 | "", 45 | " ", 46 | ] 47 | ) 48 | 49 | 50 | def test_marks_from_yaml(testdir): 51 | """Test marks from test-settings.yaml.""" 52 | directory = testdir.copy_example("tests/data/test_marks_from_yaml") 53 | 54 | # check tests detection 55 | result = testdir.runpytest(directory / "tests-inputs") 56 | assert_outcomes(result, passed=2, skipped=1) 57 | 58 | # select tests not with mark1 59 | result = testdir.runpytest( 60 | directory / "tests-inputs", "--collect-only", "-m not mark1" 61 | ) 62 | assert result.parseoutcomes()["deselected"] == 3 63 | 64 | 65 | def test_output_directory_already_exists(testdir): 66 | """Test create_output_dir fixture for existing directory error.""" 67 | directory = testdir.copy_example("tests/data/test_output_dir_fixture") 68 | result = testdir.runpytest(directory / "tests-inputs") 69 | # error because directory already exists 70 | assert_outcomes(result, errors=1) 71 | result.stdout.fnmatch_lines( 72 | [ 73 | "E FileExistsError", 74 | "", 75 | "During handling of the above exception, another exception occurred:", 76 | 'E FileExistsError: the output directory "*" already exists: ' 77 | "either remove it manually or use the --exe-clean-output option to " 78 | "remove it or use the --exe-overwrite-output to overwrite it", 79 | ] 80 | ) 81 | 82 | 83 | def test___init__(testdir): 84 | """Test error handling when missing __init__.py.""" 85 | testdir.copy_example("tests/data/test___init__") 86 | result = testdir.runpytest_subprocess() 87 | assert_outcomes(result, errors=1) 88 | result.stdout.fnmatch_lines( 89 | [ 90 | "*/tests-inputs/case2", 91 | "shall have a __init__.py because test_dummy.py exists in other " 92 | "directories", 93 | ] 94 | ) 95 | 96 | 97 | def test_cli_check_clash(testdir): 98 | """Test CLI arguments clash.""" 99 | directory = testdir.copy_example("tests/data/test_cli_check") 100 | result = testdir.runpytest_subprocess( 101 | directory, "--exe-clean-output", "--exe-overwrite-output" 102 | ) 103 | result.stderr.fnmatch_lines( 104 | [ 105 | "ERROR: options --exe-clean-output and --exe-overwrite-output " 106 | "are not compatible" 107 | ] 108 | ) 109 | 110 | 111 | @pytest.mark.parametrize( 112 | "option_name", 113 | ( 114 | "--exe-runner", 115 | "--exe-default-settings", 116 | "--exe-regression-root", 117 | "--exe-report-generator", 118 | "--exe-test-module", 119 | ), 120 | ) 121 | def test_cli_check(testdir, option_name): 122 | """Test CLI arguments paths.""" 123 | result = testdir.runpytest_subprocess(option_name, "dummy") 124 | result.stderr.fnmatch_lines( 125 | [f"ERROR: argument {option_name}: no such file or directory: dummy"] 126 | ) 127 | -------------------------------------------------------------------------------- /tests/test_plugin/test_report.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Test the report feature.""" 18 | from __future__ import annotations 19 | 20 | import stat 21 | from pathlib import Path 22 | 23 | import pytest 24 | 25 | # deal with old pytest not having no_re_match_line 26 | # TODO: remove this once old pytest is no longer supported 27 | OLD_PYTEST = pytest.__version__ < "5.3.0" 28 | 29 | 30 | def test_report_no_report_generator(testdir): 31 | """Test no report title in the output when no report option is used.""" 32 | directory = testdir.copy_example("tests/data/test_report") 33 | result = testdir.runpytest(directory / "tests-inputs") 34 | # skip runner because no --exe-runner 35 | result.assert_outcomes(skipped=1) 36 | if OLD_PYTEST: 37 | assert "report generation" not in result.stdout.str() 38 | else: 39 | result.stdout.no_re_match_line("report generation") 40 | 41 | 42 | def fix_execute_permission(script_path: str) -> None: 43 | """Pytest testdir fixture does not copy the execution bit.""" 44 | path = Path(script_path) 45 | permission = stat.S_IMODE(path.stat().st_mode) 46 | path.chmod(permission | stat.S_IXUSR) 47 | 48 | 49 | def test_report_generator_internal_error(testdir): 50 | """Test error when generator has internal error.""" 51 | directory = testdir.copy_example("tests/data/test_report") 52 | generator_path = directory / "report/generator-ko.sh" 53 | fix_execute_permission(generator_path) 54 | result = testdir.runpytest( 55 | directory / "tests-inputs", 56 | "--exe-report-generator", 57 | generator_path, 58 | ) 59 | # skip runner because no --exe-runner 60 | result.assert_outcomes(skipped=1) 61 | result.stdout.re_match_lines( 62 | [ 63 | ".*starting report generation", 64 | "Command '.*/test_report_generator_internal_error0/report/generator-ko.sh' " 65 | "returned non-zero exit status 1.", 66 | ".*report generation failed", 67 | ] 68 | ) 69 | 70 | 71 | def test_report_generator_ok(testdir): 72 | """Test error when generator work ok.""" 73 | directory = testdir.copy_example("tests/data/test_report") 74 | generator_path = directory / "report/generator.sh" 75 | fix_execute_permission(generator_path) 76 | result = testdir.runpytest( 77 | directory / "tests-inputs", "--exe-report-generator", generator_path 78 | ) 79 | # skip runner because no --exe-runner 80 | result.assert_outcomes(skipped=1) 81 | result.stdout.re_match_lines( 82 | [".*starting report generation", ".*report generation done"] 83 | ) 84 | 85 | 86 | def test_no_test_no_report(testdir): 87 | """Test no report generator is called when there is no test run.""" 88 | directory = testdir.copy_example("tests/data/test_report") 89 | generator_path = directory / "report/generator.sh" 90 | fix_execute_permission(generator_path) 91 | result = testdir.runpytest( 92 | directory / "tests-inputs/empty-case", "--exe-report-generator", generator_path 93 | ) 94 | result.assert_outcomes() 95 | if OLD_PYTEST: 96 | assert "report generation" not in result.stdout.str() 97 | else: 98 | result.stdout.no_re_match_line("report generation") 99 | -------------------------------------------------------------------------------- /tests/test_report.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tests for the report feature.""" 18 | from __future__ import annotations 19 | 20 | import shutil 21 | from collections import defaultdict 22 | 23 | import pytest 24 | import yaml 25 | from pytest_executable.report import create 26 | from pytest_executable.report import dump 27 | from pytest_executable.report import merge 28 | 29 | from . import ROOT_DATA_DIR 30 | 31 | DATA_DIR = ROOT_DATA_DIR / "report" 32 | 33 | 34 | class _TestReport: 35 | def __init__(self, dir_path: str, message: str) -> None: 36 | self.path = dir_path 37 | self.longreprtext = message 38 | 39 | 40 | class TerminalReporter: 41 | """Mock of pytest TerminalReporter class.""" 42 | 43 | def __init__(self, report_data: list[list[str]]) -> None: 44 | self.stats: dict[str, list[_TestReport]] = defaultdict(list) 45 | for status, dir_path, message in report_data: 46 | self.stats[status] += [_TestReport(dir_path, message)] 47 | 48 | 49 | # report data with all cases 50 | REPORT_DATA = [ 51 | ["skipped", "root/path", ""], 52 | ["passed", "root/path", ""], 53 | ["failed", "root/path", "failure message 0"], 54 | ["error", "root/path", "error message 0"], 55 | ["skipped", "root/dir1/path", ""], 56 | ["passed", "root/dir1/path", ""], 57 | ["failed", "root/dir1/path", "failure message 1"], 58 | ["skipped", "root/dir2/path", ""], 59 | ["passed", "root/dir2/path", ""], 60 | ["skipped", "root/dir3/path", ""], 61 | ["skipped", "root/dir3/path", ""], 62 | ] 63 | 64 | 65 | @pytest.mark.parametrize( 66 | "report_data,expected", 67 | ( 68 | # tests with one path 69 | ( 70 | REPORT_DATA[0:4], 71 | # error priority 72 | { 73 | ".": { 74 | "status": "error", 75 | "messages": ["error message 0", "failure message 0"], 76 | } 77 | }, 78 | ), 79 | ( 80 | REPORT_DATA[4:7], 81 | # failed priority 82 | {"dir1": {"status": "failed", "messages": ["failure message 1"]}}, 83 | ), 84 | ( 85 | REPORT_DATA[7:9], 86 | # passed if at least one passed 87 | {"dir2": {"status": "passed", "messages": []}}, 88 | ), 89 | ( 90 | REPORT_DATA[9:], 91 | # skipped if all skipped 92 | {"dir3": {"status": "skipped", "messages": []}}, 93 | ), 94 | # tests with several paths 95 | ( 96 | REPORT_DATA, 97 | # error priority 98 | { 99 | ".": { 100 | "status": "error", 101 | "messages": ["error message 0", "failure message 0"], 102 | }, 103 | "dir1": {"status": "failed", "messages": ["failure message 1"]}, 104 | "dir2": {"status": "passed", "messages": []}, 105 | "dir3": {"status": "skipped", "messages": []}, 106 | }, 107 | ), 108 | ), 109 | ) 110 | def test_create(report_data, expected): 111 | """Test create function.""" 112 | assert create(TerminalReporter(report_data)) == expected 113 | 114 | 115 | @pytest.mark.parametrize( 116 | "db,expected_db", 117 | ( 118 | ( 119 | { 120 | # existing entry is overwritten 121 | ".": None, 122 | # new entry is added 123 | "dir4": None, 124 | }, 125 | { 126 | ".": None, 127 | "dir1": {"status": "failed", "messages": ["failure message 1"]}, 128 | "dir2": {"status": "passed", "messages": []}, 129 | "dir3": {"status": "skipped", "messages": []}, 130 | "dir4": None, 131 | }, 132 | ), 133 | ), 134 | ) 135 | def test_merge(db, expected_db): 136 | """Test the merge function.""" 137 | db_path = DATA_DIR / "report_db.yaml" 138 | new_db = merge(db_path, db) 139 | assert new_db == expected_db 140 | 141 | 142 | def test_dump_new(tmp_path): 143 | """Test dump function without existing db file.""" 144 | report_path = tmp_path / "report_db.yaml" 145 | dump(report_path, TerminalReporter(REPORT_DATA)) 146 | 147 | with report_path.open() as file_: 148 | db = yaml.safe_load(file_) 149 | 150 | with (DATA_DIR / "report_db.yaml").open() as file_: 151 | excepted_db = yaml.safe_load(file_) 152 | 153 | assert db == excepted_db 154 | 155 | 156 | def test_dump_existing(tmp_path): 157 | """Test dump function with existing db file.""" 158 | expected_path = DATA_DIR / "report_db.yaml" 159 | report_path = tmp_path / "report_db.yaml" 160 | shutil.copy(expected_path, report_path) 161 | 162 | # with the same test outcomes, we shall get the same db 163 | dump(report_path, TerminalReporter(REPORT_DATA)) 164 | 165 | with report_path.open() as file_: 166 | db = yaml.safe_load(file_) 167 | 168 | with expected_path.open() as file_: 169 | excepted_db = yaml.safe_load(file_) 170 | 171 | assert db == excepted_db 172 | -------------------------------------------------------------------------------- /tests/test_script_runner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tests for ScriptRunner.""" 18 | from __future__ import annotations 19 | 20 | import re 21 | import subprocess 22 | from pathlib import Path 23 | 24 | import pytest 25 | from pytest_executable.script_runner import ScriptExecutionError 26 | from pytest_executable.script_runner import ScriptRunner 27 | 28 | from . import ROOT_DATA_DIR 29 | 30 | DATA_DIR = ROOT_DATA_DIR / "runners" 31 | 32 | 33 | def test_error_with_missing_setting(tmp_path): 34 | """Test error when a placeholder cannot be replaced.""" 35 | error_msg = "in .*tests/data/runners/nproc.sh: 'nproc' is undefined" 36 | with pytest.raises(ValueError, match=error_msg): 37 | ScriptRunner(DATA_DIR / "nproc.sh", {}, tmp_path) 38 | 39 | 40 | def test_error_with_unreadable_script(tmp_path): 41 | """Test error when the script is not readable.""" 42 | error_msg = "cannot read the script .*/bin/bash" 43 | with pytest.raises(TypeError, match=error_msg): 44 | ScriptRunner(Path("/bin/bash"), {}, tmp_path) 45 | 46 | 47 | def test_execution_with_setting(tmp_path): 48 | """Test script execution with placeholder replaced.""" 49 | script_path = DATA_DIR / "nproc.sh" 50 | runner = ScriptRunner(script_path, {"nproc": "100"}, tmp_path) 51 | runner.run() 52 | _assertions(tmp_path / script_path.name, "echo 100", "100", "") 53 | 54 | 55 | def test_execution_with_timeout(tmp_path): 56 | """Test script execution with timeout.""" 57 | # with enough time 58 | script_path = DATA_DIR / "timeout.sh" 59 | runner = ScriptRunner(script_path, {"timeout": "2s"}, tmp_path) 60 | runner.run() 61 | 62 | # without enough time 63 | runner = ScriptRunner(script_path, {"timeout": "0.1s"}, tmp_path) 64 | error_msg = ( 65 | r"Command '\['/usr/bin/env', 'bash', 'timeout\.sh'\]' timed out after " 66 | ".* seconds" 67 | ) 68 | with pytest.raises(subprocess.TimeoutExpired, match=error_msg): 69 | runner.run() 70 | 71 | 72 | def test_execution_error(tmp_path): 73 | """Test error when the script execution fails.""" 74 | error_msg = "execution failure, see the stdout and stderr files in /" 75 | script_path = DATA_DIR / "error.sh" 76 | runner = ScriptRunner(script_path, {}, tmp_path) 77 | with pytest.raises(ScriptExecutionError, match=error_msg): 78 | runner.run() 79 | 80 | _assertions( 81 | tmp_path / script_path.name, 82 | "ls non-existing-file", 83 | "", 84 | "ls: (?:cannot access )?'?non-existing-file'?: No such file or directory", 85 | ) 86 | 87 | 88 | def _assertions(runner_path, script, stdout, stderr_regex): 89 | # check the content of the script, stdout and stderr files 90 | with runner_path.open() as file_: 91 | assert file_.read().strip() == script 92 | with (runner_path.with_suffix(".sh.stdout")).open() as file_: 93 | assert file_.read().strip() == stdout 94 | with (runner_path.with_suffix(".sh.stderr")).open() as file_: 95 | assert re.match(stderr_regex, file_.read()) 96 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020, CS Systemes d'Information, http://www.c-s.fr 2 | # 3 | # This file is part of pytest-executable 4 | # https://www.github.com/CS-SI/pytest-executable 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tests for the settings container.""" 18 | from __future__ import annotations 19 | 20 | import pytest 21 | from jsonschema import ValidationError 22 | from pytest_executable.plugin import SETTINGS_PATH as DEFAULT_SETTINGS_FILE 23 | from pytest_executable.settings import Settings 24 | from pytest_executable.settings import Tolerances 25 | 26 | 27 | @pytest.fixture 28 | def default_settings(): 29 | """Fixture that returns a hand made default Settings object.""" 30 | return Settings(runner={}, marks=set(), references=set(), tolerances={}) 31 | 32 | 33 | def _test_merge(tmp_path, yaml_str, ref_settings): 34 | # helper function 35 | settings_file = tmp_path / "settings.yaml" 36 | settings_file.write_text(yaml_str) 37 | assert ( 38 | Settings.from_local_file(DEFAULT_SETTINGS_FILE, settings_file) == ref_settings 39 | ) 40 | 41 | 42 | def test_defaults(tmp_path, default_settings): 43 | """Test yaml merge with default settings.""" 44 | # default settings merged with itself 45 | yaml_str = "" 46 | _test_merge(tmp_path, yaml_str, default_settings) 47 | 48 | 49 | def test_merge(tmp_path): 50 | """Test yaml merge with default settings.""" 51 | # default settings merged with itself 52 | yaml_str = "" 53 | ref_settings = Settings.from_local_file( 54 | DEFAULT_SETTINGS_FILE, DEFAULT_SETTINGS_FILE 55 | ) 56 | _test_merge(tmp_path, yaml_str, ref_settings) 57 | 58 | 59 | def test_merge_0(tmp_path, default_settings): 60 | """Test merge existing dict item.""" 61 | yaml_str = """ 62 | tolerances: 63 | Velocity: 64 | rel: 1. 65 | """ 66 | default_settings.tolerances["Velocity"] = Tolerances() 67 | default_settings.tolerances["Velocity"].rel = 1 68 | _test_merge(tmp_path, yaml_str, default_settings) 69 | 70 | 71 | def test_merge_1(tmp_path, default_settings): 72 | """Test merge new dict item.""" 73 | yaml_str = """ 74 | tolerances: 75 | X: 76 | rel: 1. 77 | """ 78 | default_settings.tolerances["X"] = Tolerances(rel=1.0, abs=0.0) 79 | _test_merge(tmp_path, yaml_str, default_settings) 80 | 81 | 82 | def test_merge_2(tmp_path, default_settings): 83 | """Test merge new dict item.""" 84 | yaml_str = """ 85 | marks: 86 | - mark 87 | """ 88 | default_settings.marks = {"mark"} 89 | _test_merge(tmp_path, yaml_str, default_settings) 90 | 91 | 92 | @pytest.mark.parametrize( 93 | "yaml_str", 94 | ( # marks shall be unique 95 | """ 96 | marks: 97 | - x 98 | - x 99 | """, 100 | # references shall be unique 101 | """ 102 | references: 103 | - x 104 | - x 105 | """, 106 | # runner shall be an object 107 | """ 108 | runner: 0 109 | """, 110 | """ 111 | runner: [] 112 | """, 113 | # tolerances shall be rel or abs 114 | """ 115 | tolerances: 116 | quantity: 117 | X: 1. 118 | """, 119 | # rel shall be number 120 | """ 121 | tolerances: 122 | quantity: 123 | rel: x 124 | """, 125 | # rel shall be positive 126 | """ 127 | tolerances: 128 | quantity: 129 | rel: -1. 130 | """, 131 | ), 132 | ) 133 | def test_yaml_validation(tmp_path, yaml_str): 134 | """Test yaml schema validation.""" 135 | settings_file = tmp_path / "settings.yaml" 136 | settings_file.write_text(yaml_str) 137 | with pytest.raises(ValidationError): 138 | Settings.from_local_file(DEFAULT_SETTINGS_FILE, settings_file) 139 | 140 | 141 | def test_alien_item(tmp_path, default_settings): 142 | """Test that an alien item in the yaml is ignored.""" 143 | yaml_str = "dummy: ''" 144 | settings_file = tmp_path / "settings.yaml" 145 | settings_file.write_text(yaml_str) 146 | settings = Settings.from_local_file(DEFAULT_SETTINGS_FILE, settings_file) 147 | assert settings == default_settings 148 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | env_list = 3 | py{38,39,310,311,312}-pytest{5,6,7} 4 | 5 | [testenv] 6 | package = wheel 7 | wheel_build_env = {package_env} 8 | deps = 9 | pytest5: pytest==5.* 10 | pytest6: pytest==6.* 11 | pytest7: pytest==7.* 12 | coverage: pytest-cov 13 | extras = test 14 | set_env = 15 | coverage: __COVERAGE_POSARGS=--cov --cov-report=xml 16 | commands = 17 | pytest {env:__COVERAGE_POSARGS:} {posargs} 18 | 19 | [testenv:doc] 20 | deps = doc/requirements.txt 21 | use_develop = true 22 | commands = 23 | sphinx-build {posargs:-E} -b html doc dist/doc 24 | 25 | [testenv:doc-checklinks] 26 | deps = doc/requirements.txt 27 | use_develop = true 28 | commands = 29 | sphinx-build -b linkcheck doc dist/doc-checklinks 30 | 31 | [testenv:doc-spell] 32 | set_env = 33 | SPELLCHECK=1 34 | use_develop = false 35 | deps = 36 | -r{toxinidir}/doc/requirements.txt 37 | sphinxcontrib-spelling 38 | pyenchant 39 | commands = 40 | sphinx-build -b spelling doc dist/doc 41 | 42 | [testenv:create-dist] 43 | description = create the pypi distribution 44 | deps = 45 | twine 46 | build 47 | skip_install = true 48 | allowlist_externals = rm 49 | commands = 50 | rm -rf dist build 51 | python -m build 52 | twine check dist/* --------------------------------------------------------------------------------