├── .github └── workflows │ ├── coverage.yml │ ├── package.yml │ └── test.yml ├── .gitignore ├── .mailmap ├── .zenodo.json ├── CITATION.cff ├── LICENSES └── MPL-2.0.txt ├── README.md ├── doc ├── about │ ├── audience.rst │ ├── motivation.rst │ └── similar.rst ├── conf.py ├── creating │ ├── configuration.rst │ ├── example.rst │ ├── filter_options.rst │ ├── general.rst │ └── run_function.rst ├── developers │ ├── branches.rst │ └── contributing.rst ├── index.rst └── running │ ├── command_line_arguments.rst │ └── generated_files.rst ├── img ├── runtest-small.png └── runtest.png ├── pyproject.toml ├── requirements.txt └── runtest ├── __init__.py ├── check.py ├── cli.py ├── copy.py ├── exceptions.py ├── extract.py ├── filter_api.py ├── filter_constructor.py ├── run.py ├── scissors.py ├── test ├── __init__.py ├── different_length │ ├── out.txt │ └── ref.txt ├── generic │ ├── out.txt │ └── ref.txt ├── ignore_order │ ├── out.txt │ └── ref.txt ├── ignore_order_and_sign │ ├── out.txt │ └── ref.txt ├── integers │ ├── out.txt │ └── ref.txt └── only_string │ ├── out.txt │ └── ref.txt ├── tuple_comparison.py └── version.py /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Coverage 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | python-version: ["3.13"] 16 | os: [ubuntu-latest] 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v5 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install Python dependencies 26 | run: pip install -r requirements.txt 27 | - name: Obtain coverage analysis 28 | run: pytest -v --cov runtest runtest/*.py 29 | - name: Coveralls 30 | env: 31 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} 32 | run: | 33 | coveralls 34 | -------------------------------------------------------------------------------- /.github/workflows/package.yml: -------------------------------------------------------------------------------- 1 | name: Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | build: 9 | permissions: write-all 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | python-version: ["3.13"] 14 | os: [ubuntu-latest] 15 | 16 | steps: 17 | - name: Switch branch 18 | uses: actions/checkout@v4 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install Python dependencies 24 | run: | 25 | which python 26 | python -m pip install --upgrade pip 27 | pip install -r requirements.txt 28 | - name: Flit publish 29 | run: 30 | flit publish 31 | env: 32 | FLIT_USERNAME: __token__ 33 | FLIT_PASSWORD: ${{ secrets.PYPI_TOKEN }} 34 | # FLIT_INDEX_URL: https://test.pypi.org/legacy/ 35 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | python-version: ["3.10", "3.11", "3.12", "3.13"] 16 | os: [ubuntu-latest, macOS-latest, windows-latest] 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v5 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install Python dependencies 26 | run: pip install -r requirements.txt 27 | - name: Run tests 28 | run: | 29 | pytest -v runtest/check.py 30 | pytest -v runtest/extract.py 31 | pytest -v runtest/scissors.py 32 | pytest -v runtest/tuple_comparison.py 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | _build/ 3 | *.diff 4 | *.filtered 5 | *.reference 6 | .coverage 7 | .cache/ 8 | venv/ 9 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Radovan Bast 2 | -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "creators": [ 3 | { 4 | "name": "Bast, Radovan", 5 | "orcid": "0000-0002-7658-1847" 6 | } 7 | ], 8 | "publication_date": "2023-06-04", 9 | "title": "runtest: Numerically tolerant end-to-end test library for research software", 10 | "version": "2.3.5" 11 | } 12 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: Bast 5 | given-names: Radovan 6 | orcid: https://orcid.org/0000-0002-7658-1847 7 | title: "runtest: Numerically tolerant end-to-end test library for research software" 8 | version: 2.3.5 9 | doi: 10.5281/zenodo.1069004 10 | date-released: 2023-06-04 11 | -------------------------------------------------------------------------------- /LICENSES/MPL-2.0.txt: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at https://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![image](https://github.com/bast/runtest/workflows/Test/badge.svg)](https://github.com/bast/runtest/actions) 2 | [![image](https://coveralls.io/repos/bast/runtest/badge.png?branch=main)](https://coveralls.io/r/bast/runtest?branch=main) 3 | [![image](https://readthedocs.org/projects/runtest/badge/?version=latest)](http://runtest.readthedocs.io) 4 | [![image](https://img.shields.io/badge/license-%20MPL--v2.0-blue.svg)](LICENSES/MPL-2.0.txt) 5 | [![image](https://zenodo.org/badge/DOI/10.5281/zenodo.1069004.svg)](https://doi.org/10.5281/zenodo.1069004) 6 | [![image](https://badge.fury.io/py/runtest.svg)](https://badge.fury.io/py/runtest) 7 | 8 | 9 | # runtest 10 | 11 | Numerically tolerant end-to-end test library for research software. 12 | 13 | ![image of a hardware circuit with red and green light bulbs](img/runtest-small.png) 14 | 15 | Image: [Midjourney](https://midjourney.com/), [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/legalcode) 16 | 17 | 18 | ## Installation 19 | 20 | ``` 21 | $ pip install runtest 22 | ``` 23 | 24 | 25 | ## Supported Python versions 26 | 27 | The library is tested with Python 3.10 - 3.13. If you want to test 28 | runtest itself on your computer, you can follow what we do in the [CI 29 | workflow](https://github.com/bast/runtest/blob/main/.github/workflows/test.yml). 30 | 31 | 32 | ## Documentation 33 | 34 | - [Latest code](http://runtest.readthedocs.io/en/latest/) (main branch) 35 | 36 | 37 | Past versions 38 | - [1.3.z](http://runtest.readthedocs.io/en/release-1.3.z/) ([release-1.3.z branch](https://github.com/bast/runtest/tree/release-1.3.z)) 39 | 40 | 41 | ## Citation 42 | 43 | For a recommended citation, please check the at the bottom-right of 44 | . 45 | 46 | 47 | ## Projects using runtest 48 | 49 | - [DIRAC](http://diracprogram.org) 50 | - [Dalton](http://daltonprogram.org) and [LSDalton](http://daltonprogram.org) 51 | - [GIMIC](https://github.com/qmcurrents/gimic) 52 | - [OpenRSP](http://openrsp.org) 53 | - [MRChem](https://mrchem.readthedocs.io/en/latest/) 54 | - GRASP (General-purpose Relativistic Atomic Structure Program) 55 | - [eT](https://etprogram.org) 56 | 57 | If you use runtest, please add a link to your project via a pull 58 | request. 59 | 60 | 61 | ## Similar projects 62 | 63 | - [testcode](http://testcode.readthedocs.io) is a python module for 64 | testing for regression errors in numerical (principally scientific) 65 | software. 66 | -------------------------------------------------------------------------------- /doc/about/audience.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Audience 4 | ======== 5 | 6 | 7 | Explain runtest in one sentence 8 | ------------------------------- 9 | 10 | Runtest will assist you in running an entire calculation/simulation, extracting portions for the simulation outputs, 11 | and comparing these portions with reference outputs and scream if the results have changed above a predefined numerical 12 | tolerance. 13 | 14 | 15 | When should one use runtest? 16 | ---------------------------- 17 | 18 | - You compute numerical results. 19 | - You want a library that understands that floating point precision is limited. 20 | - You want to be able to update tests by updating reference outputs. 21 | - You look for an end-to-end testing support. 22 | 23 | 24 | When should one not use runtest? 25 | -------------------------------- 26 | 27 | - You look for a unit test library which tests single functions. Much better alternatives exist for this. 28 | -------------------------------------------------------------------------------- /doc/about/motivation.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Motivation 4 | ========== 5 | 6 | 7 | Scope 8 | ----- 9 | 10 | When testing numerical codes against functionality regression, you typically 11 | cannot use a plain diff against the reference outputs due to numerical noise in 12 | the digits and because there may be many numbers that change all the time and 13 | that you do not want to test (e.g. date and time of execution). 14 | 15 | The aim of this library is to make the testing and maintenance of tests easy. 16 | The library allows to extract portions of the program output(s) which are 17 | automatically compared to reference outputs with a relative or absolute 18 | numerical tolerance to compensate for numerical noise due to machine precision. 19 | 20 | 21 | Design decisions 22 | ---------------- 23 | 24 | The library is designed to play well with CTest, to be convenient when used 25 | interactively, and to work without trouble on Linux, Mac, and Windows. It 26 | offers a basic argument parsing for test scripts. 27 | -------------------------------------------------------------------------------- /doc/about/similar.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Similar projects 4 | ================ 5 | 6 | - http://testcode.readthedocs.io: testcode is a python module for 7 | testing for regression errors in numerical (principally scientific) software. 8 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # runtest documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Feb 27 14:54:39 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import datetime 18 | 19 | # If extensions (or modules to document with autodoc) are in another directory, 20 | # add these directories to sys.path here. If the directory is relative to the 21 | # documentation root, use os.path.abspath to make it absolute, like shown here. 22 | sys.path.insert(0, os.path.abspath("..")) 23 | from runtest import __version__ as _version 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | # extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] 34 | extensions = [ 35 | "sphinx.ext.todo", 36 | ] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ["_templates"] 40 | 41 | # The suffix(es) of source filenames. 42 | # You can specify multiple suffix as a list of string: 43 | # source_suffix = ['.rst', '.md'] 44 | source_suffix = ".rst" 45 | 46 | # The encoding of source files. 47 | # source_encoding = 'utf-8-sig' 48 | 49 | # The master toctree document. 50 | master_doc = "index" 51 | 52 | # General information about the project. 53 | project = "runtest" 54 | copyright = f"{datetime.datetime.now().year}, Radovan Bast" 55 | author = "Radovan Bast" 56 | 57 | # The version info for the project you're documenting, acts as replacement for 58 | # |version| and |release|, also used in various other places throughout the 59 | # built documents. 60 | # 61 | # The full version, including alpha/beta/rc tags. 62 | release = _version 63 | # The short X.Y version. 64 | version = ".".join(release.split(".")[0:2]) 65 | 66 | # The language for content autogenerated by Sphinx. Refer to documentation 67 | # for a list of supported languages. 68 | # 69 | # This is also used if you do content translation via gettext catalogs. 70 | # Usually you set "language" from the command line for these cases. 71 | language = None 72 | 73 | # There are two options for replacing |today|: either, you set today to some 74 | # non-false value, then it is used: 75 | # today = '' 76 | # Else, today_fmt is used as the format for a strftime call. 77 | # today_fmt = '%B %d, %Y' 78 | 79 | # List of patterns, relative to source directory, that match files and 80 | # directories to ignore when looking for source files. 81 | exclude_patterns = ["_build"] 82 | 83 | # The reST default role (used for this markup: `text`) to use for all 84 | # documents. 85 | # default_role = None 86 | 87 | # If true, '()' will be appended to :func: etc. cross-reference text. 88 | # add_function_parentheses = True 89 | 90 | # If true, the current module name will be prepended to all description 91 | # unit titles (such as .. function::). 92 | # add_module_names = True 93 | 94 | # If true, sectionauthor and moduleauthor directives will be shown in the 95 | # output. They are ignored by default. 96 | # show_authors = False 97 | 98 | # The name of the Pygments (syntax highlighting) style to use. 99 | pygments_style = "sphinx" 100 | 101 | # A list of ignored prefixes for module index sorting. 102 | # modindex_common_prefix = [] 103 | 104 | # If true, keep warnings as "system message" paragraphs in the built documents. 105 | # keep_warnings = False 106 | 107 | # If true, `todo` and `todoList` produce output, else they produce nothing. 108 | todo_include_todos = True 109 | 110 | 111 | # -- Options for HTML output ---------------------------------------------- 112 | 113 | # The theme to use for HTML and HTML Help pages. See the documentation for 114 | # a list of builtin themes. 115 | # html_theme = 'default' 116 | 117 | # on_rtd is whether we are on readthedocs.org 118 | on_rtd = os.environ.get("READTHEDOCS", None) == "True" 119 | if not on_rtd: # only import and set the theme if we're building docs locally 120 | import sphinx_rtd_theme 121 | 122 | html_theme = "sphinx_rtd_theme" 123 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 124 | 125 | # Theme options are theme-specific and customize the look and feel of a theme 126 | # further. For a list of options available for each theme, see the 127 | # documentation. 128 | # html_theme_options = {} 129 | 130 | # Add any paths that contain custom themes here, relative to this directory. 131 | # html_theme_path = [] 132 | 133 | # The name for this set of Sphinx documents. If None, it defaults to 134 | # " v documentation". 135 | # html_title = None 136 | 137 | # A shorter title for the navigation bar. Default is the same as html_title. 138 | # html_short_title = None 139 | 140 | # The name of an image file (relative to this directory) to place at the top 141 | # of the sidebar. 142 | # html_logo = None 143 | 144 | # The name of an image file (within the static path) to use as favicon of the 145 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 146 | # pixels large. 147 | # html_favicon = None 148 | 149 | # Add any paths that contain custom static files (such as style sheets) here, 150 | # relative to this directory. They are copied after the builtin static files, 151 | # so a file named "default.css" will overwrite the builtin "default.css". 152 | # html_static_path = ['_static'] 153 | 154 | # Add any extra paths that contain custom files (such as robots.txt or 155 | # .htaccess) here, relative to this directory. These files are copied 156 | # directly to the root of the documentation. 157 | # html_extra_path = [] 158 | 159 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 160 | # using the given strftime format. 161 | # html_last_updated_fmt = '%b %d, %Y' 162 | 163 | # If true, SmartyPants will be used to convert quotes and dashes to 164 | # typographically correct entities. 165 | # html_use_smartypants = True 166 | 167 | # Custom sidebar templates, maps document names to template names. 168 | # html_sidebars = {} 169 | 170 | # Additional templates that should be rendered to pages, maps page names to 171 | # template names. 172 | # html_additional_pages = {} 173 | 174 | # If false, no module index is generated. 175 | # html_domain_indices = True 176 | 177 | # If false, no index is generated. 178 | # html_use_index = True 179 | 180 | # If true, the index is split into individual pages for each letter. 181 | # html_split_index = False 182 | 183 | # If true, links to the reST sources are added to the pages. 184 | # html_show_sourcelink = True 185 | 186 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 187 | # html_show_sphinx = True 188 | 189 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 190 | # html_show_copyright = True 191 | 192 | # If true, an OpenSearch description file will be output, and all pages will 193 | # contain a tag referring to it. The value of this option must be the 194 | # base URL from which the finished HTML is served. 195 | # html_use_opensearch = '' 196 | 197 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 198 | # html_file_suffix = None 199 | 200 | # Language to be used for generating the HTML full-text search index. 201 | # Sphinx supports the following languages: 202 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 203 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 204 | # html_search_language = 'en' 205 | 206 | # A dictionary with options for the search language support, empty by default. 207 | # Now only 'ja' uses this config value 208 | # html_search_options = {'type': 'default'} 209 | 210 | # The name of a javascript file (relative to the configuration directory) that 211 | # implements a search results scorer. If empty, the default will be used. 212 | # html_search_scorer = 'scorer.js' 213 | 214 | # Output file base name for HTML help builder. 215 | htmlhelp_basename = "runtestdoc" 216 | 217 | # -- Options for LaTeX output --------------------------------------------- 218 | 219 | latex_elements = { 220 | # The paper size ('letterpaper' or 'a4paper'). 221 | #'papersize': 'letterpaper', 222 | # The font size ('10pt', '11pt' or '12pt'). 223 | #'pointsize': '10pt', 224 | # Additional stuff for the LaTeX preamble. 225 | #'preamble': '', 226 | # Latex figure (float) alignment 227 | #'figure_align': 'htbp', 228 | } 229 | 230 | # Grouping the document tree into LaTeX files. List of tuples 231 | # (source start file, target name, title, 232 | # author, documentclass [howto, manual, or own class]). 233 | latex_documents = [ 234 | ("index", "runtest.tex", "runtest Documentation", author, "manual"), 235 | ] 236 | 237 | # The name of an image file (relative to this directory) to place at the top of 238 | # the title page. 239 | # latex_logo = None 240 | 241 | # For "manual" documents, if this is true, then toplevel headings are parts, 242 | # not chapters. 243 | # latex_use_parts = False 244 | 245 | # If true, show page references after internal links. 246 | # latex_show_pagerefs = False 247 | 248 | # If true, show URL addresses after external links. 249 | # latex_show_urls = False 250 | 251 | # Documents to append as an appendix to all manuals. 252 | # latex_appendices = [] 253 | 254 | # If false, no module index is generated. 255 | # latex_domain_indices = True 256 | 257 | 258 | # -- Options for manual page output --------------------------------------- 259 | 260 | # One entry per manual page. List of tuples 261 | # (source start file, name, description, authors, manual section). 262 | man_pages = [("index", "runtest", "runtest Documentation", [author], 1)] 263 | 264 | # If true, show URL addresses after external links. 265 | # man_show_urls = False 266 | 267 | 268 | # -- Options for Texinfo output ------------------------------------------- 269 | 270 | # Grouping the document tree into Texinfo files. List of tuples 271 | # (source start file, target name, title, author, 272 | # dir menu entry, description, category) 273 | texinfo_documents = [ 274 | ( 275 | "index", 276 | "runtest", 277 | "runtest Documentation", 278 | author, 279 | "runtest", 280 | "One line description of project.", 281 | "Miscellaneous", 282 | ), 283 | ] 284 | 285 | # Documents to append as an appendix to all manuals. 286 | # texinfo_appendices = [] 287 | 288 | # If false, no module index is generated. 289 | # texinfo_domain_indices = True 290 | 291 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 292 | # texinfo_show_urls = 'footnote' 293 | -------------------------------------------------------------------------------- /doc/creating/configuration.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | How to hook up runtest with your code 4 | ===================================== 5 | 6 | The runtest library is a low-level program-independent library that provides 7 | infrastructure for running calculations and extracting and comparing numbers 8 | against reference outputs. The library does not know anything about your code. 9 | 10 | In order to tell the library how to run your code, the library requires that 11 | you define a configure function which defines how to handle a list of input 12 | files and extra arguments. This configure function also defines the launcher 13 | script or binary for your code, the full launch command, the output prefix, and 14 | relative reference path where reference outputs are stored. 15 | The output prefix can also be ``None``. 16 | 17 | Here is an example module ``runtest_config.py`` which defines such a function: 18 | 19 | .. code-block:: python 20 | 21 | def configure(options, input_files, extra_args): 22 | """ 23 | This function is used by runtest to configure runtest 24 | at runtime for code specific launch command and file naming. 25 | """ 26 | 27 | from os import path 28 | from sys import platform 29 | 30 | launcher = 'pam' 31 | launcher_full_path = path.normpath(path.join(options.binary_dir, launcher)) 32 | 33 | (inp, mol) = input_files 34 | 35 | if platform == "win32": 36 | exe = 'dirac.x.exe' 37 | else: 38 | exe = 'dirac.x' 39 | 40 | command = [] 41 | command.append('python {0}'.format(launcher_full_path)) 42 | command.append('--dirac={0}'.format(path.join(options.binary_dir, exe))) 43 | command.append('--noarch --nobackup') 44 | command.append('--inp={0} --mol={1}'.format(inp, mol)) 45 | if extra_args is not None: 46 | command.append(extra_args) 47 | 48 | full_command = ' '.join(command) 49 | 50 | inp_no_suffix = path.splitext(inp)[0] 51 | mol_no_suffix = path.splitext(mol)[0] 52 | 53 | output_prefix = '{0}_{1}'.format(inp_no_suffix, mol_no_suffix) 54 | 55 | relative_reference_path = 'result' 56 | 57 | return launcher, full_command, output_prefix, relative_reference_path 58 | 59 | The function is expected to return ``launcher``, ``full_command``, 60 | ``output_prefix``, and ``relative_reference_path``. 61 | -------------------------------------------------------------------------------- /doc/creating/example.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _example-test-script: 3 | 4 | Example test script 5 | =================== 6 | 7 | Let us consider a relatively simple annotated example. 8 | 9 | First we import modules that we need (highlighted lines): 10 | 11 | .. code-block:: python 12 | :emphasize-lines: 3-16 13 | 14 | #!/usr/bin/env python 15 | 16 | # provides os.path.join 17 | import os 18 | 19 | # provides exit 20 | import sys 21 | 22 | # we make sure we can import runtest and runtest_config 23 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 24 | 25 | # we import essential functions from the runtest library 26 | from runtest import version_info, get_filter, cli, run 27 | 28 | # this tells runtest how to run your code 29 | from runtest_config import configure 30 | 31 | # we stop the script if the major version is not compatible 32 | assert version_info.major == 2 33 | 34 | # construct a filter list which contains two filters 35 | f = [ 36 | get_filter(from_string='@ Elements of the electric dipole', 37 | to_string='@ anisotropy', 38 | rel_tolerance=1.0e-5), 39 | get_filter(from_string='************ Expectation values', 40 | to_string='s0 = T : Expectation value', 41 | rel_tolerance=1.0e-5), 42 | ] 43 | 44 | # invoke the command line interface parser which returns options 45 | options = cli() 46 | 47 | ierr = 0 48 | for inp in ['PBE0gracLB94.inp', 'GLLBsaopLBalpha.inp']: 49 | for mol in ['Ne.mol']: 50 | # the run function runs the code and filters the outputs 51 | ierr += run(options, 52 | configure, 53 | input_files=[inp, mol], 54 | filters={'out': f}) 55 | 56 | sys.exit(ierr) 57 | 58 | Then we construct a list of filters. We can construct as many lists as we like 59 | and they can contain as many filters as we like. The list does not have to be 60 | called "f". Give it a name that is meaningful to you. 61 | 62 | .. code-block:: python 63 | :emphasize-lines: 21-29 64 | 65 | #!/usr/bin/env python 66 | 67 | # provides os.path.join 68 | import os 69 | 70 | # provides exit 71 | import sys 72 | 73 | # we make sure we can import runtest and runtest_config 74 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 75 | 76 | # we import essential functions from the runtest library 77 | from runtest import version_info, get_filter, cli, run 78 | 79 | # this tells runtest how to run your code 80 | from runtest_config import configure 81 | 82 | # we stop the script if the major version is not compatible 83 | assert version_info.major == 2 84 | 85 | # construct a filter list which contains two filters 86 | f = [ 87 | get_filter(from_string='@ Elements of the electric dipole', 88 | to_string='@ anisotropy', 89 | rel_tolerance=1.0e-5), 90 | get_filter(from_string='************ Expectation values', 91 | to_string='s0 = T : Expectation value', 92 | rel_tolerance=1.0e-5), 93 | ] 94 | 95 | # invoke the command line interface parser which returns options 96 | options = cli() 97 | 98 | ierr = 0 99 | for inp in ['PBE0gracLB94.inp', 'GLLBsaopLBalpha.inp']: 100 | for mol in ['Ne.mol']: 101 | # the run function runs the code and filters the outputs 102 | ierr += run(options, 103 | configure, 104 | input_files=[inp, mol], 105 | filters={'out': f}) 106 | 107 | sys.exit(ierr) 108 | 109 | After we use the command line interface to generate options, we really run the 110 | test. Note how we pass the configure function to the run function. Also note how 111 | we pass the filter list as a dictionary. If we omit to pass it, then the 112 | calculations will be run but not verified. This is useful for multi-step jobs. 113 | From the dictionary, the library knows that it should execute the filter list 114 | "f" on output files with the suffix "out". It is no problem to apply different 115 | filters to different output files, for this add entries to the `filters` 116 | dictionary. 117 | -------------------------------------------------------------------------------- /doc/creating/filter_options.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Filter options 4 | ============== 5 | 6 | 7 | Relative tolerance 8 | ------------------ 9 | 10 | There is no default. You have to select either relative or absolute tolerance 11 | for each test when testing floats. You cannot select both at the same time. 12 | 13 | In this example we set the relative tolerance to 1.0e-10: 14 | 15 | .. code-block:: python 16 | 17 | get_filter(from_string='Electronic energy', 18 | num_lines=8, 19 | rel_tolerance=1.0e-10) 20 | 21 | 22 | Absolute tolerance 23 | ------------------ 24 | 25 | There is no default. You have to select either relative or absolute tolerance 26 | for each test when testing floats. You cannot select both at the same time. 27 | 28 | In this example we set the absolute tolerance to 1.0e-10: 29 | 30 | .. code-block:: python 31 | 32 | get_filter(from_string='Electronic energy', 33 | num_lines=8, 34 | abs_tolerance=1.0e-10) 35 | 36 | 37 | How to check entire file 38 | ------------------------ 39 | 40 | By default all lines are tested so if you omit any string anchors and number of 41 | lines we will compare numbers from the entire file. 42 | 43 | Example: 44 | 45 | .. code-block:: python 46 | 47 | get_filter(rel_tolerance=1.0e-10) 48 | 49 | 50 | Filtering between two anchor strings 51 | ------------------------------------ 52 | 53 | Example: 54 | 55 | .. code-block:: python 56 | 57 | get_filter(from_string='@ Elements of the electric dipole', 58 | to_string='@ anisotropy', 59 | rel_tolerance=1.0e-10) 60 | 61 | This will extract all floats between these strings including the lines of the 62 | strings. 63 | 64 | The start/end strings can be regular expressions, for this use from_re or 65 | to_re. Any combination containing from_string/from_re and to_string/to_re is 66 | possible. 67 | 68 | 69 | Filtering a number of lines starting with string/regex 70 | ------------------------------------------------------ 71 | 72 | Example: 73 | 74 | .. code-block:: python 75 | 76 | get_filter(from_string='Electronic energy', 77 | num_lines=8, # here we compare 8 lines 78 | abs_tolerance=1.0e-10) 79 | 80 | The start string can be a string (from_string) or a regular expression 81 | (from_re). In the above example we extract and compare all lines that start 82 | with 'Electronic energy' including the following 7 lines. 83 | 84 | 85 | Extracting single lines 86 | ----------------------- 87 | 88 | This example will compare all lines which contain 'Electronic energy': 89 | 90 | .. code-block:: python 91 | 92 | get_filter(string='Electronic energy', 93 | abs_tolerance=1.0e-10) 94 | 95 | This will match the string in a *case-sensitive* fashion. 96 | 97 | Instead of single string we can give a single regular expression (re). 98 | 99 | .. code-block:: python 100 | 101 | get_filter(re='Electronic energy', 102 | abs_tolerance=1.0e-10) 103 | 104 | Regexes follow the `Python syntax `_. 105 | For example, to match in a *case-insensitive* fashion: 106 | 107 | .. code-block:: python 108 | 109 | get_filter(re=r'(?i)Electronic energy', 110 | abs_tolerance=1.0e-10) 111 | 112 | It is not possible to use Python regex objects directly. 113 | 114 | 115 | How to ignore sign 116 | ------------------ 117 | 118 | Sometimes the sign is not predictable. For this set ``ignore_sign=True``. 119 | 120 | 121 | How to ignore the order of numbers 122 | ---------------------------------- 123 | 124 | Setting ``ignore_order=True`` will sort the numbers (as they appear consecutively 125 | between anchors, one after another) before comparing them. 126 | This is useful for tests where some numbers can change place. 127 | 128 | 129 | How to ignore very small or very large numbers 130 | ---------------------------------------------- 131 | 132 | You can ignore very small numbers with skip_below. 133 | Default is 1.0e-40. Ignore all floats that are smaller than this number 134 | (this option ignores the sign). 135 | 136 | As an example consider the following result tensor:: 137 | 138 | 3716173.43448289 0.00000264 -0.00000346 139 | -0.00008183 75047.79698485 0.00000328 140 | 0.00003493 -0.00000668 75047.79698251 141 | 142 | 0.00023164 -153158.24017016 -0.00000493 143 | 90142.70952070 -0.00000602 0.00000574 144 | 0.00001946 -0.00000028 0.00000052 145 | 146 | 0.00005844 -0.00000113 -153158.24017263 147 | -0.00005667 0.00000015 -0.00000022 148 | 90142.70952022 0.00000056 0.00000696 149 | 150 | The small numbers are actually numerical noise and we do not want to test them 151 | at all. In this case it is useful to set ``skip_below=1.0e-4``. 152 | 153 | Alternatively one could use absolute tolerance to avoid checking the noisy 154 | zeros. 155 | 156 | You can ignore very large numbers with skip_above (also this option ignores 157 | the sign). 158 | 159 | 160 | How to ignore certain numbers 161 | ----------------------------- 162 | 163 | The keyword mask is useful if you extract lines which contain both interesting 164 | and uninteresting numbers (like timings which change from run to run). 165 | 166 | Example: 167 | 168 | .. code-block:: python 169 | 170 | get_filter(from_string='no. eigenvalue (eV) mean-res.', 171 | num_lines=4, 172 | rel_tolerance=1.0e-4, 173 | mask=[1, 2, 3]) 174 | 175 | Here we use only the first 3 floats in each line. Counting starts with 1. 176 | -------------------------------------------------------------------------------- /doc/creating/general.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | General tips 4 | ============ 5 | 6 | How to add a new test 7 | --------------------- 8 | 9 | Test scripts are python scripts which return zero (success) 10 | or non-zero (failure). You define what success or failure means. 11 | The runtest library helps you with basic tasks but you are free 12 | to go beyond and define own tests with arbitrary complexity. 13 | 14 | 15 | Strive for portability 16 | ---------------------- 17 | 18 | Avoid shell programming or symlinks in test scripts otherwise the tests are not 19 | portable to Windows. Therefore do not use ``os.system()`` or ``os.symlink()``. Do not 20 | use explicit forward slashes for paths, instead use ``os.path.join()``. 21 | 22 | 23 | Always test that the test really works 24 | -------------------------------------- 25 | 26 | It is easy to make a mistake and create a test which is always "successful". 27 | Test that your test catches mistakes. Verify whether it extracts the right 28 | numbers. 29 | 30 | 31 | Never commit functionality to the main development line without tests 32 | --------------------------------------------------------------------- 33 | 34 | If you commit functionality to the main development line without tests then 35 | this functionality will break sooner or later and we have no automatic 36 | mechanism to detect it. Committing new code without tests is bad karma. 37 | 38 | 39 | Never add inputs to the test directories which are never run 40 | ------------------------------------------------------------ 41 | 42 | We want all inputs and outputs to be accessile by the default test 43 | suite. Otherwise we have no automatic way to detect that some inputs or outputs 44 | have degraded. Degraded inputs and outputs are useless and confusing. 45 | -------------------------------------------------------------------------------- /doc/creating/run_function.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Run function arguments 4 | ====================== 5 | 6 | The ``run`` function has the following signature: 7 | 8 | .. code-block:: python 9 | 10 | def run(options, 11 | configure, 12 | input_files, 13 | extra_args=None, 14 | filters=None, 15 | accepted_errors=None): 16 | ... 17 | 18 | ``options`` is set by the command line interface (by the user executing runtest). 19 | 20 | ``configure`` is specific to the code at hand (see the :ref:`example-test-script`). 21 | 22 | ``input_files`` contains the input files passed to the code launcher. The data structure of 23 | ``input_files`` is set by the ``configure`` function (in other words by the code using runtest). 24 | 25 | There are three more optional arguments to the ``run`` function which by default are set to ``None``: 26 | 27 | ``extra_args`` contains extra arguments. Again, its data structure of 28 | is set by the ``configure`` function (in other words by the code using runtest). 29 | 30 | ``filters`` is a dictionary of suffix and filter list pairs and contains 31 | filters to apply to the results. If we omit to pass it, then the calculations 32 | will be run but not verified. This is useful for multi-step jobs. See also the 33 | :ref:`example-test-script`. If the ``output_prefix`` in the ``configure`` function is set to None, 34 | then the filters are applied to the file names literally. 35 | -------------------------------------------------------------------------------- /doc/developers/branches.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Branching model 4 | =============== 5 | 6 | We follow the semantic branching model: https://dev-cafe.github.io/branching-model/ 7 | -------------------------------------------------------------------------------- /doc/developers/contributing.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Contributing 4 | ============ 5 | 6 | Yes please! Please follow this excellent guide: 7 | http://www.contribution-guide.org. We do not require any formal copyright 8 | assignment or contributor license agreement. Any contributions intentionally 9 | sent upstream are presumed to be offered under terms of the Mozilla Public License Version 2.0. 10 | 11 | Methods, and variables that start with underscore are private. 12 | 13 | Please keep the default output as silent as possible. 14 | 15 | 16 | Where to contribute 17 | ------------------- 18 | 19 | Here are some ideas: 20 | 21 | - Improve documentation 22 | - Fix typos 23 | - Make it possible to install this package using pip 24 | - Make this package distributable via PyPI 25 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | runtest 4 | ======= 5 | 6 | Numerically tolerant end-to-end test library for research software. 7 | 8 | This documents the latest code on the ``main`` branch. 9 | The ``release-1.3.z`` code is documented here: http://runtest.readthedocs.io/en/release-1.3.z/. 10 | 11 | 12 | .. toctree:: 13 | :maxdepth: 1 14 | :caption: About 15 | 16 | about/motivation.rst 17 | about/audience.rst 18 | about/similar.rst 19 | 20 | 21 | .. toctree:: 22 | :maxdepth: 1 23 | :caption: Creating tests 24 | 25 | creating/general.rst 26 | creating/configuration.rst 27 | creating/example.rst 28 | creating/run_function.rst 29 | creating/filter_options.rst 30 | 31 | 32 | .. toctree:: 33 | :maxdepth: 1 34 | :caption: Running tests 35 | 36 | running/command_line_arguments.rst 37 | running/generated_files.rst 38 | 39 | 40 | .. toctree:: 41 | :maxdepth: 1 42 | :caption: Developers 43 | 44 | developers/contributing.rst 45 | developers/branches.rst 46 | -------------------------------------------------------------------------------- /doc/running/command_line_arguments.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Command-line arguments 4 | ====================== 5 | 6 | 7 | -h, --help 8 | ---------- 9 | 10 | Show help message and exit. 11 | 12 | 13 | -b BINARY_DIR, --binary-dir=BINARY_DIR 14 | -------------------------------------- 15 | 16 | Directory containing the binary/launcher. 17 | By default it is the directory of the test script which is executed. 18 | 19 | 20 | -w WORK_DIR, --work-dir=WORK_DIR 21 | -------------------------------- 22 | 23 | Working directory where all generated files will be written to. 24 | By default it is the directory of the test script which is executed. 25 | 26 | 27 | -l LAUNCH_AGENT, --launch-agent=LAUNCH_AGENT 28 | -------------------------------------------- 29 | 30 | Prepend a launch agent command (e.g. "mpirun -np 8" or 31 | "valgrind --leak-check=yes"). 32 | By default no launch agent is prepended. 33 | 34 | 35 | -v, --verbose 36 | ------------- 37 | 38 | Give more verbose output upon test failure (by default False). 39 | 40 | 41 | -s, --skip-run 42 | -------------- 43 | 44 | Skip actual calculation(s), only compare numbers. This is useful 45 | to adjust the test script for long calculations. 46 | 47 | 48 | -n, --no-verification 49 | ----------------------- 50 | 51 | Run calculation(s) but do not verify results. This is useful to 52 | generate outputs for the first time. 53 | -------------------------------------------------------------------------------- /doc/running/generated_files.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Generated files 4 | =============== 5 | 6 | 7 | The test script generates three files per run with the suffixes 8 | ".diff", ".filtered", and ".reference". 9 | 10 | The ".filtered" file contains the extracted numbers from the present run. 11 | 12 | The ".reference" file contains the extracted numbers from the reference file. 13 | 14 | If the test passes, the ".diff" file is an empty file. If the test fails, it contains 15 | information about the difference between the present run and the reference file. 16 | -------------------------------------------------------------------------------- /img/runtest-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bast/runtest/9a8fcbd6521dfe87f6fcd9dcb07d97a9bb823f62/img/runtest-small.png -------------------------------------------------------------------------------- /img/runtest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bast/runtest/9a8fcbd6521dfe87f6fcd9dcb07d97a9bb823f62/img/runtest.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [tool.flit.metadata] 6 | module = "runtest" 7 | author = "Radovan Bast" 8 | author-email = "radovan.bast@uit.no" 9 | home-page = "https://github.com/bast/runtest" 10 | description-file="README.md" 11 | classifiers = ["License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)"] 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | coverage==4.5.4 3 | pytest-cov<2.6.0 4 | python-coveralls 5 | flit 6 | fsfe-reuse 7 | cffconvert 8 | -------------------------------------------------------------------------------- /runtest/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | """ 6 | runtest: Numerically tolerant end-to-end test library for research software. 7 | """ 8 | 9 | from .filter_constructor import get_filter 10 | from .run import run 11 | from .version import version_info, __version__ 12 | from .cli import cli 13 | 14 | __author__ = "Radovan Bast " 15 | 16 | __all__ = [ 17 | "get_filter", 18 | "version_info", 19 | "run", 20 | "cli", 21 | __version__, 22 | ] 23 | -------------------------------------------------------------------------------- /runtest/check.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | from .exceptions import FilterKeywordError, FailedTestError, BadFilterError 6 | from .extract import extract_numbers 7 | from .filter_api import recognized_kw 8 | from .filter_constructor import get_filter 9 | from .scissors import cut_sections 10 | from .tuple_comparison import tuple_matches 11 | import os 12 | 13 | 14 | def check(filter_list, out_name, ref_name, log_dir, verbose=False): 15 | """ 16 | Compares output with reference applying all filters tasks from the list of 17 | filters. 18 | 19 | Input: 20 | - filter_list -- list of filters 21 | - out_name -- actual output file name 22 | - ref_name -- reference output file name 23 | - log_dir -- directory which will hold logs 24 | - verbose -- give verbose output upon failure 25 | 26 | Returns: 27 | - nothing 28 | 29 | Generates the following files in log_dir: 30 | - out_name.filtered -- numbers extracted from output 31 | - out_name.reference -- numbers extracted from reference 32 | - out_name.diff -- difference between the two above 33 | 34 | Raises: 35 | - FailedTestError 36 | """ 37 | 38 | def _tuple_matches(t): 39 | if f.tolerance_is_relative: 40 | error_definition = "relative" 41 | else: 42 | error_definition = "absolute" 43 | return tuple_matches( 44 | t, 45 | tolerance=f.tolerance, 46 | error_definition=error_definition, 47 | ignore_sign=f.ignore_sign, 48 | skip_below=f.skip_below, 49 | skip_above=f.skip_above, 50 | ) 51 | 52 | name_out = os.path.join(log_dir, out_name + ".filtered") 53 | name_ref = os.path.join(log_dir, out_name + ".reference") 54 | name_diff = os.path.join(log_dir, out_name + ".diff") 55 | 56 | with open(name_out, "w") as log_out: 57 | with open(name_ref, "w") as log_ref: 58 | with open(name_diff, "w") as log_diff: 59 | for f in filter_list: 60 | out_filtered = cut_sections( 61 | open(out_name).readlines(), 62 | from_string=f.from_string, 63 | from_is_re=f.from_is_re, 64 | to_string=f.to_string, 65 | to_is_re=f.to_is_re, 66 | num_lines=f.num_lines, 67 | ) 68 | if out_filtered == []: 69 | if f.num_lines > 0: 70 | r = '[%i lines from "%s"]' % (f.num_lines, f.from_string) 71 | else: 72 | r = '["%s" ... "%s"]' % (f.from_string, f.to_string) 73 | message = ( 74 | "ERROR: filter %s did not extract anything from file %s\n" 75 | % (r, out_name) 76 | ) 77 | raise BadFilterError(message) 78 | 79 | log_out.write("".join(out_filtered)) 80 | out_numbers, out_locations = extract_numbers(out_filtered, f.mask) 81 | if f.mask is not None and out_numbers == []: 82 | raise FilterKeywordError( 83 | "ERROR: mask %s did not extract any numbers\n" % f.mask 84 | ) 85 | 86 | ref_filtered = cut_sections( 87 | open(ref_name).readlines(), 88 | from_string=f.from_string, 89 | from_is_re=f.from_is_re, 90 | to_string=f.to_string, 91 | to_is_re=f.to_is_re, 92 | num_lines=f.num_lines, 93 | ) 94 | if ref_filtered == []: 95 | if f.num_lines > 0: 96 | r = '[%i lines from "%s"]' % (f.num_lines, f.from_string) 97 | else: 98 | r = '["%s" ... "%s"]' % (f.from_string, f.to_string) 99 | message = ( 100 | "ERROR: filter %s did not extract anything from file %s\n" 101 | % (r, ref_name) 102 | ) 103 | raise BadFilterError(message) 104 | 105 | log_ref.write("".join(ref_filtered)) 106 | ref_numbers, _ = extract_numbers(ref_filtered, f.mask) 107 | if f.mask is not None and ref_numbers == []: 108 | raise FilterKeywordError( 109 | "ERROR: mask %s did not extract any numbers\n" % f.mask 110 | ) 111 | 112 | if f.ignore_sign: 113 | out_numbers = list(map(abs, out_numbers)) 114 | ref_numbers = list(map(abs, ref_numbers)) 115 | 116 | if f.ignore_order: 117 | out_numbers = sorted(out_numbers) 118 | ref_numbers = sorted(ref_numbers) 119 | 120 | if out_numbers == [] and ref_numbers == []: 121 | # no numbers are extracted 122 | if out_filtered != ref_filtered: 123 | log_diff.write("ERROR: extracted strings do not match\n") 124 | log_diff.write("own gave:\n") 125 | log_diff.write("".join(out_filtered) + "\n") 126 | log_diff.write("reference gave:\n") 127 | log_diff.write("".join(ref_filtered) + "\n") 128 | 129 | # we need to check for len(out_numbers) > 0 130 | # for pure strings len(out_numbers) is 0 131 | # TODO need to consider what to do with pure strings in future versions 132 | if len(out_numbers) == len(ref_numbers) and len(out_numbers) > 0: 133 | if not f.tolerance_is_set and ( 134 | any(map(lambda x: isinstance(x, float), out_numbers)) 135 | or any(map(lambda x: isinstance(x, float), ref_numbers)) 136 | ): 137 | raise FilterKeywordError( 138 | "ERROR: for floats you have to specify either rel_tolerance or abs_tolerance\n" 139 | ) 140 | list_of_tuples = map( 141 | _tuple_matches, zip(out_numbers, ref_numbers) 142 | ) 143 | matching, errors = zip( 144 | *list_of_tuples 145 | ) # unzip tuples to two lists 146 | if not all(matching): 147 | log_diff.write("\n") 148 | for k, line in enumerate(out_filtered): 149 | log_diff.write(". %s" % line) 150 | for i, _ in enumerate(out_numbers): 151 | (line_num, start_char, length) = out_locations[i] 152 | if line_num == k: 153 | if errors[i]: 154 | log_diff.write( 155 | "ERROR %s%s %s\n" 156 | % ( 157 | " " * start_char, 158 | "#" * length, 159 | errors[i], 160 | ) 161 | ) 162 | 163 | if len(out_numbers) != len(ref_numbers): 164 | log_diff.write("ERROR: extracted sizes do not match\n") 165 | log_diff.write("own gave %i numbers:\n" % len(out_numbers)) 166 | log_diff.write("".join(out_filtered) + "\n") 167 | log_diff.write( 168 | "reference gave %i numbers:\n" % len(ref_numbers) 169 | ) 170 | log_diff.write("".join(ref_filtered) + "\n") 171 | 172 | if os.path.getsize("%s.diff" % out_name) > 0: 173 | log_diff = open("%s.diff" % out_name, "r") 174 | diff = "" 175 | for line in log_diff.readlines(): 176 | diff += line 177 | log_diff.close() 178 | message = "ERROR: test %s failed\n" % out_name 179 | if verbose: 180 | message += diff 181 | raise FailedTestError(message) 182 | 183 | 184 | def _test_setup(folder, filters): 185 | _here = os.path.abspath(os.path.dirname(__file__)) 186 | test_dir = os.path.join(_here, "test", folder) 187 | out_name = os.path.join(test_dir, "out.txt") 188 | ref_name = os.path.join(test_dir, "ref.txt") 189 | log_dir = test_dir 190 | check( 191 | filter_list=filters, 192 | out_name=out_name, 193 | ref_name=ref_name, 194 | log_dir=log_dir, 195 | verbose=False, 196 | ) 197 | 198 | 199 | def test_check(): 200 | import pytest 201 | 202 | _here = os.path.abspath(os.path.dirname(__file__)) 203 | test_dir = os.path.join(_here, "test", "generic") 204 | out_name = os.path.join(test_dir, "out.txt") 205 | 206 | _test_setup(folder="generic", filters=[get_filter(abs_tolerance=0.1)]) 207 | 208 | with pytest.raises(FilterKeywordError) as e: 209 | _test_setup(folder="generic", filters=[get_filter()]) 210 | assert ( 211 | "ERROR: for floats you have to specify either rel_tolerance or abs_tolerance\n" 212 | in str(e.value) 213 | ) 214 | 215 | with pytest.raises(FailedTestError) as e: 216 | _test_setup(folder="generic", filters=[get_filter(rel_tolerance=0.01)]) 217 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 218 | with open(os.path.join(test_dir, "out.txt.diff"), "r") as f: 219 | assert ( 220 | f.read() 221 | == """ 222 | . 1.0 2.0 3.0 223 | ERROR ### expected: 3.05 (rel diff: 1.64e-02)\n""" 224 | ) 225 | 226 | with pytest.raises(FailedTestError) as e: 227 | _test_setup(folder="generic", filters=[get_filter(abs_tolerance=0.01)]) 228 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 229 | with open(os.path.join(test_dir, "out.txt.diff"), "r") as f: 230 | assert ( 231 | f.read() 232 | == """ 233 | . 1.0 2.0 3.0 234 | ERROR ### expected: 3.05 (abs diff: 5.00e-02)\n""" 235 | ) 236 | 237 | with pytest.raises(FailedTestError) as e: 238 | _test_setup( 239 | folder="generic", filters=[get_filter(abs_tolerance=0.01, ignore_sign=True)] 240 | ) 241 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 242 | with open(os.path.join(test_dir, "out.txt.diff"), "r") as f: 243 | assert ( 244 | f.read() 245 | == """ 246 | . 1.0 2.0 3.0 247 | ERROR ### expected: 3.05 (abs diff: 5.00e-02 ignoring signs)\n""" 248 | ) 249 | 250 | 251 | def test_check_bad_filter(): 252 | import pytest 253 | 254 | _here = os.path.abspath(os.path.dirname(__file__)) 255 | test_dir = os.path.join(_here, "test", "generic") 256 | out_name = os.path.join(test_dir, "out.txt") 257 | 258 | with pytest.raises(BadFilterError) as e: 259 | _test_setup( 260 | folder="generic", 261 | filters=[get_filter(from_string="does not exist", num_lines=4)], 262 | ) 263 | assert ( 264 | 'ERROR: filter [4 lines from "does not exist"] did not extract anything from file %s\n' 265 | % out_name 266 | in str(e.value) 267 | ) 268 | 269 | with pytest.raises(BadFilterError) as e: 270 | _test_setup( 271 | folder="generic", 272 | filters=[get_filter(from_string="does not exist", to_string="either")], 273 | ) 274 | assert ( 275 | 'ERROR: filter ["does not exist" ... "either"] did not extract anything from file %s\n' 276 | % out_name 277 | in str(e.value) 278 | ) 279 | 280 | 281 | def test_check_different_length(): 282 | import pytest 283 | 284 | _here = os.path.abspath(os.path.dirname(__file__)) 285 | test_dir = os.path.join(_here, "test", "different_length") 286 | out_name = os.path.join(test_dir, "out.txt") 287 | 288 | with pytest.raises(FailedTestError) as e: 289 | _test_setup(folder="different_length", filters=[get_filter(abs_tolerance=0.1)]) 290 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 291 | with open(os.path.join(test_dir, "out.txt.diff"), "r") as f: 292 | assert ( 293 | f.read() 294 | == """ERROR: extracted sizes do not match 295 | own gave 4 numbers: 296 | 1.0 2.0 3.0 4.0 297 | 298 | reference gave 3 numbers: 299 | 1.0 2.0 3.05 300 | \n""" 301 | ) 302 | 303 | 304 | def test_check_ignore_order(): 305 | import pytest 306 | 307 | _here = os.path.abspath(os.path.dirname(__file__)) 308 | test_dir = os.path.join(_here, "test", "ignore_order") 309 | out_name = os.path.join(test_dir, "out.txt") 310 | 311 | with pytest.raises(FailedTestError) as e: 312 | _test_setup(folder="ignore_order", filters=[get_filter(abs_tolerance=0.1)]) 313 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 314 | 315 | _test_setup( 316 | folder="ignore_order", 317 | filters=[get_filter(abs_tolerance=0.1, ignore_order=True)], 318 | ) 319 | 320 | 321 | def test_check_ignore_order_and_sign(): 322 | import pytest 323 | 324 | _here = os.path.abspath(os.path.dirname(__file__)) 325 | test_dir = os.path.join(_here, "test", "ignore_order_and_sign") 326 | out_name = os.path.join(test_dir, "out.txt") 327 | 328 | with pytest.raises(FailedTestError) as e: 329 | _test_setup( 330 | folder="ignore_order_and_sign", filters=[get_filter(abs_tolerance=0.1)] 331 | ) 332 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 333 | 334 | _test_setup( 335 | folder="ignore_order_and_sign", 336 | filters=[get_filter(abs_tolerance=0.1, ignore_order=True, ignore_sign=True)], 337 | ) 338 | 339 | 340 | def test_bad_keywords(): 341 | import pytest 342 | 343 | with pytest.raises(FilterKeywordError) as e: 344 | _ = get_filter(raboof=0, foo=1) 345 | exception = """ERROR: keyword(s) (foo, raboof) not recognized 346 | available keywords: ({0})\n""".format(", ".join(recognized_kw)) 347 | assert exception in str(e.value) 348 | 349 | with pytest.raises(FilterKeywordError) as e: 350 | _ = get_filter(from_string="foo", from_re="foo", to_string="foo", to_re="foo") 351 | assert ( 352 | "ERROR: incompatible keyword pairs: [('from_re', 'from_string'), ('to_re', 'to_string')]\n" 353 | in str(e.value) 354 | ) 355 | 356 | 357 | def test_only_string(): 358 | import pytest 359 | 360 | _here = os.path.abspath(os.path.dirname(__file__)) 361 | test_dir = os.path.join(_here, "test", "only_string") 362 | out_name = os.path.join(test_dir, "out.txt") 363 | 364 | _test_setup(folder="only_string", filters=[get_filter(string="raboof")]) 365 | 366 | with pytest.raises(BadFilterError) as e: 367 | _test_setup(folder="only_string", filters=[get_filter(string="foo")]) 368 | assert ( 369 | 'ERROR: filter [1 lines from "foo"] did not extract anything from file %s\n' 370 | % out_name 371 | in str(e.value) 372 | ) 373 | 374 | 375 | def test_check_integers(): 376 | import pytest 377 | 378 | _here = os.path.abspath(os.path.dirname(__file__)) 379 | test_dir = os.path.join(_here, "test", "integers") 380 | out_name = os.path.join(test_dir, "out.txt") 381 | 382 | # both integer and float input should work here 383 | _test_setup(folder="integers", filters=[get_filter(abs_tolerance=2)]) 384 | _test_setup(folder="integers", filters=[get_filter(abs_tolerance=2.0)]) 385 | 386 | with pytest.raises(FailedTestError) as e: 387 | _test_setup(folder="integers", filters=[get_filter(abs_tolerance=1)]) 388 | assert "ERROR: test %s failed\n" % out_name in str(e.value) 389 | 390 | _test_setup(folder="integers", filters=[get_filter(rel_tolerance=1.0)]) 391 | -------------------------------------------------------------------------------- /runtest/cli.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | from optparse import OptionParser 6 | import sys 7 | import os 8 | import inspect 9 | from .version import __version__ 10 | 11 | 12 | def cli(): 13 | frame = inspect.stack()[-1] 14 | module = inspect.getmodule(frame[0]) 15 | caller_file = module.__file__ 16 | caller_dir = os.path.dirname(os.path.realpath(caller_file)) 17 | 18 | parser = OptionParser( 19 | description="runtest {0} - Numerically tolerant end-to-end test library for research software.".format( 20 | __version__ 21 | ) 22 | ) 23 | 24 | parser.add_option( 25 | "--binary-dir", 26 | "-b", 27 | action="store", 28 | default=caller_dir, 29 | help="directory containing the binary/runscript [default: %default]", 30 | ) 31 | parser.add_option( 32 | "--work-dir", 33 | "-w", 34 | action="store", 35 | default=caller_dir, 36 | help="working directory [default: %default]", 37 | ) 38 | parser.add_option( 39 | "--launch-agent", 40 | "-l", 41 | action="store", 42 | default=None, 43 | help='prepend a launch agent command (e.g. "mpirun -np 8" or "valgrind --leak-check=yes") [default: %default]', 44 | ) 45 | parser.add_option( 46 | "--verbose", 47 | "-v", 48 | action="store_true", 49 | default=False, 50 | help="give more verbose output upon test failure [default: %default]", 51 | ) 52 | parser.add_option( 53 | "--skip-run", 54 | "-s", 55 | action="store_true", 56 | default=False, 57 | help="skip actual calculation(s) [default: %default]", 58 | ) 59 | parser.add_option( 60 | "--no-verification", 61 | "-n", 62 | action="store_true", 63 | default=False, 64 | help="run calculation(s) but do not verify results [default: %default]", 65 | ) 66 | 67 | (options, _args) = parser.parse_args(args=sys.argv[1:]) 68 | 69 | return options 70 | -------------------------------------------------------------------------------- /runtest/copy.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | from shutil import copy 6 | import os 7 | 8 | 9 | def copy_path(root_src_dir, root_dst_dir): 10 | for src_dir, _dirs, files in os.walk(root_src_dir): 11 | dst_dir = src_dir.replace(root_src_dir, root_dst_dir) 12 | if not os.path.exists(dst_dir): 13 | os.makedirs(dst_dir) 14 | for f in files: 15 | src_file = os.path.join(src_dir, f) 16 | dst_file = os.path.join(dst_dir, f) 17 | copy(src_file, dst_file) 18 | -------------------------------------------------------------------------------- /runtest/exceptions.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | 6 | class FilterKeywordError(Exception): 7 | pass 8 | 9 | 10 | class FailedTestError(Exception): 11 | pass 12 | 13 | 14 | class BadFilterError(Exception): 15 | pass 16 | -------------------------------------------------------------------------------- /runtest/extract.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | import re 6 | 7 | 8 | def extract_numbers(text, mask=None): 9 | """ 10 | Extracts floats and integers from string text. 11 | 12 | Returns: 13 | numbers - list of numbers 14 | locations - locations of each number as list of triples (line, start position, length) 15 | """ 16 | numeric_const_pattern = r""" 17 | [-+]? # optional sign 18 | (?: 19 | (?: \d* \. \d+ ) # .1 .12 .123 etc 9.1 etc 98.1 etc 20 | | 21 | (?: \d+ \.? ) # 1. 12. 123. etc 1 12 123 etc 22 | ) 23 | # followed by optional exponent part if desired 24 | (?: [EeDd] [+-]? \d+ ) ? 25 | """ 26 | pattern_number_and_separator = re.compile(r"^[0-9\.eEdD\+\-]+[,]?$", re.VERBOSE) 27 | pattern_int = re.compile(r"-?[0-9]+", re.VERBOSE) 28 | pattern_float = re.compile(numeric_const_pattern, re.VERBOSE) 29 | pattern_d = re.compile(r"[dD]") 30 | 31 | numbers = [] 32 | locations = [] 33 | 34 | for n, line in enumerate(text): 35 | n_matches = 0 36 | for w in line.split(): 37 | # do not consider words like TzB1g 38 | if not re.match(pattern_number_and_separator, w): 39 | continue 40 | 41 | n_matches += 1 42 | if mask is not None and n_matches not in mask: 43 | continue 44 | 45 | is_integer = False 46 | matched_floats = pattern_float.findall(w) 47 | 48 | if len(matched_floats) > 0: 49 | is_integer = matched_floats == pattern_int.findall(w) 50 | 51 | # apply floating point regex 52 | for m in matched_floats: 53 | index = line.index(m) 54 | # substitute dD by e 55 | if is_integer: 56 | numbers.append(int(m)) 57 | else: 58 | m = pattern_d.sub("e", m) 59 | numbers.append(float(m)) 60 | locations.append((n, index, len(m))) 61 | 62 | return numbers, locations 63 | 64 | 65 | def test_extract_numbers(): 66 | text = """<> - linear response function (real): 67 | ----------------------------------------------------------------------------------------------- 68 | A - Z-Dipole length B1u T+ 69 | B - Z-Dipole length B1u T+ 70 | ----------------------------------------------------------------------------------------------- 71 | Frequency (real) Real part Convergence 72 | ----------------------------------------------------------------------------------------------- 73 | 0.00000000 a.u. -1.901357604797 a.u. 3.04E-07 (converged) 74 | ----------------------------------------------------------------------------------------------- 75 | ---------------------------------------------------------------------------- 76 | 77 | 78 | +--------------------------------+ 79 | ! Electric dipole polarizability ! 80 | +--------------------------------+ 81 | 82 | 83 | 1 a.u = 0.14818471 angstrom**3 84 | 85 | 86 | @ Elements of the electric dipole polarizability tensor 87 | 88 | @ xx 1.90135760 a.u. (converged) 89 | @ yy 1.90135760 a.u. (converged) 90 | @ zz 1.90135760 a.u. (converged) 91 | 92 | @ average 1.90135760 a.u. 93 | @ anisotropy 0.000 a.u. 94 | 95 | @ xx 0.28175212 angstrom**3 96 | @ yy 0.28175212 angstrom**3 97 | @ zz 0.28175212 angstrom**3 98 | 99 | @ average 0.28175212 angstrom**3 100 | @ anisotropy 0.000 angstrom**3""" 101 | 102 | numbers, locations = extract_numbers(text.splitlines()) 103 | 104 | assert numbers == [ 105 | 0.0, 106 | -1.901357604797, 107 | 3.04e-07, 108 | 1, 109 | 0.14818471, 110 | 1.9013576, 111 | 1.9013576, 112 | 1.9013576, 113 | 1.9013576, 114 | 0.0, 115 | 0.28175212, 116 | 0.28175212, 117 | 0.28175212, 118 | 0.28175212, 119 | 0.0, 120 | ] 121 | assert locations == [ 122 | (7, 2, 10), 123 | (7, 20, 15), 124 | (7, 63, 8), 125 | (17, 1, 1), 126 | (17, 11, 10), 127 | (22, 18, 10), 128 | (23, 18, 10), 129 | (24, 18, 10), 130 | (26, 18, 10), 131 | (27, 18, 5), 132 | (29, 18, 10), 133 | (30, 18, 10), 134 | (31, 18, 10), 135 | (33, 18, 10), 136 | (34, 18, 5), 137 | ] 138 | 139 | 140 | def test_extract_numbers_mask(): 141 | text = """1.0 2.0 3.0 4.0 142 | 1.0 2.0 3.0 4.0 143 | 1.0 2.0 3.0 4.0""" 144 | 145 | numbers, locations = extract_numbers(text.splitlines(), mask=[1, 4]) 146 | 147 | assert numbers == [1.0, 4.0, 1.0, 4.0, 1.0, 4.0] 148 | assert locations == [ 149 | (0, 0, 3), 150 | (0, 12, 3), 151 | (1, 0, 3), 152 | (1, 12, 3), 153 | (2, 0, 3), 154 | (2, 12, 3), 155 | ] 156 | 157 | 158 | def test_extract_comma_separated_numbers_mask(): 159 | text = """1.0, 2.0, 3.0, 4.0 160 | 1.0, 2.0, 3.0, 4.0 161 | 12, 22, 32, 42 162 | 1.0, 2.0, 3.0, 4.0 163 | 12, 22, 32, 42""" 164 | 165 | numbers, locations = extract_numbers(text.splitlines(), mask=[1, 3]) 166 | 167 | assert numbers == [1.0, 3.0, 1.0, 3.0, 12, 32, 1.0, 3.0, 12, 32] 168 | assert locations == [ 169 | (0, 0, 3), 170 | (0, 10, 3), 171 | (1, 0, 3), 172 | (1, 10, 3), 173 | (2, 0, 2), 174 | (2, 8, 2), 175 | (3, 0, 3), 176 | (3, 10, 3), 177 | (4, 0, 2), 178 | (4, 8, 2), 179 | ] 180 | 181 | 182 | def test_extract_separated_numbers_mask(): 183 | text = """1.0, 2.0' 3.0, 4.0 184 | 1.0, 2.0- 3.0, 4.0 185 | 1.0, 2.0? 3.0, 4.0 186 | 12, 22' 32, 42 187 | 12, 22- 32, 42""" 188 | 189 | numbers, locations = extract_numbers(text.splitlines(), mask=[1, 3]) 190 | 191 | assert numbers == [1.0, 4.0, 1.0, 3.0, 1.0, 4.0, 12, 42, 12, 32] 192 | assert locations == [ 193 | (0, 0, 3), 194 | (0, 15, 3), 195 | (1, 0, 3), 196 | (1, 10, 3), 197 | (2, 0, 3), 198 | (2, 15, 3), 199 | (3, 0, 2), 200 | (3, 12, 2), 201 | (4, 0, 2), 202 | (4, 8, 2), 203 | ] 204 | -------------------------------------------------------------------------------- /runtest/filter_api.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | recognized_kw = [ 6 | "from_re", 7 | "to_re", 8 | "re", 9 | "from_string", 10 | "to_string", 11 | "string", 12 | "skip_below", 13 | "skip_above", 14 | "ignore_sign", 15 | "ignore_order", 16 | "mask", 17 | "num_lines", 18 | "rel_tolerance", 19 | "abs_tolerance", 20 | ] 21 | 22 | incompatible_pairs = [ 23 | ("from_re", "from_string"), 24 | ("to_re", "to_string"), 25 | ("to_string", "num_lines"), 26 | ("to_re", "num_lines"), 27 | ("string", "from_string"), 28 | ("string", "to_string"), 29 | ("string", "from_re"), 30 | ("string", "to_re"), 31 | ("string", "num_lines"), 32 | ("re", "from_string"), 33 | ("re", "to_string"), 34 | ("re", "from_re"), 35 | ("re", "to_re"), 36 | ("re", "num_lines"), 37 | ("rel_tolerance", "abs_tolerance"), 38 | ] 39 | -------------------------------------------------------------------------------- /runtest/filter_constructor.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | import sys 6 | from collections import namedtuple 7 | from .exceptions import FilterKeywordError 8 | from .filter_api import recognized_kw, incompatible_pairs 9 | 10 | 11 | def get_filter(**kwargs): 12 | _filter = namedtuple( 13 | "_filter", 14 | [ 15 | "from_is_re", 16 | "from_string", 17 | "skip_above", 18 | "skip_below", 19 | "ignore_sign", 20 | "ignore_order", 21 | "mask", 22 | "num_lines", 23 | "to_is_re", 24 | "to_string", 25 | "tolerance", 26 | "tolerance_is_relative", 27 | "tolerance_is_set", 28 | ], 29 | ) 30 | 31 | unrecoginzed_kw = [kw for kw in kwargs.keys() if kw not in recognized_kw] 32 | if unrecoginzed_kw != []: 33 | error = """ERROR: keyword(s) ({unrecognized}) not recognized 34 | available keywords: ({available})\n""".format( 35 | unrecognized=(", ").join(sorted(unrecoginzed_kw)), 36 | available=(", ").join(recognized_kw), 37 | ) 38 | raise FilterKeywordError(error) 39 | 40 | incompatible_kw = [ 41 | (kw1, kw2) 42 | for (kw1, kw2) in incompatible_pairs 43 | if kw1 in kwargs.keys() and kw2 in kwargs.keys() 44 | ] 45 | if incompatible_kw != []: 46 | error = "ERROR: incompatible keyword pairs: {0}\n".format(incompatible_kw) 47 | raise FilterKeywordError(error) 48 | 49 | # now continue with keywords 50 | _filter.from_string = kwargs.get("from_string", None) 51 | _filter.to_string = kwargs.get("to_string", None) 52 | _filter.ignore_sign = kwargs.get("ignore_sign", False) 53 | _filter.ignore_order = kwargs.get("ignore_order", False) 54 | _filter.skip_below = kwargs.get("skip_below", sys.float_info.min) 55 | _filter.skip_above = kwargs.get("skip_above", sys.float_info.max) 56 | _filter.num_lines = kwargs.get("num_lines", 0) 57 | 58 | if "rel_tolerance" in kwargs.keys(): 59 | _filter.tolerance = kwargs.get("rel_tolerance") 60 | _filter.tolerance_is_relative = True 61 | _filter.tolerance_is_set = True 62 | elif "abs_tolerance" in kwargs.keys(): 63 | _filter.tolerance = kwargs.get("abs_tolerance") 64 | _filter.tolerance_is_relative = False 65 | _filter.tolerance_is_set = True 66 | else: 67 | _filter.tolerance_is_set = False 68 | 69 | _filter.mask = kwargs.get("mask", None) 70 | 71 | _filter.from_is_re = False 72 | from_re = kwargs.get("from_re", "") 73 | if from_re != "": 74 | _filter.from_string = from_re 75 | _filter.from_is_re = True 76 | 77 | _filter.to_is_re = False 78 | to_re = kwargs.get("to_re", "") 79 | if to_re != "": 80 | _filter.to_string = to_re 81 | _filter.to_is_re = True 82 | 83 | only_string = kwargs.get("string", "") 84 | if only_string != "": 85 | _filter.from_string = only_string 86 | _filter.num_lines = 1 87 | 88 | only_re = kwargs.get("re", "") 89 | if only_re != "": 90 | _filter.from_string = only_re 91 | _filter.num_lines = 1 92 | _filter.from_is_re = True 93 | 94 | return _filter 95 | -------------------------------------------------------------------------------- /runtest/run.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | import os 6 | import sys 7 | import inspect 8 | import shlex 9 | import subprocess 10 | from .exceptions import FailedTestError, BadFilterError, FilterKeywordError 11 | from .copy import copy_path 12 | from .check import check 13 | 14 | 15 | def run( 16 | options, configure, input_files, extra_args=None, filters=None, accepted_errors=None 17 | ): 18 | # here we find out where the test script sits 19 | frame = inspect.stack()[-1] 20 | module = inspect.getmodule(frame[0]) 21 | caller_file = module.__file__ 22 | caller_dir = os.path.dirname(os.path.realpath(caller_file)) 23 | 24 | # if the work_dir is different from caller_dir 25 | # we copy all files under caller_dir to work_dir 26 | if options.work_dir != caller_dir: 27 | copy_path(caller_dir, options.work_dir) 28 | 29 | launcher, command, output_prefix, relative_reference_path = configure( 30 | options, input_files, extra_args 31 | ) 32 | 33 | if options.launch_agent is not None: 34 | command = "{0} {1}".format(options.launch_agent, command) 35 | 36 | launch_script_path = os.path.normpath(os.path.join(options.binary_dir, launcher)) 37 | 38 | if not options.skip_run and not os.path.exists(launch_script_path): 39 | sys.stderr.write( 40 | "ERROR: launch script/binary {0} not found in {1}\n".format( 41 | launcher, options.binary_dir 42 | ) 43 | ) 44 | sys.stderr.write(" have you set the correct --binary-dir (or -b)?\n") 45 | sys.stderr.write(" try also --help\n") 46 | sys.exit(-1) 47 | 48 | sys.stdout.write( 49 | "\nrunning test with input files {0} and args {1}\n".format( 50 | input_files, extra_args 51 | ) 52 | ) 53 | 54 | if options.skip_run: 55 | sys.stdout.write("(skipped run with -s|--skip-run)\n") 56 | else: 57 | if sys.platform != "win32": 58 | command = shlex.split(command) 59 | 60 | process = subprocess.Popen( 61 | command, 62 | cwd=options.work_dir, 63 | stdin=subprocess.PIPE, 64 | stdout=subprocess.PIPE, 65 | stderr=subprocess.PIPE, 66 | universal_newlines=True, 67 | ) 68 | stdout, stderr = process.communicate() 69 | 70 | if output_prefix is None: 71 | _output_prefix = os.path.join(options.work_dir, "") 72 | else: 73 | _output_prefix = os.path.join(options.work_dir, output_prefix) + "." 74 | with open("{0}{1}".format(_output_prefix, "stdout"), "w") as f: 75 | try: 76 | _s = stdout.decode("UTF-8") 77 | except AttributeError: 78 | _s = stdout 79 | f.write(_s) 80 | 81 | with open("{0}{1}".format(_output_prefix, "stderr"), "w") as f: 82 | try: 83 | _s = stderr.decode("UTF-8") 84 | except AttributeError: 85 | _s = stderr 86 | f.write(_s) 87 | 88 | found_accepted_errors = False 89 | if accepted_errors is not None: 90 | for error in accepted_errors: 91 | if error in stderr: 92 | # we found an error that we expect/accept 93 | sys.stdout.write( 94 | "found error which is expected/accepted: {0}\n".format(error) 95 | ) 96 | found_accepted_errors = True 97 | 98 | if process.returncode != 0: 99 | if found_accepted_errors: 100 | return 0 101 | else: 102 | sys.stdout.write("ERROR: crash during {0}\n{1}".format(command, stderr)) 103 | return 1 104 | 105 | if filters is None: 106 | sys.stdout.write("finished (no reference)\n") 107 | elif options.no_verification: 108 | sys.stdout.write("finished (verification skipped)\n") 109 | else: 110 | try: 111 | for suffix in filters: 112 | if output_prefix is None: 113 | output = suffix 114 | else: 115 | output = "{0}.{1}".format(output_prefix, suffix) 116 | check( 117 | filter_list=filters[suffix], 118 | out_name=os.path.join(options.work_dir, output), 119 | ref_name=os.path.join( 120 | options.work_dir, relative_reference_path, output 121 | ), 122 | log_dir=options.work_dir, 123 | verbose=options.verbose, 124 | ) 125 | sys.stdout.write("passed\n") 126 | except IOError as e: 127 | sys.stderr.write("ERROR: could not open file {0}\n".format(e.filename)) 128 | sys.exit(1) 129 | except FailedTestError as e: 130 | sys.stderr.write(str(e)) 131 | return 1 132 | except BadFilterError as e: 133 | sys.stderr.write(str(e)) 134 | sys.exit(1) 135 | except FilterKeywordError as e: 136 | sys.stderr.write(str(e)) 137 | sys.exit(1) 138 | return 0 139 | -------------------------------------------------------------------------------- /runtest/scissors.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | import re 6 | 7 | 8 | def cut_sections( 9 | text, 10 | from_string=None, 11 | from_is_re=False, 12 | to_string=None, 13 | to_is_re=False, 14 | num_lines=0, 15 | ): 16 | """ 17 | Cuts out sections of the text between anchors. 18 | 19 | Returns: 20 | output - list of remaining lines 21 | """ 22 | output = [] 23 | 24 | for i, _ in enumerate(text): 25 | start_line_matches = False 26 | if from_is_re: 27 | start_line_matches = re.match(r".*{0}".format(from_string), text[i]) 28 | else: 29 | if from_string is None: 30 | # we are comparing entire file 31 | return text 32 | else: 33 | start_line_matches = from_string in text[i] 34 | 35 | if start_line_matches: 36 | if num_lines > 0: 37 | for n in range(i, i + num_lines): 38 | output.append(text[n]) 39 | else: 40 | for j in range(i, len(text)): 41 | end_line_matches = False 42 | if to_is_re: 43 | end_line_matches = re.match(r".*{0}".format(to_string), text[j]) 44 | else: 45 | end_line_matches = to_string in text[j] 46 | 47 | if end_line_matches: 48 | for n in range(i, j + 1): 49 | output.append(text[n]) 50 | break 51 | 52 | return output 53 | 54 | 55 | def test_cut_sections(): 56 | text = """ 57 | 1.0 2.0 3.0 58 | 1.0 2.0 3.0 59 | 1.0 2.0 3.0 60 | 1.0 2.0 3.0 61 | 1.0 2.0 3.0 62 | 1.0 2.0 3.0 63 | 1.0 2.0 3.0 64 | raboof 1.0 3.0 7.0 65 | 1.0 3.0 7.0 66 | 1.0 3.0 7.0 67 | 1.0 3.0 7.0 68 | 1.0 3.0 7.0 69 | 1.0 3.0 7.0 70 | 1.0 3.0 7.0 71 | 1.0 3.0 7.0""" 72 | 73 | res = cut_sections(text=text.splitlines(), from_string="raboof", num_lines=5) 74 | 75 | assert res == [ 76 | "raboof 1.0 3.0 7.0", 77 | " 1.0 3.0 7.0", 78 | " 1.0 3.0 7.0", 79 | " 1.0 3.0 7.0", 80 | " 1.0 3.0 7.0", 81 | ] 82 | 83 | 84 | def test_cut_sections_re(): 85 | text = """ 86 | 1.0 87 | 1.0 88 | raboof 89 | 2.0 90 | 2.0 91 | raboof2 92 | 3.0 93 | 3.0""" 94 | 95 | res = cut_sections( 96 | text=text.splitlines(), 97 | from_string="r.*f", 98 | from_is_re=True, 99 | to_string="r.*f2", 100 | to_is_re=True, 101 | ) 102 | 103 | assert res == [" raboof", "2.0", "2.0", " raboof2", " raboof2"] 104 | 105 | 106 | def test_cut_sections_all(): 107 | text = """first line 108 | 1.0 2.0 3.0 109 | 1.0 2.0 3.0 110 | 1.0 2.0 3.0 111 | last line""" 112 | 113 | res = cut_sections(text=text.splitlines()) 114 | 115 | assert res == [ 116 | "first line", 117 | "1.0 2.0 3.0", 118 | "1.0 2.0 3.0", 119 | "1.0 2.0 3.0", 120 | "last line", 121 | ] 122 | 123 | 124 | def test_cut_sections_from_string_to_string_2_matches(): 125 | text = """first line 126 | 1.0 2.0 3.0 127 | start 128 | 0.1234 129 | end 130 | start 131 | 1.2345 132 | end 133 | 1.0 2.0 3.0 134 | last line""" 135 | 136 | res = cut_sections( 137 | text=text.splitlines(), 138 | from_string="start", 139 | to_string="end", 140 | ) 141 | 142 | assert res == [ 143 | "start", 144 | "0.1234", 145 | "end", 146 | "start", 147 | "1.2345", 148 | "end", 149 | ] 150 | 151 | 152 | def test_cut_sections_from_re_to_re_2_matches(): 153 | text = """first line 154 | 1.0 2.0 3.0 155 | raboof 156 | 0.1234 157 | raboof2 158 | raboof 159 | 1.2345 160 | 0.2 161 | raboof2 162 | 1.0 2.0 3.0 163 | last line""" 164 | 165 | res = cut_sections( 166 | text=text.splitlines(), 167 | from_string="r.*f", 168 | from_is_re=True, 169 | to_string="r.*f2", 170 | to_is_re=True, 171 | ) 172 | print("Result from cutting:") 173 | print(res) 174 | 175 | assert res == [ 176 | " raboof", 177 | "0.1234", 178 | " raboof2", 179 | " raboof2", 180 | " raboof", 181 | "1.2345", 182 | "0.2", 183 | " raboof2", 184 | " raboof2", 185 | ] 186 | -------------------------------------------------------------------------------- /runtest/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bast/runtest/9a8fcbd6521dfe87f6fcd9dcb07d97a9bb823f62/runtest/test/__init__.py -------------------------------------------------------------------------------- /runtest/test/different_length/out.txt: -------------------------------------------------------------------------------- 1 | 1.0 2.0 3.0 4.0 2 | -------------------------------------------------------------------------------- /runtest/test/different_length/ref.txt: -------------------------------------------------------------------------------- 1 | 1.0 2.0 3.05 2 | -------------------------------------------------------------------------------- /runtest/test/generic/out.txt: -------------------------------------------------------------------------------- 1 | 1.0 2.0 3.0 2 | -------------------------------------------------------------------------------- /runtest/test/generic/ref.txt: -------------------------------------------------------------------------------- 1 | 1.0 2.0 3.05 2 | -------------------------------------------------------------------------------- /runtest/test/ignore_order/out.txt: -------------------------------------------------------------------------------- 1 | 4.0 3.0 2.0 3.0 1.2 2 | -------------------------------------------------------------------------------- /runtest/test/ignore_order/ref.txt: -------------------------------------------------------------------------------- 1 | 4.0 2.0 3.0 3.0 1.2 2 | -------------------------------------------------------------------------------- /runtest/test/ignore_order_and_sign/out.txt: -------------------------------------------------------------------------------- 1 | 2.0 -3.0 -5.0 6.0 -1.0 4.0 2 | -------------------------------------------------------------------------------- /runtest/test/ignore_order_and_sign/ref.txt: -------------------------------------------------------------------------------- 1 | 1.0 -2.0 3.0 -4.0 5.0 -6.0 2 | -------------------------------------------------------------------------------- /runtest/test/integers/out.txt: -------------------------------------------------------------------------------- 1 | 1 4 3 1.0 4.0 3.0 2 | -------------------------------------------------------------------------------- /runtest/test/integers/ref.txt: -------------------------------------------------------------------------------- 1 | 1 2 3 1.0 2.0 3.0 2 | -------------------------------------------------------------------------------- /runtest/test/only_string/out.txt: -------------------------------------------------------------------------------- 1 | raboof 2 | -------------------------------------------------------------------------------- /runtest/test/only_string/ref.txt: -------------------------------------------------------------------------------- 1 | raboof 2 | -------------------------------------------------------------------------------- /runtest/tuple_comparison.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | from sys import float_info 6 | 7 | 8 | def tuple_matches( 9 | t, 10 | tolerance=1.0e-8, 11 | error_definition="relative", 12 | ignore_sign=False, 13 | skip_below=float_info.min, 14 | skip_above=float_info.max, 15 | ): 16 | """ 17 | Checks if tuple matches based on tolerance settings. 18 | 19 | Returns: 20 | (tuple matches, error message) - error message is None if there is no error 21 | """ 22 | 23 | if isinstance(tolerance, int) and error_definition == "relative": 24 | return (False, "relative tolerance cannot be integer") 25 | 26 | assert error_definition in ["relative", "absolute"] 27 | 28 | x, x_ref = t 29 | 30 | if abs(x_ref) < skip_below: 31 | return (True, None) 32 | 33 | if abs(x_ref) > skip_above: 34 | return (True, None) 35 | 36 | error = x - x_ref 37 | if error_definition == "relative": 38 | error /= x_ref 39 | 40 | if abs(error) <= tolerance: 41 | return (True, None) 42 | else: 43 | if isinstance(error, int): 44 | error_message = "expected: {0} ({1} diff: {2:d}".format( 45 | x_ref, error_definition[:3], abs(error) 46 | ) 47 | else: 48 | error_message = "expected: {0} ({1} diff: {2:6.2e}".format( 49 | x_ref, error_definition[:3], abs(error) 50 | ) 51 | if ignore_sign: 52 | error_message += " ignoring signs" 53 | error_message += ")" 54 | return (False, error_message) 55 | 56 | 57 | def test_tuple_matches(): 58 | assert tuple_matches((13, 13)) == (True, None) 59 | assert tuple_matches((13, 13), tolerance=1.0e-10, error_definition="absolute") == ( 60 | True, 61 | None, 62 | ) 63 | assert tuple_matches((3.45, 3.46), tolerance=0.01, error_definition="absolute") == ( 64 | True, 65 | None, 66 | ) 67 | assert tuple_matches((13, 13), tolerance=1.0e-10, error_definition="relative") == ( 68 | True, 69 | None, 70 | ) 71 | assert tuple_matches((13, 13), tolerance=1, error_definition="relative") == ( 72 | False, 73 | "relative tolerance cannot be integer", 74 | ) 75 | assert tuple_matches((13, 14), tolerance=1, error_definition="absolute") == ( 76 | True, 77 | None, 78 | ) 79 | assert tuple_matches((1.0 + 1.0e-9, 1.0)) == (True, None) 80 | assert tuple_matches((1.0 + 1.0e-9, 1.0), tolerance=1.0e-10) == ( 81 | False, 82 | "expected: 1.0 (rel diff: 1.00e-09)", 83 | ) 84 | assert tuple_matches((1.0 + 1.0e-7, 1.0)) == ( 85 | False, 86 | "expected: 1.0 (rel diff: 1.00e-07)", 87 | ) 88 | assert tuple_matches((0.01, 0.02), error_definition="absolute") == ( 89 | False, 90 | "expected: 0.02 (abs diff: 1.00e-02)", 91 | ) 92 | assert tuple_matches( 93 | (0.01, 0.0002), error_definition="absolute", skip_below=0.001 94 | ) == (True, None) 95 | assert tuple_matches( 96 | (0.01, 2000.0), error_definition="absolute", skip_above=100.0 97 | ) == (True, None) 98 | assert tuple_matches((10.0 + 1.0e-9, -10.0), error_definition="absolute") == ( 99 | False, 100 | "expected: -10.0 (abs diff: 2.00e+01)", 101 | ) 102 | assert tuple_matches((17, 18)) == (False, "expected: 18 (rel diff: 5.56e-02)") 103 | assert tuple_matches((17, 18), error_definition="absolute") == ( 104 | False, 105 | "expected: 18 (abs diff: 1)", 106 | ) 107 | -------------------------------------------------------------------------------- /runtest/version.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2023 Radovan Bast 2 | # 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | from collections import namedtuple 6 | 7 | __version__ = "2.3.5" 8 | 9 | version_info = namedtuple("version_info", ["major", "minor", "micro", "releaselevel"]) 10 | 11 | major_minor_micro = __version__.split("-")[0] 12 | 13 | s = major_minor_micro.split(".") 14 | 15 | version_info.major = int(s[0]) 16 | version_info.minor = int(s[1]) 17 | version_info.micro = int(s[2]) 18 | 19 | version_info.releaselevel = __version__[len(major_minor_micro) + 1 :] 20 | --------------------------------------------------------------------------------