├── .github ├── release-drafter-template.yml └── workflows │ ├── python-package.yml │ ├── python-publish.yml │ └── release-drafter.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── COMMITTEE.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── docker └── Dockerfile ├── docs ├── Makefile ├── README.md ├── _config.yml ├── _toc.yml ├── chapter1.ipynb ├── chapter2.ipynb ├── chapter3.ipynb ├── chapter4.ipynb ├── chapter5.ipynb ├── chapter6.ipynb ├── make.bat ├── publish.sh ├── sci-unit-tag.png └── source │ ├── README.md │ ├── basics.rst │ ├── commandline.rst │ ├── conf.py │ ├── index.rst │ ├── modules.rst │ ├── quick_tutorial.rst │ ├── sciunit.models.rst │ ├── sciunit.rst │ ├── sciunit.scores.rst │ ├── sciunit.unit_test.rst │ └── setup.rst ├── pyproject.toml ├── requirements.txt ├── sciunit ├── __init__.py ├── __main__.py ├── base.py ├── capabilities.py ├── config.json ├── converters.py ├── errors.py ├── models │ ├── __init__.py │ ├── backends.py │ ├── base.py │ ├── examples.py │ └── runnable.py ├── scores │ ├── __init__.py │ ├── base.py │ ├── collections.py │ ├── collections_m2m.py │ ├── complete.py │ └── incomplete.py ├── style.css ├── suites.py ├── tests.py ├── unit_test │ ├── __init__.py │ ├── __main__.py │ ├── active.py │ ├── backend_tests.ipynb │ ├── backend_tests.py │ ├── base.py │ ├── base_tests.py │ ├── command_line_tests.py │ ├── config_tests.py │ ├── converter_tests.py │ ├── doc_tests.py │ ├── error_tests.py │ ├── import_tests.py │ ├── model_tests.py │ ├── observation_tests.py │ ├── score_tests.py │ ├── test_only_lower_triangle.ipynb │ ├── test_tests.py │ ├── utils_tests.py │ ├── validate_observation.ipynb │ └── validator_tests.py ├── utils.py └── validators.py ├── setup.cfg ├── setup.py └── test.sh /.github/release-drafter-template.yml: -------------------------------------------------------------------------------- 1 | template: | 2 | ## What’s Changed 3 | 4 | $CHANGES 5 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: ["3.7", "3.8", "3.9", "3.10"] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies and the package 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install . 30 | - name: Test unit test 31 | run: | 32 | git clone -b cosmosuite http://github.com/scidash/scidash ../scidash 33 | python -m sciunit.unit_test buffer 34 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [published] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | python setup.py sdist bdist_wheel 31 | twine upload dist/* 32 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | # branches to consider in the event; optional, defaults to all 6 | branches: 7 | - master 8 | # pull_request event is required only for autolabeler 9 | pull_request: 10 | # Only following types are handled by the action, but one can default to all as well 11 | types: [opened, reopened, synchronize] 12 | 13 | jobs: 14 | update_release_draft: 15 | runs-on: ubuntu-latest 16 | steps: 17 | # (Optional) GitHub Enterprise requires GHE_HOST variable set 18 | #- name: Set GHE_HOST 19 | # run: | 20 | # echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV 21 | 22 | # Drafts your next Release notes as Pull Requests are merged into "master" 23 | - uses: release-drafter/release-drafter@v5 24 | # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml 25 | with: 26 | config-name: release-drafter-template.yml 27 | # disable-autolabeler: true 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.settings 2 | *.project 3 | *.pydevproject 4 | *.pyo 5 | *.pyc 6 | *.DS_Store 7 | *.class 8 | *~ 9 | dist 10 | ignore 11 | .ipynb_checkpoints 12 | build 13 | *.bak 14 | *.egg-info 15 | docs/chapter*.py 16 | .coverage 17 | htmlcov 18 | Untitled*.ipynb 19 | GeneratedFiles 20 | GeneratedFiles/*.py 21 | \.idea/ 22 | .vscode 23 | *scratch.ipynb* 24 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pycqa/isort 3 | rev: 5.8.0 4 | hooks: 5 | - id: isort 6 | args: ["--profile", "black"] 7 | - repo: https://github.com/psf/black 8 | rev: 21.5b0 9 | hooks: 10 | - id: black 11 | language_version: python3 12 | - repo: https://github.com/myint/autoflake 13 | rev: v1.4 14 | hooks: 15 | - id: autoflake 16 | args: 17 | [ 18 | "--in-place", 19 | "--remove-all-unused-imports", 20 | "--ignore-init-module-imports", 21 | "--remove-unused-variables", 22 | ] 23 | exclude: | 24 | (?x)^( 25 | sciunit/unit_test/active.py| 26 | sciunit/unit_test/__main__.py 27 | )$ 28 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We ask that all contributors be 6 | - Respectful and patient. 7 | - Mindful and considerate. 8 | - Consider that our actions affect others, including colleagues and the public. 9 | - Be careful in the words that you choose! 10 | - Open with tools and dissemination efforts, and transparent in all communication. 11 | - Our work, our tools, our discussions, and our findings should be open and accessible to all. 12 | - Choose freely-available, open source tools, open access dissemination practices. 13 | - Communicate over open channels. 14 | - Welcome new interactions, new views, new ideas, and new people. 15 | - Disagreements will occur in a diverse scientific community. 16 | - But they can be resolved constructively with no harmful behavior! 17 | 18 | In the interest of fostering an open and welcoming environment, we as 19 | contributors and maintainers pledge to making participation in our project and 20 | our community a harassment-free experience for everyone, regardless of age, body 21 | size, disability, ethnicity, sex characteristics, gender identity and expression, 22 | level of experience, education, socio-economic status, nationality, personal 23 | appearance, race, religion, or sexual identity and orientation. 24 | 25 | ## Our Standards 26 | 27 | Examples of behavior that contributes to creating a positive environment 28 | include: 29 | 30 | * Using welcoming and inclusive language 31 | * Being respectful of differing viewpoints and experiences 32 | * Gracefully accepting constructive criticism 33 | * Focusing on what is best for the community 34 | * Showing empathy towards other community members 35 | 36 | Examples of unacceptable behavior by participants include: 37 | 38 | * The use of sexualized language or imagery and unwelcome sexual attention or 39 | advances 40 | * Trolling, insulting/derogatory comments, and personal or political attacks 41 | * Public or private harassment 42 | * Publishing others' private information, such as a physical or electronic 43 | address, without explicit permission 44 | * Other conduct which could reasonably be considered inappropriate in a 45 | professional setting 46 | 47 | ## Our Responsibilities 48 | 49 | Project maintainers are responsible for clarifying the standards of acceptable 50 | behavior and are expected to take appropriate and fair corrective action in 51 | response to any instances of unacceptable behavior. 52 | 53 | Project maintainers have the right and responsibility to remove, edit, or 54 | reject comments, commits, code, wiki edits, issues, and other contributions 55 | that are not aligned to this Code of Conduct, or to ban temporarily or 56 | permanently any contributor for other behaviors that they deem inappropriate, 57 | threatening, offensive, or harmful. 58 | 59 | ## Scope 60 | 61 | This Code of Conduct applies both within project spaces and in public spaces 62 | when an individual is representing the project or its community. Examples of 63 | representing a project or community include using an official project e-mail 64 | address, posting via an official social media account, or acting as an appointed 65 | representative at an online or offline event. Representation of a project may be 66 | further defined and clarified by project maintainers. 67 | 68 | ## Enforcement 69 | 70 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 71 | reported by contacting the SciUnit team leader at rick@scidash.org, 72 | or any member of the [SciUnit Executive Committee](COMMITTEE.md). All 73 | complaints will be reviewed and investigated and will result in a response that 74 | is deemed necessary and appropriate to the circumstances. The project team is 75 | obligated to maintain confidentiality with regard to the reporter of an incident. 76 | Further details of specific enforcement policies may be posted separately. 77 | 78 | Project maintainers who do not follow or enforce the Code of Conduct in good 79 | faith may face temporary or permanent repercussions as determined by other 80 | members of the project's leadership. 81 | 82 | ## Attribution 83 | 84 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 85 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 86 | 87 | [homepage]: https://www.contributor-covenant.org 88 | 89 | For answers to common questions about this code of conduct, see 90 | https://www.contributor-covenant.org/faq 91 | -------------------------------------------------------------------------------- /COMMITTEE.md: -------------------------------------------------------------------------------- 1 | # SciUnit Executive Committee 2 | 3 | The executive committee steers the SciUnit project and reviews regular reports prepared by the chairperson. 4 | Concerns about project governance or code of conduct violations may be confidentially shared with any member of the committee. 5 | 6 | | Name | Title | Affiliation | 7 | |---|---|---| 8 | | Richard C. (Rick) Gerkin* | Associate Research Professor | School of Life Sciences, Arizona State Univ., USA | 9 | | Shailesh Appukuttan | Postdoctoral Researcher | CNRS, France | 10 | | Justas Birgiolas | Postdoctoral Fellow | Dept. of Psychology, Cornell University, USA | 11 | | Sharon M Crook | Professor | School of Mathematical and Statistical Sciences, Arizona State Univ., USA | 12 | | Andrew Davison | Senior Research Scientist, Group Leader, Neuroinformatics | CNRS, France | 13 | | Robin Gutzen | PhD Student | Research Centre Jülich, Germany | 14 | | Giovanni Idili | Chief Operating Officer | MetaCell, LLC, UK | 15 | | Cyrus Omar | Assistant Professor | Dept. of Computer Science, Univ. of Michigan, USA | 16 | 17 | *denotes current chairperson 18 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to SciUnit 2 | 3 | ## Reporting issues 4 | 5 | When reporting issues please include as much detail as possible about your 6 | operating system, `sciunit` version, and python version. Whenever possible, please 7 | also include a brief, self-contained code example that demonstrates the problem. 8 | 9 | ## Contributing code 10 | 11 | Thanks for your interest in contributing code to SciUnit! 12 | Our developers aim to adhere to Python Enhancement Proposals (PEP) standards to ensure that code is readable, usable, and maintainable. 13 | 14 | + If this is your first time contributing to a project on GitHub, please consider reading 15 | this [example guide](https://numpy.org/devdocs/dev/index.html) to contributing code provided by the developers of numpy. 16 | + If you have contributed to other projects on GitHub you may open a pull request directly against the `dev` branch. 17 | 18 | Either way, please be sure to provide informative commit messages 19 | and ensure that each commit contains code addressing only one development goal. 20 | 21 | Writing unit tests to cover new code is encouraged but not required. 22 | Your changes can be tested using the current set of units tests by executing `test.sh` in the root directory. 23 | 24 | The python packages `isort`, `black`, and `autoflake` are being used with the `pre-commit` framework to auto-format code. 25 | If you install `pre-commit`, you can run `pre-commit run --all-files` before a pull request or commit in your shell to apply this formatting yourself. 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2011- Richard C. Gerkin and Cyrus Omar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include LICENSE 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Python package](https://github.com/scidash/sciunit/actions/workflows/python-package.yml/badge.svg)](https://github.com/scidash/sciunit/actions/workflows/python-package.yml) 2 | [![RTFD](https://readthedocs.org/projects/sciunit/badge/?version=master&x=1)](http://sciunit.readthedocs.io/en/latest/?badge=master) 3 | [![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/scidash/sciunit/master?filepath=docs%2Fchapter1.ipynb) 4 | [![Coveralls](https://coveralls.io/repos/github/scidash/sciunit/badge.svg?branch=master&x=1)](https://coveralls.io/github/scidash/sciunit?branch=master) 5 | [![Repos using Sciunit](https://img.shields.io/librariesio/dependent-repos/pypi/sciunit.svg?x=3)](https://github.com/scidash/sciunit/network/dependents?dependent_type=REPOSITORY) 6 | ![Downloads from PyPI](https://img.shields.io/pypi/dm/sciunit?x=1) 7 | 8 | SciUnit Logo 9 | 10 | # SciUnit: A Test-Driven Framework for Formally Validating Scientific Models Against Data 11 | 12 | ## Concept 13 | [The conference paper](https://github.com/cyrus-/papers/raw/master/sciunit-icse14/sciunit-icse14.pdf) 14 | 15 | ## Documentation 16 | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/scidash/sciunit/blob/master/docs/chapter1.ipynb)
17 | [Jupyter Tutorials](https://scidash.org/sciunit/README.html)
18 | [API Documentation](http://sciunit.rtfd.io) 19 | 20 | ## Installation 21 | ```python 22 | pip install sciunit 23 | ``` 24 | or 25 | ```python 26 | conda install -c conda-forge sciunit 27 | ``` 28 | 29 | ## Basic Usage 30 | ```python 31 | my_model = MyModel(**my_args) # Instantiate a class that wraps your model of interest. 32 | my_test = MyTest(**my_params) # Instantiate a test that you write. 33 | score = my_test.judge() # Runs the test and return a rich score containing test results and more. 34 | ``` 35 | 36 | ## Domain-specific libraries and information 37 | [NeuronUnit](https://github.com/scidash/neuronunit) for neuron and ion channel physiology
38 | See others [here](https://github.com/scidash/sciunit/network/dependents?dependent_type=REPOSITORY) 39 | 40 | ## Mailing List 41 | There is a [mailing list](https://groups.google.com/forum/?fromgroups#!forum/sciunit) for announcements and discussion. 42 | Please join it if you are at all interested! 43 | 44 | ## Contributors 45 | * [Rick Gerkin](http://rick.gerk.in), Arizona State University (School of Life Science) 46 | * [Cyrus Omar](http://cs.cmu.edu/~comar), Carnegie Mellon University (Dept. of Computer Science) 47 | 48 | ## Reproducible Research ID 49 | RRID:[SCR_014528](https://scicrunch.org/resources/Any/record/nlx_144509-1/3faed1d9-6579-5da6-b4b4-75a5077656bb/search?q=sciunit&l=sciunit) 50 | 51 | ## License 52 | SciUnit is released under the permissive [MIT license](https://opensource.org/licenses/MIT), requiring only attribution in derivative works. See the LICENSE file for terms. 53 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # sciunit 2 | # author Rick Gerkin rgerkin@asu.edu 3 | FROM jupyter/datascience-notebook 4 | 5 | RUN apt-get update 6 | RUN apt-get install openssh-client -y # Needed for Versioned unit tests to pass 7 | RUN git clone http://github.com/scidash/sciunit 8 | WORKDIR sciunit 9 | RUN pip install -e . 10 | RUN git clone -b cosmosuite http://github.com/scidash/scidash ../scidash 11 | USER $NB_USER 12 | RUN sh test.sh 13 | 14 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = SciUnit 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | ### New to SciUnit? 2 | - Start with [Chapter 1](https://scidash.org/sciunit/chapter1.html) ([source](https://github.com/scidash/sciunit/blob/master/docs/chapter1.ipynb)) of the tutorial. 3 | - View [our slides](https://github.com/scidash/sciunit/blob/master/docs/workshop-tutorial.ipynb). 4 | - Or run either of the above [interactively](https://mybinder.org/v2/gh/scidash/sciunit/b491f545854040b5934b3898e7b9f7089089041f). 5 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # in _config.yml 2 | title: "SciUnit: A python library for data-driven model validation testing" 3 | logo: sci-unit-tag.png 4 | execute: 5 | execute_notebooks: "off" 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/_toc.yml: -------------------------------------------------------------------------------- 1 | # In _toc.yml 2 | format: jb-book 3 | root: README 4 | parts: 5 | - caption: Tutorial 6 | numbered: True 7 | chapters: 8 | - file: chapter1 9 | - file: chapter2 10 | - file: chapter3 11 | - file: chapter4 12 | - file: chapter5 13 | - file: chapter6 14 | - caption: Full Docs 15 | chapters: 16 | - url: https://sciunit.rtfd.io 17 | -------------------------------------------------------------------------------- /docs/chapter1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "![SciUnit Logo](https://raw.githubusercontent.com/scidash/assets/master/logos/SciUnit/sci-unit-tag.png)\n", 8 | "\n", 9 | "\n", 10 | "\"Open\n", 11 | "\n", 12 | "# Chapter 1. What is SciUnit?\n", 13 | "Everyone hopes that their model has some correspondence with reality. Usually, checking whether this is true is done informally.\n", 14 | "### SciUnit makes this formal and transparent.\n", 15 | "SciUnit is a framework for validating scientific models by creating experimental-data-driven unit tests. " 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "#### If you are using this file in Google Colab, this block of code can help you install sciunit from PyPI in Colab environment." 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "metadata": { 29 | "execution": { 30 | "iopub.execute_input": "2021-05-04T17:17:26.797848Z", 31 | "iopub.status.busy": "2021-05-04T17:17:26.796560Z", 32 | "iopub.status.idle": "2021-05-04T17:17:29.165793Z", 33 | "shell.execute_reply": "2021-05-04T17:17:29.167482Z" 34 | } 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "!pip install -q sciunit" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "After installation, let's begin with importing sciunit." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": { 52 | "execution": { 53 | "iopub.execute_input": "2021-05-04T17:17:29.175051Z", 54 | "iopub.status.busy": "2021-05-04T17:17:29.174070Z", 55 | "iopub.status.idle": "2021-05-04T17:17:30.011470Z", 56 | "shell.execute_reply": "2021-05-04T17:17:30.009992Z" 57 | } 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "import sciunit" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "### What does testing look like? \n", 69 | "We'll start with a simple example from the history of cosmology, where we know that better models displaced their predecessors.
\n", 70 | "Suppose we have a test suite called \"Saturn Suite\" that aims to test cosmological models for their correspondence to empirical data about the planet Saturn.
\n", 71 | "*Everything in this example is hypothetical, but once you understand the basic ideas you can visit the [documentation for NeuronUnit](https://github.com/scidash/neuronunit/blob/master/docs/chapter1.ipynb) to see some working, interactive examples from a different domain (neuron and ion channel models).* " 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "```python\n", 79 | "from saturnsuite.tests import position_test,velocity_test,eccentricity_test # Examples of test classes. \n", 80 | "```" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "There's nothing specific to Saturn about position, velocity, or eccentricity. They could apply to any cosmological body.\n", 88 | "### SciUnit test classes used with in one scientific domain (like cosmology) are located in a discipline-specific library. \n", 89 | "In this case, the test classes (hypothetically) come from a SciUnit library called CosmoUnit, and are instantiated with data specific to Saturn, in order to create tests specific to a model's predictions about Saturn:" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "```python\n", 97 | "'''\n", 98 | "saturnsuite/tests.py # Tests for the Saturn suite. \n", 99 | "'''\n", 100 | "from . import saturn_data # Hypothetical library containing Saturn data. \n", 101 | "from cosmounit import PositionTest, VelocityTest, EccentricityyTest # Cosmounit is an external library. \n", 102 | "position_test = PositionTest(observation=saturn_data.position)\n", 103 | "velocity_test = VelocityTest(observation=saturn_data.velocity)\n", 104 | "eccentricity_test = EccentricityTest(observation=saturn_data.eccentricity)\n", 105 | "```" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "This means the test *classes* are data-agnostic, but the test *instances* encode the data we want a model to recapitulate." 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "Next, let's load some models that aim to predict the cosmological features being assessed by the tests above." 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "```python\n", 127 | "from saturnsuite.models import ptolemy_model, copernicus_model, kepler_model, newton_model # Examples of models. \n", 128 | "```" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "Ptolemy's, Copernicus's, Kepler's, or Newton's models could similarly apply to any cosmological body.\n", 136 | "So these model classes are found in CosmoUnit, and the Saturn Suite contains model instances parameterized to emit predictions about Saturn specifically." 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": {}, 142 | "source": [ 143 | "```python\n", 144 | "'''\n", 145 | "saturnsuite/models.py # Models for the Saturn suite. \n", 146 | "'''\n", 147 | "from cosmounit import PtolemyModel, CopernicusModel, KeplerModel, NewtonModel \n", 148 | "ptolemy_model = PtolemyModel(planet='Saturn')\n", 149 | "copernicus_model = CopernicusModel(planet='Saturn')\n", 150 | "kepler_model = KeplerModel(planet='Saturn')\n", 151 | "newton_model = NewtonModel(planet='Saturn')\n", 152 | "```" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": {}, 158 | "source": [ 159 | "In the above each model takes a keyword argument 'planet' that determines about what planet the model will make predicitons." 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "### All of our tests can be organized into a suite to compare results across related tests. " 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "```python\n", 174 | "'''\n", 175 | "saturnsuite/suites.py # Tests for the Saturn suite. \n", 176 | "'''\n", 177 | "import sciunit\n", 178 | "from .tests import position_test, velocity_test, eccentricity_test\n", 179 | "saturn_motion_suite = sciunit.TestSuite([position_test, velocity_test, eccentricity_test])\n", 180 | "suites = (saturn_motion_suite,)\n", 181 | "```" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "### Now we can execute this entire test suite against our models. " 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "```python\n", 196 | "from saturn_suite.suites import saturn_motion_suite\n", 197 | "saturn_motion_suite.judge([ptolemy_model, copernicus_model, kepler_model, newton_model])\n", 198 | "```" 199 | ] 200 | }, 201 | { 202 | "cell_type": "markdown", 203 | "metadata": {}, 204 | "source": [ 205 | "The exact output will depend on your preferences (terminal, HTML, etc.) but the figure below illustrates both the results you get (center table) and the relationship between the components listed here. " 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "![Cosmo Example](https://raw.githubusercontent.com/scidash/assets/master/figures/cosmo_example.png)" 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "The figure above also refers to SciDash, an in-development portal for accessing public test results, but for the remainder of this tutorial, we will focus on model/test development, execution, and visualization on your own machine. " 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "### In the next section we'll see how to create models and tests from scratch in SciUnit. \n", 227 | "### Onto [Chapter 2](chapter2.ipynb)!" 228 | ] 229 | } 230 | ], 231 | "metadata": { 232 | "kernelspec": { 233 | "display_name": "Python 3", 234 | "language": "python", 235 | "name": "python3" 236 | }, 237 | "language_info": { 238 | "codemirror_mode": { 239 | "name": "ipython", 240 | "version": 3 241 | }, 242 | "file_extension": ".py", 243 | "mimetype": "text/x-python", 244 | "name": "python", 245 | "nbconvert_exporter": "python", 246 | "pygments_lexer": "ipython3", 247 | "version": "3.9.4" 248 | } 249 | }, 250 | "nbformat": 4, 251 | "nbformat_minor": 4 252 | } 253 | -------------------------------------------------------------------------------- /docs/chapter3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "![SciUnit Logo](https://raw.githubusercontent.com/scidash/assets/master/logos/SciUnit/sci-unit-tag.png)\n", 8 | "\n", 9 | "\"Open\n", 10 | "\n", 11 | "# Chapter 3. Testing with help from the SciUnit standard library\n", 12 | "(or [back to Chapter 2](chapter2.ipynb))" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "#### If you are using this file in Google Colab, this block of code can help you install sciunit from PyPI in Colab environment." 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 1, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "!pip install -q sciunit" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "import sciunit" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "### In this chapter we will use the same toy model in Chapter 1 but write a more interesting test with additional features included in SciUnit. " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "from sciunit.models.examples import ConstModel # One of many dummy models included for illustration. \n", 54 | "const_model_37 = ConstModel(37, name=\"Constant Model 37\")" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "Now let's write a test that validates the `observation` and returns more informative `score` type." 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "from sciunit.capabilities import ProducesNumber\n", 71 | "from sciunit.scores import ZScore # One of many SciUnit score types. \n", 72 | "from sciunit.errors import ObservationError # An exception class raised when a test is instantiated \n", 73 | " # with an invalid observation.\n", 74 | " \n", 75 | "class MeanTest(sciunit.Test):\n", 76 | " \"\"\"Tests if the model predicts \n", 77 | " the same number as the observation.\"\"\" \n", 78 | " \n", 79 | " required_capabilities = (ProducesNumber,) # The one capability required for a model to take this test. \n", 80 | " score_type = ZScore # This test's 'judge' method will return a BooleanScore. \n", 81 | " \n", 82 | " def validate_observation(self, observation):\n", 83 | " if type(observation) is not dict:\n", 84 | " raise ObservationError(\"Observation must be a python dictionary\")\n", 85 | " if 'mean' not in observation:\n", 86 | " raise ObservationError(\"Observation must contain a 'mean' entry\")\n", 87 | " \n", 88 | " def generate_prediction(self, model):\n", 89 | " return model.produce_number() # The model has this method if it inherits from the 'ProducesNumber' capability.\n", 90 | " \n", 91 | " def compute_score(self, observation, prediction):\n", 92 | " score = ZScore.compute(observation,prediction) # Compute and return a ZScore object. \n", 93 | " score.description = (\"A z-score corresponding to the normalized location of the observation \"\n", 94 | " \"relative to the predicted distribution.\")\n", 95 | " return score" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "We've done two new things here:\n", 103 | "- The optional `validate_observation` method checks the `observation` to make sure that it is the right type, that it has the right attributes, etc. This can be used to ensures that the `observation` is exactly as the other core test methods expect. If we don't provide the right kind of observation:\n", 104 | "```python\n", 105 | "-> mean_37_test = MeanTest(37, name='=37')\n", 106 | "ObservationError: Observation must be a python dictionary\n", 107 | "```\n", 108 | "then we get an error. In contrast, this is what our test was looking for:" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 5, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "observation = {'mean':37.8, 'std':2.1}\n", 118 | "mean_37_test = MeanTest(observation, name='=37')" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "- Instead of returning a `BooleanScore`, encoding a `True`/`False` value, we return a `ZScore` encoding a more quantitative summary of the relationship between the observation and the prediction. When we execute the test:" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 6, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "score = mean_37_test.judge(const_model_37)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "Then we get a more quantitative summary of the results:" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 7, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "score.summarize()" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 8, 156 | "metadata": {}, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "'A z-score corresponding to the normalized location of the observation relative to the predicted distribution.'" 162 | ] 163 | }, 164 | "execution_count": 8, 165 | "metadata": {}, 166 | "output_type": "execute_result" 167 | } 168 | ], 169 | "source": [ 170 | "score.describe()" 171 | ] 172 | } 173 | ], 174 | "metadata": { 175 | "kernelspec": { 176 | "display_name": "Python 3", 177 | "language": "python", 178 | "name": "python3" 179 | }, 180 | "language_info": { 181 | "codemirror_mode": { 182 | "name": "ipython", 183 | "version": 3 184 | }, 185 | "file_extension": ".py", 186 | "mimetype": "text/x-python", 187 | "name": "python", 188 | "nbconvert_exporter": "python", 189 | "pygments_lexer": "ipython3", 190 | "version": "3.9.4" 191 | } 192 | }, 193 | "nbformat": 4, 194 | "nbformat_minor": 4 195 | } 196 | -------------------------------------------------------------------------------- /docs/chapter4.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "![SciUnit Logo](https://raw.githubusercontent.com/scidash/assets/master/logos/SciUnit/sci-unit-tag.png)\n", 8 | "\n", 9 | "\"Open\n", 10 | "\n", 11 | "# Chapter 4. Example of RunnableModel and Backend\n", 12 | "(or [back to Chapter 3](chapter3.ipynb))" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "#### If you are using this file in Google Colab, this block of code can help you install sciunit from PyPI in Colab environment." 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 1, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "!pip install -q sciunit" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "Beside the usual model in previous sections, let’s create a model that run a Backend instance to simulate and obtain results.\n", 36 | "\n", 37 | "Firstly, import necessary components from SciUnit package." 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "import sciunit, random\n", 47 | "from sciunit import Test\n", 48 | "from sciunit.capabilities import Runnable\n", 49 | "from sciunit.scores import BooleanScore\n", 50 | "from sciunit.models import RunnableModel\n", 51 | "from sciunit.models.backends import register_backends, Backend" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "Let’s define subclasses of SciUnit Backend, Test, and Model." 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "Note that:\n", 66 | "1. A SciUnit Backend subclass should implement _backend_run method.\n", 67 | "2. A SciUnit Backend subclass should implement run method." 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 3, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "class RandomNumBackend(Backend):\n", 77 | " '''generate a random integer between min and max'''\n", 78 | "\n", 79 | " def set_run_params(self, **run_params):\n", 80 | "\n", 81 | " # get min from run_params, if not exist, then 0.\n", 82 | " self.min = run_params.get('min', 0)\n", 83 | "\n", 84 | " # get max from run_params, if not exist, then self.min + 100.\n", 85 | " self.max = run_params.get('max', self.min + 100)\n", 86 | "\n", 87 | " def _backend_run(self):\n", 88 | " # generate and return random integer between min and max.\n", 89 | " return random.randint(self.min, self.max)\n", 90 | "\n", 91 | "class RandomNumModel(RunnableModel):\n", 92 | " \"\"\"A model that always produces a constant number as output.\"\"\"\n", 93 | "\n", 94 | " def run(self):\n", 95 | " self.results = self._backend.backend_run()\n", 96 | "\n", 97 | "\n", 98 | "class RangeTest(Test):\n", 99 | " \"\"\"Tests if the model predicts the same number as the observation.\"\"\"\n", 100 | "\n", 101 | " # Default Runnable Capability for RunnableModel\n", 102 | " required_capabilities = (Runnable,)\n", 103 | "\n", 104 | " # This test's 'judge' method will return a BooleanScore.\n", 105 | " score_type = BooleanScore\n", 106 | "\n", 107 | " def generate_prediction(self, model):\n", 108 | " model.run()\n", 109 | " return model.results\n", 110 | "\n", 111 | " def compute_score(self, observation, prediction):\n", 112 | " score = BooleanScore(\n", 113 | " observation['min'] <= prediction and observation['max'] >= prediction\n", 114 | " )\n", 115 | " return score" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "Let’s define the model instance named ``model 1``." 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 4, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "model = RandomNumModel(\"model 1\")" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": {}, 137 | "source": [ 138 | "We must register any backend isntance in order to use it in model instances.\n", 139 | "\n", 140 | "``set_backend`` and ``set_run_params`` methods can help us to set the run-parameters in the model and its backend." 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 5, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "register_backends({\"Random Number\": RandomNumBackend})\n", 150 | "model.set_backend(\"Random Number\")\n", 151 | "model.set_run_params(min=1, max=10)" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "Next, create an observation that requires the generated random integer between 1 and 10 and a test instance that use the observation and against the model\n", 159 | "\n", 160 | "Then we get a more quantitative summary of the results:" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 6, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "observation = {'min': 1, 'max': 10}\n", 170 | "oneToTenTest = RangeTest(observation, \"test 1\")\n", 171 | "score = oneToTenTest.judge(model)" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "metadata": {}, 177 | "source": [ 178 | "print the score, and we can see the result." 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 7, 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "name": "stdout", 188 | "output_type": "stream", 189 | "text": [ 190 | "Pass\n" 191 | ] 192 | } 193 | ], 194 | "source": [ 195 | "print(score)" 196 | ] 197 | } 198 | ], 199 | "metadata": { 200 | "kernelspec": { 201 | "display_name": "Python 3", 202 | "language": "python", 203 | "name": "python3" 204 | }, 205 | "language_info": { 206 | "codemirror_mode": { 207 | "name": "ipython", 208 | "version": 3 209 | }, 210 | "file_extension": ".py", 211 | "mimetype": "text/x-python", 212 | "name": "python", 213 | "nbconvert_exporter": "python", 214 | "pygments_lexer": "ipython3", 215 | "version": "3.9.4" 216 | } 217 | }, 218 | "nbformat": 4, 219 | "nbformat_minor": 4 220 | } 221 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/publish.sh: -------------------------------------------------------------------------------- 1 | jupyter-book build . 2 | ghp-import -n -p -f _build/html 3 | -------------------------------------------------------------------------------- /docs/sci-unit-tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scidash/sciunit/1a2ca478512ef80e38259b8bb6e12f714c27e904/docs/sci-unit-tag.png -------------------------------------------------------------------------------- /docs/source/README.md: -------------------------------------------------------------------------------- 1 | # Recipe for building the API docs from scratch: 2 | 3 | updating .rst files with sphinx-apidoc 4 | Assuming current working directory is `sciunit/docs/source`. 5 | 6 | Linux Bash shell commands: 7 | ``` 8 | rm modules.rst sciunit.models.rst sciunit.rst sciunit.scores.rst sciunit.unit_test.rst 9 | cd .. 10 | sphinx-apidoc -o "./source" "../sciunit" 11 | sphinx-build -b html ./source ./build 12 | ``` 13 | Windows PowerShell commands: 14 | ``` 15 | rm modules.rst, sciunit.models.rst, sciunit.rst, sciunit.scores.rst, sciunit.unit_test.rst 16 | cd .. 17 | sphinx-apidoc -o "./source" "../sciunit" 18 | sphinx-build -b html ./source ./build 19 | ``` 20 | -------------------------------------------------------------------------------- /docs/source/basics.rst: -------------------------------------------------------------------------------- 1 | SciUnit basics 2 | ============== 3 | 4 | This page will give you a basic view of the SciUnit project, 5 | and you can read the quick tutorials for some simple examples. 6 | 7 | The major parts of SciUnit are Score, Test, and Model. 8 | 9 | Model 10 | ------ 11 | 12 | ``Model`` is the abstract base class for sciunit models. Generally, a model instance can 13 | generate predicted or simulated results of some scientific fact. 14 | 15 | 16 | Runnable Model 17 | 18 | ``Runnable model`` is a kind of model that implements Runnable 19 | capability, and it can be executed to simulate and output results. 20 | 21 | 22 | Backend 23 | 24 | After being registered by ``register_backends`` function, a ``Backend`` instance 25 | can be executed by a Runnable Model at the back end. It usually does some background 26 | computing for the runnable model. 27 | 28 | 29 | Score 30 | ------ 31 | 32 | ``Score`` is the abstract base class for scores. The instance of it (or its subclass) can give some types of 33 | results for test and/or test suite against the models. 34 | 35 | The complete scores type in SciUnit are ``BooleanScore``, ``ZScore``, ``CohenDScore``, 36 | ``RatioScore``, ``PercentScore``, and ``FloatScore``. 37 | 38 | Each type of score has their own features and advantage. 39 | 40 | There are also incomplete score types. These type does not contain any 41 | information regarding how good the model is, but the existing of them means there are 42 | some issues during testing or computing process. They are ``NoneScore``, ``TBDScore``, ``NAScore``, 43 | and ``InsufficientDataScore`` 44 | 45 | 46 | 47 | ScoreArray, ScoreArrayM2M 48 | 49 | Can be used like this, assuming n tests and m models: 50 | 51 | >>> sm[test] 52 | (score_1, ..., score_m) 53 | 54 | >>> sm[model] 55 | (score_1, ..., score_n) 56 | 57 | 58 | ``ScoreArray`` represents an array of scores derived from a test suite. 59 | Extends the pandas Series such that items are either 60 | models subject to a test or tests taken by a model. 61 | Also displays and computes score summaries in sciunit-specific ways. 62 | 63 | ``ScoreArrayM2M`` represents an array of scores derived from ``TestM2M``. 64 | Extends the pandas Series such that items are either 65 | models subject to a test or the test itself. 66 | 67 | ScoreMatrix, ScoreMatrixM2M 68 | 69 | Can be used like this, assuming n tests and m models: 70 | 71 | >>> sm[test] 72 | (score_1, ..., score_m) 73 | 74 | >>> sm[model] 75 | (score_1, ..., score_n) 76 | 77 | 78 | ``ScoreMatrix`` represents a matrix of scores derived from a test suite. 79 | Extends the pandas DataFrame such that tests are columns and models 80 | are the index. Also displays and compute score summaries in sciunit-specific ways. 81 | 82 | ``ScoreMatrixM2M`` represents a matrix of scores derived from ``TestM2M``. 83 | Extends the pandas DataFrame such that models/observation are both 84 | columns and the index. 85 | 86 | 87 | Test, TestM2M 88 | -------------- 89 | 90 | ``Test`` is a abstract base class for tests. 91 | 92 | ``TestM2M`` is an abstract class for handling tests involving multiple models. 93 | 94 | A test instance contains some observations which are considered as the fact. 95 | The test instance can test the model by comparing the predictions with the observations 96 | and generate a specific type of score. 97 | 98 | Enables comparison of model to model predictions, and also against 99 | experimental reference data (optional). 100 | 101 | Note: ``TestM2M`` would typically be used when handling mutliple (>2) 102 | models, with/without experimental reference data. For single model 103 | tests, you can use the 'Test' class. 104 | 105 | TestSuite 106 | ---------- 107 | 108 | A collection of tests. The instance of ``TestSuite`` can perform similar things that a test instance can do. 109 | 110 | Converter 111 | ---------- 112 | 113 | A ``Converter`` instance can be used to convert a score between two types. 114 | It can be included in a test instance. 115 | 116 | Capability 117 | ----------- 118 | 119 | ``Capability`` is the abstract base class for sciunit capabilities. 120 | A capability instance can be included in a test instance to ensure the 121 | model, which is tested by the test instance, implements some methods. 122 | -------------------------------------------------------------------------------- /docs/source/commandline.rst: -------------------------------------------------------------------------------- 1 | Config File And Using SciUnit In A Shell 2 | ============================================= 3 | 4 | Create Config File And Execute Tests In A Shell 5 | ---------------------------------------------------------------- 6 | 7 | We can build a scientific computing project with a SciUnit config file. 8 | Then, we will be able to run sciunit In A Shell 9 | 10 | Here is an example of well written SciUnit config file. 11 | This file was generated by executing ``sciunit create`` in the shell. 12 | A SciUnit config file is always named ``sciunit.ini``. 13 | 14 | .. code-block:: python 15 | 16 | [misc] 17 | config-version = 1.0 18 | nb-name = scidash 19 | 20 | [root] 21 | path = . 22 | 23 | [models] 24 | module = models 25 | 26 | [tests] 27 | module = tests 28 | 29 | [suites] 30 | module = suites 31 | 32 | ``config-version`` is the version of the config file. 33 | 34 | ``nb-name`` is the name of the IPython Notebook file that can be create with ``sciunit make-nb``. 35 | 36 | ``root`` is the root of the project. The ``path`` is the path to the project from the directory that contains this config file. 37 | 38 | ``module`` in the ``models`` section is the path from the root of the project to the file that contains ``models``, which is a list of ``Model`` instances. 39 | 40 | ``module`` in the ``tests`` section is the path the root of the project to the file that contains ``tests``, which is a list of ``Test`` instances. 41 | 42 | ``module`` in the ``suites`` section is the path the root of the project to the file that contains ``suites``, which is a list of ``TestSuite`` instances. 43 | 44 | Let's use the config file above and create corresponding files that contain definitions models, tests, and suites. 45 | 46 | In the root directory of the project, let's create three files. 47 | 48 | **tests.py** 49 | 50 | .. code-block:: python 51 | 52 | import sciunit 53 | from sciunit.scores import BooleanScore 54 | from sciunit.capabilities import ProducesNumber 55 | 56 | 57 | 58 | class EqualsTest(sciunit.Test): 59 | """Tests if the model predicts 60 | the same number as the observation.""" 61 | 62 | required_capabilities = (ProducesNumber,) 63 | score_type = BooleanScore 64 | 65 | def generate_prediction(self, model): 66 | return model.produce_number() # The model has this method if it inherits from the 'ProducesNumber' capability. 67 | 68 | def compute_score(self, observation, prediction): 69 | score = self.score_type(observation['value'] == prediction) 70 | score.description = 'Passing score if the prediction equals the observation' 71 | return score 72 | 73 | tests = [] 74 | 75 | **suites.py** 76 | 77 | .. code-block:: python 78 | 79 | import sciunit 80 | from tests import EqualsTest 81 | 82 | 83 | equals_1_test = EqualsTest({'value':1}, name='=1') 84 | equals_2_test = EqualsTest({'value':2}, name='=2') 85 | equals_37_test = EqualsTest({'value':37}, name='=37') 86 | 87 | equals_suite = sciunit.TestSuite([equals_1_test, equals_2_test, equals_37_test], name="Equals test suite") 88 | 89 | suites = [equals_suite] 90 | 91 | **models.py** 92 | 93 | .. code-block:: python 94 | 95 | import sciunit 96 | from sciunit.capabilities import ProducesNumber 97 | 98 | 99 | class ConstModel(sciunit.Model, 100 | ProducesNumber): 101 | """A model that always produces a constant number as output.""" 102 | 103 | def __init__(self, constant, name=None): 104 | self.constant = constant 105 | super(ConstModel, self).__init__(name=name, constant=constant) 106 | 107 | def produce_number(self): 108 | return self.constant 109 | 110 | const_model_1 = ConstModel(1, name='Constant Model 1') 111 | const_model_2 = ConstModel(2, name='Constant Model 2') 112 | const_model_37 = ConstModel(37, name="Constant Model 37") 113 | 114 | models = [const_model_1, const_model_2, const_model_37] 115 | 116 | 117 | We have ``tests`` at the end of ``tests.py``, ``models`` at the end of ``models.py``, 118 | and ``suites`` at the end of ``suites.py``. Since we are using test suites instead of tests, 119 | ``tests`` is an empty list in this example. They will be taken by sciunit when command 120 | ``sciunit run`` is being executing 121 | 122 | Execute ``sciunit run`` in the root directory, and then sciunit will run each test in the suites 123 | against each model and give us the result. 124 | 125 | .. code-block:: bash 126 | 127 | $ sciunit run 128 | 129 | 130 | Executing test =1 on model Constant Model 1... Score is Pass 131 | Executing test =2 on model Constant Model 1... Score is Fail 132 | Executing test =37 on model Constant Model 1... Score is Fail 133 | Executing test =1 on model Constant Model 2... Score is Fail 134 | Executing test =2 on model Constant Model 2... Score is Pass 135 | Executing test =37 on model Constant Model 2... Score is Fail 136 | Executing test =1 on model Constant Model 37... Score is Fail 137 | Executing test =2 on model Constant Model 37... Score is Fail 138 | Executing test =37 on model Constant Model 37... Score is Pass 139 | 140 | Suite Equals test suite: 141 | =1 =2 =37 142 | Constant Model 1 Pass Fail Fail 143 | Constant Model 2 Fail Pass Fail 144 | Constant Model 37 Fail Fail Pass 145 | 146 | 147 | Create and Run IPython Notebook File 148 | ------------------------------------- 149 | 150 | Next, let's move to creating and executing IPython Notebook file with 151 | ``sciunit make-nb`` and ``sciunit run-nb`` commands. 152 | 153 | Let's add a file, ``__init__.py``, to our project directory and import everything 154 | including suites, tests, and models in the file. This is necessary because the made 155 | notebook file will try to import everything in ``__init__.py`` and run each suite 156 | (a collection of tests instances) against each model. 157 | 158 | **__init__.py** 159 | 160 | .. code-block:: python 161 | 162 | from . import models 163 | from . import tests 164 | from . import suites 165 | 166 | Now, let's execute ``sciunit make-nb`` SciUnit will automatically generate 167 | a notebook file. 168 | 169 | .. code-block:: bash 170 | 171 | $ sciunit make-nb 172 | Created Jupyter notebook at: 173 | /the_path_to_the_project.../test_sciunit.ipynb 174 | 175 | The notebook file will contains two blocks of code: 176 | 177 | .. code-block:: 178 | 179 | %matplotlib inline 180 | from IPython.display import display 181 | from importlib.machinery import SourceFileLoader 182 | test_sciunit = SourceFileLoader('scidash', '/the_path_to_the_project.../__init__.py').load_module() 183 | 184 | 185 | .. code-block:: 186 | 187 | for suite in test_sciunit.suites.suites: 188 | score_matrix = suite.judge(test_sciunit.models.models, stop_on_error=True) 189 | display(score_matrix) 190 | 191 | 192 | 193 | Note: 194 | 195 | 1. the name of generated notebook file will be the value of ``nb-name`` attribute 196 | in the config file, ``sciunit.ini`` 197 | 198 | 2. The path to the project's root can be different on different machine. 199 | So, The notebook file generated usually only be valid on the machine where 200 | it is generated. If you want to execute it on different machine, try to re-generate 201 | it or change the path. 202 | 203 | Let's execute ``sciunit run-nb`` command. 204 | 205 | .. code-block:: bash 206 | 207 | $ sciunit run-nb 208 | Entering run function 209 | /the_path_to_the_project_config_file..././test_sciunit.ipynb 210 | /the_path_to_the_project_config_file.../. 211 | 212 | The result of running the notebook will be in the notebook file. 213 | You can open it by many tools like VS Code and Jupyter Lab 214 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # SciUnit documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Mar 31 23:49:49 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [ 35 | "sphinx.ext.autodoc", 36 | "sphinx.ext.intersphinx", 37 | "sphinx.ext.viewcode", 38 | "sphinx.ext.githubpages", 39 | ] 40 | 41 | # Add any paths that contain templates here, relative to this directory. 42 | templates_path = ["_templates"] 43 | 44 | # The suffix(es) of source filenames. 45 | # You can specify multiple suffix as a list of string: 46 | # 47 | # source_suffix = ['.rst', '.md'] 48 | source_suffix = ".rst" 49 | 50 | # The master toctree document. 51 | master_doc = "index" 52 | 53 | # General information about the project. 54 | project = "SciUnit" 55 | copyright = "2017, Rick Gerkin and Cyrus Omar" 56 | author = "Rick Gerkin and Cyrus Omar" 57 | 58 | # The version info for the project you're documenting, acts as replacement for 59 | # |version| and |release|, also used in various other places throughout the 60 | # built documents. 61 | # 62 | # The short X.Y version. 63 | version = "" 64 | # The full version, including alpha/beta/rc tags. 65 | release = "" 66 | 67 | # The language for content autogenerated by Sphinx. Refer to documentation 68 | # for a list of supported languages. 69 | # 70 | # This is also used if you do content translation via gettext catalogs. 71 | # Usually you set "language" from the command line for these cases. 72 | language = None 73 | 74 | # List of patterns, relative to source directory, that match files and 75 | # directories to ignore when looking for source files. 76 | # This patterns also effect to html_static_path and html_extra_path 77 | exclude_patterns = [] 78 | 79 | # The name of the Pygments (syntax highlighting) style to use. 80 | pygments_style = "sphinx" 81 | 82 | # If true, `todo` and `todoList` produce output, else they produce nothing. 83 | todo_include_todos = False 84 | 85 | 86 | # -- Options for HTML output ---------------------------------------------- 87 | 88 | # The theme to use for HTML and HTML Help pages. See the documentation for 89 | # a list of builtin themes. 90 | # 91 | # html_theme = 'bizstyle' 92 | 93 | # Theme options are theme-specific and customize the look and feel of a theme 94 | # further. For a list of options available for each theme, see the 95 | # documentation. 96 | # 97 | # html_theme_options = {} 98 | 99 | # Add any paths that contain custom static files (such as style sheets) here, 100 | # relative to this directory. They are copied after the builtin static files, 101 | # so a file named "default.css" will overwrite the builtin "default.css". 102 | html_static_path = ["_static"] 103 | 104 | 105 | # -- Options for HTMLHelp output ------------------------------------------ 106 | 107 | # Output file base name for HTML help builder. 108 | htmlhelp_basename = "SciUnitdoc" 109 | 110 | 111 | # -- Options for LaTeX output --------------------------------------------- 112 | 113 | latex_elements = { 114 | # The paper size ('letterpaper' or 'a4paper'). 115 | # 116 | # 'papersize': 'letterpaper', 117 | # The font size ('10pt', '11pt' or '12pt'). 118 | # 119 | # 'pointsize': '10pt', 120 | # Additional stuff for the LaTeX preamble. 121 | # 122 | # 'preamble': '', 123 | # Latex figure (float) alignment 124 | # 125 | # 'figure_align': 'htbp', 126 | } 127 | 128 | # Grouping the document tree into LaTeX files. List of tuples 129 | # (source start file, target name, title, 130 | # author, documentclass [howto, manual, or own class]). 131 | latex_documents = [ 132 | ( 133 | master_doc, 134 | "SciUnit.tex", 135 | "SciUnit Documentation", 136 | "Rick Gerkin and Cyrus Omar", 137 | "manual", 138 | ), 139 | ] 140 | 141 | 142 | # -- Options for manual page output --------------------------------------- 143 | 144 | # One entry per manual page. List of tuples 145 | # (source start file, name, description, authors, manual section). 146 | man_pages = [(master_doc, "sciunit", "SciUnit Documentation", [author], 1)] 147 | 148 | 149 | # -- Options for Texinfo output ------------------------------------------- 150 | 151 | # Grouping the document tree into Texinfo files. List of tuples 152 | # (source start file, target name, title, author, 153 | # dir menu entry, description, category) 154 | texinfo_documents = [ 155 | ( 156 | master_doc, 157 | "SciUnit", 158 | "SciUnit Documentation", 159 | author, 160 | "SciUnit", 161 | "One line description of project.", 162 | "Miscellaneous", 163 | ), 164 | ] 165 | 166 | 167 | # Example configuration for intersphinx: refer to the Python standard library. 168 | intersphinx_mapping = {"https://docs.python.org/": None} 169 | 170 | # Removing those three member from the documents to avoid mess. More members can be added in the future. 171 | def remove_variables(app, what, name, obj, skip, options): 172 | if name == "_url": 173 | print("-----------------------") 174 | print(what) 175 | excluded = [ 176 | "normalization_rules", 177 | "rules", 178 | "validation_rules", 179 | "__dict__", 180 | "__doc__", 181 | ] 182 | return name in excluded 183 | 184 | 185 | # Connecting remove_variables and autodoc-skip-member to setup the event handler. 186 | def setup(app): 187 | app.connect("autodoc-skip-member", remove_variables) 188 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. sciunit documentation master file, created by 2 | sphinx-quickstart on Thu Feb 13 13:47:10 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to sciunit's documentation! 7 | =================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | .. figure:: https://raw.githubusercontent.com/scidash/assets/master/logos/SciUnit/sci-unit-square-small.png 14 | :align: center 15 | 16 | ======================================================================================= 17 | 18 | Concept 19 | ------- 20 | 21 | `The conference 22 | paper `__ 23 | 24 | Tutorials With Jupyter NoteBook 25 | ------------------------------- 26 | 27 | .. image:: https://colab.research.google.com/assets/colab-badge.svg 28 | :target: https://colab.research.google.com/github/scidash/sciunit/blob/master/docs/chapter1.ipynb 29 | 30 | * `Tutorial Chapter 1`_ 31 | * `Tutorial Chapter 2`_ 32 | * `Tutorial Chapter 3`_ 33 | * `Tutorial Chapter 4`_ 34 | * `Tutorial Chapter 5`_ 35 | * `Tutorial Chapter 6`_ 36 | 37 | .. _Tutorial Chapter 1: https://github.com/scidash/sciunit/blob/master/docs/chapter1.ipynb 38 | .. _Tutorial Chapter 2: https://github.com/scidash/sciunit/blob/master/docs/chapter2.ipynb 39 | .. _Tutorial Chapter 3: https://github.com/scidash/sciunit/blob/master/docs/chapter3.ipynb 40 | .. _Tutorial Chapter 4: https://github.com/scidash/sciunit/blob/dev/docs/chapter4.ipynb 41 | .. _Tutorial Chapter 5: https://github.com/scidash/sciunit/blob/dev/docs/chapter5.ipynb 42 | .. _Tutorial Chapter 6: https://github.com/scidash/sciunit/blob/dev/docs/chapter6.ipynb 43 | 44 | Basic Usage 45 | ----------- 46 | 47 | .. code:: python 48 | 49 | my_model = MyModel(**my_args) # Instantiate a class that wraps your model of interest. 50 | my_test = MyTest(**my_params) # Instantiate a test that you write. 51 | score = my_test.judge() # Runs the test and return a rich score containing test results and more. 52 | 53 | Domain-specific libraries and information 54 | ----------------------------------------- 55 | 56 | `NeuronUnit `__ for neuron and 57 | ion channel physiology See others 58 | `here `__ 59 | 60 | Mailing List 61 | ------------ 62 | 63 | There is a `mailing 64 | list `__ for 65 | announcements and discussion. Please join it if you are at all 66 | interested! 67 | 68 | Contributors 69 | ------------ 70 | 71 | - `Rick Gerkin `__, Arizona State University 72 | (School of Life Science) 73 | - `Cyrus Omar `__, Carnegie Mellon University 74 | (Dept. of Computer Science) 75 | 76 | Reproducible Research ID 77 | ------------------------ 78 | 79 | RRID:\ `SCR\_014528 `__ 80 | 81 | License 82 | ------- 83 | 84 | SciUnit is released under the permissive `MIT 85 | license `__, requiring only 86 | attribution in derivative works. See the LICENSE file for terms. 87 | 88 | 89 | Table of Contents 90 | ----------------- 91 | 92 | .. toctree:: 93 | 94 | setup 95 | quick_tutorial 96 | basics 97 | commandline 98 | 99 | Indices and tables 100 | ================== 101 | 102 | * :ref:`genindex` 103 | * :ref:`modindex` 104 | * :ref:`search` 105 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | sciunit 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | sciunit 8 | -------------------------------------------------------------------------------- /docs/source/sciunit.models.rst: -------------------------------------------------------------------------------- 1 | sciunit.models package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | sciunit.models.backends module 8 | ------------------------------ 9 | 10 | .. automodule:: sciunit.models.backends 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sciunit.models.base module 16 | -------------------------- 17 | 18 | .. automodule:: sciunit.models.base 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sciunit.models.examples module 24 | ------------------------------ 25 | 26 | .. automodule:: sciunit.models.examples 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sciunit.models.runnable module 32 | ------------------------------ 33 | 34 | .. automodule:: sciunit.models.runnable 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: sciunit.models 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/source/sciunit.rst: -------------------------------------------------------------------------------- 1 | sciunit package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | sciunit.models 10 | sciunit.scores 11 | sciunit.unit_test 12 | 13 | Submodules 14 | ---------- 15 | 16 | sciunit.base module 17 | ------------------- 18 | 19 | .. automodule:: sciunit.base 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | 24 | sciunit.capabilities module 25 | --------------------------- 26 | 27 | .. automodule:: sciunit.capabilities 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | 32 | sciunit.converters module 33 | ------------------------- 34 | 35 | .. automodule:: sciunit.converters 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | 40 | sciunit.errors module 41 | --------------------- 42 | 43 | .. automodule:: sciunit.errors 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | 48 | sciunit.suites module 49 | --------------------- 50 | 51 | .. automodule:: sciunit.suites 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | 56 | sciunit.tests module 57 | -------------------- 58 | 59 | .. automodule:: sciunit.tests 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | 64 | sciunit.utils module 65 | -------------------- 66 | 67 | .. automodule:: sciunit.utils 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | 72 | sciunit.validators module 73 | ------------------------- 74 | 75 | .. automodule:: sciunit.validators 76 | :members: 77 | :undoc-members: 78 | :show-inheritance: 79 | 80 | sciunit.version module 81 | ---------------------- 82 | 83 | .. automodule:: sciunit.version 84 | :members: 85 | :undoc-members: 86 | :show-inheritance: 87 | 88 | 89 | Module contents 90 | --------------- 91 | 92 | .. automodule:: sciunit 93 | :members: 94 | :undoc-members: 95 | :show-inheritance: 96 | -------------------------------------------------------------------------------- /docs/source/sciunit.scores.rst: -------------------------------------------------------------------------------- 1 | sciunit.scores package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | sciunit.scores.base module 8 | -------------------------- 9 | 10 | .. automodule:: sciunit.scores.base 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sciunit.scores.collections module 16 | --------------------------------- 17 | 18 | .. automodule:: sciunit.scores.collections 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sciunit.scores.collections\_m2m module 24 | -------------------------------------- 25 | 26 | .. automodule:: sciunit.scores.collections_m2m 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sciunit.scores.complete module 32 | ------------------------------ 33 | 34 | .. automodule:: sciunit.scores.complete 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | sciunit.scores.incomplete module 40 | -------------------------------- 41 | 42 | .. automodule:: sciunit.scores.incomplete 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: sciunit.scores 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/source/sciunit.unit_test.rst: -------------------------------------------------------------------------------- 1 | sciunit.unit\_test package 2 | ========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | sciunit.unit\_test.active module 8 | -------------------------------- 9 | 10 | .. automodule:: sciunit.unit_test.active 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sciunit.unit\_test.backend\_tests module 16 | ---------------------------------------- 17 | 18 | .. automodule:: sciunit.unit_test.backend_tests 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sciunit.unit\_test.base module 24 | ------------------------------ 25 | 26 | .. automodule:: sciunit.unit_test.base 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sciunit.unit\_test.base\_tests module 32 | ------------------------------------- 33 | 34 | .. automodule:: sciunit.unit_test.base_tests 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | sciunit.unit\_test.command\_line\_tests module 40 | ---------------------------------------------- 41 | 42 | .. automodule:: sciunit.unit_test.command_line_tests 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | sciunit.unit\_test.config\_tests module 48 | --------------------------------------- 49 | 50 | .. automodule:: sciunit.unit_test.config_tests 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | sciunit.unit\_test.converter\_tests module 56 | ------------------------------------------ 57 | 58 | .. automodule:: sciunit.unit_test.converter_tests 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | sciunit.unit\_test.doc\_tests module 64 | ------------------------------------ 65 | 66 | .. automodule:: sciunit.unit_test.doc_tests 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | sciunit.unit\_test.error\_tests module 72 | -------------------------------------- 73 | 74 | .. automodule:: sciunit.unit_test.error_tests 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | sciunit.unit\_test.import\_tests module 80 | --------------------------------------- 81 | 82 | .. automodule:: sciunit.unit_test.import_tests 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | sciunit.unit\_test.model\_tests module 88 | -------------------------------------- 89 | 90 | .. automodule:: sciunit.unit_test.model_tests 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | sciunit.unit\_test.observation\_tests module 96 | -------------------------------------------- 97 | 98 | .. automodule:: sciunit.unit_test.observation_tests 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | sciunit.unit\_test.score\_tests module 104 | -------------------------------------- 105 | 106 | .. automodule:: sciunit.unit_test.score_tests 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | sciunit.unit\_test.test\_tests module 112 | ------------------------------------- 113 | 114 | .. automodule:: sciunit.unit_test.test_tests 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | sciunit.unit\_test.utils\_tests module 120 | -------------------------------------- 121 | 122 | .. automodule:: sciunit.unit_test.utils_tests 123 | :members: 124 | :undoc-members: 125 | :show-inheritance: 126 | 127 | sciunit.unit\_test.validator\_tests module 128 | ------------------------------------------ 129 | 130 | .. automodule:: sciunit.unit_test.validator_tests 131 | :members: 132 | :undoc-members: 133 | :show-inheritance: 134 | 135 | Module contents 136 | --------------- 137 | 138 | .. automodule:: sciunit.unit_test 139 | :members: 140 | :undoc-members: 141 | :show-inheritance: 142 | -------------------------------------------------------------------------------- /docs/source/setup.rst: -------------------------------------------------------------------------------- 1 | What's SciUnit and how to install it? 2 | ===================================== 3 | 4 | Everyone hopes that their model has some correspondence with reality. 5 | Usually, checking whether this is true is done informally. But SciUnit makes this formal and transparent. 6 | 7 | SciUnit is a framework for validating scientific models by creating experimental-data-driven unit tests. 8 | 9 | Installation 10 | ---------------- 11 | 12 | Note: SciUnit no longer supports Python 2. Please use Python 3. 13 | 14 | SciUnit can be installed in virtual environments using the :code:`pip` Python package installer. 15 | A virtual environment can be set up using the in-built Python :code:`venv` module, as explained `here `__, or using other tools such as `Miniconda `__. 16 | 17 | 18 | In the virtual environment, run the following command to install SciUnit as a Python package using :code:`pip`. 19 | 20 | .. code-block:: bash 21 | 22 | pip install sciunit 23 | 24 | 25 | On `Fedora Linux `__ installations, the `NeuroFedora special interest group `__ also provides SciUnit as a curated package in the Fedora community repositories. 26 | SciUnit can, therefore, be installed using the system package manager, :code:`dnf` : 27 | 28 | .. code-block:: bash 29 | 30 | sudo dnf install python3-sciunit 31 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 40.9.0", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /sciunit/__init__.py: -------------------------------------------------------------------------------- 1 | """SciUnit. 2 | 3 | A Testing Framework for Data-Driven Validation of 4 | Quantitative Scientific Models 5 | """ 6 | 7 | import json 8 | import logging 9 | import sys 10 | 11 | from .base import __version__, config, log, logger 12 | from .capabilities import Capability 13 | from .errors import Error 14 | from .models import Model 15 | from .models.backends import Backend 16 | from .scores import Score 17 | from .scores.collections import ScoreArray, ScoreMatrix 18 | from .scores.collections_m2m import ScoreArrayM2M, ScoreMatrixM2M 19 | from .suites import TestSuite 20 | from .tests import Test, TestM2M 21 | -------------------------------------------------------------------------------- /sciunit/__main__.py: -------------------------------------------------------------------------------- 1 | """SciUnit command line tools. 2 | 3 | With SciUnit installed, a sciunit.ini configuration file can be created from 4 | a *nix shell with: 5 | `sciunit create` 6 | and if models, tests, etc. are in locations specified by the configuration file 7 | then they can be executed with 8 | `sciunit run` (to run using the Python interpreter) 9 | or 10 | `sciunit make-nb` (to create Jupyter notebooks for test execution) 11 | and 12 | `sciunit run-nb` (to execute and save those notebooks) 13 | """ 14 | 15 | import argparse 16 | import codecs 17 | import configparser 18 | import os 19 | import re 20 | import sys 21 | from configparser import RawConfigParser 22 | from importlib import import_module 23 | from pathlib import Path 24 | from typing import Union 25 | 26 | import matplotlib 27 | import nbformat 28 | from nbconvert.preprocessors import ExecutePreprocessor 29 | from nbformat.v4.nbbase import new_markdown_cell, new_notebook 30 | 31 | import sciunit 32 | 33 | matplotlib.use("Agg") #: Anticipate possible headless environments 34 | 35 | NB_VERSION = 4 36 | 37 | 38 | def main(*args): 39 | """Launch the main routine.""" 40 | parser = argparse.ArgumentParser() 41 | parser.add_argument("action", help="create, check, run, make-nb, or run-nb") 42 | parser.add_argument( 43 | "--directory", 44 | "-dir", 45 | default=Path.cwd(), 46 | help="path to directory with a sciunit.ini file", 47 | ) 48 | parser.add_argument( 49 | "--stop", "-s", default=True, help="stop and raise errors, halting the program" 50 | ) 51 | parser.add_argument( 52 | "--tests", "-t", default=False, help="runs tests instead of suites" 53 | ) 54 | if args: 55 | args = parser.parse_args(args) 56 | else: 57 | args = parser.parse_args() 58 | 59 | directory = Path(args.directory) 60 | file_path = directory / "sciunit.ini" 61 | 62 | config = None 63 | if args.action == "create": 64 | create(file_path) 65 | elif args.action == "check": 66 | config = parse(file_path, show=True) 67 | print("\nNo configuration errors reported.") 68 | elif args.action == "run": 69 | config = parse(file_path) 70 | run(config, path=directory, stop_on_error=args.stop, just_tests=args.tests) 71 | elif args.action == "make-nb": 72 | config = parse(file_path) 73 | make_nb(config, path=directory, stop_on_error=args.stop, just_tests=args.tests) 74 | elif args.action == "run-nb": 75 | config = parse(file_path) 76 | run_nb(config, path=directory) 77 | else: 78 | raise NameError("No such action %s" % args.action) 79 | if config: 80 | cleanup(config, path=directory) 81 | 82 | 83 | def create(file_path: Path) -> None: 84 | """Create a default sciunit.ini config file if one does not already exist. 85 | 86 | Args: 87 | file_path (str): The path of sciunit config file that will be created. 88 | 89 | Raises: 90 | IOError: There is already a configuration file at the path. 91 | """ 92 | if file_path.exists(): 93 | raise IOError("There is already a configuration file at %s" % file_path) 94 | with open(file_path, "w") as f: 95 | config = configparser.ConfigParser() 96 | config.add_section("misc") 97 | config.set("misc", "config-version", "1.0") 98 | default_nb_name = file_path.parent.name 99 | config.set("misc", "nb-name", default_nb_name) 100 | config.add_section("root") 101 | config.set("root", "path", ".") 102 | config.add_section("models") 103 | config.set("models", "module", "models") 104 | config.add_section("tests") 105 | config.set("tests", "module", "tests") 106 | config.add_section("suites") 107 | config.set("suites", "module", "suites") 108 | config.write(f) 109 | 110 | 111 | def parse(file_path: Path = None, show: bool = False) -> RawConfigParser: 112 | """Parse a sciunit.ini config file. 113 | 114 | Args: 115 | file_path (str, optional): The path of sciunit config file that will be parsed. Defaults to None. 116 | show (bool, optional): Whether or not print the sections in the config file. Defaults to False. 117 | 118 | Raises: 119 | IOError: Raise an exception if no sciunit.ini file was found. 120 | 121 | Returns: 122 | RawConfigParser: The basic configuration object. 123 | """ 124 | if file_path is None: 125 | file_path = Path.cwd() / "sciunit.ini" 126 | if not file_path.exists(): 127 | raise IOError("No sciunit.ini file was found at %s" % file_path) 128 | 129 | # Load the configuration file 130 | config = configparser.RawConfigParser(allow_no_value=True) 131 | config.read(file_path) 132 | 133 | # List all contents 134 | for section in config.sections(): 135 | if show: 136 | print(section) 137 | for options in config.options(section): 138 | if show: 139 | print("\t%s: %s" % (options, config.get(section, options))) 140 | return config 141 | 142 | 143 | def prep( 144 | config: configparser.RawConfigParser = None, path: Union[str, Path] = None 145 | ) -> None: 146 | """Prepare to read the configuration information. 147 | 148 | Args: 149 | config (RawConfigParser, optional): The configuration object. Defaults to None. 150 | path (Union[str, Path], optional): The path of config file. Defaults to None. 151 | """ 152 | if config is None: 153 | config = parse() 154 | if path is None: 155 | path = Path.cwd() 156 | root = config.get("root", "path") 157 | root = path / root 158 | root = str(root.resolve()) 159 | os.environ["SCIDASH_HOME"] = root 160 | if sys.path[0] != root: 161 | sys.path.insert(0, root) 162 | 163 | 164 | def run( 165 | config: configparser.RawConfigParser, 166 | path: Union[str, Path] = None, 167 | stop_on_error: bool = True, 168 | just_tests: bool = False, 169 | ) -> None: 170 | """Run sciunit tests for the given configuration. 171 | 172 | Args: 173 | config (RawConfigParser): The parsed sciunit config file. 174 | path (Union[str, Path], optional): The path of sciunit config file. Defaults to None. 175 | stop_on_error (bool, optional): Stop if an error occurs. Defaults to True. 176 | just_tests (bool, optional): There are only tests, no suites. Defaults to False. 177 | """ 178 | if path is None: 179 | path = Path.cwd() 180 | prep(config, path=path) 181 | 182 | models = import_module("models") 183 | tests = import_module("tests") 184 | suites = import_module("suites") 185 | 186 | print("\n") 187 | for x in ["models", "tests", "suites"]: 188 | module = import_module(x) 189 | assert hasattr(module, x), "'%s' module requires attribute '%s'" % (x, x) 190 | 191 | if just_tests: 192 | for test in tests.tests: 193 | _run(test, models, stop_on_error) 194 | 195 | else: 196 | for suite in suites.suites: 197 | _run(suite, models, stop_on_error) 198 | 199 | 200 | def _run( 201 | test_or_suite: Union[sciunit.Test, sciunit.TestSuite], 202 | models: list, 203 | stop_on_error: bool, 204 | ) -> None: 205 | """Run a single test or suite. 206 | 207 | Args: 208 | test_or_suite (Union[Test, TestSuite]): A test or suite instance to be executed. 209 | models (list): The list of sciunit Model. 210 | stop_on_error (bool): Whether to stop on error. 211 | """ 212 | score_array_or_matrix = test_or_suite.judge( 213 | models.models, stop_on_error=stop_on_error 214 | ) 215 | kind = "Test" if isinstance(test_or_suite, sciunit.Test) else "Suite" 216 | print("\n%s %s:\n%s\n" % (kind, test_or_suite, score_array_or_matrix)) 217 | 218 | 219 | def nb_name_from_path(config: RawConfigParser, path: Union[str, Path]) -> tuple: 220 | """Get a notebook name from a path to a notebook. 221 | 222 | Args: 223 | config (RawConfigParser): The parsed sciunit config file. 224 | path (Union[str, Path]): The path of the notebook file. 225 | 226 | Returns: 227 | tuple: Notebook root node and name of the notebook. 228 | """ 229 | if path is None: 230 | path = Path.cwd() 231 | root = config.get("root", "path") 232 | root = path / root 233 | root = root.resolve() 234 | default_nb_name = root.name 235 | nb_name = config.get("misc", "nb-name", fallback=default_nb_name) 236 | return root, nb_name 237 | 238 | 239 | def make_nb( 240 | config: RawConfigParser, 241 | path: Union[str, Path] = None, 242 | stop_on_error: bool = True, 243 | just_tests: bool = False, 244 | ) -> None: 245 | """Create a Jupyter notebook sciunit tests for the given configuration. 246 | 247 | Args: 248 | config (RawConfigParser): The parsed sciunit config file. 249 | path (Union[str, Path], optional): A path to the notebook file. Defaults to None. 250 | stop_on_error (bool, optional): Whether to stop on an error. Defaults to True. 251 | just_tests (bool, optional): There are only tests, no suites. Defaults to False. 252 | """ 253 | root, nb_name = nb_name_from_path(config, path) 254 | clean = lambda varStr: re.sub("\W|^(?=\d)", "_", varStr) 255 | name = clean(nb_name) 256 | 257 | mpl_style = config.get("misc", "matplotlib", fallback="inline") 258 | cells = [new_markdown_cell("## Sciunit Testing Notebook for %s" % nb_name)] 259 | add_code_cell( 260 | cells, 261 | ( 262 | "%%matplotlib %s\n" 263 | "from IPython.display import display\n" 264 | "from importlib.machinery import SourceFileLoader\n" 265 | "%s = SourceFileLoader('scidash', '%s/__init__.py').load_module()" 266 | ) 267 | % (mpl_style, name, root), 268 | ) 269 | if just_tests: 270 | add_code_cell( 271 | cells, 272 | ( 273 | "for test in %s.tests.tests:\n" 274 | " score_array = test.judge(%s.models.models, stop_on_error=%r)\n" 275 | " display(score_array)" 276 | ) 277 | % (name, name, stop_on_error), 278 | ) 279 | else: 280 | add_code_cell( 281 | cells, 282 | ( 283 | "for suite in %s.suites.suites:\n" 284 | " score_matrix = suite.judge(" 285 | "%s.models.models, stop_on_error=%r)\n" 286 | " display(score_matrix)" 287 | ) 288 | % (name, name, stop_on_error), 289 | ) 290 | write_nb(root, nb_name, cells) 291 | 292 | 293 | def write_nb(root: str, nb_name: str, cells: list) -> None: 294 | """Write a jupyter notebook to disk. 295 | 296 | Takes a given a root directory, a notebook name, and a list of cells. 297 | 298 | Args: 299 | root (str): The root node (section) of the notebook. 300 | nb_name (str): The name of the notebook file. 301 | cells (list): The list of the cells of the notebook. 302 | """ 303 | nb = new_notebook( 304 | cells=cells, 305 | metadata={ 306 | "language": "python", 307 | }, 308 | ) 309 | nb_path = root / ("%s.ipynb" % nb_name) 310 | with codecs.open(nb_path, encoding="utf-8", mode="w") as nb_file: 311 | nbformat.write(nb, nb_file, NB_VERSION) 312 | print("Created Jupyter notebook at:\n%s" % nb_path) 313 | 314 | 315 | def run_nb(config: RawConfigParser, path: Union[str, Path] = None) -> None: 316 | """Run a notebook file. 317 | 318 | Runs the one specified by the config file, or the one at 319 | the location specificed by 'path'. 320 | 321 | Args: 322 | config (RawConfigParser): The parsed sciunit config file. 323 | path (Union[str, Path], optional): The path to the notebook file. Defaults to None. 324 | """ 325 | if path is None: 326 | path = Path.cwd() 327 | root = config.get("root", "path") 328 | root = path / root 329 | nb_name = config.get("misc", "nb-name") 330 | nb_path = root / ("%s.ipynb" % nb_name) 331 | if not nb_path.exists(): 332 | print( 333 | ("No notebook found at %s. " "Create the notebook first with make-nb?") 334 | % path 335 | ) 336 | sys.exit(0) 337 | 338 | with codecs.open(nb_path, encoding="utf-8", mode="r") as nb_file: 339 | nb = nbformat.read(nb_file, as_version=NB_VERSION) 340 | ep = ExecutePreprocessor(timeout=600) 341 | ep.preprocess(nb, {"metadata": {"path": root}}) 342 | with codecs.open(nb_path, encoding="utf-8", mode="w") as nb_file: 343 | nbformat.write(nb, nb_file, NB_VERSION) 344 | 345 | 346 | def add_code_cell(cells: list, source: str) -> None: 347 | """Add a code cell containing `source` to the notebook. 348 | 349 | Args: 350 | cells (list): The list of notebook cells. 351 | source (str): The source of the notebook cell. 352 | """ 353 | from nbformat.v4.nbbase import new_code_cell 354 | 355 | n_code_cells = len([c for c in cells if c["cell_type"] == "code"]) 356 | cells.append(new_code_cell(source=source, execution_count=n_code_cells + 1)) 357 | 358 | 359 | def cleanup(config=None, path: Union[str, Path] = None) -> None: 360 | """Cleanup by removing paths added during earlier in configuration. 361 | 362 | Args: 363 | config (RawConfigParser, optional): The parsed sciunit config file. Defaults to None. 364 | path (Union[str, Path], optional): The path to the sciunit config file. Defaults to None. 365 | """ 366 | if config is None: 367 | config = parse() 368 | if path is None: 369 | path = Path.cwd() 370 | root = config.get("root", "path") 371 | root = str(path / root) 372 | if sys.path[0] == root: 373 | sys.path.remove(root) 374 | 375 | 376 | if __name__ == "__main__": 377 | main() 378 | -------------------------------------------------------------------------------- /sciunit/capabilities.py: -------------------------------------------------------------------------------- 1 | """The base class for SciUnit capabilities. 2 | 3 | By inheriting a capability class, a model tells the test that it implements 4 | that capability and that all of its methods are safe to call. 5 | The capability must then be implemented by the modeler (i.e. all of the 6 | capabilty's methods must implemented in the model class). 7 | """ 8 | 9 | import dis 10 | import inspect 11 | import io 12 | import re 13 | import sys 14 | import warnings 15 | 16 | from .base import SciUnit, log, logger 17 | from .errors import CapabilityNotImplementedError 18 | 19 | # from sciunit.models.examples import ConstModel, UniformModel 20 | 21 | 22 | class Capability(SciUnit): 23 | """Abstract base class for sciunit capabilities.""" 24 | 25 | @classmethod 26 | def source_check(cls, model: "sciunit.Model") -> bool: 27 | required_methods = [] 28 | default_cap_methods = ["unimplemented", "__str__"] 29 | source_capable = True 30 | 31 | for key, value in vars(cls).items(): 32 | if inspect.isfunction(value) and key not in default_cap_methods: 33 | required_methods.append(key) 34 | 35 | for method in required_methods: 36 | try: 37 | stdout = sys.stdout 38 | sys.stdout = io.StringIO() 39 | 40 | dis.dis(getattr(cls, method)) 41 | 42 | dis_output = sys.stdout.getvalue() 43 | sys.stdout = stdout 44 | 45 | dis_output = re.split("\n|\s+", dis_output) 46 | dis_output = [word for word in dis_output if word] 47 | 48 | if ( 49 | "(NotImplementedError)" in dis_output 50 | or "(unimplemented)" in dis_output 51 | or "(CapabilityNotImplementedError)" in dis_output 52 | or "(NotImplemented)" in dis_output 53 | ): 54 | cap_source = inspect.getsource(getattr(cls, method)) 55 | model_source = inspect.getsource(getattr(model, method)) 56 | 57 | if cap_source == model_source: 58 | source_capable = False 59 | break 60 | 61 | except OSError: 62 | warnings.warn( 63 | """Inspect cannot get the source, and it is not guaranteed that 64 | all required methods have been implemented by the model""" 65 | ) 66 | break 67 | 68 | return source_capable 69 | 70 | @classmethod 71 | def check(cls, model: "sciunit.Model", require_extra: bool = False) -> bool: 72 | """Check whether the provided model has this capability. 73 | 74 | By default, uses isinstance. If `require_extra`, also requires that an 75 | instance check be present in `model.extra_capability_checks`. 76 | 77 | Args: 78 | model (Model): A sciunit model instance 79 | require_extra (bool, optional): Requiring that an instance check be present in 80 | `model.extra_capability_checks`. Defaults to False. 81 | 82 | Returns: 83 | bool: Whether the provided model has this capability. 84 | """ 85 | 86 | class_capable = isinstance(model, cls) 87 | source_capable = None 88 | 89 | if class_capable: 90 | source_capable = cls.source_check(model) 91 | 92 | f_name = ( 93 | model.extra_capability_checks.get(cls, None) 94 | if model.extra_capability_checks is not None 95 | else False 96 | ) 97 | 98 | if f_name: 99 | f = getattr(model, f_name) 100 | instance_capable = f() 101 | elif not require_extra: 102 | instance_capable = True 103 | else: 104 | instance_capable = False 105 | 106 | if not class_capable: 107 | log( 108 | ( 109 | "The Model class does not claim at least one Capability required by " 110 | "the Test class, so the Score is likely to be unavailable." 111 | ) 112 | ) 113 | elif not source_capable: 114 | logger.warning( 115 | ( 116 | "The model class claimed to implement all methods required by " 117 | "the Test class, but at least one was left unimplemented, " 118 | "so this model will be skipped." 119 | ) 120 | ) 121 | 122 | return class_capable and instance_capable and source_capable 123 | 124 | def unimplemented(self, message: str = "") -> None: 125 | """Raise a `CapabilityNotImplementedError` with details. 126 | 127 | Args: 128 | message (str, optional): Message for not implemented exception. Defaults to ''. 129 | 130 | Raises: 131 | CapabilityNotImplementedError: Raise a `CapabilityNotImplementedError` with details. 132 | """ 133 | from sciunit import Model 134 | 135 | capabilities = [ 136 | obj 137 | for obj in self.__class__.mro() 138 | if issubclass(obj, Capability) and not issubclass(obj, Model) 139 | ] 140 | model = self if isinstance(self, Model) else None 141 | capability = None if not capabilities else capabilities[0] 142 | raise CapabilityNotImplementedError(model, capability, message) 143 | 144 | class __metaclass__(type): 145 | @property 146 | def name(cls): 147 | return cls.__name__ 148 | 149 | def __str__(self) -> str: 150 | return self.name 151 | 152 | 153 | class ProducesNumber(Capability): 154 | """An example capability for producing some generic number.""" 155 | 156 | def produce_number(self) -> None: 157 | """Produce a number.""" 158 | self.unimplemented() 159 | 160 | 161 | class Runnable(Capability): 162 | """Capability for models that can be run, i.e. simulated.""" 163 | 164 | def run(self, **run_params) -> None: 165 | """Run, i.e. simulate the model.""" 166 | self.unimplemented() 167 | 168 | def set_run_params(self, **run_params) -> None: 169 | """Set parameters for the next run. 170 | 171 | Note these are parameters of the simulation itself, not the model. 172 | """ 173 | self.unimplemented() 174 | 175 | def set_default_run_params(self, **default_run_params) -> None: 176 | """Set default parameters for all runs. 177 | 178 | Note these are parameters of the simulation itself, not the model. 179 | """ 180 | self.unimplemented() 181 | -------------------------------------------------------------------------------- /sciunit/config.json: -------------------------------------------------------------------------------- 1 | {"cmap_high": 218, "cmap_low": 38} -------------------------------------------------------------------------------- /sciunit/converters.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes for converting from the output of a model/data comparison 3 | to the value required for particular score type. 4 | """ 5 | 6 | from string import Template 7 | from typing import Callable 8 | 9 | from .scores import BooleanScore, Score 10 | 11 | 12 | class Converter(object): 13 | """ 14 | Base converter class. 15 | Only derived classes should be used in applications. 16 | """ 17 | 18 | @property 19 | def description(self): 20 | if self.__doc__: 21 | s = " ".join([si.strip() for si in self.__doc__.split("\n")]).strip() 22 | t = Template(s) 23 | s = t.safe_substitute(self.__dict__) 24 | else: 25 | s = "No description available" 26 | return s 27 | 28 | def _convert(self, score: Score) -> None: 29 | """Takes the score attribute of a score instance and recasts it as instance of another score type. 30 | 31 | Args: 32 | score (Score): An instance of Score. 33 | 34 | Raises: 35 | NotImplementedError: Not implemented if not overrided. 36 | """ 37 | raise NotImplementedError( 38 | ( 39 | "The '_convert' method for %s " 40 | "it not implemented." % self.__class__.__name__ 41 | ) 42 | ) 43 | 44 | def convert(self, score: Score) -> Score: 45 | """Convert a type of score to another type of score. 46 | 47 | Args: 48 | score (Score): The original score. 49 | 50 | Returns: 51 | Score: The converted score. 52 | """ 53 | new_score = self._convert(score) 54 | new_score.set_raw(score.get_raw()) 55 | for key, value in score.__dict__.items(): 56 | if key not in ["score", "_raw"]: 57 | setattr(new_score, key, value) 58 | return new_score 59 | 60 | 61 | class NoConversion(Converter): 62 | """ 63 | Applies no conversion. 64 | """ 65 | 66 | def _convert(self, score: Score) -> Score: 67 | return score 68 | 69 | 70 | class LambdaConversion(Converter): 71 | """ 72 | Converts a score according to a lambda function. 73 | """ 74 | 75 | def __init__(self, f: Callable): 76 | """f should be a lambda function 77 | 78 | Args: 79 | f (Callable): The Lambda function that will be used for the score conversion. 80 | """ 81 | self.f = f 82 | 83 | def _convert(self, score: Score) -> Score: 84 | return score.__class__(self.f(score)) 85 | 86 | 87 | class AtMostToBoolean(Converter): 88 | """ 89 | Converts a score to pass if its value is at most $cutoff, otherwise False. 90 | """ 91 | 92 | def __init__(self, cutoff: int): 93 | self.cutoff = cutoff 94 | 95 | def _convert(self, score: Score) -> BooleanScore: 96 | return BooleanScore(bool(score <= self.cutoff)) 97 | 98 | 99 | class AtLeastToBoolean(Converter): 100 | """ 101 | Converts a score to Pass if its value is at least $cutoff, otherwise False. 102 | """ 103 | 104 | def __init__(self, cutoff: int): 105 | self.cutoff = cutoff 106 | 107 | def _convert(self, score: Score) -> BooleanScore: 108 | return BooleanScore(score >= self.cutoff) 109 | 110 | 111 | class RangeToBoolean(Converter): 112 | """ 113 | Converts a score to Pass if its value is within the range 114 | [$low_cutoff,$high_cutoff], otherwise Fail. 115 | """ 116 | 117 | def __init__(self, low_cutoff: int, high_cutoff: int): 118 | self.low_cutoff = low_cutoff 119 | self.high_cutoff = high_cutoff 120 | 121 | def _convert(self, score: Score) -> BooleanScore: 122 | return BooleanScore(self.low_cutoff <= score <= self.high_cutoff) 123 | -------------------------------------------------------------------------------- /sciunit/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exception classes for SciUnit 3 | """ 4 | 5 | 6 | import sciunit 7 | 8 | 9 | class Error(Exception): 10 | """Base class for errors in sciunit's core.""" 11 | 12 | 13 | class ObservationError(Error): 14 | """Raised when an observation passed to a test is invalid.""" 15 | 16 | 17 | class ParametersError(Error): 18 | """Raised when params passed to a test are invalid.""" 19 | 20 | 21 | class CapabilityError(Error): 22 | """Abstract error class for capabilities""" 23 | 24 | def __init__( 25 | self, 26 | model: "sciunit.Model", 27 | capability: "sciunit.Capability", 28 | details: str = "", 29 | ): 30 | """A constructor. 31 | Args: 32 | model (Model): A sciunit model instance. 33 | capability (Capability): a capability class. 34 | details (str, optional): Details of the error information. Defaults to ''. 35 | """ 36 | self.model = model 37 | self.capability = capability 38 | if details: 39 | details = " (%s)" % details 40 | if self.action: 41 | msg = "Model '%s' does not %s required capability: '%s'%s" % ( 42 | model.name, 43 | self.action, 44 | capability.__name__, 45 | details, 46 | ) 47 | super(CapabilityError, self).__init__(details) 48 | 49 | action = None 50 | """The action that has failed ('provide' or 'implement').""" 51 | 52 | model = None 53 | """The model instance that does not have the capability.""" 54 | 55 | capability = None 56 | """The capability class that is not provided.""" 57 | 58 | 59 | class CapabilityNotProvidedError(CapabilityError): 60 | """Error raised when a required capability is not *provided* by a model. 61 | Do not use for capabilities provided but not implemented.""" 62 | 63 | action = "provide" 64 | 65 | 66 | class CapabilityNotImplementedError(CapabilityError): 67 | """Error raised when a required capability is not *implemented* by a model. 68 | Do not use for capabilities that are not provided at all.""" 69 | 70 | action = "implement" 71 | 72 | 73 | class PredictionError(Error): 74 | """Raised when a tests's generate_prediction chokes on a model's method.""" 75 | 76 | def __init__(self, model: "sciunit.Model", method: str, **args): 77 | """Constructor of PredictionError object. 78 | 79 | Args: 80 | model (Model): A sciunit Model. 81 | method (str): The method that caused this error. 82 | """ 83 | self.model = model 84 | self.method = method 85 | self.args = args 86 | 87 | super(PredictionError, self).__init__( 88 | ( 89 | "During prediction, model '%s' could not successfully execute " 90 | "method '%s' with arguments %s" 91 | ) 92 | % (model.name, method, args) 93 | ) 94 | 95 | model = None 96 | """The model that does not have the capability.""" 97 | 98 | argument = None 99 | """The argument that could not be handled.""" 100 | 101 | 102 | class InvalidScoreError(Error): 103 | """Error raised when a score is invalid.""" 104 | 105 | 106 | class BadParameterValueError(Error): 107 | """Error raised when a model parameter value is unreasonable.""" 108 | 109 | def __init__(self, name: str, value: int): 110 | """Constructor of BadParameterValueError object. 111 | 112 | Args: 113 | name (str): Name of the parameter that caused this error. 114 | value (int): The value of the parameter. 115 | """ 116 | self.name = name 117 | self.value = value 118 | 119 | super(BadParameterValueError, self).__init__( 120 | "Parameter %s has unreasonable value of %s" % (name, value) 121 | ) 122 | -------------------------------------------------------------------------------- /sciunit/models/__init__.py: -------------------------------------------------------------------------------- 1 | """SciUnit models.""" 2 | 3 | from .base import Model 4 | from .runnable import RunnableModel 5 | -------------------------------------------------------------------------------- /sciunit/models/backends.py: -------------------------------------------------------------------------------- 1 | """Base class for simulator backends for SciUnit models.""" 2 | 3 | import inspect 4 | import pickle 5 | import shelve 6 | import tempfile 7 | from pathlib import Path 8 | from typing import Any, Union 9 | 10 | from sciunit.base import SciUnit, config 11 | 12 | available_backends = {} 13 | 14 | 15 | def register_backends(vars: dict) -> None: 16 | """Register backends for use with models. 17 | 18 | Args: 19 | vars (dict): a dictionary of variables obtained from e.g. `locals()`, 20 | at least some of which are Backend classes, e.g. from imports. 21 | """ 22 | new_backends = { 23 | x if x is None else x.replace("Backend", ""): cls 24 | for x, cls in vars.items() 25 | if inspect.isclass(cls) and issubclass(cls, Backend) 26 | } 27 | available_backends.update(new_backends) 28 | 29 | 30 | class Backend(SciUnit): 31 | """ 32 | Base class for simulator backends. 33 | 34 | Should only be used with model classes derived from `RunnableModel`. 35 | Supports caching of simulation results. 36 | Backend classes should implement simulator-specific 37 | details of modifying, running, and reading results from the simulation. 38 | """ 39 | 40 | def init_backend(self, *args, **kwargs) -> None: 41 | """Initialize the backend.""" 42 | self.model.attrs = {} 43 | 44 | self.use_memory_cache = kwargs.get("use_memory_cache", True) 45 | if self.use_memory_cache: 46 | self.init_memory_cache() 47 | self.use_disk_cache = kwargs.get("use_disk_cache", False) 48 | if self.use_disk_cache: 49 | self.init_disk_cache(location=self.use_disk_cache) 50 | self.load_model() 51 | 52 | #: Name of the backend 53 | name = None 54 | 55 | #: The function that handles running the simulation 56 | f = None 57 | 58 | #: Optional list of state variables for a backend to record. 59 | recorded_variables = None 60 | 61 | state_hide = ["memory_cache", "_results", "stdout", "exec_in_dir", "model"] 62 | 63 | def init_cache(self) -> None: 64 | """Initialize the cache.""" 65 | self.init_memory_cache() 66 | self.init_disk_cache() 67 | 68 | def init_memory_cache(self) -> None: 69 | """Initialize the in-memory version of the cache.""" 70 | self.memory_cache = {} 71 | 72 | def init_disk_cache(self, location: Union[str, Path, bool, None] = None) -> None: 73 | """Initialize the on-disk version of the cache.""" 74 | if isinstance(location, (str, Path)): 75 | location = str(location) 76 | else: 77 | # => "~/.sciunit/cache" 78 | location = str(config.path.parent / "cache") 79 | 80 | self.disk_cache_location = location 81 | 82 | def clear_disk_cache(self) -> None: 83 | """Removes the cache file from the disk if it exists. 84 | """ 85 | path = Path(self.disk_cache_location) 86 | 87 | if path.exists(): 88 | with shelve.open(str(path)) as cache: 89 | cache.clear() 90 | 91 | def get_memory_cache(self, key: str = None) -> dict: 92 | """Return result in memory cache for key 'key' or None if not found. 93 | 94 | Args: 95 | key (str, optional): [description]. Defaults to None. 96 | 97 | Returns: 98 | dict: The memory cache for key 'key' or None if not found. 99 | """ 100 | key = self.model.hash() if key is None else key 101 | if not getattr(self, "memory_cache", False): 102 | self.init_memory_cache() 103 | self._results = self.memory_cache.get(key) 104 | return self._results 105 | 106 | def get_disk_cache(self, key: str = None) -> Any: 107 | """Return result in disk cache for key 'key' or None if not found. 108 | 109 | Args: 110 | key (str, optional): keys that will be used to find cached data. Defaults to None. 111 | 112 | Returns: 113 | Any: The disk cache for key 'key' or None if not found. 114 | """ 115 | key = self.model.hash() if key is None else key 116 | if not getattr(self, "disk_cache_location", False): 117 | self.init_disk_cache() 118 | disk_cache = shelve.open(str(self.disk_cache_location)) 119 | self._results = disk_cache.get(key) 120 | disk_cache.close() 121 | return self._results 122 | 123 | def get_cache(self, key: str = None) -> Any: 124 | """Return result in disk or memory cache for key 'key' or None if not 125 | found. If both `use_disk_cache` and `use_memory_cache` are True, the 126 | memory cache is returned. 127 | 128 | Returns: 129 | Any: The cache for key 'key' or None if not found. 130 | """ 131 | if self.use_memory_cache: 132 | result = self.get_memory_cache(key=key) 133 | if result is not None: 134 | return result 135 | if self.use_disk_cache: 136 | result = self.get_disk_cache(key=key) 137 | if result is not None: 138 | return result 139 | return None 140 | 141 | def set_memory_cache(self, results: Any, key: str = None) -> None: 142 | """Store result in memory cache with key matching model state. 143 | 144 | Args: 145 | results (Any): [description] 146 | key (str, optional): [description]. Defaults to None. 147 | """ 148 | key = self.model.hash() if key is None else key 149 | if not getattr(self, "memory_cache", False): 150 | self.init_memory_cache() 151 | self.memory_cache[key] = results 152 | 153 | def set_disk_cache(self, results: Any, key: str = None) -> None: 154 | """Store result in disk cache with key matching model state. 155 | 156 | Args: 157 | results (Any): [description] 158 | key (str, optional): [description]. Defaults to None. 159 | """ 160 | if not getattr(self, "disk_cache_location", False): 161 | self.init_disk_cache() 162 | disk_cache = shelve.open(str(self.disk_cache_location)) 163 | key = self.model.hash() if key is None else key 164 | disk_cache[key] = results 165 | disk_cache.close() 166 | 167 | def set_cache(self, results: Any, key: str = None) -> bool: 168 | """Store result in disk and/or memory cache for key 'key', depending 169 | on whether `use_disk_cache` and `use_memory_cache` are True. 170 | 171 | Args: 172 | results (Any): [description] 173 | key (str, optional): [description]. Defaults to None. 174 | 175 | Returns: 176 | bool: True if cache was successfully set, else False 177 | """ 178 | if self.use_memory_cache: 179 | self.set_memory_cache(results, key=key) 180 | if self.use_disk_cache: 181 | self.set_disk_cache(results, key=key) 182 | if self.use_memory_cache or self.use_disk_cache: 183 | return True 184 | return False 185 | 186 | def load_model(self) -> None: 187 | """Load the model into memory.""" 188 | 189 | def set_attrs(self, **attrs) -> None: 190 | """Set model attributes on the backend.""" 191 | 192 | def set_run_params(self, **run_params) -> None: 193 | """Set model attributes on the backend.""" 194 | 195 | def backend_run(self) -> Any: 196 | """Check for cached results; then run the model if needed. 197 | 198 | Returns: 199 | Any: The result of running backend. 200 | """ 201 | if self.use_memory_cache or self.use_disk_cache: 202 | key = self.model.hash() 203 | if self.use_memory_cache and self.get_memory_cache(key): 204 | return self.cache_to_results(self._results) 205 | if self.use_disk_cache and self.get_disk_cache(key): 206 | return self.cache_to_results(self._results) 207 | results = self._backend_run() 208 | if self.use_memory_cache: 209 | self.set_memory_cache(self.results_to_cache(results), key) 210 | if self.use_disk_cache: 211 | self.set_disk_cache(self.results_to_cache(results), key) 212 | return results 213 | 214 | def cache_to_results(self, cache: Any) -> Any: 215 | """A method to convert cache to some hypothetical Results object. 216 | 217 | Args: 218 | cache (Any): An object returned from .get_memory_cache() or .get_disk_cache(). 219 | 220 | Returns: 221 | Any (optional): Either an object with the results of the simulation, 222 | or None (e.g. if cache_to_results() simply injects the results into some global object). 223 | """ 224 | return cache 225 | 226 | def results_to_cache(self, results: Any) -> Any: 227 | """A method to convert the results from your model run 228 | into storable cache object (usually a simple dictionary or an array). 229 | 230 | Args: 231 | results (Any): An object returned from your ._backend_run(). 232 | 233 | Returns: 234 | Any: The results in the format that's good for storing in cache. 235 | """ 236 | return results 237 | 238 | def _backend_run(self) -> Any: 239 | """Run the model via the backend.""" 240 | raise NotImplementedError("Each backend must implement '_backend_run'") 241 | 242 | def save_results(self, path: Union[str, Path] = ".") -> None: 243 | """Save results on disk. 244 | 245 | Args: 246 | path (Union[str, Path], optional): [description]. Defaults to '.'. 247 | """ 248 | with open(path, "wb") as f: 249 | pickle.dump(self.results, f) 250 | 251 | 252 | class BackendException(Exception): 253 | """Generic backend exception class.""" 254 | 255 | 256 | # Register the base class as a Backend just so that there is 257 | # always something available. This Backend won't do anything 258 | # useful other than caching. 259 | register_backends({None: Backend}) 260 | -------------------------------------------------------------------------------- /sciunit/models/base.py: -------------------------------------------------------------------------------- 1 | """Base class for SciUnit models.""" 2 | 3 | import inspect 4 | from fnmatch import fnmatchcase 5 | from typing import Union 6 | 7 | from sciunit.base import SciUnit 8 | from sciunit.capabilities import Capability 9 | 10 | 11 | class Model(SciUnit): 12 | """Abstract base class for sciunit models.""" 13 | 14 | def __init__(self, name=None, **params): 15 | """Instantiate model.""" 16 | if name is None: 17 | name = self.__class__.__name__ 18 | self.name = name 19 | self.params = params 20 | super(Model, self).__init__() 21 | self.check_params() 22 | 23 | name = None 24 | """The name of the model. Defaults to the class name.""" 25 | 26 | description = "" 27 | """A description of the model.""" 28 | 29 | params = None 30 | """The parameters to the model (a dictionary). 31 | These distinguish one model of a class from another.""" 32 | 33 | run_args = None 34 | """These are the run-time arguments for the model. 35 | Execution of run() should make use of these arguments.""" 36 | 37 | extra_capability_checks = None 38 | """Optional extra checks of capabilities on a per-instance basis.""" 39 | 40 | _backend = None 41 | """Optional model backend for executing some methods, e.g. simulations.""" 42 | 43 | state_hide = ["results", "temp_dir", "_temp_dir", "stdout"] 44 | 45 | @classmethod 46 | def get_capabilities(cls) -> list: 47 | """List the model's capabilities.""" 48 | capabilities = [] 49 | for _cls in cls.mro(): 50 | if ( 51 | issubclass(_cls, Capability) 52 | and _cls is not Capability 53 | and not issubclass(_cls, Model) 54 | ): 55 | capabilities.append(_cls) 56 | return capabilities 57 | 58 | @property 59 | def capabilities(self) -> list: 60 | return self.__class__.get_capabilities() 61 | 62 | @property 63 | def failed_extra_capabilities(self) -> list: 64 | """Check to see if instance passes its `extra_capability_checks`.""" 65 | failed = [] 66 | if self.extra_capability_checks is not None: 67 | for capability, f_name in self.extra_capability_checks.items(): 68 | f = getattr(self, f_name) 69 | instance_capable = f() 70 | if isinstance(self, capability) and not instance_capable: 71 | failed.append(capability) 72 | return failed 73 | 74 | def describe(self) -> str: 75 | """Describe the model. 76 | 77 | Returns: 78 | str: The description of the model. 79 | """ 80 | result = "No description available" 81 | if self.description: 82 | result = "%s" % self.description 83 | else: 84 | if self.__doc__: 85 | s = [] 86 | s += [self.__doc__.strip().replace("\n", "").replace(" ", " ")] 87 | result = "\n".join(s) 88 | return result 89 | 90 | def curr_method(self, back: int = 0) -> str: 91 | """Return the name of the current method (calling this one). 92 | 93 | Args: 94 | back (int, optional): [description]. Defaults to 0. 95 | 96 | Returns: 97 | str: The name of the current method that calls this one. 98 | """ 99 | return inspect.stack()[1 + back][3] 100 | 101 | def check_params(self) -> None: 102 | """Check model parameters to see if they are reasonable. 103 | 104 | For example, this method could check self.params to see if a particular 105 | value was within an acceptable range. This should be implemented 106 | as needed by specific model classes. 107 | """ 108 | 109 | def is_match(self, match: Union[str, "Model"]) -> bool: 110 | """Return whether this model is the same as `match`. 111 | 112 | Matches if the model is the same as or has the same name as `match`. 113 | 114 | Args: 115 | match (Union[str, 'Model']): [description] 116 | 117 | Returns: 118 | bool: Whether this model is the same as `match`. 119 | """ 120 | result = False 121 | if self == match: 122 | result = True 123 | elif isinstance(match, str) and fnmatchcase(self.name, match): 124 | result = True # Found by instance or name 125 | return result 126 | 127 | def __getattr__(self, attr): 128 | try: 129 | result = super(Model, self).__getattribute__(attr) 130 | except AttributeError: 131 | try: 132 | result = self._backend.__getattribute__(attr) 133 | except: 134 | raise AttributeError("Model %s has no attribute %s" % (self, attr)) 135 | return result 136 | 137 | def __str__(self): 138 | """Return the model name.""" 139 | return "%s" % self.name 140 | 141 | def __repr__(self): 142 | """Returns a representation of the model.""" 143 | return "%s (%s)" % (self.name, self.__class__.__name__) 144 | -------------------------------------------------------------------------------- /sciunit/models/examples.py: -------------------------------------------------------------------------------- 1 | """Example SciUnit model classes.""" 2 | 3 | import random 4 | from typing import Union 5 | 6 | from sciunit.capabilities import ProducesNumber 7 | from sciunit.models import Model 8 | from sciunit.utils import ( # Decorator for caching of capability method results. 9 | class_intern, 10 | method_cache, 11 | method_memoize, 12 | ) 13 | 14 | 15 | class ConstModel(Model, ProducesNumber): 16 | """A model that always produces a constant number as output.""" 17 | 18 | def __init__(self, constant: Union[int, float], name: str = None): 19 | self.constant = constant 20 | super(ConstModel, self).__init__(name=name, constant=constant) 21 | 22 | def produce_number(self) -> Union[int, float]: 23 | return self.constant 24 | 25 | 26 | class UniformModel(Model, ProducesNumber): 27 | """A model that always produces a random uniformly distributed number. 28 | in [a,b] as output.""" 29 | 30 | def __init__(self, a, b, name=None): 31 | self.a, self.b = a, b 32 | super(UniformModel, self).__init__(name=name, a=a, b=b) 33 | 34 | def produce_number(self) -> float: 35 | """Produece a number between `a` and `b`. 36 | 37 | Returns: 38 | float: The number between a and b. 39 | """ 40 | return random.uniform(self.a, self.b) 41 | 42 | 43 | ################################################################ 44 | # Here are several examples of caching and sharing can be used 45 | # to reduce the computational load of testing. 46 | ################################################################ 47 | 48 | 49 | class UniqueRandomNumberModel(Model, ProducesNumber): 50 | """An example model to ProducesNumber.""" 51 | 52 | def produce_number(self) -> float: 53 | """Each call to this method will produce a different random number. 54 | 55 | Returns: 56 | float: A random number produced. 57 | """ 58 | return random.random() 59 | 60 | 61 | class RepeatedRandomNumberModel(Model, ProducesNumber): 62 | """An example model to demonstrate ProducesNumber with cypy.lazy.""" 63 | 64 | @method_memoize 65 | def produce_number(self): 66 | """Each call to this method will produce the same random number as was returned in the first call, ensuring reproducibility and eliminating computational overhead. 67 | 68 | Returns: 69 | float: A random number produced. 70 | """ 71 | return random.random() 72 | 73 | 74 | @class_intern 75 | class SharedModel(Model): 76 | """A model that, each time it is instantiated with the same parameters, 77 | will return the same instance at the same locaiton in memory. 78 | Attributes should not be set post-instantiation 79 | unless the goal is to set those attributes on all models of this class.""" 80 | 81 | 82 | class PersistentUniformModel(UniformModel): 83 | """TODO""" 84 | 85 | def run(self) -> None: 86 | self._x = random.uniform(self.a, self.b) 87 | 88 | def produce_number(self) -> float: 89 | return self._x 90 | 91 | 92 | class CacheByInstancePersistentUniformModel(PersistentUniformModel): 93 | """TODO""" 94 | 95 | @method_cache(by="instance", method="run") 96 | def produce_number(self) -> float: 97 | return self._x 98 | 99 | 100 | class CacheByValuePersistentUniformModel(PersistentUniformModel): 101 | """TODO""" 102 | 103 | @method_cache(by="value", method="run") 104 | def produce_number(self) -> float: 105 | return self._x 106 | -------------------------------------------------------------------------------- /sciunit/models/runnable.py: -------------------------------------------------------------------------------- 1 | """Runnable model.""" 2 | 3 | import inspect 4 | import warnings 5 | from typing import Union 6 | 7 | import sciunit.capabilities as cap 8 | 9 | from .backends import Backend, available_backends 10 | from .base import Model 11 | 12 | 13 | class RunnableModel(Model, cap.Runnable): 14 | """A model which can be run to produce simulation results.""" 15 | 16 | def __init__( 17 | self, 18 | name, # Name of the model 19 | backend=None, # Backend to run the models 20 | attrs=None, # Optional dictionary of model attributes 21 | **params, 22 | ): 23 | super(RunnableModel, self).__init__(name=name, **params) 24 | self.skip_run = False # Backend can use this to skip simulation 25 | self.run_params = {} # Should be reset between tests 26 | self.print_run_params = False # Print the run parameters with each run 27 | self.default_run_params = {} # Should be applied to all tests 28 | if attrs and not isinstance(attrs, dict): 29 | raise TypeError("Model 'attrs' must be a dictionary.") 30 | self.attrs = attrs if attrs else {} 31 | self.set_backend(backend) 32 | self.use_default_run_params() 33 | 34 | def get_backend(self): 35 | """Return the simulation backend.""" 36 | return self._backend 37 | 38 | def set_backend(self, backend: Union[str, tuple, list, None]): 39 | """Set the simulation backend. 40 | 41 | Args: 42 | backend (Union[str, tuple, list, None]): One or more name(s) of backend(s). 43 | The name of backend should be registered before using. 44 | 45 | Raises: 46 | TypeError: Backend must be string, tuple, list, or None 47 | Exception: The backend was not found. 48 | """ 49 | if inspect.isclass(backend) and Backend in backend.__bases__: 50 | name = backend.__name__ 51 | args = [] 52 | kwargs = {} 53 | available_backends[name] = backend 54 | elif isinstance(backend, str): 55 | name = backend if len(backend) else None 56 | args = [] 57 | kwargs = {} 58 | elif isinstance(backend, (tuple, list)): 59 | name = "" 60 | args = [] 61 | kwargs = {} 62 | for i in range(len(backend)): 63 | if i == 0: 64 | name = backend[i] 65 | else: 66 | if isinstance(backend[i], dict): 67 | kwargs.update(backend[i]) 68 | else: 69 | args += backend[i] 70 | elif backend is None: 71 | name = None 72 | args = [] 73 | kwargs = {} 74 | else: 75 | raise TypeError( 76 | "The backend must be of type Backend, string, tuple, list, or None" 77 | ) 78 | if name in available_backends: 79 | self.backend = name 80 | self._backend = available_backends[name]() 81 | elif name is None: 82 | # The base class should not be called. 83 | warnings.warn( 84 | "The `None` backend was selected and will have limited functionality" 85 | ) 86 | else: 87 | raise Exception("Backend %s not found in backends.py" % name) 88 | self._backend.model = self 89 | self._backend.init_backend(*args, **kwargs) 90 | 91 | def run(self, **run_params) -> None: 92 | """Run the simulation (or lookup the results in the cache).""" 93 | self.use_default_run_params() 94 | self.set_run_params(**run_params) 95 | if self.print_run_params: 96 | print("Run Params:", self.run_params) 97 | self.results = self._backend.backend_run() 98 | 99 | def set_attrs(self, **attrs) -> None: 100 | """Set model attributes, e.g. input resistance of a cell.""" 101 | self.attrs.update(attrs) 102 | self._backend.set_attrs(**attrs) 103 | 104 | def set_run_params(self, **run_params) -> None: 105 | """Set run-time parameters, e.g. the somatic current to inject.""" 106 | self.run_params.update(run_params) 107 | self.check_run_params() 108 | self._backend.set_run_params(**run_params) 109 | 110 | def check_run_params(self) -> None: 111 | """Check if run parameters are reasonable for this model class. 112 | 113 | Raise a sciunit.BadParameterValueError if any of them are not. 114 | """ 115 | 116 | def reset_run_params(self) -> None: 117 | self.run_params = {} 118 | 119 | def set_default_run_params(self, **params) -> None: 120 | self.default_run_params.update(params) 121 | 122 | def use_default_run_params(self) -> None: 123 | for key, value in self.default_run_params.items(): 124 | if key not in self.run_params: 125 | self.set_run_params(**{key: value}) 126 | 127 | def reset_default_run_params(self) -> None: 128 | self.default_run_params = {} 129 | 130 | @property 131 | def state(self): 132 | return self._state(keys=["name", "url", "attrs", "run_params", "backend"]) 133 | 134 | def __del__(self) -> None: 135 | if hasattr(self, "temp_dir"): 136 | self.temp_dir.cleanup() # Delete the temporary directory 137 | s = super(RunnableModel, self) 138 | if hasattr(s, "__del__"): 139 | s.__del__() 140 | -------------------------------------------------------------------------------- /sciunit/scores/__init__.py: -------------------------------------------------------------------------------- 1 | """Contains classes for different representations of test scores. 2 | 3 | It also contains score collections such as arrays and matrices. 4 | """ 5 | 6 | from .base import ErrorScore, Score, score_logger 7 | from .complete import * 8 | from .incomplete import * 9 | -------------------------------------------------------------------------------- /sciunit/scores/collections_m2m.py: -------------------------------------------------------------------------------- 1 | """Score collections for direct comparison of models against other models.""" 2 | 3 | import warnings 4 | from typing import Any, List, Tuple, Union 5 | 6 | import pandas as pd 7 | 8 | from sciunit.base import SciUnit 9 | from sciunit.models import Model 10 | from sciunit.tests import Test 11 | 12 | 13 | class ScoreArrayM2M(pd.Series, SciUnit): 14 | """ 15 | Represents an array of scores derived from TestM2M. 16 | Extends the pandas Series such that items are either 17 | models subject to a test or the test itself. 18 | 19 | Attributes: 20 | index ([type]): [description] 21 | """ 22 | 23 | def __init__( 24 | self, test: Test, models: List[Model], scores: List["sciunit.scores.Score"] 25 | ): 26 | items = models if not test.observation else [test] + models 27 | super(ScoreArrayM2M, self).__init__(data=scores, index=items) 28 | 29 | state_hide = ['related_data', 'scores', 'norm_scores', 'style', 'plot', 'iat', 'at', 'iloc', 'loc', 'T'] 30 | 31 | def __getitem__(self, item: Union[str, callable]) -> Any: 32 | if isinstance(item, str): 33 | result = self.get_by_name(item) 34 | else: 35 | result = super(ScoreArrayM2M, self).__getitem__(item) 36 | return result 37 | 38 | def __getattr__(self, name: str) -> Any: 39 | if name in ["score", "norm_scores", "related_data"]: 40 | attr = self.apply(lambda x: getattr(x, name)) 41 | else: 42 | attr = super(ScoreArrayM2M, self).__getattribute__(name) 43 | return attr 44 | 45 | def get_by_name(self, name: str) -> str: 46 | """Get item (can be a model, observation, or test) in `index` by name. 47 | 48 | Args: 49 | name (str): name of the item. 50 | 51 | Raises: 52 | KeyError: Item not found. 53 | 54 | Returns: 55 | Any: Item found. 56 | """ 57 | for entry in self.index: 58 | if entry.name == name or name.lower() == "observation": 59 | return self.__getitem__(entry) 60 | raise KeyError( 61 | ("Doesn't match test, 'observation' or " "any model: '%s'") % name 62 | ) 63 | 64 | @property 65 | def norm_scores(self) -> pd.Series: 66 | """A series of norm scores. 67 | 68 | Returns: 69 | Series: A series of norm scores. 70 | """ 71 | return self.map(lambda x: x.norm_score) 72 | 73 | 74 | class ScoreMatrixM2M(pd.DataFrame, SciUnit): 75 | """ 76 | Represents a matrix of scores derived from TestM2M. 77 | Extends the pandas DataFrame such that models/observation are both 78 | columns and the index. 79 | """ 80 | 81 | def __init__( 82 | self, test: Test, models: List[Model], scores: List["sciunit.scores.Score"] 83 | ): 84 | if not test.observation: 85 | items = models 86 | else: 87 | # better to have header as "observation" than test.name 88 | # only affects pandas.DataFrame; not test.name in individual scores 89 | test.name = "observation" 90 | items = [test] + models 91 | super(ScoreMatrixM2M, self).__init__(data=scores, index=items, columns=items) 92 | with warnings.catch_warnings(): 93 | warnings.filterwarnings( 94 | "ignore", 95 | message=(".*Pandas doesn't allow columns " "to be created via a new "), 96 | ) 97 | self.test = test 98 | self.models = models 99 | 100 | state_hide = ['related_data', 'scores', 'norm_scores', 'style', 'plot', 'iat', 'at', 'iloc', 'loc', 'T'] 101 | 102 | def __getitem__( 103 | self, item: Union[Tuple[Test, Model], str, Tuple[list, tuple]] 104 | ) -> Any: 105 | if isinstance(item, (Test, Model)): 106 | result = ScoreArrayM2M(self.test, self.models, scores=self.loc[item, :]) 107 | elif isinstance(item, str): 108 | result = self.get_by_name(item) 109 | elif isinstance(item, (list, tuple)) and len(item) == 2: 110 | result = self.get_group(item) 111 | else: 112 | raise TypeError( 113 | ( 114 | "Expected test/'observation'; model; " 115 | "test/'observation',model; " 116 | "model,test/'observation'; or model,model" 117 | ) 118 | ) 119 | return result 120 | 121 | def get_by_name(self, name: str) -> Union[Model, Test]: 122 | """Get the model or test from the `models` or `tests` by name. 123 | 124 | Args: 125 | name (str): The name of the model or test. 126 | 127 | Raises: 128 | KeyError: Raise an exception if there is not a model or test named `name`. 129 | 130 | Returns: 131 | Union[Model, Test]: The test or model found. 132 | """ 133 | for model in self.models: 134 | if model.name == name: 135 | return self.__getitem__(model) 136 | if self.test.name == name or name.lower() == "observation": 137 | return self.__getitem__(self.test) 138 | raise KeyError( 139 | ("Doesn't match test, 'observation' or " "any model: '%s'") % name 140 | ) 141 | 142 | def get_group(self, x: list) -> Any: 143 | """[summary] 144 | 145 | Args: 146 | x (list): [description] 147 | 148 | Raises: 149 | TypeError: [description] 150 | 151 | Returns: 152 | Any: [description] 153 | """ 154 | if isinstance(x[0], (Test, Model)) and isinstance(x[1], (Test, Model)): 155 | return self.loc[x[0], x[1]] 156 | elif isinstance(x[0], str): 157 | return self.__getitem__(x[0]).__getitem__(x[1]) 158 | raise TypeError("Expected test/model pair") 159 | 160 | def __getattr__(self, name: str) -> Any: 161 | if name in ["score", "norm_score", "related_data"]: 162 | attr = self.applymap(lambda x: getattr(x, name)) 163 | else: 164 | attr = super(ScoreMatrixM2M, self).__getattribute__(name) 165 | return attr 166 | 167 | @property 168 | def norm_scores(self) -> pd.DataFrame: 169 | """Get a pandas DataFrame instance that contains norm scores. 170 | 171 | Returns: 172 | DataFrame: A pandas DataFrame instance that contains norm scores. 173 | """ 174 | return self.applymap(lambda x: x.norm_score) 175 | -------------------------------------------------------------------------------- /sciunit/scores/incomplete.py: -------------------------------------------------------------------------------- 1 | """Score types for tests that did not complete successfully. 2 | 3 | These include details about the various possible reasons 4 | that a particular combination of model and test could not be completed. 5 | """ 6 | 7 | from sciunit.errors import InvalidScoreError 8 | 9 | from .base import Score 10 | 11 | 12 | class NoneScore(Score): 13 | """A `None` score. 14 | 15 | Usually indicates that the model has not been 16 | checked to see if it has the capabilities required by the test.""" 17 | 18 | def __init__(self, score: Score, related_data: dict = None): 19 | if isinstance(score, str) or score is None: 20 | super(NoneScore, self).__init__(score, related_data=related_data) 21 | else: 22 | raise InvalidScoreError("Score must be a string or None") 23 | 24 | @property 25 | def norm_score(self) -> None: 26 | """Return None as the norm score of this NoneScore instance. 27 | 28 | Returns: 29 | None: The norm score, which is None. 30 | """ 31 | return None 32 | 33 | def __str__(self) -> str: 34 | if self.score: 35 | s = "%s (%s)" % (self.description, self.score) 36 | else: 37 | s = self.description 38 | return s 39 | 40 | 41 | class TBDScore(NoneScore): 42 | """A TBD (to be determined) score. Indicates that the model has 43 | capabilities required by the test but has not yet taken it.""" 44 | 45 | description = "None" 46 | 47 | 48 | class NAScore(NoneScore): 49 | """A N/A (not applicable) score. 50 | 51 | Indicates that the model doesn't have the 52 | capabilities that the test requires.""" 53 | 54 | description = "N/A" 55 | 56 | 57 | class InsufficientDataScore(NoneScore): 58 | """A score returned when the model or test data 59 | is insufficient to score the test.""" 60 | 61 | description = "Insufficient Data" 62 | -------------------------------------------------------------------------------- /sciunit/style.css: -------------------------------------------------------------------------------- 1 | td.red { 2 | background-color: #FF0000; 3 | border: 1px solid black; 4 | } 5 | td.green { 6 | background-color: #00FF00; 7 | border: 1px solid black; 8 | } 9 | td.grey { 10 | background-color: #AAAAAA; 11 | border: 1px solid black; 12 | } 13 | th { 14 | text-align: center; 15 | border: 1px solid black; 16 | } 17 | table td, table th { 18 | border: 10px solid black; 19 | font-size: 250%; 20 | } -------------------------------------------------------------------------------- /sciunit/unit_test/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit testing module for sciunit. 3 | The script `test.sh` will run all tests imported into `active.py`. 4 | """ 5 | -------------------------------------------------------------------------------- /sciunit/unit_test/__main__.py: -------------------------------------------------------------------------------- 1 | """All unit tests for SciUnit""" 2 | 3 | import sys 4 | import unittest 5 | 6 | from .active import * # Import all the tests from the unit_test.active module 7 | 8 | 9 | def main(): 10 | buffer = "buffer" in sys.argv 11 | sys.argv = sys.argv[:1] # :Args need to be removed for __main__ to work. 12 | unittest.main(buffer=buffer) 13 | 14 | 15 | if __name__ == "__main__": 16 | main() 17 | -------------------------------------------------------------------------------- /sciunit/unit_test/active.py: -------------------------------------------------------------------------------- 1 | """ 2 | All active unit tests for SciUnit. This module is the default target of 3 | for testing in `__main__.py`. Modify this file if you want to add or remove 4 | tests located in other modules. 5 | """ 6 | 7 | from .backend_tests import * 8 | from .base_tests import * 9 | from .command_line_tests import * 10 | from .config_tests import * 11 | from .converter_tests import * 12 | from .doc_tests import * 13 | from .error_tests import * 14 | from .import_tests import * 15 | from .model_tests import * 16 | from .observation_tests import * 17 | from .score_tests import * 18 | from .test_tests import * 19 | from .utils_tests import * 20 | from .validator_tests import * 21 | -------------------------------------------------------------------------------- /sciunit/unit_test/backend_tests.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import sciunit\n", 10 | "from sciunit.models.backends import Backend\n", 11 | "from sciunit.capabilities import Runnable\n", 12 | "from sciunit.errors import CapabilityNotImplementedError" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "class MyModel(sciunit.Model, Runnable):\n", 22 | " pass" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "m = MyModel()" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 4, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "name": "stdout", 41 | "output_type": "stream", 42 | "text": [ 43 | "MyModel \n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "try:\n", 49 | " m.run()\n", 50 | "except CapabilityNotImplementedError:\n", 51 | " pass\n", 52 | "else:\n", 53 | " raise Exception(\"This should have been an error!!!!\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 5, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "class MyModel(sciunit.models.RunnableModel):\n", 63 | " def run(self, a, b=5):\n", 64 | " pass\n", 65 | " \n", 66 | "m = MyModel(name='My Model')" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 6, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "m = sciunit.models.RunnableModel(name='smith', backend=None)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 7, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "m = sciunit.models.RunnableModel(name='jones', backend='')" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 8, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "def run(self, **kwargs):\n", 94 | " print('running')\n", 95 | "\n", 96 | "from types import MethodType\n", 97 | "m.run = MethodType(run, m)" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 9, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "name": "stdout", 107 | "output_type": "stream", 108 | "text": [ 109 | "running\n" 110 | ] 111 | } 112 | ], 113 | "source": [ 114 | "m.run()" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 10, 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "name": "stdout", 124 | "output_type": "stream", 125 | "text": [ 126 | "running\n" 127 | ] 128 | } 129 | ], 130 | "source": [ 131 | "try:\n", 132 | " m.run()\n", 133 | "except NotImplementedError:\n", 134 | " raise Exception(\"Implementation failed\")" 135 | ] 136 | } 137 | ], 138 | "metadata": { 139 | "kernelspec": { 140 | "display_name": "Python 3", 141 | "language": "python", 142 | "name": "python3" 143 | }, 144 | "language_info": { 145 | "codemirror_mode": { 146 | "name": "ipython", 147 | "version": 3 148 | }, 149 | "file_extension": ".py", 150 | "mimetype": "text/x-python", 151 | "name": "python", 152 | "nbconvert_exporter": "python", 153 | "pygments_lexer": "ipython3", 154 | "version": "3.7.4" 155 | } 156 | }, 157 | "nbformat": 4, 158 | "nbformat_minor": 4 159 | } 160 | -------------------------------------------------------------------------------- /sciunit/unit_test/backend_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for backends.""" 2 | 3 | import unittest 4 | from pathlib import Path 5 | 6 | from sciunit import Model 7 | from sciunit.models.backends import Backend 8 | from sciunit.utils import NotebookTools 9 | 10 | class BackendsTestCase(unittest.TestCase, NotebookTools): 11 | """Unit tests for the sciunit module""" 12 | 13 | path = "." 14 | 15 | def test_backends(self): 16 | """Test backends.""" 17 | self.do_notebook("backend_tests") 18 | 19 | def test_backends_init_caches(self): 20 | myModel = Model() 21 | backend = Backend() 22 | backend.model = myModel 23 | 24 | backend.init_backend(use_disk_cache=True, use_memory_cache=True) 25 | backend.init_backend(use_disk_cache=False, use_memory_cache=True) 26 | backend.init_backend(use_disk_cache=True, use_memory_cache=False) 27 | backend.init_backend(use_disk_cache=False, use_memory_cache=False) 28 | backend.init_cache() 29 | 30 | def test_backends_init_disk_caches(self): 31 | # Automatically set disk_cache location 32 | myModel = Model() 33 | backend = Backend() 34 | backend.model = myModel 35 | backend.init_backend(use_disk_cache=True, use_memory_cache=False) 36 | self.assertTrue(backend.disk_cache_location.endswith(".sciunit/cache")) 37 | 38 | # Manually set disk_cache location (a string) 39 | myModel = Model() 40 | backend = Backend() 41 | backend.model = myModel 42 | backend.init_backend(use_disk_cache="/some/good/path", use_memory_cache=False) 43 | self.assertEqual(backend.disk_cache_location, "/some/good/path") 44 | 45 | # Manually set disk_cache location (a Path) 46 | myModel = Model() 47 | backend = Backend() 48 | backend.model = myModel 49 | backend.init_backend(use_disk_cache=Path("/some/good/path"), use_memory_cache=False) 50 | self.assertEqual(backend.disk_cache_location, "/some/good/path") 51 | 52 | def test_backends_set_caches(self): 53 | myModel = Model() 54 | backend = Backend() 55 | backend.model = myModel 56 | backend.init_backend(use_disk_cache=True, use_memory_cache=True) 57 | backend.clear_disk_cache() 58 | # backend.init_memory_cache() 59 | self.assertIsNone(backend.get_disk_cache("key1")) 60 | self.assertIsNone(backend.get_disk_cache("key2")) 61 | self.assertIsNone(backend.get_memory_cache("key1")) 62 | self.assertIsNone(backend.get_memory_cache("key2")) 63 | backend.set_disk_cache("value1", "key1") 64 | backend.set_memory_cache("value1", "key1") 65 | self.assertEqual(backend.get_memory_cache("key1"), "value1") 66 | self.assertEqual(backend.get_disk_cache("key1"), "value1") 67 | backend.set_disk_cache("value2") 68 | backend.set_memory_cache("value2") 69 | self.assertEqual(backend.get_memory_cache(myModel.hash()), "value2") 70 | self.assertEqual(backend.get_disk_cache(myModel.hash()), "value2") 71 | 72 | backend.load_model() 73 | backend.set_attrs(test_attribute="test attribute") 74 | backend.set_run_params(test_param="test parameter") 75 | backend.init_backend(use_disk_cache=True, use_memory_cache=True) 76 | 77 | def test_backend_run(self): 78 | backend = Backend() 79 | self.assertRaises(NotImplementedError, backend._backend_run) 80 | 81 | class MyBackend(Backend): 82 | model = Model() 83 | 84 | def _backend_run(self) -> str: 85 | return "test result" 86 | 87 | backend = MyBackend() 88 | backend.init_backend(use_disk_cache=True, use_memory_cache=True) 89 | backend.backend_run() 90 | backend.set_disk_cache("value1", "key1") 91 | backend.set_memory_cache("value1", "key1") 92 | backend.backend_run() 93 | backend.set_disk_cache("value2") 94 | backend.set_memory_cache("value2") 95 | backend.backend_run() 96 | 97 | backend = MyBackend() 98 | backend.init_backend(use_disk_cache=False, use_memory_cache=True) 99 | backend.backend_run() 100 | backend.set_disk_cache("value1", "key1") 101 | backend.set_memory_cache("value1", "key1") 102 | backend.backend_run() 103 | backend.set_disk_cache("value2") 104 | backend.set_memory_cache("value2") 105 | backend.backend_run() 106 | 107 | backend = MyBackend() 108 | backend.init_backend(use_disk_cache=True, use_memory_cache=False) 109 | backend.backend_run() 110 | backend.set_disk_cache("value1", "key1") 111 | backend.set_memory_cache("value1", "key1") 112 | backend.backend_run() 113 | backend.set_disk_cache("value2") 114 | backend.set_memory_cache("value2") 115 | backend.backend_run() 116 | 117 | backend = MyBackend() 118 | backend.init_backend(use_disk_cache=False, use_memory_cache=False) 119 | backend.backend_run() 120 | backend.set_disk_cache("value1", "key1") 121 | backend.set_memory_cache("value1", "key1") 122 | backend.backend_run() 123 | backend.set_disk_cache("value2") 124 | backend.set_memory_cache("value2") 125 | backend.backend_run() 126 | 127 | def test_backend_cache_to_results(self): 128 | myModel = Model() 129 | class MyBackend(Backend): 130 | def cache_to_results(self, cache): 131 | return { "color": "red" } 132 | 133 | def results_to_cache(self, results): 134 | return { "color": "blue" } 135 | 136 | def _backend_run(self): 137 | return { "color": "white" } 138 | 139 | backend = MyBackend() 140 | backend.model = myModel 141 | backend.init_backend(use_disk_cache=False, use_memory_cache=True) 142 | # On first run we get the original object 143 | self.assertEqual(backend.backend_run(), { "color": "white" }) 144 | # And on consequent runs we get the object recovered from the cache 145 | self.assertEqual(backend.backend_run(), { "color": "red" }) 146 | self.assertEqual(backend.backend_run(), { "color": "red" }) 147 | 148 | if __name__ == "__main__": 149 | unittest.main() 150 | -------------------------------------------------------------------------------- /sciunit/unit_test/base.py: -------------------------------------------------------------------------------- 1 | """Common imports for many unit tests in this directory""" 2 | 3 | import sys 4 | 5 | import matplotlib as mpl 6 | 7 | OSX = sys.platform == "darwin" 8 | if OSX or "Qt" in mpl.rcParams["backend"]: 9 | mpl.use("Agg") # Avoid any problems with Macs or headless displays. 10 | 11 | 12 | class SuiteBase(object): 13 | """Abstract base class for testing suites and scores""" 14 | 15 | def setUp(self): 16 | from sciunit.models.examples import UniformModel 17 | from sciunit.tests import RangeTest 18 | 19 | self.M = UniformModel 20 | self.T = RangeTest 21 | 22 | def prep_models_and_tests(self): 23 | from sciunit import TestSuite 24 | 25 | t1 = self.T([2, 3], name="test1") 26 | t2 = self.T([5, 6]) 27 | m1 = self.M(2, 3) 28 | m2 = self.M(5, 6) 29 | ts = TestSuite([t1, t2], name="MySuite") 30 | return (ts, t1, t2, m1, m2) 31 | -------------------------------------------------------------------------------- /sciunit/unit_test/base_tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from sciunit.utils import TmpTestFolder 4 | 5 | tmp_folder = TmpTestFolder() 6 | 7 | class BaseCase(unittest.TestCase): 8 | """Unit tests for config files""" 9 | 10 | @classmethod 11 | def setUpClass(cls): 12 | tmp_folder.create() 13 | 14 | @classmethod 15 | def tearDownClass(cls): 16 | tmp_folder.delete() 17 | 18 | def test_deep_exclude(self): 19 | from sciunit.base import deep_exclude 20 | 21 | test_state = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5} 22 | test_exclude = [("a", "b"), ("c", "d")] 23 | deep_exclude(test_state, test_exclude) 24 | 25 | def test_default(self): 26 | # TODO 27 | pass 28 | 29 | def test_SciUnit(self): 30 | from sciunit.base import SciUnit 31 | 32 | sciunitObj = SciUnit() 33 | self.assertIsInstance(sciunitObj.properties(), dict) 34 | self.assertIsInstance(sciunitObj.__getstate__(), dict) 35 | self.assertIsInstance(sciunitObj.json(), str) 36 | sciunitObj.json(string=False) 37 | self.assertIsInstance(sciunitObj._class, dict) 38 | sciunitObj.testState = "testState" 39 | SciUnit.state_hide.append("testState") 40 | self.assertFalse("testState" in sciunitObj.__getstate__()) 41 | 42 | def test_Versioned(self): 43 | from git import Repo 44 | from sciunit.base import Versioned 45 | 46 | ver = Versioned() 47 | 48 | # Testing .get_remote() 49 | # 1. Checking our sciunit .git repo 50 | # (to make sure .get_remote() works with real repos too!) 51 | self.assertEqual("origin", ver.get_remote("I am not a remote").name) 52 | self.assertEqual("origin", ver.get_remote().name) 53 | # 2. Checking NO .git repo 54 | self.assertEqual(None, ver.get_remote(repo=None)) 55 | # 3. Checking a .git repo without remotes 56 | git_repo = Repo.init(tmp_folder.path / "git_repo") 57 | self.assertEqual(None, ver.get_remote(repo=git_repo)) 58 | # 4. Checking a .git repo with remotes 59 | origin = git_repo.create_remote("origin", "https://origin.com") 60 | beta = git_repo.create_remote('beta', "https://beta.com") 61 | self.assertEqual(origin, ver.get_remote(repo=git_repo)) 62 | self.assertEqual(origin, ver.get_remote("not a remote", repo=git_repo)) 63 | self.assertEqual(beta, ver.get_remote("beta", repo=git_repo)) 64 | 65 | # Testing .get_repo() 66 | self.assertIsInstance(ver.get_repo(), Repo) 67 | 68 | # Testing .get_remote_url() 69 | self.assertIsInstance(ver.get_remote_url("I am not a remote"), str) 70 | 71 | if __name__ == "__main__": 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /sciunit/unit_test/command_line_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for command line tools.""" 2 | 3 | import platform 4 | import tempfile 5 | import unittest 6 | from pathlib import Path 7 | 8 | import sciunit 9 | 10 | 11 | class CommandLineTestCase(unittest.TestCase): 12 | """Unit tests for command line tools.""" 13 | 14 | def setUp(self): 15 | from sciunit.__main__ import main 16 | 17 | self.main = main 18 | path = Path(sciunit.__path__[0]).resolve() 19 | SCIDASH_HOME = path.parent.parent 20 | self.cosmosuite_path = str(SCIDASH_HOME / "scidash") 21 | 22 | def test_sciunit_1create(self): 23 | try: 24 | self.main("--directory", self.cosmosuite_path, "create") 25 | except Exception as e: 26 | if "There is already a configuration file" not in str(e): 27 | raise e 28 | else: 29 | temp_path = tempfile.mkdtemp() 30 | self.main("--directory", temp_path, "create") 31 | 32 | def test_sciunit_2check(self): 33 | self.main("--directory", self.cosmosuite_path, "check") 34 | 35 | def test_sciunit_3run(self): 36 | self.main("--directory", self.cosmosuite_path, "run") 37 | 38 | def test_sciunit_4make_nb(self): 39 | self.main("--directory", self.cosmosuite_path, "make-nb") 40 | 41 | # Skip for python versions that don't have importlib.machinery 42 | @unittest.skipIf( 43 | platform.python_version() < "3.5", "sciunit not supported on Python < 3.5" 44 | ) 45 | def test_sciunit_5run_nb(self): 46 | self.main("--directory", self.cosmosuite_path, "run-nb") 47 | 48 | 49 | if __name__ == "__main__": 50 | test_program = unittest.main(verbosity=0, buffer=True, exit=False) 51 | -------------------------------------------------------------------------------- /sciunit/unit_test/config_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for user configuration""" 2 | 3 | import unittest 4 | from pathlib import Path 5 | 6 | import sciunit 7 | 8 | 9 | class ConfigTestCase(unittest.TestCase): 10 | """Unit tests for config files""" 11 | 12 | def test_new_config(self): 13 | sciunit.config.create() 14 | 15 | self.assertTrue(sciunit.config.path.is_file()) 16 | cmap_low = sciunit.config.get("cmap_low") 17 | 18 | self.assertTrue(isinstance(cmap_low, int)) 19 | dummy = sciunit.config.get("dummy", 37) 20 | self.assertEqual(dummy, 37) 21 | try: 22 | sciunit.config.get("dummy") 23 | except sciunit.Error as e: 24 | self.assertTrue("does not contain key" in str(e)) 25 | 26 | def test_missing_config(self): 27 | sciunit.config.path = Path("_delete.json") 28 | sciunit.config.get_from_disk() 29 | 30 | def test_bad_config(self): 31 | sciunit.config.path = Path("_delete.json") 32 | with open(sciunit.config.path, "w") as f: 33 | f.write(".......") 34 | sciunit.config.get_from_disk() 35 | 36 | 37 | if __name__ == "__main__": 38 | unittest.main() 39 | -------------------------------------------------------------------------------- /sciunit/unit_test/converter_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for score converters""" 2 | 3 | import unittest 4 | 5 | 6 | class ConvertersTestCase(unittest.TestCase): 7 | """Unit tests for Score converters""" 8 | 9 | def test_converters(self): 10 | from sciunit.converters import ( 11 | AtLeastToBoolean, 12 | AtMostToBoolean, 13 | LambdaConversion, 14 | NoConversion, 15 | RangeToBoolean, 16 | ) 17 | from sciunit.scores import BooleanScore, ZScore 18 | 19 | old_score = ZScore(1.3) 20 | new_score = NoConversion().convert(old_score) 21 | self.assertEqual(old_score, new_score) 22 | new_score = LambdaConversion(lambda x: x.score ** 2).convert(old_score) 23 | self.assertEqual(old_score.score ** 2, new_score.score) 24 | new_score = AtMostToBoolean(3).convert(old_score) 25 | self.assertEqual(new_score, BooleanScore(True)) 26 | new_score = AtMostToBoolean(1).convert(old_score) 27 | self.assertEqual(new_score, BooleanScore(False)) 28 | new_score = AtLeastToBoolean(1).convert(old_score) 29 | self.assertEqual(new_score, BooleanScore(True)) 30 | new_score = AtLeastToBoolean(3).convert(old_score) 31 | self.assertEqual(new_score, BooleanScore(False)) 32 | new_score = RangeToBoolean(1, 3).convert(old_score) 33 | self.assertEqual(new_score, BooleanScore(True)) 34 | new_score = RangeToBoolean(3, 5).convert(old_score) 35 | self.assertEqual(new_score, BooleanScore(False)) 36 | self.assertEqual(new_score.raw, str(old_score.score)) 37 | 38 | def test_converters2(self): 39 | from sciunit import Score 40 | from sciunit.converters import Converter 41 | 42 | converterObj = Converter() 43 | 44 | self.assertIsInstance(converterObj.description, str) 45 | self.assertRaises(NotImplementedError, converterObj._convert, Score(0.5)) 46 | 47 | class MyConverter(Converter): 48 | pass 49 | 50 | myConverterObj = MyConverter() 51 | self.assertEqual(myConverterObj.description, "No description available") 52 | 53 | 54 | if __name__ == "__main__": 55 | unittest.main() 56 | -------------------------------------------------------------------------------- /sciunit/unit_test/doc_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for documentation""" 2 | 3 | import unittest 4 | 5 | from sciunit.utils import NotebookTools 6 | 7 | 8 | class DocumentationTestCase(NotebookTools, unittest.TestCase): 9 | """Unit tests for documentation notebooks""" 10 | 11 | path = "../../docs" 12 | 13 | def test_chapter1(self): 14 | self.do_notebook("chapter1") 15 | 16 | def test_chapter2(self): 17 | self.do_notebook("chapter2") 18 | 19 | def test_chapter3(self): 20 | self.do_notebook("chapter3") 21 | 22 | def test_chapter4(self): 23 | self.do_notebook("chapter4") 24 | 25 | def test_chapter5(self): 26 | self.do_notebook("chapter5") 27 | 28 | @unittest.skip("Requires sympy") 29 | def test_chapter6(self): 30 | self.do_notebook("chapter6") 31 | 32 | # def test_chapter6(self): 33 | # self.do_notebook("chapter6") 34 | -------------------------------------------------------------------------------- /sciunit/unit_test/error_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for sciunit errors""" 2 | 3 | import unittest 4 | 5 | 6 | class ErrorsTestCase(unittest.TestCase): 7 | """Unit tests for various error classes""" 8 | 9 | def test_error_types(self): 10 | from sciunit import Capability, Model 11 | from sciunit.errors import ( 12 | BadParameterValueError, 13 | CapabilityError, 14 | InvalidScoreError, 15 | PredictionError, 16 | ) 17 | 18 | CapabilityError(Model(), Capability) 19 | CapabilityError(Model(), Capability, "this is a test detail") 20 | PredictionError(Model(), "foo") 21 | InvalidScoreError() 22 | BadParameterValueError("x", 3) 23 | 24 | 25 | if __name__ == "__main__": 26 | unittest.main() 27 | -------------------------------------------------------------------------------- /sciunit/unit_test/import_tests.py: -------------------------------------------------------------------------------- 1 | """Tests of imports of sciunit submodules and other dependencies""" 2 | 3 | import unittest 4 | 5 | 6 | class ImportTestCase(unittest.TestCase): 7 | """Unit tests for imports""" 8 | 9 | def test_quantities(self): 10 | import quantities as pq 11 | 12 | pq.Quantity([10, 20, 30], pq.pA) 13 | 14 | def test_import_everything(self): 15 | import sciunit 16 | from sciunit.utils import import_all_modules 17 | 18 | # Recursively import all submodules 19 | import_all_modules(sciunit) 20 | 21 | 22 | if __name__ == "__main__": 23 | unittest.main() 24 | -------------------------------------------------------------------------------- /sciunit/unit_test/model_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for models and capabilities""" 2 | 3 | import unittest 4 | 5 | 6 | class ModelsTestCase(unittest.TestCase): 7 | """Unit tests for the sciunit module""" 8 | 9 | def setUp(self): 10 | from sciunit.models.examples import UniformModel 11 | 12 | self.M = UniformModel 13 | 14 | def test_is_match(self): 15 | from sciunit import Model 16 | 17 | m = Model() 18 | m2 = Model() 19 | self.assertFalse(m.is_match(m2)) 20 | self.assertTrue(m.is_match(m)) 21 | self.assertTrue(m.is_match("Model")) 22 | 23 | def test_getattr(self): 24 | from sciunit import Model 25 | 26 | m = Model() 27 | self.assertEqual(m.name, m.__getattr__("name")) 28 | 29 | def test_curr_method(self): 30 | from sciunit import Model 31 | 32 | class TestModel(Model): 33 | def test_calling_curr_method(self): 34 | return self.curr_method() 35 | 36 | m = TestModel() 37 | test_method_name = m.test_calling_curr_method() 38 | self.assertEqual(test_method_name, "test_calling_curr_method") 39 | 40 | def test_failed_extra_capabilities(self): 41 | from sciunit import Model 42 | 43 | class TestModel(Model): 44 | def test_return_none_function(self): 45 | return None 46 | 47 | m = TestModel() 48 | m.extra_capability_checks = {TestModel: "test_return_none_function"} 49 | test_list = m.failed_extra_capabilities 50 | 51 | self.assertEqual(test_list[0], TestModel) 52 | self.assertEqual(len(test_list), 1) 53 | 54 | def test_get_model_state(self): 55 | from sciunit import Model 56 | 57 | m = Model() 58 | state = m.__getstate__() 59 | self.assertTrue(["capabilities" in state]) 60 | self.assertTrue(m.capabilities == state["capabilities"]) 61 | 62 | def test_get_model_capabilities(self): 63 | from sciunit.capabilities import ProducesNumber 64 | 65 | m = self.M(2, 3) 66 | self.assertEqual(m.capabilities, [ProducesNumber]) 67 | 68 | def test_get_model_description(self): 69 | m = self.M(2, 3) 70 | m.describe() 71 | m.description = "Lorem Ipsum" 72 | m.describe() 73 | 74 | def test_check_model_capabilities(self): 75 | from sciunit.tests import RangeTest 76 | 77 | t = RangeTest([2, 3]) 78 | m = self.M(2, 3) 79 | t.check(m) 80 | 81 | def test_check_missing_capabilities_1(self): 82 | m = self.M( 83 | 2, 3, name="Not actually runnable due to lack of capability provision" 84 | ) 85 | try: 86 | m.run() 87 | except AttributeError as e: 88 | pass 89 | else: 90 | self.fail("Unprovided capability was called and not caught") 91 | 92 | def test_check_missing_capabilities_2(self): 93 | from sciunit.capabilities import Runnable 94 | from sciunit.errors import CapabilityNotImplementedError 95 | 96 | class MyModel(self.M, Runnable): 97 | pass 98 | 99 | m = MyModel( 100 | 2, 3, name="Not actually runnable due to lack of capability implementation" 101 | ) 102 | try: 103 | m.run() 104 | except CapabilityNotImplementedError as e: 105 | pass 106 | else: 107 | self.fail("Unimplemented capability was called and not caught") 108 | 109 | def test_check_missing_capabilities_3(self): 110 | from sciunit.capabilities import Runnable 111 | 112 | class MyModel(self.M, Runnable): 113 | def run(self): 114 | print("Actually running!") 115 | 116 | m = MyModel(2, 3, name="Now actually runnable") 117 | m.run() 118 | 119 | def test_regular_models(self): 120 | from sciunit.models.examples import ( 121 | ConstModel, 122 | PersistentUniformModel, 123 | UniformModel, 124 | ) 125 | 126 | m = ConstModel(3) 127 | self.assertEqual(m.produce_number(), 3) 128 | 129 | m = UniformModel(3, 4) 130 | self.assertTrue(3 < m.produce_number() < 4) 131 | 132 | m = PersistentUniformModel(3, 4) 133 | m.run() 134 | self.assertTrue(3 < m.produce_number() < 4) 135 | 136 | def test_irregular_models(self): 137 | from sciunit.models.examples import ( 138 | CacheByInstancePersistentUniformModel, 139 | CacheByValuePersistentUniformModel, 140 | ) 141 | 142 | a = CacheByInstancePersistentUniformModel(2, 3) 143 | a1 = a.produce_number() 144 | a2 = a.produce_number() 145 | self.assertEqual(a1, a2) 146 | b = CacheByInstancePersistentUniformModel(2, 3) 147 | b1 = b.produce_number() 148 | self.assertNotEqual(b1, a2) 149 | 150 | c = CacheByValuePersistentUniformModel(2, 3) 151 | c1 = c.produce_number() 152 | c2 = c.produce_number() 153 | self.assertEqual(c1, c2) 154 | d = CacheByValuePersistentUniformModel(2, 3) 155 | d1 = d.produce_number() 156 | self.assertEqual(d1, c2) 157 | 158 | 159 | class CapabilitiesTestCase(unittest.TestCase): 160 | """Unit tests for sciunit Capability classes""" 161 | 162 | def test_capabilities(self): 163 | from sciunit import Model 164 | from sciunit.capabilities import Capability, ProducesNumber, Runnable 165 | from sciunit.models import Model 166 | from sciunit.models.examples import ( 167 | RepeatedRandomNumberModel, 168 | UniqueRandomNumberModel, 169 | ) 170 | 171 | class MyModel(Model, ProducesNumber): 172 | def produce_number(self): 173 | return 3.14 174 | 175 | m = MyModel() 176 | self.assertEqual(m.produce_number(), 3.14) 177 | 178 | m = UniqueRandomNumberModel() 179 | self.assertNotEqual(m.produce_number(), m.produce_number()) 180 | 181 | m = RepeatedRandomNumberModel() 182 | self.assertEqual(m.produce_number(), m.produce_number()) 183 | 184 | m = Runnable() 185 | self.assertRaises(BaseException, m.run) 186 | self.assertRaises(BaseException, m.set_run_params) 187 | self.assertRaises(BaseException, m.set_default_run_params) 188 | 189 | m = ProducesNumber() 190 | self.assertRaises(BaseException, m.produce_number) 191 | 192 | m = Capability() 193 | m.name = "test name" 194 | self.assertEqual(str(m), "test name") 195 | 196 | def test_source_check(self): 197 | 198 | from sciunit import Model 199 | from sciunit.capabilities import Capability 200 | from sciunit.errors import CapabilityNotImplementedError 201 | from sciunit.models import Model 202 | 203 | class MyCap1(Capability): 204 | def fn1(self): 205 | raise NotImplementedError("fn1 not implemented.") 206 | 207 | class MyCap2(Capability): 208 | def fn1(self): 209 | self.unimplemented("fn1 not implemented.") 210 | 211 | class MyCap3(Capability): 212 | def fn1(self): 213 | raise CapabilityNotImplementedError( 214 | model=self, 215 | capability=self.__class__, 216 | details="fn1 not implemented.", 217 | ) 218 | 219 | class MyModel1(Model, MyCap1): 220 | def fn1(self): 221 | return "fn1 have been implemented" 222 | 223 | class MyModel2(Model, MyCap1): 224 | pass 225 | 226 | class MyModel3(Model, MyCap2): 227 | def fn1(self): 228 | return "fn1 have been implemented" 229 | 230 | class MyModel4(Model, MyCap2): 231 | pass 232 | 233 | class MyModel5(Model, MyCap3): 234 | def fn1(self): 235 | return "fn1 have been implemented" 236 | 237 | class MyModel6(Model, MyCap3): 238 | pass 239 | 240 | self.assertTrue(MyCap1.source_check(MyModel1())) 241 | self.assertFalse(MyCap1.source_check(MyModel2())) 242 | self.assertTrue(MyCap2.source_check(MyModel3())) 243 | self.assertFalse(MyCap2.source_check(MyModel4())) 244 | self.assertTrue(MyCap3.source_check(MyModel5())) 245 | self.assertFalse(MyCap3.source_check(MyModel6())) 246 | 247 | 248 | class RunnableModelTestCase(unittest.TestCase): 249 | def test_backend(self): 250 | from sciunit.models import RunnableModel 251 | from sciunit.models.backends import Backend, register_backends 252 | 253 | self.assertRaises(TypeError, RunnableModel, name="", attrs=1) 254 | model = RunnableModel(name="test name") 255 | self.assertIsInstance(model.get_backend(), Backend) 256 | self.assertRaises(TypeError, model.set_backend, 0) 257 | 258 | model.set_backend(None) 259 | self.assertRaises(Exception, model.set_backend, "invalid backend") 260 | 261 | model.set_attrs(test_attr="test attribute") 262 | model.set_run_params(test_run_params="test runtime parameter") 263 | model.check_run_params() 264 | model.reset_run_params() 265 | model.set_default_run_params(test_run_params="test runtime parameter") 266 | model.reset_default_run_params() 267 | self.assertIsInstance(model.__getstate__(), dict) 268 | 269 | class MyBackend1(Backend): 270 | def _backend_run(self) -> str: 271 | return "test result 1" 272 | 273 | class MyBackend2(Backend): 274 | def _backend_run(self) -> str: 275 | return "test result 2" 276 | 277 | name_backend_dict = {"backend1": MyBackend2, "backend2": MyBackend2} 278 | backend_names = ["backend1", "backend2"] 279 | 280 | model = RunnableModel(name="test name") 281 | register_backends(name_backend_dict) 282 | model.set_backend(backend_names) 283 | model.print_run_params = True 284 | model.run() 285 | 286 | model = RunnableModel(name="test name") 287 | model.default_run_params = {"para1": 1} 288 | model.use_default_run_params() 289 | 290 | 291 | if __name__ == "__main__": 292 | unittest.main() 293 | -------------------------------------------------------------------------------- /sciunit/unit_test/observation_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for observations.""" 2 | 3 | import unittest 4 | 5 | from sciunit.utils import NotebookTools 6 | 7 | 8 | class ObservationsTestCase(unittest.TestCase, NotebookTools): 9 | """Unit tests for the sciunit module""" 10 | 11 | path = "." 12 | 13 | def test_observation_validation(self): 14 | """Test validation of observations against the `observation_schema`.""" 15 | self.do_notebook("validate_observation") 16 | -------------------------------------------------------------------------------- /sciunit/unit_test/score_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for scores and score collections""" 2 | 3 | import unittest 4 | 5 | import numpy as np 6 | from IPython.display import display 7 | from pandas import DataFrame 8 | from pandas.core.frame import DataFrame 9 | from pandas.core.series import Series 10 | from quantities import Quantity 11 | 12 | from sciunit import Score, ScoreArray, ScoreMatrix 13 | from sciunit.errors import InvalidScoreError 14 | from sciunit.models import Model 15 | from sciunit.scores import ( 16 | BooleanScore, 17 | CohenDScore, 18 | ErrorScore, 19 | FloatScore, 20 | InsufficientDataScore, 21 | NAScore, 22 | NoneScore, 23 | PercentScore, 24 | RandomScore, 25 | RatioScore, 26 | TBDScore, 27 | ZScore, 28 | ) 29 | from sciunit.scores.collections_m2m import ScoreArrayM2M, ScoreMatrixM2M 30 | from sciunit.tests import RangeTest, Test 31 | from sciunit.unit_test.base import SuiteBase 32 | from sciunit.utils import NotebookTools 33 | 34 | 35 | class ScoresTestCase(SuiteBase, unittest.TestCase, NotebookTools): 36 | 37 | path = "." 38 | 39 | def test_score_matrix_constructor(self): 40 | tests = [Test([1, 2, 3])] 41 | models = [Model()] 42 | scores = np.array([ZScore(1.0)]) 43 | ScoreArray(tests) 44 | scoreMatrix = ScoreMatrix(tests, models, scores) 45 | scoreMatrix = ScoreMatrix(tests, models, scores, transpose=True) 46 | 47 | tests = Test([1, 2, 3]) 48 | models = Model() 49 | ScoreMatrix(tests, models, scores) 50 | 51 | def test_score_matrix(self): 52 | t, t1, t2, m1, m2 = self.prep_models_and_tests() 53 | sm = t.judge(m1) 54 | 55 | self.assertRaises(TypeError, sm.__getitem__, 0) 56 | 57 | self.assertEqual(str(sm.get_group((t1, m1))), "Pass") 58 | self.assertEqual(str(sm.get_group((m1, t1))), "Pass") 59 | self.assertEqual(str(sm.get_group((m1.name, t1.name))), "Pass") 60 | self.assertEqual(str(sm.get_group((t1.name, m1.name))), "Pass") 61 | 62 | self.assertRaises(TypeError, sm.get_group, (0, 0)) 63 | self.assertRaises(KeyError, sm.get_by_name, "This name does not exist") 64 | 65 | self.assertIsInstance(sm.__getattr__("score"), DataFrame) 66 | self.assertIsInstance(sm.norm_scores, DataFrame) 67 | self.assertIsInstance(sm.T, ScoreMatrix) 68 | # self.assertIsInstance(sm.to_html(True, True, True), str) 69 | # self.assertIsInstance(sm.to_html(), str) 70 | 71 | self.assertTrue(type(sm) is ScoreMatrix) 72 | self.assertTrue(sm[t1][m1].score) 73 | self.assertTrue(sm["test1"][m1].score) 74 | self.assertTrue(sm[m1]["test1"].score) 75 | self.assertFalse(sm[t2][m1].score) 76 | self.assertEqual(sm[(m1, t1)].score, True) 77 | self.assertEqual(sm[(m1, t2)].score, False) 78 | sm = t.judge([m1, m2]) 79 | self.assertEqual(sm.stature(t1, m1), 1) 80 | self.assertEqual(sm.stature(t1, m2), 2) 81 | display(sm) 82 | 83 | ######### m2m ################# 84 | t1.observation = [2, 3] 85 | smm2m = ScoreMatrixM2M( 86 | test=t1, models=[m1], scores=[[Score(1), Score(1)], [Score(1), Score(1)]] 87 | ) 88 | 89 | self.assertIsInstance(smm2m.__getattr__("score"), DataFrame) 90 | self.assertIsInstance(smm2m.__getattr__("norm_scores"), DataFrame) 91 | self.assertIsInstance(smm2m.__getattr__("related_data"), DataFrame) 92 | self.assertRaises(KeyError, smm2m.get_by_name, "Not Exist") 93 | self.assertIsInstance(smm2m.norm_scores, DataFrame) 94 | self.assertRaises(KeyError, smm2m.get_by_name, "Not Exist") 95 | self.assertRaises(TypeError, smm2m.get_group, [0]) 96 | self.assertIsInstance(smm2m.get_group([m1.name, t1.name]), Score) 97 | self.assertEqual(smm2m.get_group([m1.name, t1.name]).score, 1) 98 | self.assertIsInstance(smm2m.get_group([m1, t1]), Score) 99 | self.assertEqual(smm2m.get_group([m1, t1]).score, 1) 100 | 101 | def test_score_arrays(self): 102 | t, t1, t2, m1, m2 = self.prep_models_and_tests() 103 | sm = t.judge(m1) 104 | sa = sm[m1] 105 | self.assertTrue(type(sa) is ScoreArray) 106 | self.assertIsInstance(sa.__getattr__("score"), Series) 107 | self.assertRaises(KeyError, sa.get_by_name, "This name does not exist") 108 | self.assertEqual(list(sa.norm_scores.values), [1.0, 0.0]) 109 | self.assertEqual(sa.stature(t1), 1) 110 | self.assertEqual(sa.stature(t2), 2) 111 | self.assertEqual(sa.stature(t1), 1) 112 | display(sa) 113 | 114 | ######### m2m ################# 115 | sam2m = ScoreArrayM2M( 116 | test=t1, models=[m1], scores=[[Score(1), Score(1)], [Score(1), Score(1)]] 117 | ) 118 | self.assertRaises(KeyError, sam2m.get_by_name, "Not Exist") 119 | 120 | def test_regular_score_types_1(self): 121 | self.assertEqual(PercentScore(0).norm_score, 0) 122 | self.assertEqual(PercentScore(100).norm_score, 1) 123 | 124 | score = PercentScore(42) 125 | self.assertRaises(InvalidScoreError, PercentScore, 101) 126 | self.assertRaises(InvalidScoreError, PercentScore, -1) 127 | self.assertEqual(str(score), "42.0%") 128 | self.assertEqual(score.norm_score, 0.42) 129 | 130 | self.assertEqual(1, ZScore(0.0).norm_score) 131 | self.assertEqual(0, ZScore(1e12).norm_score) 132 | self.assertEqual(0, ZScore(-1e12).norm_score) 133 | 134 | ZScore(0.7) 135 | score = ZScore.compute({"mean": 3.0, "std": 1.0}, {"value": 2.0}) 136 | 137 | self.assertIsInstance( 138 | ZScore.compute({"mean": 3.0}, {"value": 2.0}), InsufficientDataScore 139 | ) 140 | self.assertIsInstance( 141 | ZScore.compute({"mean": 3.0, "std": -1.0}, {"value": 2.0}), 142 | InsufficientDataScore, 143 | ) 144 | self.assertIsInstance( 145 | ZScore.compute({"mean": np.nan, "std": np.nan}, {"value": np.nan}), 146 | InsufficientDataScore, 147 | ) 148 | self.assertEqual(score.score, -1.0) 149 | 150 | self.assertEqual(1, CohenDScore(0.0).norm_score) 151 | self.assertEqual(0, CohenDScore(1e12).norm_score) 152 | self.assertEqual(0, CohenDScore(-1e12).norm_score) 153 | CohenDScore(-0.3) 154 | score = CohenDScore.compute( 155 | {"mean": 3.0, "std": 1.0}, {"mean": 2.0, "std": 1.0} 156 | ) 157 | 158 | self.assertAlmostEqual(-0.707, score.score, 3) 159 | self.assertEqual("D = -0.71", str(score)) 160 | 161 | score = CohenDScore.compute( 162 | {"mean": 3.0, "std": 10.0, "n": 10}, {"mean": 2.5, "std": 10.0, "n": 10} 163 | ) 164 | self.assertAlmostEqual(-0.05, score.score, 2) 165 | 166 | def test_regular_score_types_2(self): 167 | BooleanScore(True) 168 | BooleanScore(False) 169 | score = BooleanScore.compute(5, 5) 170 | self.assertEqual(score.norm_score, 1) 171 | score = BooleanScore.compute(4, 5) 172 | self.assertEqual(score.norm_score, 0) 173 | 174 | self.assertEqual(1, BooleanScore(True).norm_score) 175 | self.assertEqual(0, BooleanScore(False).norm_score) 176 | 177 | t = RangeTest([2, 3]) 178 | score.test = t 179 | score.describe() 180 | score.description = "Lorem Ipsum" 181 | score.describe() 182 | 183 | score = FloatScore(3.14) 184 | self.assertRaises( 185 | InvalidScoreError, score.check_score, Quantity([1, 2, 3], "J") 186 | ) 187 | 188 | obs = np.array([1.0, 2.0, 3.0]) 189 | pred = np.array([1.0, 2.0, 4.0]) 190 | score = FloatScore.compute_ssd(obs, pred) 191 | self.assertEqual(str(score), "1") 192 | self.assertEqual(score.score, 1.0) 193 | 194 | score = RatioScore(1.2) 195 | self.assertEqual(1, RatioScore(1.0).norm_score) 196 | self.assertEqual(0, RatioScore(1e12).norm_score) 197 | self.assertEqual(0, RatioScore(1e-12).norm_score) 198 | 199 | self.assertEqual(str(score), "Ratio = 1.20") 200 | 201 | self.assertRaises(InvalidScoreError, RatioScore, -1.0) 202 | score = RatioScore.compute({"mean": 4.0, "std": 1.0}, {"value": 2.0}) 203 | 204 | self.assertEqual(score.score, 0.5) 205 | 206 | def test_irregular_score_types(self): 207 | e = Exception("This is an error") 208 | score = ErrorScore(e) 209 | score = NAScore(None) 210 | score = TBDScore(None) 211 | score = NoneScore(None) 212 | score = NoneScore("this is a string") 213 | self.assertIsInstance(str(score), str) 214 | self.assertRaises(InvalidScoreError, NoneScore, ["this is a string list"]) 215 | 216 | score = InsufficientDataScore(None) 217 | self.assertEqual(score.norm_score, None) 218 | 219 | def test_only_lower_triangle(self): 220 | """Test validation of observations against the `observation_schema`.""" 221 | self.do_notebook("test_only_lower_triangle") 222 | 223 | def test_RandomScore(self): 224 | """Note: RandomScore is only used for debugging purposes""" 225 | score = RandomScore(0.5) 226 | self.assertEqual("0.5", str(score)) 227 | 228 | def test_Score(self): 229 | self.assertIsInstance(Score.compute({}, {}), NotImplementedError) 230 | score = Score(0.5) 231 | self.assertEqual(score.norm_score, 0.5) 232 | self.assertAlmostEqual(score.log_norm_score, -0.693, 2) 233 | self.assertAlmostEqual(score.log2_norm_score, -1.0, 1) 234 | self.assertAlmostEqual(score.log10_norm_score, -0.301, 1) 235 | self.assertIsInstance(score.raw, str) 236 | score._raw = "this is a string" 237 | self.assertIsNone(score.raw) 238 | self.assertIsInstance(score.__repr__(), str) 239 | self.assertIsInstance(score.__str__(), str) 240 | 241 | self.assertFalse(score.__ne__(score)) 242 | self.assertTrue(score.__ne__(Score(998.0))) 243 | self.assertFalse(score.__ne__(0.5)) 244 | self.assertTrue(score.__ne__(0.6)) 245 | 246 | self.assertFalse(score.__gt__(score)) 247 | self.assertTrue(score.__gt__(Score(0.2))) 248 | self.assertFalse(score.__gt__(0.5)) 249 | self.assertTrue(score.__gt__(0.2)) 250 | 251 | self.assertFalse(score.__lt__(score)) 252 | self.assertTrue(score.__lt__(Score(0.9))) 253 | self.assertFalse(score.__lt__(0.5)) 254 | self.assertTrue(score.__lt__(0.9)) 255 | 256 | self.assertTrue(score.__le__(score)) 257 | self.assertTrue(score.__le__(Score(0.5))) 258 | self.assertTrue(score.__le__(0.5)) 259 | self.assertTrue(score.__le__(0.5)) 260 | self.assertFalse(score.__le__(0.1)) 261 | self.assertFalse(score.__le__(Score(0.1))) 262 | 263 | self.assertIsInstance(score.score_type, str) 264 | 265 | def test_ErrorScore(self): 266 | score = ErrorScore(0.5) 267 | self.assertEqual(0.0, score.norm_score) 268 | self.assertIsInstance(score.summary, str) 269 | self.assertIsInstance(score._describe(), str) 270 | self.assertIsInstance(str(score), str) 271 | 272 | 273 | if __name__ == "__main__": 274 | unittest.main() 275 | -------------------------------------------------------------------------------- /sciunit/unit_test/test_only_lower_triangle.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from sciunit.models.examples import ConstModel\n", 10 | "from sciunit.tests import TestM2M\n", 11 | "from sciunit.scores import RatioScore\n", 12 | "from sciunit.capabilities import ProducesNumber" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "class MyTestM2M(TestM2M): \n", 22 | " required_capabilities = (ProducesNumber,)\n", 23 | " score_type = RatioScore \n", 24 | " \n", 25 | " def generate_prediction(self, model): \n", 26 | " return model.produce_number() " 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 11, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "models = [ConstModel(x, name='%s producer'%x) for x in range(1,4)]" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 12, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/html": [ 46 | "
\n", 47 | "\n", 60 | "\n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | "
1 producer2 producer3 producer
1 producerRatio = 1.00Ratio = 2.00Ratio = 3.00
2 producerRatio = 0.50Ratio = 1.00Ratio = 1.50
3 producerRatio = 0.33Ratio = 0.67Ratio = 1.00
\n", 90 | "
" 91 | ], 92 | "text/plain": [ 93 | " 1 producer 2 producer 3 producer\n", 94 | "1 producer Ratio = 1.00 Ratio = 2.00 Ratio = 3.00\n", 95 | "2 producer Ratio = 0.50 Ratio = 1.00 Ratio = 1.50\n", 96 | "3 producer Ratio = 0.33 Ratio = 0.67 Ratio = 1.00" 97 | ] 98 | }, 99 | "execution_count": 12, 100 | "metadata": {}, 101 | "output_type": "execute_result" 102 | } 103 | ], 104 | "source": [ 105 | "test = MyTestM2M() \n", 106 | "test.judge(models) " 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 13, 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "data": { 116 | "text/html": [ 117 | "
\n", 118 | "\n", 131 | "\n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | "
1 producer2 producer3 producer
1 producerRatio = 1.00Ratio = 2.00Ratio = 3.00
2 producerRatio = 2.00Ratio = 1.00Ratio = 1.50
3 producerRatio = 3.00Ratio = 1.50Ratio = 1.00
\n", 161 | "
" 162 | ], 163 | "text/plain": [ 164 | " 1 producer 2 producer 3 producer\n", 165 | "1 producer Ratio = 1.00 Ratio = 2.00 Ratio = 3.00\n", 166 | "2 producer Ratio = 2.00 Ratio = 1.00 Ratio = 1.50\n", 167 | "3 producer Ratio = 3.00 Ratio = 1.50 Ratio = 1.00" 168 | ] 169 | }, 170 | "execution_count": 13, 171 | "metadata": {}, 172 | "output_type": "execute_result" 173 | } 174 | ], 175 | "source": [ 176 | "test.judge(models, only_lower_triangle=True)" 177 | ] 178 | } 179 | ], 180 | "metadata": { 181 | "kernelspec": { 182 | "display_name": "Python 3", 183 | "language": "python", 184 | "name": "python3" 185 | }, 186 | "language_info": { 187 | "codemirror_mode": { 188 | "name": "ipython", 189 | "version": 3 190 | }, 191 | "file_extension": ".py", 192 | "mimetype": "text/x-python", 193 | "name": "python", 194 | "nbconvert_exporter": "python", 195 | "pygments_lexer": "ipython3", 196 | "version": "3.7.1" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 2 201 | } 202 | -------------------------------------------------------------------------------- /sciunit/unit_test/test_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for (sciunit) tests and test suites""" 2 | 3 | import unittest 4 | 5 | import quantities as pq 6 | 7 | from sciunit import Model, TestSuite, config 8 | from sciunit.capabilities import ProducesNumber 9 | from sciunit.errors import Error, InvalidScoreError, ObservationError, ParametersError 10 | from sciunit.models.examples import ConstModel, UniformModel 11 | from sciunit.scores import BooleanScore, FloatScore, ZScore 12 | from sciunit.scores.collections import ScoreMatrix 13 | from sciunit.tests import ProtocolToFeaturesTest, RangeTest, Test, TestM2M 14 | 15 | from .base import SuiteBase 16 | 17 | 18 | class TestsTestCase(unittest.TestCase): 19 | """Unit tests for the sciunit module""" 20 | 21 | def setUp(self): 22 | self.M = UniformModel 23 | self.T = RangeTest 24 | 25 | def test_get_test_description(self): 26 | t = self.T([2, 3]) 27 | t.describe() 28 | t.description = "Lorem Ipsum" 29 | t.describe() 30 | 31 | class MyTest(self.T): 32 | """Lorem Ipsum""" 33 | 34 | t = MyTest([2, 3]) 35 | t.description = None 36 | self.assertEqual(t.describe(), "Lorem Ipsum") 37 | 38 | def test_check_model_capabilities(self): 39 | t = self.T([2, 3]) 40 | m = self.M(2, 3) 41 | t.check(m) 42 | 43 | def test_rangetest(self): 44 | from sciunit.converters import NoConversion 45 | 46 | range_2_3_test = RangeTest(observation=[2, 3]) 47 | range_2_3_test.converter = NoConversion() 48 | one_model = ConstModel(2.5) 49 | self.assertTrue(range_2_3_test.check_capabilities(one_model)) 50 | score = range_2_3_test.judge(one_model) 51 | self.assertTrue(isinstance(score, BooleanScore)) 52 | self.assertEqual(score.score, True) 53 | self.assertTrue(score.test is range_2_3_test) 54 | self.assertTrue(score.model is one_model) 55 | 56 | def test_Test(self): 57 | pv = config["PREVALIDATE"] 58 | config["PREVALIDATE"] = 1 59 | with self.assertRaises(ObservationError): 60 | t = Test(None) 61 | 62 | with self.assertRaises(ObservationError): 63 | 64 | class Test2(Test): 65 | observation_schema = None 66 | score_type = ZScore 67 | units = pq.pA 68 | 69 | def generate_prediction(self): 70 | return 1 71 | 72 | t = Test2({"mean": 5 * pq.pA}) 73 | 74 | t = Test({}) 75 | self.assertRaises( 76 | ObservationError, t.validate_observation, "I am not an observation" 77 | ) 78 | t.observation_schema = {} 79 | t.validate_observation({0: 0, 1: 1}) 80 | Test.observation_schema = [{}, {}] 81 | self.assertListEqual(t.observation_schema_names(), ["Schema 1", "Schema 2"]) 82 | config["PREVALIDATE"] = pv 83 | 84 | self.assertRaises(ParametersError, t.validate_params, None) 85 | self.assertRaises(ParametersError, t.validate_params, "I am not an observation") 86 | t.params_schema = {} 87 | t.validate_params({0: 1, 1: 2}) 88 | 89 | self.assertRaises(Error, t.check_capabilities, "I am not a model") 90 | t.condition_model(Model()) 91 | self.assertRaises(NotImplementedError, t.generate_prediction, Model()) 92 | self.assertRaises(NotImplementedError, t.optimize, Model()) 93 | 94 | self.assertTrue(t.compute_score({0: 2, 1: 2}, {0: 2, 1: 2}).score) 95 | self.assertFalse(t.compute_score({0: -2, 1: 2}, {0: 2, 1: -2}).score) 96 | t.score_type = None 97 | self.assertRaises(NotImplementedError, t.compute_score, {}, {}) 98 | 99 | t.score_type = BooleanScore 100 | self.assertRaises(InvalidScoreError, t.check_score_type, FloatScore(0.5)) 101 | 102 | 103 | class TestSuitesTestCase(SuiteBase, unittest.TestCase): 104 | """Unit tests for the sciunit module""" 105 | 106 | def test_testsuite(self): 107 | t1 = self.T([2, 3]) 108 | t2 = self.T([5, 6]) 109 | m1 = self.M(2, 3) 110 | m2 = self.M(5, 6) 111 | t = TestSuite([t1, t2]) 112 | t.judge([m1, m2]) 113 | self.assertIsInstance(t.check([m1, m2]), ScoreMatrix) 114 | capa_list = t.check_capabilities(m1) 115 | self.assertTrue(capa_list[0]) 116 | self.assertTrue(capa_list[1]) 117 | 118 | t = TestSuite( 119 | {"test 1": t1, "test 2": t2, "test 3 (non-Test)": "I am not a Test"} 120 | ) 121 | self.assertRaises(TypeError, t.assert_tests, 0) 122 | self.assertRaises(TypeError, t.assert_tests, [0]) 123 | self.assertRaises(TypeError, t.assert_models, 0) 124 | self.assertRaises(TypeError, t.assert_models, [0]) 125 | self.assertRaises(NotImplementedError, t.optimize, m1) 126 | self.assertRaises(KeyError, t.__getitem__, "wrong name") 127 | self.assertIsInstance(t[0], RangeTest) 128 | 129 | t.judge([m1, m2]) 130 | t = TestSuite([t1, t2], skip_models=[m1], include_models=[m2]) 131 | t.judge([m1, m2]) 132 | 133 | def test_testsuite_hooks(self): 134 | t1 = self.T([2, 3]) 135 | t1.hook_called = False 136 | t2 = self.T([5, 6]) 137 | m = self.M(2, 3) 138 | 139 | def f(test, tests, score, a=None): 140 | self.assertEqual(score, True) 141 | self.assertEqual(a, 1) 142 | t1.hook_called = True 143 | 144 | ts = TestSuite( 145 | [t1, t2], name="MySuite", hooks={t1: {"f": f, "kwargs": {"a": 1}}} 146 | ) 147 | ts.judge(m) 148 | self.assertEqual(t1.hook_called, True) 149 | 150 | def test_testsuite_from_observations(self): 151 | m = self.M(2, 3) 152 | ts = TestSuite.from_observations( 153 | [(self.T, [2, 3]), (self.T, [5, 6])], name="MySuite" 154 | ) 155 | ts.judge(m) 156 | 157 | def test_testsuite_set_verbose(self): 158 | t1 = self.T([2, 3]) 159 | t2 = self.T([5, 6]) 160 | t = TestSuite([t1, t2]) 161 | t.set_verbose(True) 162 | self.assertEqual(t1.verbose, True) 163 | self.assertEqual(t2.verbose, True) 164 | 165 | def test_testsuite_serialize(self): 166 | tests = [RangeTest(observation=(x, x + 3)) for x in [1, 2, 3, 4]] 167 | ts = TestSuite(tests, name="RangeSuite") 168 | self.assertTrue(isinstance(ts.json(), str)) 169 | 170 | 171 | class M2MsTestCase(unittest.TestCase): 172 | """Tests for the M2M flavor of tests and test suites""" 173 | 174 | def setUp(self): 175 | self.myModel1 = ConstModel(100.0, "Model1") 176 | self.myModel2 = ConstModel(110.0, "Model2") 177 | 178 | class NumberTest_M2M(TestM2M): 179 | """Dummy Test""" 180 | 181 | score_type = FloatScore 182 | description = "Tests the parameter 'value' between two models" 183 | 184 | def __init__(self, observation=None, name="ValueTest-M2M"): 185 | TestM2M.__init__(self, observation, name) 186 | self.required_capabilities += (ProducesNumber,) 187 | 188 | def generate_prediction(self, model, verbose=False): 189 | """Implementation of sciunit.Test.generate_prediction.""" 190 | prediction = model.produce_number() 191 | return prediction 192 | 193 | def compute_score(self, prediction1, prediction2): 194 | """Implementation of sciunit.Test.score_prediction.""" 195 | score = FloatScore(prediction1 - prediction2) 196 | score.description = "Difference between model predictions" 197 | return score 198 | 199 | self.NumberTest_M2M = NumberTest_M2M 200 | 201 | def test_testm2m_with_observation(self): 202 | myTest = self.NumberTest_M2M(observation=95.0) 203 | myScore = myTest.judge([self.myModel1, self.myModel2]) 204 | 205 | # Test model vs observation 206 | self.assertEqual(myScore[myTest][self.myModel1], -5.0) 207 | self.assertEqual(myScore[self.myModel1][myTest], 5.0) 208 | self.assertEqual(myScore["observation"][self.myModel2], -15.0) 209 | self.assertEqual(myScore[self.myModel2]["observation"], 15.0) 210 | 211 | # Test model vs model 212 | self.assertEqual(myScore[self.myModel1][self.myModel2], -10.0) 213 | self.assertEqual(myScore[self.myModel2][self.myModel1], 10.0) 214 | 215 | def test_testm2m_without_observation(self): 216 | myTest = self.NumberTest_M2M(observation=None) 217 | myScore = myTest.judge([self.myModel1, self.myModel2]) 218 | 219 | # Test model vs model; different ways of specifying individual scores 220 | self.assertEqual(myScore[self.myModel1][self.myModel2], -10.0) 221 | self.assertEqual(myScore[self.myModel2][self.myModel1], 10.0) 222 | self.assertEqual(myScore["Model1"][self.myModel2], -10.0) 223 | self.assertEqual(myScore["Model2"][self.myModel1], 10.0) 224 | self.assertEqual(myScore[self.myModel1][self.myModel1], 0.0) 225 | self.assertEqual(myScore["Model2"]["Model2"], 0.0) 226 | 227 | def test_testm2m(self): 228 | myTest = TestM2M(observation=95.0) 229 | myTest.validate_observation(None) 230 | myTest.score_type = None 231 | self.assertRaises(NotImplementedError, myTest.compute_score, {}, {}) 232 | myTest.score_type = BooleanScore 233 | self.assertTrue(myTest.compute_score(95, 96)) 234 | self.assertRaises(TypeError, myTest.judge, "str") 235 | 236 | 237 | class ProtocolToFeaturesTestCase(unittest.TestCase): 238 | def test_ProtocolToFeaturesTest(self): 239 | t = ProtocolToFeaturesTest([1, 2, 3]) 240 | m = Model() 241 | m.run = lambda: 0 242 | 243 | self.assertIsInstance(t.generate_prediction(m), NotImplementedError) 244 | self.assertIsInstance(t.setup_protocol(m), NotImplementedError) 245 | self.assertIsInstance(t.get_result(m), NotImplementedError) 246 | self.assertIsInstance(t.extract_features(m, list()), NotImplementedError) 247 | 248 | 249 | if __name__ == "__main__": 250 | unittest.main() 251 | -------------------------------------------------------------------------------- /sciunit/unit_test/utils_tests.py: -------------------------------------------------------------------------------- 1 | """Unit tests for sciunit utility functions and classes""" 2 | 3 | import tempfile 4 | import unittest 5 | 6 | import sciunit 7 | from sciunit.utils import use_backend_cache 8 | import numpy as np 9 | 10 | 11 | class CacheTestCase(unittest.TestCase): 12 | 13 | def test_basic_cache(self): 14 | class dummy_test(sciunit.Test): 15 | 16 | # The name of the cache key param determines the cache key location 17 | @use_backend_cache(cache_key_param='dummy_cache_key') 18 | def create_random_matrix(self, model): 19 | # Generate a random matrix that will land in cache 20 | return np.random.randint(0, 100, size=(5,5)) 21 | 22 | class dummy_avg_test(dummy_test): 23 | 24 | default_params = {'dummy_cache_key': '1234'} 25 | 26 | @use_backend_cache 27 | def generate_prediction(self, model): 28 | return np.mean(self.create_random_matrix(model)) 29 | 30 | class dummy_std_test(dummy_test): 31 | 32 | default_params = {'dummy_cache_key': '1234'} 33 | 34 | @use_backend_cache 35 | def generate_prediction(self, model): 36 | return np.std(self.create_random_matrix(model)) 37 | 38 | @use_backend_cache 39 | def warning_function(self): 40 | return 'I am supposed to fail, because I have no model' 41 | 42 | class dummy_backend(sciunit.models.backends.Backend): 43 | pass 44 | 45 | class dummy_model(sciunit.models.RunnableModel): 46 | pass 47 | 48 | 49 | # Initialize dummy tests and models 50 | avg_corr_test1 = dummy_avg_test([], dummy_cache_key='1234') 51 | avg_corr_test2 = dummy_avg_test([], dummy_cache_key='5678') 52 | std_corr_test = dummy_std_test([]) 53 | modelA = dummy_model(name='modelA', backend=dummy_backend) 54 | modelB = dummy_model(name='modelB', backend=dummy_backend) 55 | 56 | # Run predictions for the first time 57 | avg_corrsA1 = avg_corr_test1.generate_prediction(model=modelA) 58 | avg_corrsA2 = avg_corr_test2.generate_prediction(modelA) 59 | cached_predictionA1_avg = avg_corr_test1.get_backend_cache(model=modelA) 60 | cached_predictionA2_avg = avg_corr_test2.get_backend_cache(model=modelA) 61 | dummy_matrixA1 = avg_corr_test1.get_backend_cache(model=modelA, 62 | key='1234') 63 | dummy_matrixA2 = avg_corr_test2.get_backend_cache(model=modelA, 64 | key='5678') 65 | # dummy matrix is already generated 66 | # and cached specific for modelA with key '1234' 67 | std_corrsA = std_corr_test.generate_prediction(modelA) 68 | cached_predictionA_std = std_corr_test.get_backend_cache(model=modelA) 69 | dummy_matrixA_std = std_corr_test.get_backend_cache(model=modelA, 70 | key='1234') 71 | 72 | # Check if cached predictions are equal to original computations 73 | self.assertTrue(avg_corrsA1 == cached_predictionA1_avg) 74 | self.assertTrue(std_corrsA == cached_predictionA_std) 75 | 76 | # Check that different tests yield different predictions 77 | # These are floats, unlikely to ever be the same by chance 78 | self.assertTrue(cached_predictionA1_avg != cached_predictionA2_avg) 79 | self.assertTrue(cached_predictionA1_avg != cached_predictionA_std) 80 | 81 | # Check cached matrices are the same 82 | self.assertTrue(np.any(dummy_matrixA1 != dummy_matrixA2)) 83 | self.assertTrue(np.all(dummy_matrixA1 == dummy_matrixA_std)) 84 | 85 | """Check that a different model will have a different chache""" 86 | avg_corrsB = avg_corr_test1.generate_prediction(modelB) 87 | cached_predictionB1_avg = avg_corr_test1.get_backend_cache(model=modelB) 88 | dummy_matrixB1 = avg_corr_test1.get_backend_cache(model=modelB, 89 | key='1234') 90 | self.assertTrue(cached_predictionA1_avg != cached_predictionB1_avg) 91 | self.assertTrue(np.any(dummy_matrixA1 != dummy_matrixB1)) 92 | 93 | """Test the failing cases of the decorator""" 94 | with self.assertWarns(Warning): 95 | std_corr_test.warning_function() 96 | 97 | with self.assertWarns(Warning): 98 | test = dummy_test([]) 99 | test.create_random_matrix(modelA) 100 | 101 | class UtilsTestCase(unittest.TestCase): 102 | """Unit tests for sciunit.utils""" 103 | 104 | def test_warnings_traceback(self): 105 | from sciunit.utils import set_warnings_traceback, warn_with_traceback 106 | 107 | set_warnings_traceback(True) 108 | warn_with_traceback("This is a test warning", Warning, "utils_tests.py", 13) 109 | 110 | set_warnings_traceback(False) 111 | warn_with_traceback("This is a test warning", Warning, "utils_tests.py", 16) 112 | 113 | def test_notebook(self): 114 | from sciunit.utils import NotebookTools 115 | 116 | notebookObj = NotebookTools() 117 | notebookObj.execute_notebook("../docs/chapter1") 118 | 119 | def test_log(self): 120 | from sciunit.base import log, strip_html 121 | from sciunit.utils import html_log 122 | 123 | str1 = "test log 1" 124 | str1_stripped = "test log 1" 125 | str2 = "test log 2" 126 | self.assertEqual(strip_html(str1), str1_stripped) 127 | log(str1_stripped) 128 | html_log(str1, str2) 129 | 130 | def test_assert_dimensionless(self): 131 | import quantities as pq 132 | 133 | from sciunit.utils import assert_dimensionless 134 | 135 | assert_dimensionless(3 * pq.s * pq.Hz) 136 | try: 137 | assert_dimensionless(3 * pq.s) 138 | except TypeError: 139 | pass 140 | else: 141 | raise Exception("Should have produced a type error") 142 | 143 | def test_dict_hash(self): 144 | from sciunit.utils import dict_hash 145 | 146 | d1 = {"a": 1, "b": 2, "c": 3} 147 | d2 = {"c": 3, "a": 1, "b": 2} 148 | dh1 = dict_hash(d1) 149 | dh2 = dict_hash(d2) 150 | self.assertTrue(type(dh1) is str) 151 | self.assertTrue(type(dh2) is str) 152 | self.assertEqual(d1, d2) 153 | 154 | def test_import_module_from_path(self): 155 | from sciunit.utils import import_module_from_path 156 | 157 | temp_file = tempfile.mkstemp(suffix=".py")[1] 158 | with open(temp_file, "w") as f: 159 | f.write("value = 42") 160 | module = import_module_from_path(temp_file) 161 | self.assertEqual(module.value, 42) 162 | 163 | def test_versioned(self): 164 | from sciunit.models.examples import ConstModel 165 | 166 | m = ConstModel(37) 167 | print("Commit hash is %s" % m.version) 168 | print("Remote URL is %s" % m.remote_url) 169 | self.assertTrue("sciunit" in m.remote_url) 170 | 171 | def test_MockDevice(self): 172 | from io import StringIO 173 | 174 | from sciunit.utils import MockDevice 175 | 176 | s = StringIO() 177 | myMD = MockDevice(s) 178 | myMD.write("test mock device writing") 179 | 180 | def test_memoize(self): 181 | from random import randint 182 | 183 | from sciunit.utils import memoize 184 | 185 | @memoize 186 | def f(a): 187 | return a + randint(0, 1000000) 188 | 189 | # Should be equal despite the random integer 190 | # because of memoization 191 | self.assertEqual(f(3), f(3)) 192 | 193 | def test_intern(self): 194 | from sciunit.utils import class_intern 195 | 196 | class N: 197 | def __init__(self, n): 198 | self.n = n 199 | 200 | five = N(5) 201 | five2 = N(5) 202 | self.assertNotEqual(five, five2) 203 | 204 | # Add the decorator to the class N. 205 | N = class_intern(N) 206 | 207 | five = N(5) 208 | five2 = N(5) 209 | self.assertEqual(five, five2) 210 | 211 | 212 | if __name__ == "__main__": 213 | unittest.main() 214 | -------------------------------------------------------------------------------- /sciunit/unit_test/validate_observation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import quantities as pq\n", 10 | "import sciunit\n", 11 | "from sciunit.errors import ObservationError" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "class MyTest(sciunit.Test):\n", 21 | " observation_schema = {'mean': {'type': 'float'}, \n", 22 | " 'std': {'type': 'float'}}" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "test = MyTest({'mean': 3.1, 'std': 1.4}) # Correctly formatted observation" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 4, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "name": "stdout", 41 | "output_type": "stream", 42 | "text": [ 43 | "{'observation': [{'std': ['must be of float type']}]}\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "try:\n", 49 | " test = MyTest({'mean': 3.1, 'std': '1.4'}) # Incorrectly formatted; note string value for 'std' key\n", 50 | "except ObservationError as e:\n", 51 | " print(e)" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 5, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "class QuantityTest(sciunit.Test):\n", 61 | " observation_schema = {'mean': {'type': 'quantity'},\n", 62 | " 'std': {'type': 'quantity'}}" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 6, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "test = QuantityTest({'mean': 3.1*pq.V, 'std': 1.4*pq.V}) # Correctly formatted observation" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 7, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "{'observation': [{'mean': ['must be of quantity type']}]}\n" 84 | ] 85 | } 86 | ], 87 | "source": [ 88 | "try:\n", 89 | " test = QuantityTest({'mean': 3.1, 'std': 1.4*pq.V}) # Incorrectly formatted; no quantity for 'mean' key\n", 90 | "except ObservationError as e:\n", 91 | " print(e)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 8, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "test = QuantityTest({'mean': 3.1*pq.mV, 'std': 1.4*pq.mV}) # Correctly formatted observation" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 9, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "class QuantityTest2(sciunit.Test):\n", 110 | " observation_schema = {'mean': {'units': True, 'required': True},\n", 111 | " 'std': {'units': True, 'min':0, 'required': True},\n", 112 | " 'n': {'type': 'integer', 'min': 1}}\n", 113 | " units = pq.V" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 10, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "test = QuantityTest2({'mean': 3.1*pq.mV, 'std': 1.4*pq.mV}) # Sould validate" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 11, 128 | "metadata": {}, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "{'observation': [{'sd': [\"Must have units of 'volt'\"]}]}\n" 135 | ] 136 | } 137 | ], 138 | "source": [ 139 | "try:\n", 140 | " test = QuantityTest2({'mean': 3.1*pq.mV, 'std': 1.4*pq.ms}) # Incorrect units for 'std' key\n", 141 | "except ObservationError as e:\n", 142 | " print(e)" 143 | ] 144 | } 145 | ], 146 | "metadata": { 147 | "kernelspec": { 148 | "display_name": "Python 3", 149 | "language": "python", 150 | "name": "python3" 151 | }, 152 | "language_info": { 153 | "codemirror_mode": { 154 | "name": "ipython", 155 | "version": 3 156 | }, 157 | "file_extension": ".py", 158 | "mimetype": "text/x-python", 159 | "name": "python", 160 | "nbconvert_exporter": "python", 161 | "pygments_lexer": "ipython3", 162 | "version": "3.5.2" 163 | } 164 | }, 165 | "nbformat": 4, 166 | "nbformat_minor": 2 167 | } 168 | -------------------------------------------------------------------------------- /sciunit/unit_test/validator_tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import quantities as pq 4 | 5 | 6 | class ValidatorTestCase(unittest.TestCase): 7 | def test_register(self): 8 | class TestClass: 9 | intValue = 0 10 | 11 | def getIntValue(self): 12 | return self.intValue 13 | 14 | from cerberus import TypeDefinition, Validator 15 | 16 | from sciunit.validators import register_quantity, register_type 17 | 18 | register_type(TestClass, "TestType1") 19 | q = pq.Quantity([1, 2, 3], "J") 20 | register_quantity(q, "TestType2") 21 | self.assertIsInstance(Validator.types_mapping["TestType1"], TypeDefinition) 22 | self.assertIsInstance(Validator.types_mapping["TestType2"], TypeDefinition) 23 | 24 | def test_ObservationValidator(self): 25 | import random 26 | 27 | from sciunit import Test 28 | from sciunit.validators import ObservationValidator 29 | 30 | long_test_list = [None] * 100 31 | for index in range(100): 32 | long_test_list[index] = random.randint(1, 1000) 33 | 34 | q = pq.Quantity([1, 2, 3], "ft") 35 | units = q.simplified.units 36 | units.name = "UnitName" 37 | 38 | testObj = Test({}) 39 | testObj.units = units 40 | testObj.observation = long_test_list 41 | obsVal = ObservationValidator(test=testObj) 42 | 43 | # test constructor 44 | self.assertRaises(BaseException, ObservationValidator) 45 | self.assertIsInstance(obsVal, ObservationValidator) 46 | 47 | # test _validate_iterable 48 | obsVal._validate_iterable(True, "key", long_test_list) 49 | self.assertRaises( 50 | BaseException, 51 | obsVal._validate_iterable, 52 | is_iterable=True, 53 | key="Test key", 54 | value=0, 55 | ) 56 | 57 | # test _validate_units 58 | self.assertRaises( 59 | BaseException, 60 | obsVal._validate_units, 61 | has_units=True, 62 | key="Test Key", 63 | value="I am not units", 64 | ) 65 | 66 | # units in test object is q.simplified.units 67 | obsVal._validate_units(has_units=True, key="TestKey", value=q) 68 | 69 | # units in test object is a dict 70 | testObj.units = {"TestKey": units} 71 | obsVal._validate_units(has_units=True, key="TestKey", value=q) 72 | 73 | print( 74 | "debug here..............................................................................." 75 | ) 76 | # Units dismatch 77 | q2 = pq.Quantity([1], "J") 78 | self.assertRaises( 79 | BaseException, 80 | obsVal._validate_units, 81 | has_units=True, 82 | key="TestKey", 83 | value=q2, 84 | ) 85 | 86 | def test_ParametersValidator(self): 87 | from sciunit.validators import ParametersValidator 88 | 89 | paraVal = ParametersValidator() 90 | 91 | # test validate_quantity 92 | q = pq.Quantity([1, 2, 3], "A") 93 | paraVal.validate_quantity(q) 94 | self.assertRaises( 95 | BaseException, paraVal.validate_quantity, "I am not a quantity" 96 | ) 97 | 98 | q = pq.Quantity([1, 2, 3], pq.s) 99 | self.assertTrue(paraVal._validate_type_time(q)) 100 | self.assertRaises(BaseException, paraVal._validate_type_voltage, q) 101 | self.assertRaises(BaseException, paraVal._validate_type_current, q) 102 | 103 | q = pq.Quantity([1, 2, 3], pq.V) 104 | self.assertTrue(paraVal._validate_type_voltage(q)) 105 | self.assertRaises(BaseException, paraVal._validate_type_time, q) 106 | self.assertRaises(BaseException, paraVal._validate_type_current, q) 107 | 108 | q = pq.Quantity([1, 2, 3], pq.A) 109 | self.assertTrue(paraVal._validate_type_current(q)) 110 | self.assertRaises(BaseException, paraVal._validate_type_voltage, q) 111 | self.assertRaises(BaseException, paraVal._validate_type_time, q) 112 | 113 | self.assertRaises( 114 | BaseException, paraVal._validate_type_current, "I am not a quantity" 115 | ) 116 | self.assertRaises( 117 | BaseException, paraVal._validate_type_voltage, "I am not a quantity" 118 | ) 119 | self.assertRaises( 120 | BaseException, paraVal._validate_type_time, "I am not a quantity" 121 | ) 122 | 123 | 124 | if __name__ == "__main__": 125 | unittest.main() 126 | -------------------------------------------------------------------------------- /sciunit/validators.py: -------------------------------------------------------------------------------- 1 | """Cerberus validator classes for SciUnit.""" 2 | 3 | import inspect 4 | from typing import Any 5 | 6 | import quantities as pq 7 | from cerberus import TypeDefinition, Validator 8 | 9 | 10 | def register_type(cls, name: str) -> None: 11 | """Register `name` as a type to validate as an instance of class `cls`. 12 | 13 | Args: 14 | cls: a class 15 | name (str): the name to be registered. 16 | """ 17 | x = TypeDefinition(name, (cls,), ()) 18 | Validator.types_mapping[name] = x 19 | 20 | 21 | def register_quantity(quantity: pq.Quantity, name: str) -> None: 22 | """Register `name` as a type to validate as an instance of the class of `quantity`. 23 | 24 | Args: 25 | quantity (pq.Quantity): a quantity. 26 | name (str): the name to be registered. 27 | """ 28 | 29 | x = TypeDefinition(name, (quantity.__class__,), ()) 30 | Validator.types_mapping[name] = x 31 | 32 | 33 | class ObservationValidator(Validator): 34 | """Cerberus validator class for observations. 35 | 36 | Attributes: 37 | test (Test): A sciunit test instance to be validated. 38 | _error (str, str): The error that happens during the validating process. 39 | """ 40 | 41 | def __init__(self, *args, **kwargs): 42 | """Constructor of ObservationValidator. 43 | 44 | Must pass `test` as a keyword argument. Cannot be a positional argument without modifications to cerberus. 45 | 46 | Raises: 47 | Exception: "Observation validator constructor must have a `test` keyword argument." 48 | """ 49 | 50 | try: 51 | self.test = kwargs["test"] 52 | except KeyError: 53 | raise Exception( 54 | ( 55 | "Observation validator constructor must have " 56 | "a `test` keyword argument" 57 | ) 58 | ) 59 | super(ObservationValidator, self).__init__(*args, **kwargs) 60 | register_type(pq.quantity.Quantity, "quantity") 61 | 62 | def _validate_iterable(self, is_iterable: bool, key: str, value: Any) -> None: 63 | """Validate fields with `iterable` key in schema set to True 64 | The rule's arguments are validated against this schema: 65 | {'type': 'boolean'} 66 | """ 67 | if is_iterable: 68 | try: 69 | iter(value) 70 | except TypeError: 71 | self._error(key, "Must be iterable (e.g. a list or array)") 72 | 73 | def _validate_units(self, has_units: bool, key: str, value: Any) -> None: 74 | """Validate fields with `units` key in schema set to True. 75 | The rule's arguments are validated against this schema: 76 | {'type': 'boolean'} 77 | """ 78 | if has_units: 79 | if isinstance(self.test.units, dict): 80 | required_units = self.test.units[key] 81 | else: 82 | required_units = self.test.units 83 | if not isinstance(value, pq.quantity.Quantity): 84 | #self._error(key, "Must be a python quantity") 85 | value *= pq.dimensionless 86 | provided_units = value.simplified.units 87 | if not isinstance(required_units, pq.Dimensionless): 88 | required_units = required_units.simplified.units 89 | else: 90 | required_units = required_units.units 91 | if not required_units == provided_units: 92 | self._error(key, "Must have units of '%s'" % self.test.units.name) 93 | 94 | 95 | class ParametersValidator(Validator): 96 | """Cerberus validator class for observations. 97 | 98 | Attributes: 99 | units_type ([type]): The type of Python quantity's unit. 100 | _error (str, str): value is not a Python quantity instance. 101 | """ 102 | 103 | # doc is needed here 104 | units_map = {"time": "s", "voltage": "V", "current": "A"} 105 | 106 | def validate_quantity(self, value: pq.quantity.Quantity) -> None: 107 | """Validate that the value is of the `Quantity` type. 108 | 109 | Args: 110 | value (pq.quantity.Quantity): The Quantity instance to be validated. 111 | """ 112 | if not isinstance(value, pq.quantity.Quantity): 113 | self._error("%s" % value, "Must be a Python quantity.") 114 | 115 | def validate_units(self, value: pq.quantity.Quantity) -> bool: 116 | """Validate units, assuming that it was called by _validate_type_*. 117 | 118 | Args: 119 | value (pq.quantity.Quantity): A python quantity instance. 120 | 121 | Returns: 122 | bool: Whether it is valid. 123 | """ 124 | self.validate_quantity(value) 125 | self.units_type = inspect.stack()[1][3].split("_")[-1] 126 | assert self.units_type, ( 127 | "`validate_units` should not be called " 128 | "directly. It should be called by a " 129 | "_validate_type_* methods that sets " 130 | "`units_type`" 131 | ) 132 | units = getattr(pq, self.units_map[self.units_type]) 133 | if not value.simplified.units == units: 134 | self._error("%s" % value, "Must have dimensions of %s." % self.units_type) 135 | return True 136 | 137 | def _validate_type_time(self, value: pq.quantity.Quantity) -> bool: 138 | """Validate fields requiring `units` of seconds. 139 | 140 | Args: 141 | value (pq.quantity.Quantity): A python quantity instance. 142 | 143 | Returns: 144 | bool: Whether it is valid. 145 | """ 146 | return self.validate_units(value) 147 | 148 | def _validate_type_voltage(self, value: pq.quantity.Quantity) -> bool: 149 | """Validate fields requiring `units` of volts. 150 | 151 | Args: 152 | value (pq.quantity.Quantity): A python quantity instance. 153 | 154 | Returns: 155 | bool: Whether it is valid. 156 | """ 157 | return self.validate_units(value) 158 | 159 | def _validate_type_current(self, value: pq.quantity.Quantity) -> bool: 160 | """Validate fields requiring `units` of amps. 161 | 162 | Args: 163 | value (pq.quantity.Quantity): A python quantity instance. 164 | 165 | Returns: 166 | bool: Whether it is valid. 167 | """ 168 | return self.validate_units(value) 169 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = sciunit 3 | version = 0.2.8 4 | description = A test-driven framework for formally validating scientific models against data. 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | url = http://sciunit.scidash.org 8 | 9 | author = Rick Gerkin 10 | author_email = rgerkin@asu.edu 11 | license = MIT 12 | classifiers = 13 | License :: OSI Approved :: MIT License 14 | Programming Language :: Python :: 3 15 | Programming Language :: Python :: 3.7 16 | Programming Language :: Python :: 3.8 17 | Programming Language :: Python :: 3.9 18 | 19 | 20 | [options] 21 | zip_safe = False 22 | packages = find: 23 | python_requires = >=3.7 24 | install_requires = 25 | bs4 26 | cerberus>=1.2 27 | deepdiff 28 | gitpython 29 | importlib-metadata 30 | ipykernel 31 | ipython 32 | jsonpickle 33 | lxml 34 | matplotlib 35 | nbconvert 36 | nbformat 37 | pandas>=0.18 38 | quantities>=0.13.0 39 | 40 | [options.entry_points] 41 | console_scripts = sciunit=sciunit.__main__:main 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import site 3 | import sys 4 | 5 | site.ENABLE_USER_SITE = "--user" in sys.argv[1:] 6 | 7 | setuptools.setup() 8 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | pip -q install coveralls 2 | if [ ! -d "../scidash" ]; then 3 | git clone -b cosmosuite http://github.com/scidash/scidash ../scidash 4 | fi 5 | # All tests listed in sciunit/unit_test/active.py will be run 6 | UNIT_TEST_SUITE="sciunit.unit_test buffer" 7 | coverage run -m --source=. --omit=*unit_test*,setup.py,.eggs $UNIT_TEST_SUITE 8 | --------------------------------------------------------------------------------