├── .github
├── release-drafter-template.yml
└── workflows
│ ├── python-package.yml
│ ├── python-publish.yml
│ └── release-drafter.yml
├── .gitignore
├── .pre-commit-config.yaml
├── CODE_OF_CONDUCT.md
├── COMMITTEE.md
├── CONTRIBUTING.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── docker
└── Dockerfile
├── docs
├── Makefile
├── README.md
├── _config.yml
├── _toc.yml
├── chapter1.ipynb
├── chapter2.ipynb
├── chapter3.ipynb
├── chapter4.ipynb
├── chapter5.ipynb
├── chapter6.ipynb
├── make.bat
├── publish.sh
├── sci-unit-tag.png
└── source
│ ├── README.md
│ ├── basics.rst
│ ├── commandline.rst
│ ├── conf.py
│ ├── index.rst
│ ├── modules.rst
│ ├── quick_tutorial.rst
│ ├── sciunit.models.rst
│ ├── sciunit.rst
│ ├── sciunit.scores.rst
│ ├── sciunit.unit_test.rst
│ └── setup.rst
├── pyproject.toml
├── requirements.txt
├── sciunit
├── __init__.py
├── __main__.py
├── base.py
├── capabilities.py
├── config.json
├── converters.py
├── errors.py
├── models
│ ├── __init__.py
│ ├── backends.py
│ ├── base.py
│ ├── examples.py
│ └── runnable.py
├── scores
│ ├── __init__.py
│ ├── base.py
│ ├── collections.py
│ ├── collections_m2m.py
│ ├── complete.py
│ └── incomplete.py
├── style.css
├── suites.py
├── tests.py
├── unit_test
│ ├── __init__.py
│ ├── __main__.py
│ ├── active.py
│ ├── backend_tests.ipynb
│ ├── backend_tests.py
│ ├── base.py
│ ├── base_tests.py
│ ├── command_line_tests.py
│ ├── config_tests.py
│ ├── converter_tests.py
│ ├── doc_tests.py
│ ├── error_tests.py
│ ├── import_tests.py
│ ├── model_tests.py
│ ├── observation_tests.py
│ ├── score_tests.py
│ ├── test_only_lower_triangle.ipynb
│ ├── test_tests.py
│ ├── utils_tests.py
│ ├── validate_observation.ipynb
│ └── validator_tests.py
├── utils.py
└── validators.py
├── setup.cfg
├── setup.py
└── test.sh
/.github/release-drafter-template.yml:
--------------------------------------------------------------------------------
1 | template: |
2 | ## What’s Changed
3 |
4 | $CHANGES
5 |
--------------------------------------------------------------------------------
/.github/workflows/python-package.yml:
--------------------------------------------------------------------------------
1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3 |
4 | name: Python package
5 |
6 | on:
7 | push:
8 | branches: [ master ]
9 | pull_request:
10 | branches: [ master ]
11 |
12 | jobs:
13 | build:
14 |
15 | runs-on: ubuntu-latest
16 | strategy:
17 | matrix:
18 | python-version: ["3.7", "3.8", "3.9", "3.10"]
19 |
20 | steps:
21 | - uses: actions/checkout@v2
22 | - name: Set up Python ${{ matrix.python-version }}
23 | uses: actions/setup-python@v2
24 | with:
25 | python-version: ${{ matrix.python-version }}
26 | - name: Install dependencies and the package
27 | run: |
28 | python -m pip install --upgrade pip
29 | pip install .
30 | - name: Test unit test
31 | run: |
32 | git clone -b cosmosuite http://github.com/scidash/scidash ../scidash
33 | python -m sciunit.unit_test buffer
34 |
--------------------------------------------------------------------------------
/.github/workflows/python-publish.yml:
--------------------------------------------------------------------------------
1 | # This workflow will upload a Python Package using Twine when a release is created
2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
3 |
4 | name: Upload Python Package
5 |
6 | on:
7 | release:
8 | types: [published]
9 |
10 | jobs:
11 | deploy:
12 |
13 | runs-on: ubuntu-latest
14 |
15 | steps:
16 | - uses: actions/checkout@v2
17 | - name: Set up Python
18 | uses: actions/setup-python@v2
19 | with:
20 | python-version: '3.x'
21 | - name: Install dependencies
22 | run: |
23 | python -m pip install --upgrade pip
24 | pip install setuptools wheel twine
25 | - name: Build and publish
26 | env:
27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
29 | run: |
30 | python setup.py sdist bdist_wheel
31 | twine upload dist/*
32 |
--------------------------------------------------------------------------------
/.github/workflows/release-drafter.yml:
--------------------------------------------------------------------------------
1 | name: Release Drafter
2 |
3 | on:
4 | push:
5 | # branches to consider in the event; optional, defaults to all
6 | branches:
7 | - master
8 | # pull_request event is required only for autolabeler
9 | pull_request:
10 | # Only following types are handled by the action, but one can default to all as well
11 | types: [opened, reopened, synchronize]
12 |
13 | jobs:
14 | update_release_draft:
15 | runs-on: ubuntu-latest
16 | steps:
17 | # (Optional) GitHub Enterprise requires GHE_HOST variable set
18 | #- name: Set GHE_HOST
19 | # run: |
20 | # echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV
21 |
22 | # Drafts your next Release notes as Pull Requests are merged into "master"
23 | - uses: release-drafter/release-drafter@v5
24 | # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
25 | with:
26 | config-name: release-drafter-template.yml
27 | # disable-autolabeler: true
28 | env:
29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
30 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.settings
2 | *.project
3 | *.pydevproject
4 | *.pyo
5 | *.pyc
6 | *.DS_Store
7 | *.class
8 | *~
9 | dist
10 | ignore
11 | .ipynb_checkpoints
12 | build
13 | *.bak
14 | *.egg-info
15 | docs/chapter*.py
16 | .coverage
17 | htmlcov
18 | Untitled*.ipynb
19 | GeneratedFiles
20 | GeneratedFiles/*.py
21 | \.idea/
22 | .vscode
23 | *scratch.ipynb*
24 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | repos:
2 | - repo: https://github.com/pycqa/isort
3 | rev: 5.8.0
4 | hooks:
5 | - id: isort
6 | args: ["--profile", "black"]
7 | - repo: https://github.com/psf/black
8 | rev: 21.5b0
9 | hooks:
10 | - id: black
11 | language_version: python3
12 | - repo: https://github.com/myint/autoflake
13 | rev: v1.4
14 | hooks:
15 | - id: autoflake
16 | args:
17 | [
18 | "--in-place",
19 | "--remove-all-unused-imports",
20 | "--ignore-init-module-imports",
21 | "--remove-unused-variables",
22 | ]
23 | exclude: |
24 | (?x)^(
25 | sciunit/unit_test/active.py|
26 | sciunit/unit_test/__main__.py
27 | )$
28 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | We ask that all contributors be
6 | - Respectful and patient.
7 | - Mindful and considerate.
8 | - Consider that our actions affect others, including colleagues and the public.
9 | - Be careful in the words that you choose!
10 | - Open with tools and dissemination efforts, and transparent in all communication.
11 | - Our work, our tools, our discussions, and our findings should be open and accessible to all.
12 | - Choose freely-available, open source tools, open access dissemination practices.
13 | - Communicate over open channels.
14 | - Welcome new interactions, new views, new ideas, and new people.
15 | - Disagreements will occur in a diverse scientific community.
16 | - But they can be resolved constructively with no harmful behavior!
17 |
18 | In the interest of fostering an open and welcoming environment, we as
19 | contributors and maintainers pledge to making participation in our project and
20 | our community a harassment-free experience for everyone, regardless of age, body
21 | size, disability, ethnicity, sex characteristics, gender identity and expression,
22 | level of experience, education, socio-economic status, nationality, personal
23 | appearance, race, religion, or sexual identity and orientation.
24 |
25 | ## Our Standards
26 |
27 | Examples of behavior that contributes to creating a positive environment
28 | include:
29 |
30 | * Using welcoming and inclusive language
31 | * Being respectful of differing viewpoints and experiences
32 | * Gracefully accepting constructive criticism
33 | * Focusing on what is best for the community
34 | * Showing empathy towards other community members
35 |
36 | Examples of unacceptable behavior by participants include:
37 |
38 | * The use of sexualized language or imagery and unwelcome sexual attention or
39 | advances
40 | * Trolling, insulting/derogatory comments, and personal or political attacks
41 | * Public or private harassment
42 | * Publishing others' private information, such as a physical or electronic
43 | address, without explicit permission
44 | * Other conduct which could reasonably be considered inappropriate in a
45 | professional setting
46 |
47 | ## Our Responsibilities
48 |
49 | Project maintainers are responsible for clarifying the standards of acceptable
50 | behavior and are expected to take appropriate and fair corrective action in
51 | response to any instances of unacceptable behavior.
52 |
53 | Project maintainers have the right and responsibility to remove, edit, or
54 | reject comments, commits, code, wiki edits, issues, and other contributions
55 | that are not aligned to this Code of Conduct, or to ban temporarily or
56 | permanently any contributor for other behaviors that they deem inappropriate,
57 | threatening, offensive, or harmful.
58 |
59 | ## Scope
60 |
61 | This Code of Conduct applies both within project spaces and in public spaces
62 | when an individual is representing the project or its community. Examples of
63 | representing a project or community include using an official project e-mail
64 | address, posting via an official social media account, or acting as an appointed
65 | representative at an online or offline event. Representation of a project may be
66 | further defined and clarified by project maintainers.
67 |
68 | ## Enforcement
69 |
70 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
71 | reported by contacting the SciUnit team leader at rick@scidash.org,
72 | or any member of the [SciUnit Executive Committee](COMMITTEE.md). All
73 | complaints will be reviewed and investigated and will result in a response that
74 | is deemed necessary and appropriate to the circumstances. The project team is
75 | obligated to maintain confidentiality with regard to the reporter of an incident.
76 | Further details of specific enforcement policies may be posted separately.
77 |
78 | Project maintainers who do not follow or enforce the Code of Conduct in good
79 | faith may face temporary or permanent repercussions as determined by other
80 | members of the project's leadership.
81 |
82 | ## Attribution
83 |
84 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
85 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
86 |
87 | [homepage]: https://www.contributor-covenant.org
88 |
89 | For answers to common questions about this code of conduct, see
90 | https://www.contributor-covenant.org/faq
91 |
--------------------------------------------------------------------------------
/COMMITTEE.md:
--------------------------------------------------------------------------------
1 | # SciUnit Executive Committee
2 |
3 | The executive committee steers the SciUnit project and reviews regular reports prepared by the chairperson.
4 | Concerns about project governance or code of conduct violations may be confidentially shared with any member of the committee.
5 |
6 | | Name | Title | Affiliation |
7 | |---|---|---|
8 | | Richard C. (Rick) Gerkin* | Associate Research Professor | School of Life Sciences, Arizona State Univ., USA |
9 | | Shailesh Appukuttan | Postdoctoral Researcher | CNRS, France |
10 | | Justas Birgiolas | Postdoctoral Fellow | Dept. of Psychology, Cornell University, USA |
11 | | Sharon M Crook | Professor | School of Mathematical and Statistical Sciences, Arizona State Univ., USA |
12 | | Andrew Davison | Senior Research Scientist, Group Leader, Neuroinformatics | CNRS, France |
13 | | Robin Gutzen | PhD Student | Research Centre Jülich, Germany |
14 | | Giovanni Idili | Chief Operating Officer | MetaCell, LLC, UK |
15 | | Cyrus Omar | Assistant Professor | Dept. of Computer Science, Univ. of Michigan, USA |
16 |
17 | *denotes current chairperson
18 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing to SciUnit
2 |
3 | ## Reporting issues
4 |
5 | When reporting issues please include as much detail as possible about your
6 | operating system, `sciunit` version, and python version. Whenever possible, please
7 | also include a brief, self-contained code example that demonstrates the problem.
8 |
9 | ## Contributing code
10 |
11 | Thanks for your interest in contributing code to SciUnit!
12 | Our developers aim to adhere to Python Enhancement Proposals (PEP) standards to ensure that code is readable, usable, and maintainable.
13 |
14 | + If this is your first time contributing to a project on GitHub, please consider reading
15 | this [example guide](https://numpy.org/devdocs/dev/index.html) to contributing code provided by the developers of numpy.
16 | + If you have contributed to other projects on GitHub you may open a pull request directly against the `dev` branch.
17 |
18 | Either way, please be sure to provide informative commit messages
19 | and ensure that each commit contains code addressing only one development goal.
20 |
21 | Writing unit tests to cover new code is encouraged but not required.
22 | Your changes can be tested using the current set of units tests by executing `test.sh` in the root directory.
23 |
24 | The python packages `isort`, `black`, and `autoflake` are being used with the `pre-commit` framework to auto-format code.
25 | If you install `pre-commit`, you can run `pre-commit run --all-files` before a pull request or commit in your shell to apply this formatting yourself.
26 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2011- Richard C. Gerkin and Cyrus Omar
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include requirements.txt
2 | include LICENSE
3 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://github.com/scidash/sciunit/actions/workflows/python-package.yml)
2 | [](http://sciunit.readthedocs.io/en/latest/?badge=master)
3 | [](https://mybinder.org/v2/gh/scidash/sciunit/master?filepath=docs%2Fchapter1.ipynb)
4 | [](https://coveralls.io/github/scidash/sciunit?branch=master)
5 | [](https://github.com/scidash/sciunit/network/dependents?dependent_type=REPOSITORY)
6 | 
7 |
8 |
9 |
10 | # SciUnit: A Test-Driven Framework for Formally Validating Scientific Models Against Data
11 |
12 | ## Concept
13 | [The conference paper](https://github.com/cyrus-/papers/raw/master/sciunit-icse14/sciunit-icse14.pdf)
14 |
15 | ## Documentation
16 | [](https://colab.research.google.com/github/scidash/sciunit/blob/master/docs/chapter1.ipynb)
17 | [Jupyter Tutorials](https://scidash.org/sciunit/README.html)
18 | [API Documentation](http://sciunit.rtfd.io)
19 |
20 | ## Installation
21 | ```python
22 | pip install sciunit
23 | ```
24 | or
25 | ```python
26 | conda install -c conda-forge sciunit
27 | ```
28 |
29 | ## Basic Usage
30 | ```python
31 | my_model = MyModel(**my_args) # Instantiate a class that wraps your model of interest.
32 | my_test = MyTest(**my_params) # Instantiate a test that you write.
33 | score = my_test.judge() # Runs the test and return a rich score containing test results and more.
34 | ```
35 |
36 | ## Domain-specific libraries and information
37 | [NeuronUnit](https://github.com/scidash/neuronunit) for neuron and ion channel physiology
38 | See others [here](https://github.com/scidash/sciunit/network/dependents?dependent_type=REPOSITORY)
39 |
40 | ## Mailing List
41 | There is a [mailing list](https://groups.google.com/forum/?fromgroups#!forum/sciunit) for announcements and discussion.
42 | Please join it if you are at all interested!
43 |
44 | ## Contributors
45 | * [Rick Gerkin](http://rick.gerk.in), Arizona State University (School of Life Science)
46 | * [Cyrus Omar](http://cs.cmu.edu/~comar), Carnegie Mellon University (Dept. of Computer Science)
47 |
48 | ## Reproducible Research ID
49 | RRID:[SCR_014528](https://scicrunch.org/resources/Any/record/nlx_144509-1/3faed1d9-6579-5da6-b4b4-75a5077656bb/search?q=sciunit&l=sciunit)
50 |
51 | ## License
52 | SciUnit is released under the permissive [MIT license](https://opensource.org/licenses/MIT), requiring only attribution in derivative works. See the LICENSE file for terms.
53 |
--------------------------------------------------------------------------------
/docker/Dockerfile:
--------------------------------------------------------------------------------
1 | # sciunit
2 | # author Rick Gerkin rgerkin@asu.edu
3 | FROM jupyter/datascience-notebook
4 |
5 | RUN apt-get update
6 | RUN apt-get install openssh-client -y # Needed for Versioned unit tests to pass
7 | RUN git clone http://github.com/scidash/sciunit
8 | WORKDIR sciunit
9 | RUN pip install -e .
10 | RUN git clone -b cosmosuite http://github.com/scidash/scidash ../scidash
11 | USER $NB_USER
12 | RUN sh test.sh
13 |
14 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Minimal makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line.
5 | SPHINXOPTS =
6 | SPHINXBUILD = sphinx-build
7 | SPHINXPROJ = SciUnit
8 | SOURCEDIR = source
9 | BUILDDIR = build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
--------------------------------------------------------------------------------
/docs/README.md:
--------------------------------------------------------------------------------
1 | ### New to SciUnit?
2 | - Start with [Chapter 1](https://scidash.org/sciunit/chapter1.html) ([source](https://github.com/scidash/sciunit/blob/master/docs/chapter1.ipynb)) of the tutorial.
3 | - View [our slides](https://github.com/scidash/sciunit/blob/master/docs/workshop-tutorial.ipynb).
4 | - Or run either of the above [interactively](https://mybinder.org/v2/gh/scidash/sciunit/b491f545854040b5934b3898e7b9f7089089041f).
5 |
--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | # in _config.yml
2 | title: "SciUnit: A python library for data-driven model validation testing"
3 | logo: sci-unit-tag.png
4 | execute:
5 | execute_notebooks: "off"
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/docs/_toc.yml:
--------------------------------------------------------------------------------
1 | # In _toc.yml
2 | format: jb-book
3 | root: README
4 | parts:
5 | - caption: Tutorial
6 | numbered: True
7 | chapters:
8 | - file: chapter1
9 | - file: chapter2
10 | - file: chapter3
11 | - file: chapter4
12 | - file: chapter5
13 | - file: chapter6
14 | - caption: Full Docs
15 | chapters:
16 | - url: https://sciunit.rtfd.io
17 |
--------------------------------------------------------------------------------
/docs/chapter1.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "\n",
8 | "\n",
9 | "\n",
10 | "
\n",
11 | "\n",
12 | "# Chapter 1. What is SciUnit?\n",
13 | "Everyone hopes that their model has some correspondence with reality. Usually, checking whether this is true is done informally.\n",
14 | "### SciUnit makes this formal and transparent.\n",
15 | "SciUnit is a framework for validating scientific models by creating experimental-data-driven unit tests. "
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {},
21 | "source": [
22 | "#### If you are using this file in Google Colab, this block of code can help you install sciunit from PyPI in Colab environment."
23 | ]
24 | },
25 | {
26 | "cell_type": "code",
27 | "execution_count": 1,
28 | "metadata": {
29 | "execution": {
30 | "iopub.execute_input": "2021-05-04T17:17:26.797848Z",
31 | "iopub.status.busy": "2021-05-04T17:17:26.796560Z",
32 | "iopub.status.idle": "2021-05-04T17:17:29.165793Z",
33 | "shell.execute_reply": "2021-05-04T17:17:29.167482Z"
34 | }
35 | },
36 | "outputs": [],
37 | "source": [
38 | "!pip install -q sciunit"
39 | ]
40 | },
41 | {
42 | "cell_type": "markdown",
43 | "metadata": {},
44 | "source": [
45 | "After installation, let's begin with importing sciunit."
46 | ]
47 | },
48 | {
49 | "cell_type": "code",
50 | "execution_count": 2,
51 | "metadata": {
52 | "execution": {
53 | "iopub.execute_input": "2021-05-04T17:17:29.175051Z",
54 | "iopub.status.busy": "2021-05-04T17:17:29.174070Z",
55 | "iopub.status.idle": "2021-05-04T17:17:30.011470Z",
56 | "shell.execute_reply": "2021-05-04T17:17:30.009992Z"
57 | }
58 | },
59 | "outputs": [],
60 | "source": [
61 | "import sciunit"
62 | ]
63 | },
64 | {
65 | "cell_type": "markdown",
66 | "metadata": {},
67 | "source": [
68 | "### What does testing look like? \n",
69 | "We'll start with a simple example from the history of cosmology, where we know that better models displaced their predecessors.
\n",
70 | "Suppose we have a test suite called \"Saturn Suite\" that aims to test cosmological models for their correspondence to empirical data about the planet Saturn.
\n",
71 | "*Everything in this example is hypothetical, but once you understand the basic ideas you can visit the [documentation for NeuronUnit](https://github.com/scidash/neuronunit/blob/master/docs/chapter1.ipynb) to see some working, interactive examples from a different domain (neuron and ion channel models).* "
72 | ]
73 | },
74 | {
75 | "cell_type": "markdown",
76 | "metadata": {},
77 | "source": [
78 | "```python\n",
79 | "from saturnsuite.tests import position_test,velocity_test,eccentricity_test # Examples of test classes. \n",
80 | "```"
81 | ]
82 | },
83 | {
84 | "cell_type": "markdown",
85 | "metadata": {},
86 | "source": [
87 | "There's nothing specific to Saturn about position, velocity, or eccentricity. They could apply to any cosmological body.\n",
88 | "### SciUnit test classes used with in one scientific domain (like cosmology) are located in a discipline-specific library. \n",
89 | "In this case, the test classes (hypothetically) come from a SciUnit library called CosmoUnit, and are instantiated with data specific to Saturn, in order to create tests specific to a model's predictions about Saturn:"
90 | ]
91 | },
92 | {
93 | "cell_type": "markdown",
94 | "metadata": {},
95 | "source": [
96 | "```python\n",
97 | "'''\n",
98 | "saturnsuite/tests.py # Tests for the Saturn suite. \n",
99 | "'''\n",
100 | "from . import saturn_data # Hypothetical library containing Saturn data. \n",
101 | "from cosmounit import PositionTest, VelocityTest, EccentricityyTest # Cosmounit is an external library. \n",
102 | "position_test = PositionTest(observation=saturn_data.position)\n",
103 | "velocity_test = VelocityTest(observation=saturn_data.velocity)\n",
104 | "eccentricity_test = EccentricityTest(observation=saturn_data.eccentricity)\n",
105 | "```"
106 | ]
107 | },
108 | {
109 | "cell_type": "markdown",
110 | "metadata": {},
111 | "source": [
112 | "This means the test *classes* are data-agnostic, but the test *instances* encode the data we want a model to recapitulate."
113 | ]
114 | },
115 | {
116 | "cell_type": "markdown",
117 | "metadata": {},
118 | "source": [
119 | "Next, let's load some models that aim to predict the cosmological features being assessed by the tests above."
120 | ]
121 | },
122 | {
123 | "cell_type": "markdown",
124 | "metadata": {},
125 | "source": [
126 | "```python\n",
127 | "from saturnsuite.models import ptolemy_model, copernicus_model, kepler_model, newton_model # Examples of models. \n",
128 | "```"
129 | ]
130 | },
131 | {
132 | "cell_type": "markdown",
133 | "metadata": {},
134 | "source": [
135 | "Ptolemy's, Copernicus's, Kepler's, or Newton's models could similarly apply to any cosmological body.\n",
136 | "So these model classes are found in CosmoUnit, and the Saturn Suite contains model instances parameterized to emit predictions about Saturn specifically."
137 | ]
138 | },
139 | {
140 | "cell_type": "markdown",
141 | "metadata": {},
142 | "source": [
143 | "```python\n",
144 | "'''\n",
145 | "saturnsuite/models.py # Models for the Saturn suite. \n",
146 | "'''\n",
147 | "from cosmounit import PtolemyModel, CopernicusModel, KeplerModel, NewtonModel \n",
148 | "ptolemy_model = PtolemyModel(planet='Saturn')\n",
149 | "copernicus_model = CopernicusModel(planet='Saturn')\n",
150 | "kepler_model = KeplerModel(planet='Saturn')\n",
151 | "newton_model = NewtonModel(planet='Saturn')\n",
152 | "```"
153 | ]
154 | },
155 | {
156 | "cell_type": "markdown",
157 | "metadata": {},
158 | "source": [
159 | "In the above each model takes a keyword argument 'planet' that determines about what planet the model will make predicitons."
160 | ]
161 | },
162 | {
163 | "cell_type": "markdown",
164 | "metadata": {},
165 | "source": [
166 | "### All of our tests can be organized into a suite to compare results across related tests. "
167 | ]
168 | },
169 | {
170 | "cell_type": "markdown",
171 | "metadata": {},
172 | "source": [
173 | "```python\n",
174 | "'''\n",
175 | "saturnsuite/suites.py # Tests for the Saturn suite. \n",
176 | "'''\n",
177 | "import sciunit\n",
178 | "from .tests import position_test, velocity_test, eccentricity_test\n",
179 | "saturn_motion_suite = sciunit.TestSuite([position_test, velocity_test, eccentricity_test])\n",
180 | "suites = (saturn_motion_suite,)\n",
181 | "```"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "### Now we can execute this entire test suite against our models. "
189 | ]
190 | },
191 | {
192 | "cell_type": "markdown",
193 | "metadata": {},
194 | "source": [
195 | "```python\n",
196 | "from saturn_suite.suites import saturn_motion_suite\n",
197 | "saturn_motion_suite.judge([ptolemy_model, copernicus_model, kepler_model, newton_model])\n",
198 | "```"
199 | ]
200 | },
201 | {
202 | "cell_type": "markdown",
203 | "metadata": {},
204 | "source": [
205 | "The exact output will depend on your preferences (terminal, HTML, etc.) but the figure below illustrates both the results you get (center table) and the relationship between the components listed here. "
206 | ]
207 | },
208 | {
209 | "cell_type": "markdown",
210 | "metadata": {},
211 | "source": [
212 | ""
213 | ]
214 | },
215 | {
216 | "cell_type": "markdown",
217 | "metadata": {},
218 | "source": [
219 | "The figure above also refers to SciDash, an in-development portal for accessing public test results, but for the remainder of this tutorial, we will focus on model/test development, execution, and visualization on your own machine. "
220 | ]
221 | },
222 | {
223 | "cell_type": "markdown",
224 | "metadata": {},
225 | "source": [
226 | "### In the next section we'll see how to create models and tests from scratch in SciUnit. \n",
227 | "### Onto [Chapter 2](chapter2.ipynb)!"
228 | ]
229 | }
230 | ],
231 | "metadata": {
232 | "kernelspec": {
233 | "display_name": "Python 3",
234 | "language": "python",
235 | "name": "python3"
236 | },
237 | "language_info": {
238 | "codemirror_mode": {
239 | "name": "ipython",
240 | "version": 3
241 | },
242 | "file_extension": ".py",
243 | "mimetype": "text/x-python",
244 | "name": "python",
245 | "nbconvert_exporter": "python",
246 | "pygments_lexer": "ipython3",
247 | "version": "3.9.4"
248 | }
249 | },
250 | "nbformat": 4,
251 | "nbformat_minor": 4
252 | }
253 |
--------------------------------------------------------------------------------
/docs/chapter3.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "\n",
8 | "\n",
9 | "
\n",
10 | "\n",
11 | "# Chapter 3. Testing with help from the SciUnit standard library\n",
12 | "(or [back to Chapter 2](chapter2.ipynb))"
13 | ]
14 | },
15 | {
16 | "cell_type": "markdown",
17 | "metadata": {},
18 | "source": [
19 | "#### If you are using this file in Google Colab, this block of code can help you install sciunit from PyPI in Colab environment."
20 | ]
21 | },
22 | {
23 | "cell_type": "code",
24 | "execution_count": 1,
25 | "metadata": {},
26 | "outputs": [],
27 | "source": [
28 | "!pip install -q sciunit"
29 | ]
30 | },
31 | {
32 | "cell_type": "code",
33 | "execution_count": 2,
34 | "metadata": {},
35 | "outputs": [],
36 | "source": [
37 | "import sciunit"
38 | ]
39 | },
40 | {
41 | "cell_type": "markdown",
42 | "metadata": {},
43 | "source": [
44 | "### In this chapter we will use the same toy model in Chapter 1 but write a more interesting test with additional features included in SciUnit. "
45 | ]
46 | },
47 | {
48 | "cell_type": "code",
49 | "execution_count": 3,
50 | "metadata": {},
51 | "outputs": [],
52 | "source": [
53 | "from sciunit.models.examples import ConstModel # One of many dummy models included for illustration. \n",
54 | "const_model_37 = ConstModel(37, name=\"Constant Model 37\")"
55 | ]
56 | },
57 | {
58 | "cell_type": "markdown",
59 | "metadata": {},
60 | "source": [
61 | "Now let's write a test that validates the `observation` and returns more informative `score` type."
62 | ]
63 | },
64 | {
65 | "cell_type": "code",
66 | "execution_count": 4,
67 | "metadata": {},
68 | "outputs": [],
69 | "source": [
70 | "from sciunit.capabilities import ProducesNumber\n",
71 | "from sciunit.scores import ZScore # One of many SciUnit score types. \n",
72 | "from sciunit.errors import ObservationError # An exception class raised when a test is instantiated \n",
73 | " # with an invalid observation.\n",
74 | " \n",
75 | "class MeanTest(sciunit.Test):\n",
76 | " \"\"\"Tests if the model predicts \n",
77 | " the same number as the observation.\"\"\" \n",
78 | " \n",
79 | " required_capabilities = (ProducesNumber,) # The one capability required for a model to take this test. \n",
80 | " score_type = ZScore # This test's 'judge' method will return a BooleanScore. \n",
81 | " \n",
82 | " def validate_observation(self, observation):\n",
83 | " if type(observation) is not dict:\n",
84 | " raise ObservationError(\"Observation must be a python dictionary\")\n",
85 | " if 'mean' not in observation:\n",
86 | " raise ObservationError(\"Observation must contain a 'mean' entry\")\n",
87 | " \n",
88 | " def generate_prediction(self, model):\n",
89 | " return model.produce_number() # The model has this method if it inherits from the 'ProducesNumber' capability.\n",
90 | " \n",
91 | " def compute_score(self, observation, prediction):\n",
92 | " score = ZScore.compute(observation,prediction) # Compute and return a ZScore object. \n",
93 | " score.description = (\"A z-score corresponding to the normalized location of the observation \"\n",
94 | " \"relative to the predicted distribution.\")\n",
95 | " return score"
96 | ]
97 | },
98 | {
99 | "cell_type": "markdown",
100 | "metadata": {},
101 | "source": [
102 | "We've done two new things here:\n",
103 | "- The optional `validate_observation` method checks the `observation` to make sure that it is the right type, that it has the right attributes, etc. This can be used to ensures that the `observation` is exactly as the other core test methods expect. If we don't provide the right kind of observation:\n",
104 | "```python\n",
105 | "-> mean_37_test = MeanTest(37, name='=37')\n",
106 | "ObservationError: Observation must be a python dictionary\n",
107 | "```\n",
108 | "then we get an error. In contrast, this is what our test was looking for:"
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": 5,
114 | "metadata": {},
115 | "outputs": [],
116 | "source": [
117 | "observation = {'mean':37.8, 'std':2.1}\n",
118 | "mean_37_test = MeanTest(observation, name='=37')"
119 | ]
120 | },
121 | {
122 | "cell_type": "markdown",
123 | "metadata": {},
124 | "source": [
125 | "- Instead of returning a `BooleanScore`, encoding a `True`/`False` value, we return a `ZScore` encoding a more quantitative summary of the relationship between the observation and the prediction. When we execute the test:"
126 | ]
127 | },
128 | {
129 | "cell_type": "code",
130 | "execution_count": 6,
131 | "metadata": {},
132 | "outputs": [],
133 | "source": [
134 | "score = mean_37_test.judge(const_model_37)"
135 | ]
136 | },
137 | {
138 | "cell_type": "markdown",
139 | "metadata": {},
140 | "source": [
141 | "Then we get a more quantitative summary of the results:"
142 | ]
143 | },
144 | {
145 | "cell_type": "code",
146 | "execution_count": 7,
147 | "metadata": {},
148 | "outputs": [],
149 | "source": [
150 | "score.summarize()"
151 | ]
152 | },
153 | {
154 | "cell_type": "code",
155 | "execution_count": 8,
156 | "metadata": {},
157 | "outputs": [
158 | {
159 | "data": {
160 | "text/plain": [
161 | "'A z-score corresponding to the normalized location of the observation relative to the predicted distribution.'"
162 | ]
163 | },
164 | "execution_count": 8,
165 | "metadata": {},
166 | "output_type": "execute_result"
167 | }
168 | ],
169 | "source": [
170 | "score.describe()"
171 | ]
172 | }
173 | ],
174 | "metadata": {
175 | "kernelspec": {
176 | "display_name": "Python 3",
177 | "language": "python",
178 | "name": "python3"
179 | },
180 | "language_info": {
181 | "codemirror_mode": {
182 | "name": "ipython",
183 | "version": 3
184 | },
185 | "file_extension": ".py",
186 | "mimetype": "text/x-python",
187 | "name": "python",
188 | "nbconvert_exporter": "python",
189 | "pygments_lexer": "ipython3",
190 | "version": "3.9.4"
191 | }
192 | },
193 | "nbformat": 4,
194 | "nbformat_minor": 4
195 | }
196 |
--------------------------------------------------------------------------------
/docs/chapter4.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "\n",
8 | "\n",
9 | "
\n",
10 | "\n",
11 | "# Chapter 4. Example of RunnableModel and Backend\n",
12 | "(or [back to Chapter 3](chapter3.ipynb))"
13 | ]
14 | },
15 | {
16 | "cell_type": "markdown",
17 | "metadata": {},
18 | "source": [
19 | "#### If you are using this file in Google Colab, this block of code can help you install sciunit from PyPI in Colab environment."
20 | ]
21 | },
22 | {
23 | "cell_type": "code",
24 | "execution_count": 1,
25 | "metadata": {},
26 | "outputs": [],
27 | "source": [
28 | "!pip install -q sciunit"
29 | ]
30 | },
31 | {
32 | "cell_type": "markdown",
33 | "metadata": {},
34 | "source": [
35 | "Beside the usual model in previous sections, let’s create a model that run a Backend instance to simulate and obtain results.\n",
36 | "\n",
37 | "Firstly, import necessary components from SciUnit package."
38 | ]
39 | },
40 | {
41 | "cell_type": "code",
42 | "execution_count": 2,
43 | "metadata": {},
44 | "outputs": [],
45 | "source": [
46 | "import sciunit, random\n",
47 | "from sciunit import Test\n",
48 | "from sciunit.capabilities import Runnable\n",
49 | "from sciunit.scores import BooleanScore\n",
50 | "from sciunit.models import RunnableModel\n",
51 | "from sciunit.models.backends import register_backends, Backend"
52 | ]
53 | },
54 | {
55 | "cell_type": "markdown",
56 | "metadata": {},
57 | "source": [
58 | "Let’s define subclasses of SciUnit Backend, Test, and Model."
59 | ]
60 | },
61 | {
62 | "cell_type": "markdown",
63 | "metadata": {},
64 | "source": [
65 | "Note that:\n",
66 | "1. A SciUnit Backend subclass should implement _backend_run method.\n",
67 | "2. A SciUnit Backend subclass should implement run method."
68 | ]
69 | },
70 | {
71 | "cell_type": "code",
72 | "execution_count": 3,
73 | "metadata": {},
74 | "outputs": [],
75 | "source": [
76 | "class RandomNumBackend(Backend):\n",
77 | " '''generate a random integer between min and max'''\n",
78 | "\n",
79 | " def set_run_params(self, **run_params):\n",
80 | "\n",
81 | " # get min from run_params, if not exist, then 0.\n",
82 | " self.min = run_params.get('min', 0)\n",
83 | "\n",
84 | " # get max from run_params, if not exist, then self.min + 100.\n",
85 | " self.max = run_params.get('max', self.min + 100)\n",
86 | "\n",
87 | " def _backend_run(self):\n",
88 | " # generate and return random integer between min and max.\n",
89 | " return random.randint(self.min, self.max)\n",
90 | "\n",
91 | "class RandomNumModel(RunnableModel):\n",
92 | " \"\"\"A model that always produces a constant number as output.\"\"\"\n",
93 | "\n",
94 | " def run(self):\n",
95 | " self.results = self._backend.backend_run()\n",
96 | "\n",
97 | "\n",
98 | "class RangeTest(Test):\n",
99 | " \"\"\"Tests if the model predicts the same number as the observation.\"\"\"\n",
100 | "\n",
101 | " # Default Runnable Capability for RunnableModel\n",
102 | " required_capabilities = (Runnable,)\n",
103 | "\n",
104 | " # This test's 'judge' method will return a BooleanScore.\n",
105 | " score_type = BooleanScore\n",
106 | "\n",
107 | " def generate_prediction(self, model):\n",
108 | " model.run()\n",
109 | " return model.results\n",
110 | "\n",
111 | " def compute_score(self, observation, prediction):\n",
112 | " score = BooleanScore(\n",
113 | " observation['min'] <= prediction and observation['max'] >= prediction\n",
114 | " )\n",
115 | " return score"
116 | ]
117 | },
118 | {
119 | "cell_type": "markdown",
120 | "metadata": {},
121 | "source": [
122 | "Let’s define the model instance named ``model 1``."
123 | ]
124 | },
125 | {
126 | "cell_type": "code",
127 | "execution_count": 4,
128 | "metadata": {},
129 | "outputs": [],
130 | "source": [
131 | "model = RandomNumModel(\"model 1\")"
132 | ]
133 | },
134 | {
135 | "cell_type": "markdown",
136 | "metadata": {},
137 | "source": [
138 | "We must register any backend isntance in order to use it in model instances.\n",
139 | "\n",
140 | "``set_backend`` and ``set_run_params`` methods can help us to set the run-parameters in the model and its backend."
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": 5,
146 | "metadata": {},
147 | "outputs": [],
148 | "source": [
149 | "register_backends({\"Random Number\": RandomNumBackend})\n",
150 | "model.set_backend(\"Random Number\")\n",
151 | "model.set_run_params(min=1, max=10)"
152 | ]
153 | },
154 | {
155 | "cell_type": "markdown",
156 | "metadata": {},
157 | "source": [
158 | "Next, create an observation that requires the generated random integer between 1 and 10 and a test instance that use the observation and against the model\n",
159 | "\n",
160 | "Then we get a more quantitative summary of the results:"
161 | ]
162 | },
163 | {
164 | "cell_type": "code",
165 | "execution_count": 6,
166 | "metadata": {},
167 | "outputs": [],
168 | "source": [
169 | "observation = {'min': 1, 'max': 10}\n",
170 | "oneToTenTest = RangeTest(observation, \"test 1\")\n",
171 | "score = oneToTenTest.judge(model)"
172 | ]
173 | },
174 | {
175 | "cell_type": "markdown",
176 | "metadata": {},
177 | "source": [
178 | "print the score, and we can see the result."
179 | ]
180 | },
181 | {
182 | "cell_type": "code",
183 | "execution_count": 7,
184 | "metadata": {},
185 | "outputs": [
186 | {
187 | "name": "stdout",
188 | "output_type": "stream",
189 | "text": [
190 | "Pass\n"
191 | ]
192 | }
193 | ],
194 | "source": [
195 | "print(score)"
196 | ]
197 | }
198 | ],
199 | "metadata": {
200 | "kernelspec": {
201 | "display_name": "Python 3",
202 | "language": "python",
203 | "name": "python3"
204 | },
205 | "language_info": {
206 | "codemirror_mode": {
207 | "name": "ipython",
208 | "version": 3
209 | },
210 | "file_extension": ".py",
211 | "mimetype": "text/x-python",
212 | "name": "python",
213 | "nbconvert_exporter": "python",
214 | "pygments_lexer": "ipython3",
215 | "version": "3.9.4"
216 | }
217 | },
218 | "nbformat": 4,
219 | "nbformat_minor": 4
220 | }
221 |
--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | pushd %~dp0
4 |
5 | REM Command file for Sphinx documentation
6 |
7 | if "%SPHINXBUILD%" == "" (
8 | set SPHINXBUILD=sphinx-build
9 | )
10 | set SOURCEDIR=source
11 | set BUILDDIR=build
12 |
13 | if "%1" == "" goto help
14 |
15 | %SPHINXBUILD% >NUL 2>NUL
16 | if errorlevel 9009 (
17 | echo.
18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
19 | echo.installed, then set the SPHINXBUILD environment variable to point
20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you
21 | echo.may add the Sphinx directory to PATH.
22 | echo.
23 | echo.If you don't have Sphinx installed, grab it from
24 | echo.http://sphinx-doc.org/
25 | exit /b 1
26 | )
27 |
28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
29 | goto end
30 |
31 | :help
32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
33 |
34 | :end
35 | popd
36 |
--------------------------------------------------------------------------------
/docs/publish.sh:
--------------------------------------------------------------------------------
1 | jupyter-book build .
2 | ghp-import -n -p -f _build/html
3 |
--------------------------------------------------------------------------------
/docs/sci-unit-tag.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scidash/sciunit/1a2ca478512ef80e38259b8bb6e12f714c27e904/docs/sci-unit-tag.png
--------------------------------------------------------------------------------
/docs/source/README.md:
--------------------------------------------------------------------------------
1 | # Recipe for building the API docs from scratch:
2 |
3 | updating .rst files with sphinx-apidoc
4 | Assuming current working directory is `sciunit/docs/source`.
5 |
6 | Linux Bash shell commands:
7 | ```
8 | rm modules.rst sciunit.models.rst sciunit.rst sciunit.scores.rst sciunit.unit_test.rst
9 | cd ..
10 | sphinx-apidoc -o "./source" "../sciunit"
11 | sphinx-build -b html ./source ./build
12 | ```
13 | Windows PowerShell commands:
14 | ```
15 | rm modules.rst, sciunit.models.rst, sciunit.rst, sciunit.scores.rst, sciunit.unit_test.rst
16 | cd ..
17 | sphinx-apidoc -o "./source" "../sciunit"
18 | sphinx-build -b html ./source ./build
19 | ```
20 |
--------------------------------------------------------------------------------
/docs/source/basics.rst:
--------------------------------------------------------------------------------
1 | SciUnit basics
2 | ==============
3 |
4 | This page will give you a basic view of the SciUnit project,
5 | and you can read the quick tutorials for some simple examples.
6 |
7 | The major parts of SciUnit are Score, Test, and Model.
8 |
9 | Model
10 | ------
11 |
12 | ``Model`` is the abstract base class for sciunit models. Generally, a model instance can
13 | generate predicted or simulated results of some scientific fact.
14 |
15 |
16 | Runnable Model
17 |
18 | ``Runnable model`` is a kind of model that implements Runnable
19 | capability, and it can be executed to simulate and output results.
20 |
21 |
22 | Backend
23 |
24 | After being registered by ``register_backends`` function, a ``Backend`` instance
25 | can be executed by a Runnable Model at the back end. It usually does some background
26 | computing for the runnable model.
27 |
28 |
29 | Score
30 | ------
31 |
32 | ``Score`` is the abstract base class for scores. The instance of it (or its subclass) can give some types of
33 | results for test and/or test suite against the models.
34 |
35 | The complete scores type in SciUnit are ``BooleanScore``, ``ZScore``, ``CohenDScore``,
36 | ``RatioScore``, ``PercentScore``, and ``FloatScore``.
37 |
38 | Each type of score has their own features and advantage.
39 |
40 | There are also incomplete score types. These type does not contain any
41 | information regarding how good the model is, but the existing of them means there are
42 | some issues during testing or computing process. They are ``NoneScore``, ``TBDScore``, ``NAScore``,
43 | and ``InsufficientDataScore``
44 |
45 |
46 |
47 | ScoreArray, ScoreArrayM2M
48 |
49 | Can be used like this, assuming n tests and m models:
50 |
51 | >>> sm[test]
52 | (score_1, ..., score_m)
53 |
54 | >>> sm[model]
55 | (score_1, ..., score_n)
56 |
57 |
58 | ``ScoreArray`` represents an array of scores derived from a test suite.
59 | Extends the pandas Series such that items are either
60 | models subject to a test or tests taken by a model.
61 | Also displays and computes score summaries in sciunit-specific ways.
62 |
63 | ``ScoreArrayM2M`` represents an array of scores derived from ``TestM2M``.
64 | Extends the pandas Series such that items are either
65 | models subject to a test or the test itself.
66 |
67 | ScoreMatrix, ScoreMatrixM2M
68 |
69 | Can be used like this, assuming n tests and m models:
70 |
71 | >>> sm[test]
72 | (score_1, ..., score_m)
73 |
74 | >>> sm[model]
75 | (score_1, ..., score_n)
76 |
77 |
78 | ``ScoreMatrix`` represents a matrix of scores derived from a test suite.
79 | Extends the pandas DataFrame such that tests are columns and models
80 | are the index. Also displays and compute score summaries in sciunit-specific ways.
81 |
82 | ``ScoreMatrixM2M`` represents a matrix of scores derived from ``TestM2M``.
83 | Extends the pandas DataFrame such that models/observation are both
84 | columns and the index.
85 |
86 |
87 | Test, TestM2M
88 | --------------
89 |
90 | ``Test`` is a abstract base class for tests.
91 |
92 | ``TestM2M`` is an abstract class for handling tests involving multiple models.
93 |
94 | A test instance contains some observations which are considered as the fact.
95 | The test instance can test the model by comparing the predictions with the observations
96 | and generate a specific type of score.
97 |
98 | Enables comparison of model to model predictions, and also against
99 | experimental reference data (optional).
100 |
101 | Note: ``TestM2M`` would typically be used when handling mutliple (>2)
102 | models, with/without experimental reference data. For single model
103 | tests, you can use the 'Test' class.
104 |
105 | TestSuite
106 | ----------
107 |
108 | A collection of tests. The instance of ``TestSuite`` can perform similar things that a test instance can do.
109 |
110 | Converter
111 | ----------
112 |
113 | A ``Converter`` instance can be used to convert a score between two types.
114 | It can be included in a test instance.
115 |
116 | Capability
117 | -----------
118 |
119 | ``Capability`` is the abstract base class for sciunit capabilities.
120 | A capability instance can be included in a test instance to ensure the
121 | model, which is tested by the test instance, implements some methods.
122 |
--------------------------------------------------------------------------------
/docs/source/commandline.rst:
--------------------------------------------------------------------------------
1 | Config File And Using SciUnit In A Shell
2 | =============================================
3 |
4 | Create Config File And Execute Tests In A Shell
5 | ----------------------------------------------------------------
6 |
7 | We can build a scientific computing project with a SciUnit config file.
8 | Then, we will be able to run sciunit In A Shell
9 |
10 | Here is an example of well written SciUnit config file.
11 | This file was generated by executing ``sciunit create`` in the shell.
12 | A SciUnit config file is always named ``sciunit.ini``.
13 |
14 | .. code-block:: python
15 |
16 | [misc]
17 | config-version = 1.0
18 | nb-name = scidash
19 |
20 | [root]
21 | path = .
22 |
23 | [models]
24 | module = models
25 |
26 | [tests]
27 | module = tests
28 |
29 | [suites]
30 | module = suites
31 |
32 | ``config-version`` is the version of the config file.
33 |
34 | ``nb-name`` is the name of the IPython Notebook file that can be create with ``sciunit make-nb``.
35 |
36 | ``root`` is the root of the project. The ``path`` is the path to the project from the directory that contains this config file.
37 |
38 | ``module`` in the ``models`` section is the path from the root of the project to the file that contains ``models``, which is a list of ``Model`` instances.
39 |
40 | ``module`` in the ``tests`` section is the path the root of the project to the file that contains ``tests``, which is a list of ``Test`` instances.
41 |
42 | ``module`` in the ``suites`` section is the path the root of the project to the file that contains ``suites``, which is a list of ``TestSuite`` instances.
43 |
44 | Let's use the config file above and create corresponding files that contain definitions models, tests, and suites.
45 |
46 | In the root directory of the project, let's create three files.
47 |
48 | **tests.py**
49 |
50 | .. code-block:: python
51 |
52 | import sciunit
53 | from sciunit.scores import BooleanScore
54 | from sciunit.capabilities import ProducesNumber
55 |
56 |
57 |
58 | class EqualsTest(sciunit.Test):
59 | """Tests if the model predicts
60 | the same number as the observation."""
61 |
62 | required_capabilities = (ProducesNumber,)
63 | score_type = BooleanScore
64 |
65 | def generate_prediction(self, model):
66 | return model.produce_number() # The model has this method if it inherits from the 'ProducesNumber' capability.
67 |
68 | def compute_score(self, observation, prediction):
69 | score = self.score_type(observation['value'] == prediction)
70 | score.description = 'Passing score if the prediction equals the observation'
71 | return score
72 |
73 | tests = []
74 |
75 | **suites.py**
76 |
77 | .. code-block:: python
78 |
79 | import sciunit
80 | from tests import EqualsTest
81 |
82 |
83 | equals_1_test = EqualsTest({'value':1}, name='=1')
84 | equals_2_test = EqualsTest({'value':2}, name='=2')
85 | equals_37_test = EqualsTest({'value':37}, name='=37')
86 |
87 | equals_suite = sciunit.TestSuite([equals_1_test, equals_2_test, equals_37_test], name="Equals test suite")
88 |
89 | suites = [equals_suite]
90 |
91 | **models.py**
92 |
93 | .. code-block:: python
94 |
95 | import sciunit
96 | from sciunit.capabilities import ProducesNumber
97 |
98 |
99 | class ConstModel(sciunit.Model,
100 | ProducesNumber):
101 | """A model that always produces a constant number as output."""
102 |
103 | def __init__(self, constant, name=None):
104 | self.constant = constant
105 | super(ConstModel, self).__init__(name=name, constant=constant)
106 |
107 | def produce_number(self):
108 | return self.constant
109 |
110 | const_model_1 = ConstModel(1, name='Constant Model 1')
111 | const_model_2 = ConstModel(2, name='Constant Model 2')
112 | const_model_37 = ConstModel(37, name="Constant Model 37")
113 |
114 | models = [const_model_1, const_model_2, const_model_37]
115 |
116 |
117 | We have ``tests`` at the end of ``tests.py``, ``models`` at the end of ``models.py``,
118 | and ``suites`` at the end of ``suites.py``. Since we are using test suites instead of tests,
119 | ``tests`` is an empty list in this example. They will be taken by sciunit when command
120 | ``sciunit run`` is being executing
121 |
122 | Execute ``sciunit run`` in the root directory, and then sciunit will run each test in the suites
123 | against each model and give us the result.
124 |
125 | .. code-block:: bash
126 |
127 | $ sciunit run
128 |
129 |
130 | Executing test =1 on model Constant Model 1... Score is Pass
131 | Executing test =2 on model Constant Model 1... Score is Fail
132 | Executing test =37 on model Constant Model 1... Score is Fail
133 | Executing test =1 on model Constant Model 2... Score is Fail
134 | Executing test =2 on model Constant Model 2... Score is Pass
135 | Executing test =37 on model Constant Model 2... Score is Fail
136 | Executing test =1 on model Constant Model 37... Score is Fail
137 | Executing test =2 on model Constant Model 37... Score is Fail
138 | Executing test =37 on model Constant Model 37... Score is Pass
139 |
140 | Suite Equals test suite:
141 | =1 =2 =37
142 | Constant Model 1 Pass Fail Fail
143 | Constant Model 2 Fail Pass Fail
144 | Constant Model 37 Fail Fail Pass
145 |
146 |
147 | Create and Run IPython Notebook File
148 | -------------------------------------
149 |
150 | Next, let's move to creating and executing IPython Notebook file with
151 | ``sciunit make-nb`` and ``sciunit run-nb`` commands.
152 |
153 | Let's add a file, ``__init__.py``, to our project directory and import everything
154 | including suites, tests, and models in the file. This is necessary because the made
155 | notebook file will try to import everything in ``__init__.py`` and run each suite
156 | (a collection of tests instances) against each model.
157 |
158 | **__init__.py**
159 |
160 | .. code-block:: python
161 |
162 | from . import models
163 | from . import tests
164 | from . import suites
165 |
166 | Now, let's execute ``sciunit make-nb`` SciUnit will automatically generate
167 | a notebook file.
168 |
169 | .. code-block:: bash
170 |
171 | $ sciunit make-nb
172 | Created Jupyter notebook at:
173 | /the_path_to_the_project.../test_sciunit.ipynb
174 |
175 | The notebook file will contains two blocks of code:
176 |
177 | .. code-block::
178 |
179 | %matplotlib inline
180 | from IPython.display import display
181 | from importlib.machinery import SourceFileLoader
182 | test_sciunit = SourceFileLoader('scidash', '/the_path_to_the_project.../__init__.py').load_module()
183 |
184 |
185 | .. code-block::
186 |
187 | for suite in test_sciunit.suites.suites:
188 | score_matrix = suite.judge(test_sciunit.models.models, stop_on_error=True)
189 | display(score_matrix)
190 |
191 |
192 |
193 | Note:
194 |
195 | 1. the name of generated notebook file will be the value of ``nb-name`` attribute
196 | in the config file, ``sciunit.ini``
197 |
198 | 2. The path to the project's root can be different on different machine.
199 | So, The notebook file generated usually only be valid on the machine where
200 | it is generated. If you want to execute it on different machine, try to re-generate
201 | it or change the path.
202 |
203 | Let's execute ``sciunit run-nb`` command.
204 |
205 | .. code-block:: bash
206 |
207 | $ sciunit run-nb
208 | Entering run function
209 | /the_path_to_the_project_config_file..././test_sciunit.ipynb
210 | /the_path_to_the_project_config_file.../.
211 |
212 | The result of running the notebook will be in the notebook file.
213 | You can open it by many tools like VS Code and Jupyter Lab
214 |
--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 | #
4 | # SciUnit documentation build configuration file, created by
5 | # sphinx-quickstart on Fri Mar 31 23:49:49 2017.
6 | #
7 | # This file is execfile()d with the current directory set to its
8 | # containing dir.
9 | #
10 | # Note that not all possible configuration values are present in this
11 | # autogenerated file.
12 | #
13 | # All configuration values have a default; values that are commented out
14 | # serve to show the default.
15 |
16 | # If extensions (or modules to document with autodoc) are in another directory,
17 | # add these directories to sys.path here. If the directory is relative to the
18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
19 | #
20 | # import os
21 | # import sys
22 | # sys.path.insert(0, os.path.abspath('.'))
23 |
24 |
25 | # -- General configuration ------------------------------------------------
26 |
27 | # If your documentation needs a minimal Sphinx version, state it here.
28 | #
29 | # needs_sphinx = '1.0'
30 |
31 | # Add any Sphinx extension module names here, as strings. They can be
32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
33 | # ones.
34 | extensions = [
35 | "sphinx.ext.autodoc",
36 | "sphinx.ext.intersphinx",
37 | "sphinx.ext.viewcode",
38 | "sphinx.ext.githubpages",
39 | ]
40 |
41 | # Add any paths that contain templates here, relative to this directory.
42 | templates_path = ["_templates"]
43 |
44 | # The suffix(es) of source filenames.
45 | # You can specify multiple suffix as a list of string:
46 | #
47 | # source_suffix = ['.rst', '.md']
48 | source_suffix = ".rst"
49 |
50 | # The master toctree document.
51 | master_doc = "index"
52 |
53 | # General information about the project.
54 | project = "SciUnit"
55 | copyright = "2017, Rick Gerkin and Cyrus Omar"
56 | author = "Rick Gerkin and Cyrus Omar"
57 |
58 | # The version info for the project you're documenting, acts as replacement for
59 | # |version| and |release|, also used in various other places throughout the
60 | # built documents.
61 | #
62 | # The short X.Y version.
63 | version = ""
64 | # The full version, including alpha/beta/rc tags.
65 | release = ""
66 |
67 | # The language for content autogenerated by Sphinx. Refer to documentation
68 | # for a list of supported languages.
69 | #
70 | # This is also used if you do content translation via gettext catalogs.
71 | # Usually you set "language" from the command line for these cases.
72 | language = None
73 |
74 | # List of patterns, relative to source directory, that match files and
75 | # directories to ignore when looking for source files.
76 | # This patterns also effect to html_static_path and html_extra_path
77 | exclude_patterns = []
78 |
79 | # The name of the Pygments (syntax highlighting) style to use.
80 | pygments_style = "sphinx"
81 |
82 | # If true, `todo` and `todoList` produce output, else they produce nothing.
83 | todo_include_todos = False
84 |
85 |
86 | # -- Options for HTML output ----------------------------------------------
87 |
88 | # The theme to use for HTML and HTML Help pages. See the documentation for
89 | # a list of builtin themes.
90 | #
91 | # html_theme = 'bizstyle'
92 |
93 | # Theme options are theme-specific and customize the look and feel of a theme
94 | # further. For a list of options available for each theme, see the
95 | # documentation.
96 | #
97 | # html_theme_options = {}
98 |
99 | # Add any paths that contain custom static files (such as style sheets) here,
100 | # relative to this directory. They are copied after the builtin static files,
101 | # so a file named "default.css" will overwrite the builtin "default.css".
102 | html_static_path = ["_static"]
103 |
104 |
105 | # -- Options for HTMLHelp output ------------------------------------------
106 |
107 | # Output file base name for HTML help builder.
108 | htmlhelp_basename = "SciUnitdoc"
109 |
110 |
111 | # -- Options for LaTeX output ---------------------------------------------
112 |
113 | latex_elements = {
114 | # The paper size ('letterpaper' or 'a4paper').
115 | #
116 | # 'papersize': 'letterpaper',
117 | # The font size ('10pt', '11pt' or '12pt').
118 | #
119 | # 'pointsize': '10pt',
120 | # Additional stuff for the LaTeX preamble.
121 | #
122 | # 'preamble': '',
123 | # Latex figure (float) alignment
124 | #
125 | # 'figure_align': 'htbp',
126 | }
127 |
128 | # Grouping the document tree into LaTeX files. List of tuples
129 | # (source start file, target name, title,
130 | # author, documentclass [howto, manual, or own class]).
131 | latex_documents = [
132 | (
133 | master_doc,
134 | "SciUnit.tex",
135 | "SciUnit Documentation",
136 | "Rick Gerkin and Cyrus Omar",
137 | "manual",
138 | ),
139 | ]
140 |
141 |
142 | # -- Options for manual page output ---------------------------------------
143 |
144 | # One entry per manual page. List of tuples
145 | # (source start file, name, description, authors, manual section).
146 | man_pages = [(master_doc, "sciunit", "SciUnit Documentation", [author], 1)]
147 |
148 |
149 | # -- Options for Texinfo output -------------------------------------------
150 |
151 | # Grouping the document tree into Texinfo files. List of tuples
152 | # (source start file, target name, title, author,
153 | # dir menu entry, description, category)
154 | texinfo_documents = [
155 | (
156 | master_doc,
157 | "SciUnit",
158 | "SciUnit Documentation",
159 | author,
160 | "SciUnit",
161 | "One line description of project.",
162 | "Miscellaneous",
163 | ),
164 | ]
165 |
166 |
167 | # Example configuration for intersphinx: refer to the Python standard library.
168 | intersphinx_mapping = {"https://docs.python.org/": None}
169 |
170 | # Removing those three member from the documents to avoid mess. More members can be added in the future.
171 | def remove_variables(app, what, name, obj, skip, options):
172 | if name == "_url":
173 | print("-----------------------")
174 | print(what)
175 | excluded = [
176 | "normalization_rules",
177 | "rules",
178 | "validation_rules",
179 | "__dict__",
180 | "__doc__",
181 | ]
182 | return name in excluded
183 |
184 |
185 | # Connecting remove_variables and autodoc-skip-member to setup the event handler.
186 | def setup(app):
187 | app.connect("autodoc-skip-member", remove_variables)
188 |
--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
1 | .. sciunit documentation master file, created by
2 | sphinx-quickstart on Thu Feb 13 13:47:10 2020.
3 | You can adapt this file completely to your liking, but it should at least
4 | contain the root `toctree` directive.
5 |
6 | Welcome to sciunit's documentation!
7 | ===================================
8 |
9 | .. toctree::
10 | :maxdepth: 2
11 | :caption: Contents:
12 |
13 | .. figure:: https://raw.githubusercontent.com/scidash/assets/master/logos/SciUnit/sci-unit-square-small.png
14 | :align: center
15 |
16 | =======================================================================================
17 |
18 | Concept
19 | -------
20 |
21 | `The conference
22 | paper `__
23 |
24 | Tutorials With Jupyter NoteBook
25 | -------------------------------
26 |
27 | .. image:: https://colab.research.google.com/assets/colab-badge.svg
28 | :target: https://colab.research.google.com/github/scidash/sciunit/blob/master/docs/chapter1.ipynb
29 |
30 | * `Tutorial Chapter 1`_
31 | * `Tutorial Chapter 2`_
32 | * `Tutorial Chapter 3`_
33 | * `Tutorial Chapter 4`_
34 | * `Tutorial Chapter 5`_
35 | * `Tutorial Chapter 6`_
36 |
37 | .. _Tutorial Chapter 1: https://github.com/scidash/sciunit/blob/master/docs/chapter1.ipynb
38 | .. _Tutorial Chapter 2: https://github.com/scidash/sciunit/blob/master/docs/chapter2.ipynb
39 | .. _Tutorial Chapter 3: https://github.com/scidash/sciunit/blob/master/docs/chapter3.ipynb
40 | .. _Tutorial Chapter 4: https://github.com/scidash/sciunit/blob/dev/docs/chapter4.ipynb
41 | .. _Tutorial Chapter 5: https://github.com/scidash/sciunit/blob/dev/docs/chapter5.ipynb
42 | .. _Tutorial Chapter 6: https://github.com/scidash/sciunit/blob/dev/docs/chapter6.ipynb
43 |
44 | Basic Usage
45 | -----------
46 |
47 | .. code:: python
48 |
49 | my_model = MyModel(**my_args) # Instantiate a class that wraps your model of interest.
50 | my_test = MyTest(**my_params) # Instantiate a test that you write.
51 | score = my_test.judge() # Runs the test and return a rich score containing test results and more.
52 |
53 | Domain-specific libraries and information
54 | -----------------------------------------
55 |
56 | `NeuronUnit `__ for neuron and
57 | ion channel physiology See others
58 | `here `__
59 |
60 | Mailing List
61 | ------------
62 |
63 | There is a `mailing
64 | list `__ for
65 | announcements and discussion. Please join it if you are at all
66 | interested!
67 |
68 | Contributors
69 | ------------
70 |
71 | - `Rick Gerkin `__, Arizona State University
72 | (School of Life Science)
73 | - `Cyrus Omar `__, Carnegie Mellon University
74 | (Dept. of Computer Science)
75 |
76 | Reproducible Research ID
77 | ------------------------
78 |
79 | RRID:\ `SCR\_014528 `__
80 |
81 | License
82 | -------
83 |
84 | SciUnit is released under the permissive `MIT
85 | license `__, requiring only
86 | attribution in derivative works. See the LICENSE file for terms.
87 |
88 |
89 | Table of Contents
90 | -----------------
91 |
92 | .. toctree::
93 |
94 | setup
95 | quick_tutorial
96 | basics
97 | commandline
98 |
99 | Indices and tables
100 | ==================
101 |
102 | * :ref:`genindex`
103 | * :ref:`modindex`
104 | * :ref:`search`
105 |
--------------------------------------------------------------------------------
/docs/source/modules.rst:
--------------------------------------------------------------------------------
1 | sciunit
2 | =======
3 |
4 | .. toctree::
5 | :maxdepth: 4
6 |
7 | sciunit
8 |
--------------------------------------------------------------------------------
/docs/source/sciunit.models.rst:
--------------------------------------------------------------------------------
1 | sciunit.models package
2 | ======================
3 |
4 | Submodules
5 | ----------
6 |
7 | sciunit.models.backends module
8 | ------------------------------
9 |
10 | .. automodule:: sciunit.models.backends
11 | :members:
12 | :undoc-members:
13 | :show-inheritance:
14 |
15 | sciunit.models.base module
16 | --------------------------
17 |
18 | .. automodule:: sciunit.models.base
19 | :members:
20 | :undoc-members:
21 | :show-inheritance:
22 |
23 | sciunit.models.examples module
24 | ------------------------------
25 |
26 | .. automodule:: sciunit.models.examples
27 | :members:
28 | :undoc-members:
29 | :show-inheritance:
30 |
31 | sciunit.models.runnable module
32 | ------------------------------
33 |
34 | .. automodule:: sciunit.models.runnable
35 | :members:
36 | :undoc-members:
37 | :show-inheritance:
38 |
39 |
40 | Module contents
41 | ---------------
42 |
43 | .. automodule:: sciunit.models
44 | :members:
45 | :undoc-members:
46 | :show-inheritance:
47 |
--------------------------------------------------------------------------------
/docs/source/sciunit.rst:
--------------------------------------------------------------------------------
1 | sciunit package
2 | ===============
3 |
4 | Subpackages
5 | -----------
6 |
7 | .. toctree::
8 |
9 | sciunit.models
10 | sciunit.scores
11 | sciunit.unit_test
12 |
13 | Submodules
14 | ----------
15 |
16 | sciunit.base module
17 | -------------------
18 |
19 | .. automodule:: sciunit.base
20 | :members:
21 | :undoc-members:
22 | :show-inheritance:
23 |
24 | sciunit.capabilities module
25 | ---------------------------
26 |
27 | .. automodule:: sciunit.capabilities
28 | :members:
29 | :undoc-members:
30 | :show-inheritance:
31 |
32 | sciunit.converters module
33 | -------------------------
34 |
35 | .. automodule:: sciunit.converters
36 | :members:
37 | :undoc-members:
38 | :show-inheritance:
39 |
40 | sciunit.errors module
41 | ---------------------
42 |
43 | .. automodule:: sciunit.errors
44 | :members:
45 | :undoc-members:
46 | :show-inheritance:
47 |
48 | sciunit.suites module
49 | ---------------------
50 |
51 | .. automodule:: sciunit.suites
52 | :members:
53 | :undoc-members:
54 | :show-inheritance:
55 |
56 | sciunit.tests module
57 | --------------------
58 |
59 | .. automodule:: sciunit.tests
60 | :members:
61 | :undoc-members:
62 | :show-inheritance:
63 |
64 | sciunit.utils module
65 | --------------------
66 |
67 | .. automodule:: sciunit.utils
68 | :members:
69 | :undoc-members:
70 | :show-inheritance:
71 |
72 | sciunit.validators module
73 | -------------------------
74 |
75 | .. automodule:: sciunit.validators
76 | :members:
77 | :undoc-members:
78 | :show-inheritance:
79 |
80 | sciunit.version module
81 | ----------------------
82 |
83 | .. automodule:: sciunit.version
84 | :members:
85 | :undoc-members:
86 | :show-inheritance:
87 |
88 |
89 | Module contents
90 | ---------------
91 |
92 | .. automodule:: sciunit
93 | :members:
94 | :undoc-members:
95 | :show-inheritance:
96 |
--------------------------------------------------------------------------------
/docs/source/sciunit.scores.rst:
--------------------------------------------------------------------------------
1 | sciunit.scores package
2 | ======================
3 |
4 | Submodules
5 | ----------
6 |
7 | sciunit.scores.base module
8 | --------------------------
9 |
10 | .. automodule:: sciunit.scores.base
11 | :members:
12 | :undoc-members:
13 | :show-inheritance:
14 |
15 | sciunit.scores.collections module
16 | ---------------------------------
17 |
18 | .. automodule:: sciunit.scores.collections
19 | :members:
20 | :undoc-members:
21 | :show-inheritance:
22 |
23 | sciunit.scores.collections\_m2m module
24 | --------------------------------------
25 |
26 | .. automodule:: sciunit.scores.collections_m2m
27 | :members:
28 | :undoc-members:
29 | :show-inheritance:
30 |
31 | sciunit.scores.complete module
32 | ------------------------------
33 |
34 | .. automodule:: sciunit.scores.complete
35 | :members:
36 | :undoc-members:
37 | :show-inheritance:
38 |
39 | sciunit.scores.incomplete module
40 | --------------------------------
41 |
42 | .. automodule:: sciunit.scores.incomplete
43 | :members:
44 | :undoc-members:
45 | :show-inheritance:
46 |
47 |
48 | Module contents
49 | ---------------
50 |
51 | .. automodule:: sciunit.scores
52 | :members:
53 | :undoc-members:
54 | :show-inheritance:
55 |
--------------------------------------------------------------------------------
/docs/source/sciunit.unit_test.rst:
--------------------------------------------------------------------------------
1 | sciunit.unit\_test package
2 | ==========================
3 |
4 | Submodules
5 | ----------
6 |
7 | sciunit.unit\_test.active module
8 | --------------------------------
9 |
10 | .. automodule:: sciunit.unit_test.active
11 | :members:
12 | :undoc-members:
13 | :show-inheritance:
14 |
15 | sciunit.unit\_test.backend\_tests module
16 | ----------------------------------------
17 |
18 | .. automodule:: sciunit.unit_test.backend_tests
19 | :members:
20 | :undoc-members:
21 | :show-inheritance:
22 |
23 | sciunit.unit\_test.base module
24 | ------------------------------
25 |
26 | .. automodule:: sciunit.unit_test.base
27 | :members:
28 | :undoc-members:
29 | :show-inheritance:
30 |
31 | sciunit.unit\_test.base\_tests module
32 | -------------------------------------
33 |
34 | .. automodule:: sciunit.unit_test.base_tests
35 | :members:
36 | :undoc-members:
37 | :show-inheritance:
38 |
39 | sciunit.unit\_test.command\_line\_tests module
40 | ----------------------------------------------
41 |
42 | .. automodule:: sciunit.unit_test.command_line_tests
43 | :members:
44 | :undoc-members:
45 | :show-inheritance:
46 |
47 | sciunit.unit\_test.config\_tests module
48 | ---------------------------------------
49 |
50 | .. automodule:: sciunit.unit_test.config_tests
51 | :members:
52 | :undoc-members:
53 | :show-inheritance:
54 |
55 | sciunit.unit\_test.converter\_tests module
56 | ------------------------------------------
57 |
58 | .. automodule:: sciunit.unit_test.converter_tests
59 | :members:
60 | :undoc-members:
61 | :show-inheritance:
62 |
63 | sciunit.unit\_test.doc\_tests module
64 | ------------------------------------
65 |
66 | .. automodule:: sciunit.unit_test.doc_tests
67 | :members:
68 | :undoc-members:
69 | :show-inheritance:
70 |
71 | sciunit.unit\_test.error\_tests module
72 | --------------------------------------
73 |
74 | .. automodule:: sciunit.unit_test.error_tests
75 | :members:
76 | :undoc-members:
77 | :show-inheritance:
78 |
79 | sciunit.unit\_test.import\_tests module
80 | ---------------------------------------
81 |
82 | .. automodule:: sciunit.unit_test.import_tests
83 | :members:
84 | :undoc-members:
85 | :show-inheritance:
86 |
87 | sciunit.unit\_test.model\_tests module
88 | --------------------------------------
89 |
90 | .. automodule:: sciunit.unit_test.model_tests
91 | :members:
92 | :undoc-members:
93 | :show-inheritance:
94 |
95 | sciunit.unit\_test.observation\_tests module
96 | --------------------------------------------
97 |
98 | .. automodule:: sciunit.unit_test.observation_tests
99 | :members:
100 | :undoc-members:
101 | :show-inheritance:
102 |
103 | sciunit.unit\_test.score\_tests module
104 | --------------------------------------
105 |
106 | .. automodule:: sciunit.unit_test.score_tests
107 | :members:
108 | :undoc-members:
109 | :show-inheritance:
110 |
111 | sciunit.unit\_test.test\_tests module
112 | -------------------------------------
113 |
114 | .. automodule:: sciunit.unit_test.test_tests
115 | :members:
116 | :undoc-members:
117 | :show-inheritance:
118 |
119 | sciunit.unit\_test.utils\_tests module
120 | --------------------------------------
121 |
122 | .. automodule:: sciunit.unit_test.utils_tests
123 | :members:
124 | :undoc-members:
125 | :show-inheritance:
126 |
127 | sciunit.unit\_test.validator\_tests module
128 | ------------------------------------------
129 |
130 | .. automodule:: sciunit.unit_test.validator_tests
131 | :members:
132 | :undoc-members:
133 | :show-inheritance:
134 |
135 | Module contents
136 | ---------------
137 |
138 | .. automodule:: sciunit.unit_test
139 | :members:
140 | :undoc-members:
141 | :show-inheritance:
142 |
--------------------------------------------------------------------------------
/docs/source/setup.rst:
--------------------------------------------------------------------------------
1 | What's SciUnit and how to install it?
2 | =====================================
3 |
4 | Everyone hopes that their model has some correspondence with reality.
5 | Usually, checking whether this is true is done informally. But SciUnit makes this formal and transparent.
6 |
7 | SciUnit is a framework for validating scientific models by creating experimental-data-driven unit tests.
8 |
9 | Installation
10 | ----------------
11 |
12 | Note: SciUnit no longer supports Python 2. Please use Python 3.
13 |
14 | SciUnit can be installed in virtual environments using the :code:`pip` Python package installer.
15 | A virtual environment can be set up using the in-built Python :code:`venv` module, as explained `here `__, or using other tools such as `Miniconda `__.
16 |
17 |
18 | In the virtual environment, run the following command to install SciUnit as a Python package using :code:`pip`.
19 |
20 | .. code-block:: bash
21 |
22 | pip install sciunit
23 |
24 |
25 | On `Fedora Linux `__ installations, the `NeuroFedora special interest group `__ also provides SciUnit as a curated package in the Fedora community repositories.
26 | SciUnit can, therefore, be installed using the system package manager, :code:`dnf` :
27 |
28 | .. code-block:: bash
29 |
30 | sudo dnf install python3-sciunit
31 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = [
3 | "setuptools >= 40.9.0",
4 | "wheel"
5 | ]
6 | build-backend = "setuptools.build_meta"
7 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/sciunit/__init__.py:
--------------------------------------------------------------------------------
1 | """SciUnit.
2 |
3 | A Testing Framework for Data-Driven Validation of
4 | Quantitative Scientific Models
5 | """
6 |
7 | import json
8 | import logging
9 | import sys
10 |
11 | from .base import __version__, config, log, logger
12 | from .capabilities import Capability
13 | from .errors import Error
14 | from .models import Model
15 | from .models.backends import Backend
16 | from .scores import Score
17 | from .scores.collections import ScoreArray, ScoreMatrix
18 | from .scores.collections_m2m import ScoreArrayM2M, ScoreMatrixM2M
19 | from .suites import TestSuite
20 | from .tests import Test, TestM2M
21 |
--------------------------------------------------------------------------------
/sciunit/__main__.py:
--------------------------------------------------------------------------------
1 | """SciUnit command line tools.
2 |
3 | With SciUnit installed, a sciunit.ini configuration file can be created from
4 | a *nix shell with:
5 | `sciunit create`
6 | and if models, tests, etc. are in locations specified by the configuration file
7 | then they can be executed with
8 | `sciunit run` (to run using the Python interpreter)
9 | or
10 | `sciunit make-nb` (to create Jupyter notebooks for test execution)
11 | and
12 | `sciunit run-nb` (to execute and save those notebooks)
13 | """
14 |
15 | import argparse
16 | import codecs
17 | import configparser
18 | import os
19 | import re
20 | import sys
21 | from configparser import RawConfigParser
22 | from importlib import import_module
23 | from pathlib import Path
24 | from typing import Union
25 |
26 | import matplotlib
27 | import nbformat
28 | from nbconvert.preprocessors import ExecutePreprocessor
29 | from nbformat.v4.nbbase import new_markdown_cell, new_notebook
30 |
31 | import sciunit
32 |
33 | matplotlib.use("Agg") #: Anticipate possible headless environments
34 |
35 | NB_VERSION = 4
36 |
37 |
38 | def main(*args):
39 | """Launch the main routine."""
40 | parser = argparse.ArgumentParser()
41 | parser.add_argument("action", help="create, check, run, make-nb, or run-nb")
42 | parser.add_argument(
43 | "--directory",
44 | "-dir",
45 | default=Path.cwd(),
46 | help="path to directory with a sciunit.ini file",
47 | )
48 | parser.add_argument(
49 | "--stop", "-s", default=True, help="stop and raise errors, halting the program"
50 | )
51 | parser.add_argument(
52 | "--tests", "-t", default=False, help="runs tests instead of suites"
53 | )
54 | if args:
55 | args = parser.parse_args(args)
56 | else:
57 | args = parser.parse_args()
58 |
59 | directory = Path(args.directory)
60 | file_path = directory / "sciunit.ini"
61 |
62 | config = None
63 | if args.action == "create":
64 | create(file_path)
65 | elif args.action == "check":
66 | config = parse(file_path, show=True)
67 | print("\nNo configuration errors reported.")
68 | elif args.action == "run":
69 | config = parse(file_path)
70 | run(config, path=directory, stop_on_error=args.stop, just_tests=args.tests)
71 | elif args.action == "make-nb":
72 | config = parse(file_path)
73 | make_nb(config, path=directory, stop_on_error=args.stop, just_tests=args.tests)
74 | elif args.action == "run-nb":
75 | config = parse(file_path)
76 | run_nb(config, path=directory)
77 | else:
78 | raise NameError("No such action %s" % args.action)
79 | if config:
80 | cleanup(config, path=directory)
81 |
82 |
83 | def create(file_path: Path) -> None:
84 | """Create a default sciunit.ini config file if one does not already exist.
85 |
86 | Args:
87 | file_path (str): The path of sciunit config file that will be created.
88 |
89 | Raises:
90 | IOError: There is already a configuration file at the path.
91 | """
92 | if file_path.exists():
93 | raise IOError("There is already a configuration file at %s" % file_path)
94 | with open(file_path, "w") as f:
95 | config = configparser.ConfigParser()
96 | config.add_section("misc")
97 | config.set("misc", "config-version", "1.0")
98 | default_nb_name = file_path.parent.name
99 | config.set("misc", "nb-name", default_nb_name)
100 | config.add_section("root")
101 | config.set("root", "path", ".")
102 | config.add_section("models")
103 | config.set("models", "module", "models")
104 | config.add_section("tests")
105 | config.set("tests", "module", "tests")
106 | config.add_section("suites")
107 | config.set("suites", "module", "suites")
108 | config.write(f)
109 |
110 |
111 | def parse(file_path: Path = None, show: bool = False) -> RawConfigParser:
112 | """Parse a sciunit.ini config file.
113 |
114 | Args:
115 | file_path (str, optional): The path of sciunit config file that will be parsed. Defaults to None.
116 | show (bool, optional): Whether or not print the sections in the config file. Defaults to False.
117 |
118 | Raises:
119 | IOError: Raise an exception if no sciunit.ini file was found.
120 |
121 | Returns:
122 | RawConfigParser: The basic configuration object.
123 | """
124 | if file_path is None:
125 | file_path = Path.cwd() / "sciunit.ini"
126 | if not file_path.exists():
127 | raise IOError("No sciunit.ini file was found at %s" % file_path)
128 |
129 | # Load the configuration file
130 | config = configparser.RawConfigParser(allow_no_value=True)
131 | config.read(file_path)
132 |
133 | # List all contents
134 | for section in config.sections():
135 | if show:
136 | print(section)
137 | for options in config.options(section):
138 | if show:
139 | print("\t%s: %s" % (options, config.get(section, options)))
140 | return config
141 |
142 |
143 | def prep(
144 | config: configparser.RawConfigParser = None, path: Union[str, Path] = None
145 | ) -> None:
146 | """Prepare to read the configuration information.
147 |
148 | Args:
149 | config (RawConfigParser, optional): The configuration object. Defaults to None.
150 | path (Union[str, Path], optional): The path of config file. Defaults to None.
151 | """
152 | if config is None:
153 | config = parse()
154 | if path is None:
155 | path = Path.cwd()
156 | root = config.get("root", "path")
157 | root = path / root
158 | root = str(root.resolve())
159 | os.environ["SCIDASH_HOME"] = root
160 | if sys.path[0] != root:
161 | sys.path.insert(0, root)
162 |
163 |
164 | def run(
165 | config: configparser.RawConfigParser,
166 | path: Union[str, Path] = None,
167 | stop_on_error: bool = True,
168 | just_tests: bool = False,
169 | ) -> None:
170 | """Run sciunit tests for the given configuration.
171 |
172 | Args:
173 | config (RawConfigParser): The parsed sciunit config file.
174 | path (Union[str, Path], optional): The path of sciunit config file. Defaults to None.
175 | stop_on_error (bool, optional): Stop if an error occurs. Defaults to True.
176 | just_tests (bool, optional): There are only tests, no suites. Defaults to False.
177 | """
178 | if path is None:
179 | path = Path.cwd()
180 | prep(config, path=path)
181 |
182 | models = import_module("models")
183 | tests = import_module("tests")
184 | suites = import_module("suites")
185 |
186 | print("\n")
187 | for x in ["models", "tests", "suites"]:
188 | module = import_module(x)
189 | assert hasattr(module, x), "'%s' module requires attribute '%s'" % (x, x)
190 |
191 | if just_tests:
192 | for test in tests.tests:
193 | _run(test, models, stop_on_error)
194 |
195 | else:
196 | for suite in suites.suites:
197 | _run(suite, models, stop_on_error)
198 |
199 |
200 | def _run(
201 | test_or_suite: Union[sciunit.Test, sciunit.TestSuite],
202 | models: list,
203 | stop_on_error: bool,
204 | ) -> None:
205 | """Run a single test or suite.
206 |
207 | Args:
208 | test_or_suite (Union[Test, TestSuite]): A test or suite instance to be executed.
209 | models (list): The list of sciunit Model.
210 | stop_on_error (bool): Whether to stop on error.
211 | """
212 | score_array_or_matrix = test_or_suite.judge(
213 | models.models, stop_on_error=stop_on_error
214 | )
215 | kind = "Test" if isinstance(test_or_suite, sciunit.Test) else "Suite"
216 | print("\n%s %s:\n%s\n" % (kind, test_or_suite, score_array_or_matrix))
217 |
218 |
219 | def nb_name_from_path(config: RawConfigParser, path: Union[str, Path]) -> tuple:
220 | """Get a notebook name from a path to a notebook.
221 |
222 | Args:
223 | config (RawConfigParser): The parsed sciunit config file.
224 | path (Union[str, Path]): The path of the notebook file.
225 |
226 | Returns:
227 | tuple: Notebook root node and name of the notebook.
228 | """
229 | if path is None:
230 | path = Path.cwd()
231 | root = config.get("root", "path")
232 | root = path / root
233 | root = root.resolve()
234 | default_nb_name = root.name
235 | nb_name = config.get("misc", "nb-name", fallback=default_nb_name)
236 | return root, nb_name
237 |
238 |
239 | def make_nb(
240 | config: RawConfigParser,
241 | path: Union[str, Path] = None,
242 | stop_on_error: bool = True,
243 | just_tests: bool = False,
244 | ) -> None:
245 | """Create a Jupyter notebook sciunit tests for the given configuration.
246 |
247 | Args:
248 | config (RawConfigParser): The parsed sciunit config file.
249 | path (Union[str, Path], optional): A path to the notebook file. Defaults to None.
250 | stop_on_error (bool, optional): Whether to stop on an error. Defaults to True.
251 | just_tests (bool, optional): There are only tests, no suites. Defaults to False.
252 | """
253 | root, nb_name = nb_name_from_path(config, path)
254 | clean = lambda varStr: re.sub("\W|^(?=\d)", "_", varStr)
255 | name = clean(nb_name)
256 |
257 | mpl_style = config.get("misc", "matplotlib", fallback="inline")
258 | cells = [new_markdown_cell("## Sciunit Testing Notebook for %s" % nb_name)]
259 | add_code_cell(
260 | cells,
261 | (
262 | "%%matplotlib %s\n"
263 | "from IPython.display import display\n"
264 | "from importlib.machinery import SourceFileLoader\n"
265 | "%s = SourceFileLoader('scidash', '%s/__init__.py').load_module()"
266 | )
267 | % (mpl_style, name, root),
268 | )
269 | if just_tests:
270 | add_code_cell(
271 | cells,
272 | (
273 | "for test in %s.tests.tests:\n"
274 | " score_array = test.judge(%s.models.models, stop_on_error=%r)\n"
275 | " display(score_array)"
276 | )
277 | % (name, name, stop_on_error),
278 | )
279 | else:
280 | add_code_cell(
281 | cells,
282 | (
283 | "for suite in %s.suites.suites:\n"
284 | " score_matrix = suite.judge("
285 | "%s.models.models, stop_on_error=%r)\n"
286 | " display(score_matrix)"
287 | )
288 | % (name, name, stop_on_error),
289 | )
290 | write_nb(root, nb_name, cells)
291 |
292 |
293 | def write_nb(root: str, nb_name: str, cells: list) -> None:
294 | """Write a jupyter notebook to disk.
295 |
296 | Takes a given a root directory, a notebook name, and a list of cells.
297 |
298 | Args:
299 | root (str): The root node (section) of the notebook.
300 | nb_name (str): The name of the notebook file.
301 | cells (list): The list of the cells of the notebook.
302 | """
303 | nb = new_notebook(
304 | cells=cells,
305 | metadata={
306 | "language": "python",
307 | },
308 | )
309 | nb_path = root / ("%s.ipynb" % nb_name)
310 | with codecs.open(nb_path, encoding="utf-8", mode="w") as nb_file:
311 | nbformat.write(nb, nb_file, NB_VERSION)
312 | print("Created Jupyter notebook at:\n%s" % nb_path)
313 |
314 |
315 | def run_nb(config: RawConfigParser, path: Union[str, Path] = None) -> None:
316 | """Run a notebook file.
317 |
318 | Runs the one specified by the config file, or the one at
319 | the location specificed by 'path'.
320 |
321 | Args:
322 | config (RawConfigParser): The parsed sciunit config file.
323 | path (Union[str, Path], optional): The path to the notebook file. Defaults to None.
324 | """
325 | if path is None:
326 | path = Path.cwd()
327 | root = config.get("root", "path")
328 | root = path / root
329 | nb_name = config.get("misc", "nb-name")
330 | nb_path = root / ("%s.ipynb" % nb_name)
331 | if not nb_path.exists():
332 | print(
333 | ("No notebook found at %s. " "Create the notebook first with make-nb?")
334 | % path
335 | )
336 | sys.exit(0)
337 |
338 | with codecs.open(nb_path, encoding="utf-8", mode="r") as nb_file:
339 | nb = nbformat.read(nb_file, as_version=NB_VERSION)
340 | ep = ExecutePreprocessor(timeout=600)
341 | ep.preprocess(nb, {"metadata": {"path": root}})
342 | with codecs.open(nb_path, encoding="utf-8", mode="w") as nb_file:
343 | nbformat.write(nb, nb_file, NB_VERSION)
344 |
345 |
346 | def add_code_cell(cells: list, source: str) -> None:
347 | """Add a code cell containing `source` to the notebook.
348 |
349 | Args:
350 | cells (list): The list of notebook cells.
351 | source (str): The source of the notebook cell.
352 | """
353 | from nbformat.v4.nbbase import new_code_cell
354 |
355 | n_code_cells = len([c for c in cells if c["cell_type"] == "code"])
356 | cells.append(new_code_cell(source=source, execution_count=n_code_cells + 1))
357 |
358 |
359 | def cleanup(config=None, path: Union[str, Path] = None) -> None:
360 | """Cleanup by removing paths added during earlier in configuration.
361 |
362 | Args:
363 | config (RawConfigParser, optional): The parsed sciunit config file. Defaults to None.
364 | path (Union[str, Path], optional): The path to the sciunit config file. Defaults to None.
365 | """
366 | if config is None:
367 | config = parse()
368 | if path is None:
369 | path = Path.cwd()
370 | root = config.get("root", "path")
371 | root = str(path / root)
372 | if sys.path[0] == root:
373 | sys.path.remove(root)
374 |
375 |
376 | if __name__ == "__main__":
377 | main()
378 |
--------------------------------------------------------------------------------
/sciunit/capabilities.py:
--------------------------------------------------------------------------------
1 | """The base class for SciUnit capabilities.
2 |
3 | By inheriting a capability class, a model tells the test that it implements
4 | that capability and that all of its methods are safe to call.
5 | The capability must then be implemented by the modeler (i.e. all of the
6 | capabilty's methods must implemented in the model class).
7 | """
8 |
9 | import dis
10 | import inspect
11 | import io
12 | import re
13 | import sys
14 | import warnings
15 |
16 | from .base import SciUnit, log, logger
17 | from .errors import CapabilityNotImplementedError
18 |
19 | # from sciunit.models.examples import ConstModel, UniformModel
20 |
21 |
22 | class Capability(SciUnit):
23 | """Abstract base class for sciunit capabilities."""
24 |
25 | @classmethod
26 | def source_check(cls, model: "sciunit.Model") -> bool:
27 | required_methods = []
28 | default_cap_methods = ["unimplemented", "__str__"]
29 | source_capable = True
30 |
31 | for key, value in vars(cls).items():
32 | if inspect.isfunction(value) and key not in default_cap_methods:
33 | required_methods.append(key)
34 |
35 | for method in required_methods:
36 | try:
37 | stdout = sys.stdout
38 | sys.stdout = io.StringIO()
39 |
40 | dis.dis(getattr(cls, method))
41 |
42 | dis_output = sys.stdout.getvalue()
43 | sys.stdout = stdout
44 |
45 | dis_output = re.split("\n|\s+", dis_output)
46 | dis_output = [word for word in dis_output if word]
47 |
48 | if (
49 | "(NotImplementedError)" in dis_output
50 | or "(unimplemented)" in dis_output
51 | or "(CapabilityNotImplementedError)" in dis_output
52 | or "(NotImplemented)" in dis_output
53 | ):
54 | cap_source = inspect.getsource(getattr(cls, method))
55 | model_source = inspect.getsource(getattr(model, method))
56 |
57 | if cap_source == model_source:
58 | source_capable = False
59 | break
60 |
61 | except OSError:
62 | warnings.warn(
63 | """Inspect cannot get the source, and it is not guaranteed that
64 | all required methods have been implemented by the model"""
65 | )
66 | break
67 |
68 | return source_capable
69 |
70 | @classmethod
71 | def check(cls, model: "sciunit.Model", require_extra: bool = False) -> bool:
72 | """Check whether the provided model has this capability.
73 |
74 | By default, uses isinstance. If `require_extra`, also requires that an
75 | instance check be present in `model.extra_capability_checks`.
76 |
77 | Args:
78 | model (Model): A sciunit model instance
79 | require_extra (bool, optional): Requiring that an instance check be present in
80 | `model.extra_capability_checks`. Defaults to False.
81 |
82 | Returns:
83 | bool: Whether the provided model has this capability.
84 | """
85 |
86 | class_capable = isinstance(model, cls)
87 | source_capable = None
88 |
89 | if class_capable:
90 | source_capable = cls.source_check(model)
91 |
92 | f_name = (
93 | model.extra_capability_checks.get(cls, None)
94 | if model.extra_capability_checks is not None
95 | else False
96 | )
97 |
98 | if f_name:
99 | f = getattr(model, f_name)
100 | instance_capable = f()
101 | elif not require_extra:
102 | instance_capable = True
103 | else:
104 | instance_capable = False
105 |
106 | if not class_capable:
107 | log(
108 | (
109 | "The Model class does not claim at least one Capability required by "
110 | "the Test class, so the Score is likely to be unavailable."
111 | )
112 | )
113 | elif not source_capable:
114 | logger.warning(
115 | (
116 | "The model class claimed to implement all methods required by "
117 | "the Test class, but at least one was left unimplemented, "
118 | "so this model will be skipped."
119 | )
120 | )
121 |
122 | return class_capable and instance_capable and source_capable
123 |
124 | def unimplemented(self, message: str = "") -> None:
125 | """Raise a `CapabilityNotImplementedError` with details.
126 |
127 | Args:
128 | message (str, optional): Message for not implemented exception. Defaults to ''.
129 |
130 | Raises:
131 | CapabilityNotImplementedError: Raise a `CapabilityNotImplementedError` with details.
132 | """
133 | from sciunit import Model
134 |
135 | capabilities = [
136 | obj
137 | for obj in self.__class__.mro()
138 | if issubclass(obj, Capability) and not issubclass(obj, Model)
139 | ]
140 | model = self if isinstance(self, Model) else None
141 | capability = None if not capabilities else capabilities[0]
142 | raise CapabilityNotImplementedError(model, capability, message)
143 |
144 | class __metaclass__(type):
145 | @property
146 | def name(cls):
147 | return cls.__name__
148 |
149 | def __str__(self) -> str:
150 | return self.name
151 |
152 |
153 | class ProducesNumber(Capability):
154 | """An example capability for producing some generic number."""
155 |
156 | def produce_number(self) -> None:
157 | """Produce a number."""
158 | self.unimplemented()
159 |
160 |
161 | class Runnable(Capability):
162 | """Capability for models that can be run, i.e. simulated."""
163 |
164 | def run(self, **run_params) -> None:
165 | """Run, i.e. simulate the model."""
166 | self.unimplemented()
167 |
168 | def set_run_params(self, **run_params) -> None:
169 | """Set parameters for the next run.
170 |
171 | Note these are parameters of the simulation itself, not the model.
172 | """
173 | self.unimplemented()
174 |
175 | def set_default_run_params(self, **default_run_params) -> None:
176 | """Set default parameters for all runs.
177 |
178 | Note these are parameters of the simulation itself, not the model.
179 | """
180 | self.unimplemented()
181 |
--------------------------------------------------------------------------------
/sciunit/config.json:
--------------------------------------------------------------------------------
1 | {"cmap_high": 218, "cmap_low": 38}
--------------------------------------------------------------------------------
/sciunit/converters.py:
--------------------------------------------------------------------------------
1 | """
2 | Classes for converting from the output of a model/data comparison
3 | to the value required for particular score type.
4 | """
5 |
6 | from string import Template
7 | from typing import Callable
8 |
9 | from .scores import BooleanScore, Score
10 |
11 |
12 | class Converter(object):
13 | """
14 | Base converter class.
15 | Only derived classes should be used in applications.
16 | """
17 |
18 | @property
19 | def description(self):
20 | if self.__doc__:
21 | s = " ".join([si.strip() for si in self.__doc__.split("\n")]).strip()
22 | t = Template(s)
23 | s = t.safe_substitute(self.__dict__)
24 | else:
25 | s = "No description available"
26 | return s
27 |
28 | def _convert(self, score: Score) -> None:
29 | """Takes the score attribute of a score instance and recasts it as instance of another score type.
30 |
31 | Args:
32 | score (Score): An instance of Score.
33 |
34 | Raises:
35 | NotImplementedError: Not implemented if not overrided.
36 | """
37 | raise NotImplementedError(
38 | (
39 | "The '_convert' method for %s "
40 | "it not implemented." % self.__class__.__name__
41 | )
42 | )
43 |
44 | def convert(self, score: Score) -> Score:
45 | """Convert a type of score to another type of score.
46 |
47 | Args:
48 | score (Score): The original score.
49 |
50 | Returns:
51 | Score: The converted score.
52 | """
53 | new_score = self._convert(score)
54 | new_score.set_raw(score.get_raw())
55 | for key, value in score.__dict__.items():
56 | if key not in ["score", "_raw"]:
57 | setattr(new_score, key, value)
58 | return new_score
59 |
60 |
61 | class NoConversion(Converter):
62 | """
63 | Applies no conversion.
64 | """
65 |
66 | def _convert(self, score: Score) -> Score:
67 | return score
68 |
69 |
70 | class LambdaConversion(Converter):
71 | """
72 | Converts a score according to a lambda function.
73 | """
74 |
75 | def __init__(self, f: Callable):
76 | """f should be a lambda function
77 |
78 | Args:
79 | f (Callable): The Lambda function that will be used for the score conversion.
80 | """
81 | self.f = f
82 |
83 | def _convert(self, score: Score) -> Score:
84 | return score.__class__(self.f(score))
85 |
86 |
87 | class AtMostToBoolean(Converter):
88 | """
89 | Converts a score to pass if its value is at most $cutoff, otherwise False.
90 | """
91 |
92 | def __init__(self, cutoff: int):
93 | self.cutoff = cutoff
94 |
95 | def _convert(self, score: Score) -> BooleanScore:
96 | return BooleanScore(bool(score <= self.cutoff))
97 |
98 |
99 | class AtLeastToBoolean(Converter):
100 | """
101 | Converts a score to Pass if its value is at least $cutoff, otherwise False.
102 | """
103 |
104 | def __init__(self, cutoff: int):
105 | self.cutoff = cutoff
106 |
107 | def _convert(self, score: Score) -> BooleanScore:
108 | return BooleanScore(score >= self.cutoff)
109 |
110 |
111 | class RangeToBoolean(Converter):
112 | """
113 | Converts a score to Pass if its value is within the range
114 | [$low_cutoff,$high_cutoff], otherwise Fail.
115 | """
116 |
117 | def __init__(self, low_cutoff: int, high_cutoff: int):
118 | self.low_cutoff = low_cutoff
119 | self.high_cutoff = high_cutoff
120 |
121 | def _convert(self, score: Score) -> BooleanScore:
122 | return BooleanScore(self.low_cutoff <= score <= self.high_cutoff)
123 |
--------------------------------------------------------------------------------
/sciunit/errors.py:
--------------------------------------------------------------------------------
1 | """
2 | Exception classes for SciUnit
3 | """
4 |
5 |
6 | import sciunit
7 |
8 |
9 | class Error(Exception):
10 | """Base class for errors in sciunit's core."""
11 |
12 |
13 | class ObservationError(Error):
14 | """Raised when an observation passed to a test is invalid."""
15 |
16 |
17 | class ParametersError(Error):
18 | """Raised when params passed to a test are invalid."""
19 |
20 |
21 | class CapabilityError(Error):
22 | """Abstract error class for capabilities"""
23 |
24 | def __init__(
25 | self,
26 | model: "sciunit.Model",
27 | capability: "sciunit.Capability",
28 | details: str = "",
29 | ):
30 | """A constructor.
31 | Args:
32 | model (Model): A sciunit model instance.
33 | capability (Capability): a capability class.
34 | details (str, optional): Details of the error information. Defaults to ''.
35 | """
36 | self.model = model
37 | self.capability = capability
38 | if details:
39 | details = " (%s)" % details
40 | if self.action:
41 | msg = "Model '%s' does not %s required capability: '%s'%s" % (
42 | model.name,
43 | self.action,
44 | capability.__name__,
45 | details,
46 | )
47 | super(CapabilityError, self).__init__(details)
48 |
49 | action = None
50 | """The action that has failed ('provide' or 'implement')."""
51 |
52 | model = None
53 | """The model instance that does not have the capability."""
54 |
55 | capability = None
56 | """The capability class that is not provided."""
57 |
58 |
59 | class CapabilityNotProvidedError(CapabilityError):
60 | """Error raised when a required capability is not *provided* by a model.
61 | Do not use for capabilities provided but not implemented."""
62 |
63 | action = "provide"
64 |
65 |
66 | class CapabilityNotImplementedError(CapabilityError):
67 | """Error raised when a required capability is not *implemented* by a model.
68 | Do not use for capabilities that are not provided at all."""
69 |
70 | action = "implement"
71 |
72 |
73 | class PredictionError(Error):
74 | """Raised when a tests's generate_prediction chokes on a model's method."""
75 |
76 | def __init__(self, model: "sciunit.Model", method: str, **args):
77 | """Constructor of PredictionError object.
78 |
79 | Args:
80 | model (Model): A sciunit Model.
81 | method (str): The method that caused this error.
82 | """
83 | self.model = model
84 | self.method = method
85 | self.args = args
86 |
87 | super(PredictionError, self).__init__(
88 | (
89 | "During prediction, model '%s' could not successfully execute "
90 | "method '%s' with arguments %s"
91 | )
92 | % (model.name, method, args)
93 | )
94 |
95 | model = None
96 | """The model that does not have the capability."""
97 |
98 | argument = None
99 | """The argument that could not be handled."""
100 |
101 |
102 | class InvalidScoreError(Error):
103 | """Error raised when a score is invalid."""
104 |
105 |
106 | class BadParameterValueError(Error):
107 | """Error raised when a model parameter value is unreasonable."""
108 |
109 | def __init__(self, name: str, value: int):
110 | """Constructor of BadParameterValueError object.
111 |
112 | Args:
113 | name (str): Name of the parameter that caused this error.
114 | value (int): The value of the parameter.
115 | """
116 | self.name = name
117 | self.value = value
118 |
119 | super(BadParameterValueError, self).__init__(
120 | "Parameter %s has unreasonable value of %s" % (name, value)
121 | )
122 |
--------------------------------------------------------------------------------
/sciunit/models/__init__.py:
--------------------------------------------------------------------------------
1 | """SciUnit models."""
2 |
3 | from .base import Model
4 | from .runnable import RunnableModel
5 |
--------------------------------------------------------------------------------
/sciunit/models/backends.py:
--------------------------------------------------------------------------------
1 | """Base class for simulator backends for SciUnit models."""
2 |
3 | import inspect
4 | import pickle
5 | import shelve
6 | import tempfile
7 | from pathlib import Path
8 | from typing import Any, Union
9 |
10 | from sciunit.base import SciUnit, config
11 |
12 | available_backends = {}
13 |
14 |
15 | def register_backends(vars: dict) -> None:
16 | """Register backends for use with models.
17 |
18 | Args:
19 | vars (dict): a dictionary of variables obtained from e.g. `locals()`,
20 | at least some of which are Backend classes, e.g. from imports.
21 | """
22 | new_backends = {
23 | x if x is None else x.replace("Backend", ""): cls
24 | for x, cls in vars.items()
25 | if inspect.isclass(cls) and issubclass(cls, Backend)
26 | }
27 | available_backends.update(new_backends)
28 |
29 |
30 | class Backend(SciUnit):
31 | """
32 | Base class for simulator backends.
33 |
34 | Should only be used with model classes derived from `RunnableModel`.
35 | Supports caching of simulation results.
36 | Backend classes should implement simulator-specific
37 | details of modifying, running, and reading results from the simulation.
38 | """
39 |
40 | def init_backend(self, *args, **kwargs) -> None:
41 | """Initialize the backend."""
42 | self.model.attrs = {}
43 |
44 | self.use_memory_cache = kwargs.get("use_memory_cache", True)
45 | if self.use_memory_cache:
46 | self.init_memory_cache()
47 | self.use_disk_cache = kwargs.get("use_disk_cache", False)
48 | if self.use_disk_cache:
49 | self.init_disk_cache(location=self.use_disk_cache)
50 | self.load_model()
51 |
52 | #: Name of the backend
53 | name = None
54 |
55 | #: The function that handles running the simulation
56 | f = None
57 |
58 | #: Optional list of state variables for a backend to record.
59 | recorded_variables = None
60 |
61 | state_hide = ["memory_cache", "_results", "stdout", "exec_in_dir", "model"]
62 |
63 | def init_cache(self) -> None:
64 | """Initialize the cache."""
65 | self.init_memory_cache()
66 | self.init_disk_cache()
67 |
68 | def init_memory_cache(self) -> None:
69 | """Initialize the in-memory version of the cache."""
70 | self.memory_cache = {}
71 |
72 | def init_disk_cache(self, location: Union[str, Path, bool, None] = None) -> None:
73 | """Initialize the on-disk version of the cache."""
74 | if isinstance(location, (str, Path)):
75 | location = str(location)
76 | else:
77 | # => "~/.sciunit/cache"
78 | location = str(config.path.parent / "cache")
79 |
80 | self.disk_cache_location = location
81 |
82 | def clear_disk_cache(self) -> None:
83 | """Removes the cache file from the disk if it exists.
84 | """
85 | path = Path(self.disk_cache_location)
86 |
87 | if path.exists():
88 | with shelve.open(str(path)) as cache:
89 | cache.clear()
90 |
91 | def get_memory_cache(self, key: str = None) -> dict:
92 | """Return result in memory cache for key 'key' or None if not found.
93 |
94 | Args:
95 | key (str, optional): [description]. Defaults to None.
96 |
97 | Returns:
98 | dict: The memory cache for key 'key' or None if not found.
99 | """
100 | key = self.model.hash() if key is None else key
101 | if not getattr(self, "memory_cache", False):
102 | self.init_memory_cache()
103 | self._results = self.memory_cache.get(key)
104 | return self._results
105 |
106 | def get_disk_cache(self, key: str = None) -> Any:
107 | """Return result in disk cache for key 'key' or None if not found.
108 |
109 | Args:
110 | key (str, optional): keys that will be used to find cached data. Defaults to None.
111 |
112 | Returns:
113 | Any: The disk cache for key 'key' or None if not found.
114 | """
115 | key = self.model.hash() if key is None else key
116 | if not getattr(self, "disk_cache_location", False):
117 | self.init_disk_cache()
118 | disk_cache = shelve.open(str(self.disk_cache_location))
119 | self._results = disk_cache.get(key)
120 | disk_cache.close()
121 | return self._results
122 |
123 | def get_cache(self, key: str = None) -> Any:
124 | """Return result in disk or memory cache for key 'key' or None if not
125 | found. If both `use_disk_cache` and `use_memory_cache` are True, the
126 | memory cache is returned.
127 |
128 | Returns:
129 | Any: The cache for key 'key' or None if not found.
130 | """
131 | if self.use_memory_cache:
132 | result = self.get_memory_cache(key=key)
133 | if result is not None:
134 | return result
135 | if self.use_disk_cache:
136 | result = self.get_disk_cache(key=key)
137 | if result is not None:
138 | return result
139 | return None
140 |
141 | def set_memory_cache(self, results: Any, key: str = None) -> None:
142 | """Store result in memory cache with key matching model state.
143 |
144 | Args:
145 | results (Any): [description]
146 | key (str, optional): [description]. Defaults to None.
147 | """
148 | key = self.model.hash() if key is None else key
149 | if not getattr(self, "memory_cache", False):
150 | self.init_memory_cache()
151 | self.memory_cache[key] = results
152 |
153 | def set_disk_cache(self, results: Any, key: str = None) -> None:
154 | """Store result in disk cache with key matching model state.
155 |
156 | Args:
157 | results (Any): [description]
158 | key (str, optional): [description]. Defaults to None.
159 | """
160 | if not getattr(self, "disk_cache_location", False):
161 | self.init_disk_cache()
162 | disk_cache = shelve.open(str(self.disk_cache_location))
163 | key = self.model.hash() if key is None else key
164 | disk_cache[key] = results
165 | disk_cache.close()
166 |
167 | def set_cache(self, results: Any, key: str = None) -> bool:
168 | """Store result in disk and/or memory cache for key 'key', depending
169 | on whether `use_disk_cache` and `use_memory_cache` are True.
170 |
171 | Args:
172 | results (Any): [description]
173 | key (str, optional): [description]. Defaults to None.
174 |
175 | Returns:
176 | bool: True if cache was successfully set, else False
177 | """
178 | if self.use_memory_cache:
179 | self.set_memory_cache(results, key=key)
180 | if self.use_disk_cache:
181 | self.set_disk_cache(results, key=key)
182 | if self.use_memory_cache or self.use_disk_cache:
183 | return True
184 | return False
185 |
186 | def load_model(self) -> None:
187 | """Load the model into memory."""
188 |
189 | def set_attrs(self, **attrs) -> None:
190 | """Set model attributes on the backend."""
191 |
192 | def set_run_params(self, **run_params) -> None:
193 | """Set model attributes on the backend."""
194 |
195 | def backend_run(self) -> Any:
196 | """Check for cached results; then run the model if needed.
197 |
198 | Returns:
199 | Any: The result of running backend.
200 | """
201 | if self.use_memory_cache or self.use_disk_cache:
202 | key = self.model.hash()
203 | if self.use_memory_cache and self.get_memory_cache(key):
204 | return self.cache_to_results(self._results)
205 | if self.use_disk_cache and self.get_disk_cache(key):
206 | return self.cache_to_results(self._results)
207 | results = self._backend_run()
208 | if self.use_memory_cache:
209 | self.set_memory_cache(self.results_to_cache(results), key)
210 | if self.use_disk_cache:
211 | self.set_disk_cache(self.results_to_cache(results), key)
212 | return results
213 |
214 | def cache_to_results(self, cache: Any) -> Any:
215 | """A method to convert cache to some hypothetical Results object.
216 |
217 | Args:
218 | cache (Any): An object returned from .get_memory_cache() or .get_disk_cache().
219 |
220 | Returns:
221 | Any (optional): Either an object with the results of the simulation,
222 | or None (e.g. if cache_to_results() simply injects the results into some global object).
223 | """
224 | return cache
225 |
226 | def results_to_cache(self, results: Any) -> Any:
227 | """A method to convert the results from your model run
228 | into storable cache object (usually a simple dictionary or an array).
229 |
230 | Args:
231 | results (Any): An object returned from your ._backend_run().
232 |
233 | Returns:
234 | Any: The results in the format that's good for storing in cache.
235 | """
236 | return results
237 |
238 | def _backend_run(self) -> Any:
239 | """Run the model via the backend."""
240 | raise NotImplementedError("Each backend must implement '_backend_run'")
241 |
242 | def save_results(self, path: Union[str, Path] = ".") -> None:
243 | """Save results on disk.
244 |
245 | Args:
246 | path (Union[str, Path], optional): [description]. Defaults to '.'.
247 | """
248 | with open(path, "wb") as f:
249 | pickle.dump(self.results, f)
250 |
251 |
252 | class BackendException(Exception):
253 | """Generic backend exception class."""
254 |
255 |
256 | # Register the base class as a Backend just so that there is
257 | # always something available. This Backend won't do anything
258 | # useful other than caching.
259 | register_backends({None: Backend})
260 |
--------------------------------------------------------------------------------
/sciunit/models/base.py:
--------------------------------------------------------------------------------
1 | """Base class for SciUnit models."""
2 |
3 | import inspect
4 | from fnmatch import fnmatchcase
5 | from typing import Union
6 |
7 | from sciunit.base import SciUnit
8 | from sciunit.capabilities import Capability
9 |
10 |
11 | class Model(SciUnit):
12 | """Abstract base class for sciunit models."""
13 |
14 | def __init__(self, name=None, **params):
15 | """Instantiate model."""
16 | if name is None:
17 | name = self.__class__.__name__
18 | self.name = name
19 | self.params = params
20 | super(Model, self).__init__()
21 | self.check_params()
22 |
23 | name = None
24 | """The name of the model. Defaults to the class name."""
25 |
26 | description = ""
27 | """A description of the model."""
28 |
29 | params = None
30 | """The parameters to the model (a dictionary).
31 | These distinguish one model of a class from another."""
32 |
33 | run_args = None
34 | """These are the run-time arguments for the model.
35 | Execution of run() should make use of these arguments."""
36 |
37 | extra_capability_checks = None
38 | """Optional extra checks of capabilities on a per-instance basis."""
39 |
40 | _backend = None
41 | """Optional model backend for executing some methods, e.g. simulations."""
42 |
43 | state_hide = ["results", "temp_dir", "_temp_dir", "stdout"]
44 |
45 | @classmethod
46 | def get_capabilities(cls) -> list:
47 | """List the model's capabilities."""
48 | capabilities = []
49 | for _cls in cls.mro():
50 | if (
51 | issubclass(_cls, Capability)
52 | and _cls is not Capability
53 | and not issubclass(_cls, Model)
54 | ):
55 | capabilities.append(_cls)
56 | return capabilities
57 |
58 | @property
59 | def capabilities(self) -> list:
60 | return self.__class__.get_capabilities()
61 |
62 | @property
63 | def failed_extra_capabilities(self) -> list:
64 | """Check to see if instance passes its `extra_capability_checks`."""
65 | failed = []
66 | if self.extra_capability_checks is not None:
67 | for capability, f_name in self.extra_capability_checks.items():
68 | f = getattr(self, f_name)
69 | instance_capable = f()
70 | if isinstance(self, capability) and not instance_capable:
71 | failed.append(capability)
72 | return failed
73 |
74 | def describe(self) -> str:
75 | """Describe the model.
76 |
77 | Returns:
78 | str: The description of the model.
79 | """
80 | result = "No description available"
81 | if self.description:
82 | result = "%s" % self.description
83 | else:
84 | if self.__doc__:
85 | s = []
86 | s += [self.__doc__.strip().replace("\n", "").replace(" ", " ")]
87 | result = "\n".join(s)
88 | return result
89 |
90 | def curr_method(self, back: int = 0) -> str:
91 | """Return the name of the current method (calling this one).
92 |
93 | Args:
94 | back (int, optional): [description]. Defaults to 0.
95 |
96 | Returns:
97 | str: The name of the current method that calls this one.
98 | """
99 | return inspect.stack()[1 + back][3]
100 |
101 | def check_params(self) -> None:
102 | """Check model parameters to see if they are reasonable.
103 |
104 | For example, this method could check self.params to see if a particular
105 | value was within an acceptable range. This should be implemented
106 | as needed by specific model classes.
107 | """
108 |
109 | def is_match(self, match: Union[str, "Model"]) -> bool:
110 | """Return whether this model is the same as `match`.
111 |
112 | Matches if the model is the same as or has the same name as `match`.
113 |
114 | Args:
115 | match (Union[str, 'Model']): [description]
116 |
117 | Returns:
118 | bool: Whether this model is the same as `match`.
119 | """
120 | result = False
121 | if self == match:
122 | result = True
123 | elif isinstance(match, str) and fnmatchcase(self.name, match):
124 | result = True # Found by instance or name
125 | return result
126 |
127 | def __getattr__(self, attr):
128 | try:
129 | result = super(Model, self).__getattribute__(attr)
130 | except AttributeError:
131 | try:
132 | result = self._backend.__getattribute__(attr)
133 | except:
134 | raise AttributeError("Model %s has no attribute %s" % (self, attr))
135 | return result
136 |
137 | def __str__(self):
138 | """Return the model name."""
139 | return "%s" % self.name
140 |
141 | def __repr__(self):
142 | """Returns a representation of the model."""
143 | return "%s (%s)" % (self.name, self.__class__.__name__)
144 |
--------------------------------------------------------------------------------
/sciunit/models/examples.py:
--------------------------------------------------------------------------------
1 | """Example SciUnit model classes."""
2 |
3 | import random
4 | from typing import Union
5 |
6 | from sciunit.capabilities import ProducesNumber
7 | from sciunit.models import Model
8 | from sciunit.utils import ( # Decorator for caching of capability method results.
9 | class_intern,
10 | method_cache,
11 | method_memoize,
12 | )
13 |
14 |
15 | class ConstModel(Model, ProducesNumber):
16 | """A model that always produces a constant number as output."""
17 |
18 | def __init__(self, constant: Union[int, float], name: str = None):
19 | self.constant = constant
20 | super(ConstModel, self).__init__(name=name, constant=constant)
21 |
22 | def produce_number(self) -> Union[int, float]:
23 | return self.constant
24 |
25 |
26 | class UniformModel(Model, ProducesNumber):
27 | """A model that always produces a random uniformly distributed number.
28 | in [a,b] as output."""
29 |
30 | def __init__(self, a, b, name=None):
31 | self.a, self.b = a, b
32 | super(UniformModel, self).__init__(name=name, a=a, b=b)
33 |
34 | def produce_number(self) -> float:
35 | """Produece a number between `a` and `b`.
36 |
37 | Returns:
38 | float: The number between a and b.
39 | """
40 | return random.uniform(self.a, self.b)
41 |
42 |
43 | ################################################################
44 | # Here are several examples of caching and sharing can be used
45 | # to reduce the computational load of testing.
46 | ################################################################
47 |
48 |
49 | class UniqueRandomNumberModel(Model, ProducesNumber):
50 | """An example model to ProducesNumber."""
51 |
52 | def produce_number(self) -> float:
53 | """Each call to this method will produce a different random number.
54 |
55 | Returns:
56 | float: A random number produced.
57 | """
58 | return random.random()
59 |
60 |
61 | class RepeatedRandomNumberModel(Model, ProducesNumber):
62 | """An example model to demonstrate ProducesNumber with cypy.lazy."""
63 |
64 | @method_memoize
65 | def produce_number(self):
66 | """Each call to this method will produce the same random number as was returned in the first call, ensuring reproducibility and eliminating computational overhead.
67 |
68 | Returns:
69 | float: A random number produced.
70 | """
71 | return random.random()
72 |
73 |
74 | @class_intern
75 | class SharedModel(Model):
76 | """A model that, each time it is instantiated with the same parameters,
77 | will return the same instance at the same locaiton in memory.
78 | Attributes should not be set post-instantiation
79 | unless the goal is to set those attributes on all models of this class."""
80 |
81 |
82 | class PersistentUniformModel(UniformModel):
83 | """TODO"""
84 |
85 | def run(self) -> None:
86 | self._x = random.uniform(self.a, self.b)
87 |
88 | def produce_number(self) -> float:
89 | return self._x
90 |
91 |
92 | class CacheByInstancePersistentUniformModel(PersistentUniformModel):
93 | """TODO"""
94 |
95 | @method_cache(by="instance", method="run")
96 | def produce_number(self) -> float:
97 | return self._x
98 |
99 |
100 | class CacheByValuePersistentUniformModel(PersistentUniformModel):
101 | """TODO"""
102 |
103 | @method_cache(by="value", method="run")
104 | def produce_number(self) -> float:
105 | return self._x
106 |
--------------------------------------------------------------------------------
/sciunit/models/runnable.py:
--------------------------------------------------------------------------------
1 | """Runnable model."""
2 |
3 | import inspect
4 | import warnings
5 | from typing import Union
6 |
7 | import sciunit.capabilities as cap
8 |
9 | from .backends import Backend, available_backends
10 | from .base import Model
11 |
12 |
13 | class RunnableModel(Model, cap.Runnable):
14 | """A model which can be run to produce simulation results."""
15 |
16 | def __init__(
17 | self,
18 | name, # Name of the model
19 | backend=None, # Backend to run the models
20 | attrs=None, # Optional dictionary of model attributes
21 | **params,
22 | ):
23 | super(RunnableModel, self).__init__(name=name, **params)
24 | self.skip_run = False # Backend can use this to skip simulation
25 | self.run_params = {} # Should be reset between tests
26 | self.print_run_params = False # Print the run parameters with each run
27 | self.default_run_params = {} # Should be applied to all tests
28 | if attrs and not isinstance(attrs, dict):
29 | raise TypeError("Model 'attrs' must be a dictionary.")
30 | self.attrs = attrs if attrs else {}
31 | self.set_backend(backend)
32 | self.use_default_run_params()
33 |
34 | def get_backend(self):
35 | """Return the simulation backend."""
36 | return self._backend
37 |
38 | def set_backend(self, backend: Union[str, tuple, list, None]):
39 | """Set the simulation backend.
40 |
41 | Args:
42 | backend (Union[str, tuple, list, None]): One or more name(s) of backend(s).
43 | The name of backend should be registered before using.
44 |
45 | Raises:
46 | TypeError: Backend must be string, tuple, list, or None
47 | Exception: The backend was not found.
48 | """
49 | if inspect.isclass(backend) and Backend in backend.__bases__:
50 | name = backend.__name__
51 | args = []
52 | kwargs = {}
53 | available_backends[name] = backend
54 | elif isinstance(backend, str):
55 | name = backend if len(backend) else None
56 | args = []
57 | kwargs = {}
58 | elif isinstance(backend, (tuple, list)):
59 | name = ""
60 | args = []
61 | kwargs = {}
62 | for i in range(len(backend)):
63 | if i == 0:
64 | name = backend[i]
65 | else:
66 | if isinstance(backend[i], dict):
67 | kwargs.update(backend[i])
68 | else:
69 | args += backend[i]
70 | elif backend is None:
71 | name = None
72 | args = []
73 | kwargs = {}
74 | else:
75 | raise TypeError(
76 | "The backend must be of type Backend, string, tuple, list, or None"
77 | )
78 | if name in available_backends:
79 | self.backend = name
80 | self._backend = available_backends[name]()
81 | elif name is None:
82 | # The base class should not be called.
83 | warnings.warn(
84 | "The `None` backend was selected and will have limited functionality"
85 | )
86 | else:
87 | raise Exception("Backend %s not found in backends.py" % name)
88 | self._backend.model = self
89 | self._backend.init_backend(*args, **kwargs)
90 |
91 | def run(self, **run_params) -> None:
92 | """Run the simulation (or lookup the results in the cache)."""
93 | self.use_default_run_params()
94 | self.set_run_params(**run_params)
95 | if self.print_run_params:
96 | print("Run Params:", self.run_params)
97 | self.results = self._backend.backend_run()
98 |
99 | def set_attrs(self, **attrs) -> None:
100 | """Set model attributes, e.g. input resistance of a cell."""
101 | self.attrs.update(attrs)
102 | self._backend.set_attrs(**attrs)
103 |
104 | def set_run_params(self, **run_params) -> None:
105 | """Set run-time parameters, e.g. the somatic current to inject."""
106 | self.run_params.update(run_params)
107 | self.check_run_params()
108 | self._backend.set_run_params(**run_params)
109 |
110 | def check_run_params(self) -> None:
111 | """Check if run parameters are reasonable for this model class.
112 |
113 | Raise a sciunit.BadParameterValueError if any of them are not.
114 | """
115 |
116 | def reset_run_params(self) -> None:
117 | self.run_params = {}
118 |
119 | def set_default_run_params(self, **params) -> None:
120 | self.default_run_params.update(params)
121 |
122 | def use_default_run_params(self) -> None:
123 | for key, value in self.default_run_params.items():
124 | if key not in self.run_params:
125 | self.set_run_params(**{key: value})
126 |
127 | def reset_default_run_params(self) -> None:
128 | self.default_run_params = {}
129 |
130 | @property
131 | def state(self):
132 | return self._state(keys=["name", "url", "attrs", "run_params", "backend"])
133 |
134 | def __del__(self) -> None:
135 | if hasattr(self, "temp_dir"):
136 | self.temp_dir.cleanup() # Delete the temporary directory
137 | s = super(RunnableModel, self)
138 | if hasattr(s, "__del__"):
139 | s.__del__()
140 |
--------------------------------------------------------------------------------
/sciunit/scores/__init__.py:
--------------------------------------------------------------------------------
1 | """Contains classes for different representations of test scores.
2 |
3 | It also contains score collections such as arrays and matrices.
4 | """
5 |
6 | from .base import ErrorScore, Score, score_logger
7 | from .complete import *
8 | from .incomplete import *
9 |
--------------------------------------------------------------------------------
/sciunit/scores/collections_m2m.py:
--------------------------------------------------------------------------------
1 | """Score collections for direct comparison of models against other models."""
2 |
3 | import warnings
4 | from typing import Any, List, Tuple, Union
5 |
6 | import pandas as pd
7 |
8 | from sciunit.base import SciUnit
9 | from sciunit.models import Model
10 | from sciunit.tests import Test
11 |
12 |
13 | class ScoreArrayM2M(pd.Series, SciUnit):
14 | """
15 | Represents an array of scores derived from TestM2M.
16 | Extends the pandas Series such that items are either
17 | models subject to a test or the test itself.
18 |
19 | Attributes:
20 | index ([type]): [description]
21 | """
22 |
23 | def __init__(
24 | self, test: Test, models: List[Model], scores: List["sciunit.scores.Score"]
25 | ):
26 | items = models if not test.observation else [test] + models
27 | super(ScoreArrayM2M, self).__init__(data=scores, index=items)
28 |
29 | state_hide = ['related_data', 'scores', 'norm_scores', 'style', 'plot', 'iat', 'at', 'iloc', 'loc', 'T']
30 |
31 | def __getitem__(self, item: Union[str, callable]) -> Any:
32 | if isinstance(item, str):
33 | result = self.get_by_name(item)
34 | else:
35 | result = super(ScoreArrayM2M, self).__getitem__(item)
36 | return result
37 |
38 | def __getattr__(self, name: str) -> Any:
39 | if name in ["score", "norm_scores", "related_data"]:
40 | attr = self.apply(lambda x: getattr(x, name))
41 | else:
42 | attr = super(ScoreArrayM2M, self).__getattribute__(name)
43 | return attr
44 |
45 | def get_by_name(self, name: str) -> str:
46 | """Get item (can be a model, observation, or test) in `index` by name.
47 |
48 | Args:
49 | name (str): name of the item.
50 |
51 | Raises:
52 | KeyError: Item not found.
53 |
54 | Returns:
55 | Any: Item found.
56 | """
57 | for entry in self.index:
58 | if entry.name == name or name.lower() == "observation":
59 | return self.__getitem__(entry)
60 | raise KeyError(
61 | ("Doesn't match test, 'observation' or " "any model: '%s'") % name
62 | )
63 |
64 | @property
65 | def norm_scores(self) -> pd.Series:
66 | """A series of norm scores.
67 |
68 | Returns:
69 | Series: A series of norm scores.
70 | """
71 | return self.map(lambda x: x.norm_score)
72 |
73 |
74 | class ScoreMatrixM2M(pd.DataFrame, SciUnit):
75 | """
76 | Represents a matrix of scores derived from TestM2M.
77 | Extends the pandas DataFrame such that models/observation are both
78 | columns and the index.
79 | """
80 |
81 | def __init__(
82 | self, test: Test, models: List[Model], scores: List["sciunit.scores.Score"]
83 | ):
84 | if not test.observation:
85 | items = models
86 | else:
87 | # better to have header as "observation" than test.name
88 | # only affects pandas.DataFrame; not test.name in individual scores
89 | test.name = "observation"
90 | items = [test] + models
91 | super(ScoreMatrixM2M, self).__init__(data=scores, index=items, columns=items)
92 | with warnings.catch_warnings():
93 | warnings.filterwarnings(
94 | "ignore",
95 | message=(".*Pandas doesn't allow columns " "to be created via a new "),
96 | )
97 | self.test = test
98 | self.models = models
99 |
100 | state_hide = ['related_data', 'scores', 'norm_scores', 'style', 'plot', 'iat', 'at', 'iloc', 'loc', 'T']
101 |
102 | def __getitem__(
103 | self, item: Union[Tuple[Test, Model], str, Tuple[list, tuple]]
104 | ) -> Any:
105 | if isinstance(item, (Test, Model)):
106 | result = ScoreArrayM2M(self.test, self.models, scores=self.loc[item, :])
107 | elif isinstance(item, str):
108 | result = self.get_by_name(item)
109 | elif isinstance(item, (list, tuple)) and len(item) == 2:
110 | result = self.get_group(item)
111 | else:
112 | raise TypeError(
113 | (
114 | "Expected test/'observation'; model; "
115 | "test/'observation',model; "
116 | "model,test/'observation'; or model,model"
117 | )
118 | )
119 | return result
120 |
121 | def get_by_name(self, name: str) -> Union[Model, Test]:
122 | """Get the model or test from the `models` or `tests` by name.
123 |
124 | Args:
125 | name (str): The name of the model or test.
126 |
127 | Raises:
128 | KeyError: Raise an exception if there is not a model or test named `name`.
129 |
130 | Returns:
131 | Union[Model, Test]: The test or model found.
132 | """
133 | for model in self.models:
134 | if model.name == name:
135 | return self.__getitem__(model)
136 | if self.test.name == name or name.lower() == "observation":
137 | return self.__getitem__(self.test)
138 | raise KeyError(
139 | ("Doesn't match test, 'observation' or " "any model: '%s'") % name
140 | )
141 |
142 | def get_group(self, x: list) -> Any:
143 | """[summary]
144 |
145 | Args:
146 | x (list): [description]
147 |
148 | Raises:
149 | TypeError: [description]
150 |
151 | Returns:
152 | Any: [description]
153 | """
154 | if isinstance(x[0], (Test, Model)) and isinstance(x[1], (Test, Model)):
155 | return self.loc[x[0], x[1]]
156 | elif isinstance(x[0], str):
157 | return self.__getitem__(x[0]).__getitem__(x[1])
158 | raise TypeError("Expected test/model pair")
159 |
160 | def __getattr__(self, name: str) -> Any:
161 | if name in ["score", "norm_score", "related_data"]:
162 | attr = self.applymap(lambda x: getattr(x, name))
163 | else:
164 | attr = super(ScoreMatrixM2M, self).__getattribute__(name)
165 | return attr
166 |
167 | @property
168 | def norm_scores(self) -> pd.DataFrame:
169 | """Get a pandas DataFrame instance that contains norm scores.
170 |
171 | Returns:
172 | DataFrame: A pandas DataFrame instance that contains norm scores.
173 | """
174 | return self.applymap(lambda x: x.norm_score)
175 |
--------------------------------------------------------------------------------
/sciunit/scores/incomplete.py:
--------------------------------------------------------------------------------
1 | """Score types for tests that did not complete successfully.
2 |
3 | These include details about the various possible reasons
4 | that a particular combination of model and test could not be completed.
5 | """
6 |
7 | from sciunit.errors import InvalidScoreError
8 |
9 | from .base import Score
10 |
11 |
12 | class NoneScore(Score):
13 | """A `None` score.
14 |
15 | Usually indicates that the model has not been
16 | checked to see if it has the capabilities required by the test."""
17 |
18 | def __init__(self, score: Score, related_data: dict = None):
19 | if isinstance(score, str) or score is None:
20 | super(NoneScore, self).__init__(score, related_data=related_data)
21 | else:
22 | raise InvalidScoreError("Score must be a string or None")
23 |
24 | @property
25 | def norm_score(self) -> None:
26 | """Return None as the norm score of this NoneScore instance.
27 |
28 | Returns:
29 | None: The norm score, which is None.
30 | """
31 | return None
32 |
33 | def __str__(self) -> str:
34 | if self.score:
35 | s = "%s (%s)" % (self.description, self.score)
36 | else:
37 | s = self.description
38 | return s
39 |
40 |
41 | class TBDScore(NoneScore):
42 | """A TBD (to be determined) score. Indicates that the model has
43 | capabilities required by the test but has not yet taken it."""
44 |
45 | description = "None"
46 |
47 |
48 | class NAScore(NoneScore):
49 | """A N/A (not applicable) score.
50 |
51 | Indicates that the model doesn't have the
52 | capabilities that the test requires."""
53 |
54 | description = "N/A"
55 |
56 |
57 | class InsufficientDataScore(NoneScore):
58 | """A score returned when the model or test data
59 | is insufficient to score the test."""
60 |
61 | description = "Insufficient Data"
62 |
--------------------------------------------------------------------------------
/sciunit/style.css:
--------------------------------------------------------------------------------
1 | td.red {
2 | background-color: #FF0000;
3 | border: 1px solid black;
4 | }
5 | td.green {
6 | background-color: #00FF00;
7 | border: 1px solid black;
8 | }
9 | td.grey {
10 | background-color: #AAAAAA;
11 | border: 1px solid black;
12 | }
13 | th {
14 | text-align: center;
15 | border: 1px solid black;
16 | }
17 | table td, table th {
18 | border: 10px solid black;
19 | font-size: 250%;
20 | }
--------------------------------------------------------------------------------
/sciunit/unit_test/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Unit testing module for sciunit.
3 | The script `test.sh` will run all tests imported into `active.py`.
4 | """
5 |
--------------------------------------------------------------------------------
/sciunit/unit_test/__main__.py:
--------------------------------------------------------------------------------
1 | """All unit tests for SciUnit"""
2 |
3 | import sys
4 | import unittest
5 |
6 | from .active import * # Import all the tests from the unit_test.active module
7 |
8 |
9 | def main():
10 | buffer = "buffer" in sys.argv
11 | sys.argv = sys.argv[:1] # :Args need to be removed for __main__ to work.
12 | unittest.main(buffer=buffer)
13 |
14 |
15 | if __name__ == "__main__":
16 | main()
17 |
--------------------------------------------------------------------------------
/sciunit/unit_test/active.py:
--------------------------------------------------------------------------------
1 | """
2 | All active unit tests for SciUnit. This module is the default target of
3 | for testing in `__main__.py`. Modify this file if you want to add or remove
4 | tests located in other modules.
5 | """
6 |
7 | from .backend_tests import *
8 | from .base_tests import *
9 | from .command_line_tests import *
10 | from .config_tests import *
11 | from .converter_tests import *
12 | from .doc_tests import *
13 | from .error_tests import *
14 | from .import_tests import *
15 | from .model_tests import *
16 | from .observation_tests import *
17 | from .score_tests import *
18 | from .test_tests import *
19 | from .utils_tests import *
20 | from .validator_tests import *
21 |
--------------------------------------------------------------------------------
/sciunit/unit_test/backend_tests.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "import sciunit\n",
10 | "from sciunit.models.backends import Backend\n",
11 | "from sciunit.capabilities import Runnable\n",
12 | "from sciunit.errors import CapabilityNotImplementedError"
13 | ]
14 | },
15 | {
16 | "cell_type": "code",
17 | "execution_count": 2,
18 | "metadata": {},
19 | "outputs": [],
20 | "source": [
21 | "class MyModel(sciunit.Model, Runnable):\n",
22 | " pass"
23 | ]
24 | },
25 | {
26 | "cell_type": "code",
27 | "execution_count": 3,
28 | "metadata": {},
29 | "outputs": [],
30 | "source": [
31 | "m = MyModel()"
32 | ]
33 | },
34 | {
35 | "cell_type": "code",
36 | "execution_count": 4,
37 | "metadata": {},
38 | "outputs": [
39 | {
40 | "name": "stdout",
41 | "output_type": "stream",
42 | "text": [
43 | "MyModel \n"
44 | ]
45 | }
46 | ],
47 | "source": [
48 | "try:\n",
49 | " m.run()\n",
50 | "except CapabilityNotImplementedError:\n",
51 | " pass\n",
52 | "else:\n",
53 | " raise Exception(\"This should have been an error!!!!\")"
54 | ]
55 | },
56 | {
57 | "cell_type": "code",
58 | "execution_count": 5,
59 | "metadata": {},
60 | "outputs": [],
61 | "source": [
62 | "class MyModel(sciunit.models.RunnableModel):\n",
63 | " def run(self, a, b=5):\n",
64 | " pass\n",
65 | " \n",
66 | "m = MyModel(name='My Model')"
67 | ]
68 | },
69 | {
70 | "cell_type": "code",
71 | "execution_count": 6,
72 | "metadata": {},
73 | "outputs": [],
74 | "source": [
75 | "m = sciunit.models.RunnableModel(name='smith', backend=None)"
76 | ]
77 | },
78 | {
79 | "cell_type": "code",
80 | "execution_count": 7,
81 | "metadata": {},
82 | "outputs": [],
83 | "source": [
84 | "m = sciunit.models.RunnableModel(name='jones', backend='')"
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "execution_count": 8,
90 | "metadata": {},
91 | "outputs": [],
92 | "source": [
93 | "def run(self, **kwargs):\n",
94 | " print('running')\n",
95 | "\n",
96 | "from types import MethodType\n",
97 | "m.run = MethodType(run, m)"
98 | ]
99 | },
100 | {
101 | "cell_type": "code",
102 | "execution_count": 9,
103 | "metadata": {},
104 | "outputs": [
105 | {
106 | "name": "stdout",
107 | "output_type": "stream",
108 | "text": [
109 | "running\n"
110 | ]
111 | }
112 | ],
113 | "source": [
114 | "m.run()"
115 | ]
116 | },
117 | {
118 | "cell_type": "code",
119 | "execution_count": 10,
120 | "metadata": {},
121 | "outputs": [
122 | {
123 | "name": "stdout",
124 | "output_type": "stream",
125 | "text": [
126 | "running\n"
127 | ]
128 | }
129 | ],
130 | "source": [
131 | "try:\n",
132 | " m.run()\n",
133 | "except NotImplementedError:\n",
134 | " raise Exception(\"Implementation failed\")"
135 | ]
136 | }
137 | ],
138 | "metadata": {
139 | "kernelspec": {
140 | "display_name": "Python 3",
141 | "language": "python",
142 | "name": "python3"
143 | },
144 | "language_info": {
145 | "codemirror_mode": {
146 | "name": "ipython",
147 | "version": 3
148 | },
149 | "file_extension": ".py",
150 | "mimetype": "text/x-python",
151 | "name": "python",
152 | "nbconvert_exporter": "python",
153 | "pygments_lexer": "ipython3",
154 | "version": "3.7.4"
155 | }
156 | },
157 | "nbformat": 4,
158 | "nbformat_minor": 4
159 | }
160 |
--------------------------------------------------------------------------------
/sciunit/unit_test/backend_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for backends."""
2 |
3 | import unittest
4 | from pathlib import Path
5 |
6 | from sciunit import Model
7 | from sciunit.models.backends import Backend
8 | from sciunit.utils import NotebookTools
9 |
10 | class BackendsTestCase(unittest.TestCase, NotebookTools):
11 | """Unit tests for the sciunit module"""
12 |
13 | path = "."
14 |
15 | def test_backends(self):
16 | """Test backends."""
17 | self.do_notebook("backend_tests")
18 |
19 | def test_backends_init_caches(self):
20 | myModel = Model()
21 | backend = Backend()
22 | backend.model = myModel
23 |
24 | backend.init_backend(use_disk_cache=True, use_memory_cache=True)
25 | backend.init_backend(use_disk_cache=False, use_memory_cache=True)
26 | backend.init_backend(use_disk_cache=True, use_memory_cache=False)
27 | backend.init_backend(use_disk_cache=False, use_memory_cache=False)
28 | backend.init_cache()
29 |
30 | def test_backends_init_disk_caches(self):
31 | # Automatically set disk_cache location
32 | myModel = Model()
33 | backend = Backend()
34 | backend.model = myModel
35 | backend.init_backend(use_disk_cache=True, use_memory_cache=False)
36 | self.assertTrue(backend.disk_cache_location.endswith(".sciunit/cache"))
37 |
38 | # Manually set disk_cache location (a string)
39 | myModel = Model()
40 | backend = Backend()
41 | backend.model = myModel
42 | backend.init_backend(use_disk_cache="/some/good/path", use_memory_cache=False)
43 | self.assertEqual(backend.disk_cache_location, "/some/good/path")
44 |
45 | # Manually set disk_cache location (a Path)
46 | myModel = Model()
47 | backend = Backend()
48 | backend.model = myModel
49 | backend.init_backend(use_disk_cache=Path("/some/good/path"), use_memory_cache=False)
50 | self.assertEqual(backend.disk_cache_location, "/some/good/path")
51 |
52 | def test_backends_set_caches(self):
53 | myModel = Model()
54 | backend = Backend()
55 | backend.model = myModel
56 | backend.init_backend(use_disk_cache=True, use_memory_cache=True)
57 | backend.clear_disk_cache()
58 | # backend.init_memory_cache()
59 | self.assertIsNone(backend.get_disk_cache("key1"))
60 | self.assertIsNone(backend.get_disk_cache("key2"))
61 | self.assertIsNone(backend.get_memory_cache("key1"))
62 | self.assertIsNone(backend.get_memory_cache("key2"))
63 | backend.set_disk_cache("value1", "key1")
64 | backend.set_memory_cache("value1", "key1")
65 | self.assertEqual(backend.get_memory_cache("key1"), "value1")
66 | self.assertEqual(backend.get_disk_cache("key1"), "value1")
67 | backend.set_disk_cache("value2")
68 | backend.set_memory_cache("value2")
69 | self.assertEqual(backend.get_memory_cache(myModel.hash()), "value2")
70 | self.assertEqual(backend.get_disk_cache(myModel.hash()), "value2")
71 |
72 | backend.load_model()
73 | backend.set_attrs(test_attribute="test attribute")
74 | backend.set_run_params(test_param="test parameter")
75 | backend.init_backend(use_disk_cache=True, use_memory_cache=True)
76 |
77 | def test_backend_run(self):
78 | backend = Backend()
79 | self.assertRaises(NotImplementedError, backend._backend_run)
80 |
81 | class MyBackend(Backend):
82 | model = Model()
83 |
84 | def _backend_run(self) -> str:
85 | return "test result"
86 |
87 | backend = MyBackend()
88 | backend.init_backend(use_disk_cache=True, use_memory_cache=True)
89 | backend.backend_run()
90 | backend.set_disk_cache("value1", "key1")
91 | backend.set_memory_cache("value1", "key1")
92 | backend.backend_run()
93 | backend.set_disk_cache("value2")
94 | backend.set_memory_cache("value2")
95 | backend.backend_run()
96 |
97 | backend = MyBackend()
98 | backend.init_backend(use_disk_cache=False, use_memory_cache=True)
99 | backend.backend_run()
100 | backend.set_disk_cache("value1", "key1")
101 | backend.set_memory_cache("value1", "key1")
102 | backend.backend_run()
103 | backend.set_disk_cache("value2")
104 | backend.set_memory_cache("value2")
105 | backend.backend_run()
106 |
107 | backend = MyBackend()
108 | backend.init_backend(use_disk_cache=True, use_memory_cache=False)
109 | backend.backend_run()
110 | backend.set_disk_cache("value1", "key1")
111 | backend.set_memory_cache("value1", "key1")
112 | backend.backend_run()
113 | backend.set_disk_cache("value2")
114 | backend.set_memory_cache("value2")
115 | backend.backend_run()
116 |
117 | backend = MyBackend()
118 | backend.init_backend(use_disk_cache=False, use_memory_cache=False)
119 | backend.backend_run()
120 | backend.set_disk_cache("value1", "key1")
121 | backend.set_memory_cache("value1", "key1")
122 | backend.backend_run()
123 | backend.set_disk_cache("value2")
124 | backend.set_memory_cache("value2")
125 | backend.backend_run()
126 |
127 | def test_backend_cache_to_results(self):
128 | myModel = Model()
129 | class MyBackend(Backend):
130 | def cache_to_results(self, cache):
131 | return { "color": "red" }
132 |
133 | def results_to_cache(self, results):
134 | return { "color": "blue" }
135 |
136 | def _backend_run(self):
137 | return { "color": "white" }
138 |
139 | backend = MyBackend()
140 | backend.model = myModel
141 | backend.init_backend(use_disk_cache=False, use_memory_cache=True)
142 | # On first run we get the original object
143 | self.assertEqual(backend.backend_run(), { "color": "white" })
144 | # And on consequent runs we get the object recovered from the cache
145 | self.assertEqual(backend.backend_run(), { "color": "red" })
146 | self.assertEqual(backend.backend_run(), { "color": "red" })
147 |
148 | if __name__ == "__main__":
149 | unittest.main()
150 |
--------------------------------------------------------------------------------
/sciunit/unit_test/base.py:
--------------------------------------------------------------------------------
1 | """Common imports for many unit tests in this directory"""
2 |
3 | import sys
4 |
5 | import matplotlib as mpl
6 |
7 | OSX = sys.platform == "darwin"
8 | if OSX or "Qt" in mpl.rcParams["backend"]:
9 | mpl.use("Agg") # Avoid any problems with Macs or headless displays.
10 |
11 |
12 | class SuiteBase(object):
13 | """Abstract base class for testing suites and scores"""
14 |
15 | def setUp(self):
16 | from sciunit.models.examples import UniformModel
17 | from sciunit.tests import RangeTest
18 |
19 | self.M = UniformModel
20 | self.T = RangeTest
21 |
22 | def prep_models_and_tests(self):
23 | from sciunit import TestSuite
24 |
25 | t1 = self.T([2, 3], name="test1")
26 | t2 = self.T([5, 6])
27 | m1 = self.M(2, 3)
28 | m2 = self.M(5, 6)
29 | ts = TestSuite([t1, t2], name="MySuite")
30 | return (ts, t1, t2, m1, m2)
31 |
--------------------------------------------------------------------------------
/sciunit/unit_test/base_tests.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | from sciunit.utils import TmpTestFolder
4 |
5 | tmp_folder = TmpTestFolder()
6 |
7 | class BaseCase(unittest.TestCase):
8 | """Unit tests for config files"""
9 |
10 | @classmethod
11 | def setUpClass(cls):
12 | tmp_folder.create()
13 |
14 | @classmethod
15 | def tearDownClass(cls):
16 | tmp_folder.delete()
17 |
18 | def test_deep_exclude(self):
19 | from sciunit.base import deep_exclude
20 |
21 | test_state = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
22 | test_exclude = [("a", "b"), ("c", "d")]
23 | deep_exclude(test_state, test_exclude)
24 |
25 | def test_default(self):
26 | # TODO
27 | pass
28 |
29 | def test_SciUnit(self):
30 | from sciunit.base import SciUnit
31 |
32 | sciunitObj = SciUnit()
33 | self.assertIsInstance(sciunitObj.properties(), dict)
34 | self.assertIsInstance(sciunitObj.__getstate__(), dict)
35 | self.assertIsInstance(sciunitObj.json(), str)
36 | sciunitObj.json(string=False)
37 | self.assertIsInstance(sciunitObj._class, dict)
38 | sciunitObj.testState = "testState"
39 | SciUnit.state_hide.append("testState")
40 | self.assertFalse("testState" in sciunitObj.__getstate__())
41 |
42 | def test_Versioned(self):
43 | from git import Repo
44 | from sciunit.base import Versioned
45 |
46 | ver = Versioned()
47 |
48 | # Testing .get_remote()
49 | # 1. Checking our sciunit .git repo
50 | # (to make sure .get_remote() works with real repos too!)
51 | self.assertEqual("origin", ver.get_remote("I am not a remote").name)
52 | self.assertEqual("origin", ver.get_remote().name)
53 | # 2. Checking NO .git repo
54 | self.assertEqual(None, ver.get_remote(repo=None))
55 | # 3. Checking a .git repo without remotes
56 | git_repo = Repo.init(tmp_folder.path / "git_repo")
57 | self.assertEqual(None, ver.get_remote(repo=git_repo))
58 | # 4. Checking a .git repo with remotes
59 | origin = git_repo.create_remote("origin", "https://origin.com")
60 | beta = git_repo.create_remote('beta', "https://beta.com")
61 | self.assertEqual(origin, ver.get_remote(repo=git_repo))
62 | self.assertEqual(origin, ver.get_remote("not a remote", repo=git_repo))
63 | self.assertEqual(beta, ver.get_remote("beta", repo=git_repo))
64 |
65 | # Testing .get_repo()
66 | self.assertIsInstance(ver.get_repo(), Repo)
67 |
68 | # Testing .get_remote_url()
69 | self.assertIsInstance(ver.get_remote_url("I am not a remote"), str)
70 |
71 | if __name__ == "__main__":
72 | unittest.main()
73 |
--------------------------------------------------------------------------------
/sciunit/unit_test/command_line_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for command line tools."""
2 |
3 | import platform
4 | import tempfile
5 | import unittest
6 | from pathlib import Path
7 |
8 | import sciunit
9 |
10 |
11 | class CommandLineTestCase(unittest.TestCase):
12 | """Unit tests for command line tools."""
13 |
14 | def setUp(self):
15 | from sciunit.__main__ import main
16 |
17 | self.main = main
18 | path = Path(sciunit.__path__[0]).resolve()
19 | SCIDASH_HOME = path.parent.parent
20 | self.cosmosuite_path = str(SCIDASH_HOME / "scidash")
21 |
22 | def test_sciunit_1create(self):
23 | try:
24 | self.main("--directory", self.cosmosuite_path, "create")
25 | except Exception as e:
26 | if "There is already a configuration file" not in str(e):
27 | raise e
28 | else:
29 | temp_path = tempfile.mkdtemp()
30 | self.main("--directory", temp_path, "create")
31 |
32 | def test_sciunit_2check(self):
33 | self.main("--directory", self.cosmosuite_path, "check")
34 |
35 | def test_sciunit_3run(self):
36 | self.main("--directory", self.cosmosuite_path, "run")
37 |
38 | def test_sciunit_4make_nb(self):
39 | self.main("--directory", self.cosmosuite_path, "make-nb")
40 |
41 | # Skip for python versions that don't have importlib.machinery
42 | @unittest.skipIf(
43 | platform.python_version() < "3.5", "sciunit not supported on Python < 3.5"
44 | )
45 | def test_sciunit_5run_nb(self):
46 | self.main("--directory", self.cosmosuite_path, "run-nb")
47 |
48 |
49 | if __name__ == "__main__":
50 | test_program = unittest.main(verbosity=0, buffer=True, exit=False)
51 |
--------------------------------------------------------------------------------
/sciunit/unit_test/config_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for user configuration"""
2 |
3 | import unittest
4 | from pathlib import Path
5 |
6 | import sciunit
7 |
8 |
9 | class ConfigTestCase(unittest.TestCase):
10 | """Unit tests for config files"""
11 |
12 | def test_new_config(self):
13 | sciunit.config.create()
14 |
15 | self.assertTrue(sciunit.config.path.is_file())
16 | cmap_low = sciunit.config.get("cmap_low")
17 |
18 | self.assertTrue(isinstance(cmap_low, int))
19 | dummy = sciunit.config.get("dummy", 37)
20 | self.assertEqual(dummy, 37)
21 | try:
22 | sciunit.config.get("dummy")
23 | except sciunit.Error as e:
24 | self.assertTrue("does not contain key" in str(e))
25 |
26 | def test_missing_config(self):
27 | sciunit.config.path = Path("_delete.json")
28 | sciunit.config.get_from_disk()
29 |
30 | def test_bad_config(self):
31 | sciunit.config.path = Path("_delete.json")
32 | with open(sciunit.config.path, "w") as f:
33 | f.write(".......")
34 | sciunit.config.get_from_disk()
35 |
36 |
37 | if __name__ == "__main__":
38 | unittest.main()
39 |
--------------------------------------------------------------------------------
/sciunit/unit_test/converter_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for score converters"""
2 |
3 | import unittest
4 |
5 |
6 | class ConvertersTestCase(unittest.TestCase):
7 | """Unit tests for Score converters"""
8 |
9 | def test_converters(self):
10 | from sciunit.converters import (
11 | AtLeastToBoolean,
12 | AtMostToBoolean,
13 | LambdaConversion,
14 | NoConversion,
15 | RangeToBoolean,
16 | )
17 | from sciunit.scores import BooleanScore, ZScore
18 |
19 | old_score = ZScore(1.3)
20 | new_score = NoConversion().convert(old_score)
21 | self.assertEqual(old_score, new_score)
22 | new_score = LambdaConversion(lambda x: x.score ** 2).convert(old_score)
23 | self.assertEqual(old_score.score ** 2, new_score.score)
24 | new_score = AtMostToBoolean(3).convert(old_score)
25 | self.assertEqual(new_score, BooleanScore(True))
26 | new_score = AtMostToBoolean(1).convert(old_score)
27 | self.assertEqual(new_score, BooleanScore(False))
28 | new_score = AtLeastToBoolean(1).convert(old_score)
29 | self.assertEqual(new_score, BooleanScore(True))
30 | new_score = AtLeastToBoolean(3).convert(old_score)
31 | self.assertEqual(new_score, BooleanScore(False))
32 | new_score = RangeToBoolean(1, 3).convert(old_score)
33 | self.assertEqual(new_score, BooleanScore(True))
34 | new_score = RangeToBoolean(3, 5).convert(old_score)
35 | self.assertEqual(new_score, BooleanScore(False))
36 | self.assertEqual(new_score.raw, str(old_score.score))
37 |
38 | def test_converters2(self):
39 | from sciunit import Score
40 | from sciunit.converters import Converter
41 |
42 | converterObj = Converter()
43 |
44 | self.assertIsInstance(converterObj.description, str)
45 | self.assertRaises(NotImplementedError, converterObj._convert, Score(0.5))
46 |
47 | class MyConverter(Converter):
48 | pass
49 |
50 | myConverterObj = MyConverter()
51 | self.assertEqual(myConverterObj.description, "No description available")
52 |
53 |
54 | if __name__ == "__main__":
55 | unittest.main()
56 |
--------------------------------------------------------------------------------
/sciunit/unit_test/doc_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for documentation"""
2 |
3 | import unittest
4 |
5 | from sciunit.utils import NotebookTools
6 |
7 |
8 | class DocumentationTestCase(NotebookTools, unittest.TestCase):
9 | """Unit tests for documentation notebooks"""
10 |
11 | path = "../../docs"
12 |
13 | def test_chapter1(self):
14 | self.do_notebook("chapter1")
15 |
16 | def test_chapter2(self):
17 | self.do_notebook("chapter2")
18 |
19 | def test_chapter3(self):
20 | self.do_notebook("chapter3")
21 |
22 | def test_chapter4(self):
23 | self.do_notebook("chapter4")
24 |
25 | def test_chapter5(self):
26 | self.do_notebook("chapter5")
27 |
28 | @unittest.skip("Requires sympy")
29 | def test_chapter6(self):
30 | self.do_notebook("chapter6")
31 |
32 | # def test_chapter6(self):
33 | # self.do_notebook("chapter6")
34 |
--------------------------------------------------------------------------------
/sciunit/unit_test/error_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for sciunit errors"""
2 |
3 | import unittest
4 |
5 |
6 | class ErrorsTestCase(unittest.TestCase):
7 | """Unit tests for various error classes"""
8 |
9 | def test_error_types(self):
10 | from sciunit import Capability, Model
11 | from sciunit.errors import (
12 | BadParameterValueError,
13 | CapabilityError,
14 | InvalidScoreError,
15 | PredictionError,
16 | )
17 |
18 | CapabilityError(Model(), Capability)
19 | CapabilityError(Model(), Capability, "this is a test detail")
20 | PredictionError(Model(), "foo")
21 | InvalidScoreError()
22 | BadParameterValueError("x", 3)
23 |
24 |
25 | if __name__ == "__main__":
26 | unittest.main()
27 |
--------------------------------------------------------------------------------
/sciunit/unit_test/import_tests.py:
--------------------------------------------------------------------------------
1 | """Tests of imports of sciunit submodules and other dependencies"""
2 |
3 | import unittest
4 |
5 |
6 | class ImportTestCase(unittest.TestCase):
7 | """Unit tests for imports"""
8 |
9 | def test_quantities(self):
10 | import quantities as pq
11 |
12 | pq.Quantity([10, 20, 30], pq.pA)
13 |
14 | def test_import_everything(self):
15 | import sciunit
16 | from sciunit.utils import import_all_modules
17 |
18 | # Recursively import all submodules
19 | import_all_modules(sciunit)
20 |
21 |
22 | if __name__ == "__main__":
23 | unittest.main()
24 |
--------------------------------------------------------------------------------
/sciunit/unit_test/model_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for models and capabilities"""
2 |
3 | import unittest
4 |
5 |
6 | class ModelsTestCase(unittest.TestCase):
7 | """Unit tests for the sciunit module"""
8 |
9 | def setUp(self):
10 | from sciunit.models.examples import UniformModel
11 |
12 | self.M = UniformModel
13 |
14 | def test_is_match(self):
15 | from sciunit import Model
16 |
17 | m = Model()
18 | m2 = Model()
19 | self.assertFalse(m.is_match(m2))
20 | self.assertTrue(m.is_match(m))
21 | self.assertTrue(m.is_match("Model"))
22 |
23 | def test_getattr(self):
24 | from sciunit import Model
25 |
26 | m = Model()
27 | self.assertEqual(m.name, m.__getattr__("name"))
28 |
29 | def test_curr_method(self):
30 | from sciunit import Model
31 |
32 | class TestModel(Model):
33 | def test_calling_curr_method(self):
34 | return self.curr_method()
35 |
36 | m = TestModel()
37 | test_method_name = m.test_calling_curr_method()
38 | self.assertEqual(test_method_name, "test_calling_curr_method")
39 |
40 | def test_failed_extra_capabilities(self):
41 | from sciunit import Model
42 |
43 | class TestModel(Model):
44 | def test_return_none_function(self):
45 | return None
46 |
47 | m = TestModel()
48 | m.extra_capability_checks = {TestModel: "test_return_none_function"}
49 | test_list = m.failed_extra_capabilities
50 |
51 | self.assertEqual(test_list[0], TestModel)
52 | self.assertEqual(len(test_list), 1)
53 |
54 | def test_get_model_state(self):
55 | from sciunit import Model
56 |
57 | m = Model()
58 | state = m.__getstate__()
59 | self.assertTrue(["capabilities" in state])
60 | self.assertTrue(m.capabilities == state["capabilities"])
61 |
62 | def test_get_model_capabilities(self):
63 | from sciunit.capabilities import ProducesNumber
64 |
65 | m = self.M(2, 3)
66 | self.assertEqual(m.capabilities, [ProducesNumber])
67 |
68 | def test_get_model_description(self):
69 | m = self.M(2, 3)
70 | m.describe()
71 | m.description = "Lorem Ipsum"
72 | m.describe()
73 |
74 | def test_check_model_capabilities(self):
75 | from sciunit.tests import RangeTest
76 |
77 | t = RangeTest([2, 3])
78 | m = self.M(2, 3)
79 | t.check(m)
80 |
81 | def test_check_missing_capabilities_1(self):
82 | m = self.M(
83 | 2, 3, name="Not actually runnable due to lack of capability provision"
84 | )
85 | try:
86 | m.run()
87 | except AttributeError as e:
88 | pass
89 | else:
90 | self.fail("Unprovided capability was called and not caught")
91 |
92 | def test_check_missing_capabilities_2(self):
93 | from sciunit.capabilities import Runnable
94 | from sciunit.errors import CapabilityNotImplementedError
95 |
96 | class MyModel(self.M, Runnable):
97 | pass
98 |
99 | m = MyModel(
100 | 2, 3, name="Not actually runnable due to lack of capability implementation"
101 | )
102 | try:
103 | m.run()
104 | except CapabilityNotImplementedError as e:
105 | pass
106 | else:
107 | self.fail("Unimplemented capability was called and not caught")
108 |
109 | def test_check_missing_capabilities_3(self):
110 | from sciunit.capabilities import Runnable
111 |
112 | class MyModel(self.M, Runnable):
113 | def run(self):
114 | print("Actually running!")
115 |
116 | m = MyModel(2, 3, name="Now actually runnable")
117 | m.run()
118 |
119 | def test_regular_models(self):
120 | from sciunit.models.examples import (
121 | ConstModel,
122 | PersistentUniformModel,
123 | UniformModel,
124 | )
125 |
126 | m = ConstModel(3)
127 | self.assertEqual(m.produce_number(), 3)
128 |
129 | m = UniformModel(3, 4)
130 | self.assertTrue(3 < m.produce_number() < 4)
131 |
132 | m = PersistentUniformModel(3, 4)
133 | m.run()
134 | self.assertTrue(3 < m.produce_number() < 4)
135 |
136 | def test_irregular_models(self):
137 | from sciunit.models.examples import (
138 | CacheByInstancePersistentUniformModel,
139 | CacheByValuePersistentUniformModel,
140 | )
141 |
142 | a = CacheByInstancePersistentUniformModel(2, 3)
143 | a1 = a.produce_number()
144 | a2 = a.produce_number()
145 | self.assertEqual(a1, a2)
146 | b = CacheByInstancePersistentUniformModel(2, 3)
147 | b1 = b.produce_number()
148 | self.assertNotEqual(b1, a2)
149 |
150 | c = CacheByValuePersistentUniformModel(2, 3)
151 | c1 = c.produce_number()
152 | c2 = c.produce_number()
153 | self.assertEqual(c1, c2)
154 | d = CacheByValuePersistentUniformModel(2, 3)
155 | d1 = d.produce_number()
156 | self.assertEqual(d1, c2)
157 |
158 |
159 | class CapabilitiesTestCase(unittest.TestCase):
160 | """Unit tests for sciunit Capability classes"""
161 |
162 | def test_capabilities(self):
163 | from sciunit import Model
164 | from sciunit.capabilities import Capability, ProducesNumber, Runnable
165 | from sciunit.models import Model
166 | from sciunit.models.examples import (
167 | RepeatedRandomNumberModel,
168 | UniqueRandomNumberModel,
169 | )
170 |
171 | class MyModel(Model, ProducesNumber):
172 | def produce_number(self):
173 | return 3.14
174 |
175 | m = MyModel()
176 | self.assertEqual(m.produce_number(), 3.14)
177 |
178 | m = UniqueRandomNumberModel()
179 | self.assertNotEqual(m.produce_number(), m.produce_number())
180 |
181 | m = RepeatedRandomNumberModel()
182 | self.assertEqual(m.produce_number(), m.produce_number())
183 |
184 | m = Runnable()
185 | self.assertRaises(BaseException, m.run)
186 | self.assertRaises(BaseException, m.set_run_params)
187 | self.assertRaises(BaseException, m.set_default_run_params)
188 |
189 | m = ProducesNumber()
190 | self.assertRaises(BaseException, m.produce_number)
191 |
192 | m = Capability()
193 | m.name = "test name"
194 | self.assertEqual(str(m), "test name")
195 |
196 | def test_source_check(self):
197 |
198 | from sciunit import Model
199 | from sciunit.capabilities import Capability
200 | from sciunit.errors import CapabilityNotImplementedError
201 | from sciunit.models import Model
202 |
203 | class MyCap1(Capability):
204 | def fn1(self):
205 | raise NotImplementedError("fn1 not implemented.")
206 |
207 | class MyCap2(Capability):
208 | def fn1(self):
209 | self.unimplemented("fn1 not implemented.")
210 |
211 | class MyCap3(Capability):
212 | def fn1(self):
213 | raise CapabilityNotImplementedError(
214 | model=self,
215 | capability=self.__class__,
216 | details="fn1 not implemented.",
217 | )
218 |
219 | class MyModel1(Model, MyCap1):
220 | def fn1(self):
221 | return "fn1 have been implemented"
222 |
223 | class MyModel2(Model, MyCap1):
224 | pass
225 |
226 | class MyModel3(Model, MyCap2):
227 | def fn1(self):
228 | return "fn1 have been implemented"
229 |
230 | class MyModel4(Model, MyCap2):
231 | pass
232 |
233 | class MyModel5(Model, MyCap3):
234 | def fn1(self):
235 | return "fn1 have been implemented"
236 |
237 | class MyModel6(Model, MyCap3):
238 | pass
239 |
240 | self.assertTrue(MyCap1.source_check(MyModel1()))
241 | self.assertFalse(MyCap1.source_check(MyModel2()))
242 | self.assertTrue(MyCap2.source_check(MyModel3()))
243 | self.assertFalse(MyCap2.source_check(MyModel4()))
244 | self.assertTrue(MyCap3.source_check(MyModel5()))
245 | self.assertFalse(MyCap3.source_check(MyModel6()))
246 |
247 |
248 | class RunnableModelTestCase(unittest.TestCase):
249 | def test_backend(self):
250 | from sciunit.models import RunnableModel
251 | from sciunit.models.backends import Backend, register_backends
252 |
253 | self.assertRaises(TypeError, RunnableModel, name="", attrs=1)
254 | model = RunnableModel(name="test name")
255 | self.assertIsInstance(model.get_backend(), Backend)
256 | self.assertRaises(TypeError, model.set_backend, 0)
257 |
258 | model.set_backend(None)
259 | self.assertRaises(Exception, model.set_backend, "invalid backend")
260 |
261 | model.set_attrs(test_attr="test attribute")
262 | model.set_run_params(test_run_params="test runtime parameter")
263 | model.check_run_params()
264 | model.reset_run_params()
265 | model.set_default_run_params(test_run_params="test runtime parameter")
266 | model.reset_default_run_params()
267 | self.assertIsInstance(model.__getstate__(), dict)
268 |
269 | class MyBackend1(Backend):
270 | def _backend_run(self) -> str:
271 | return "test result 1"
272 |
273 | class MyBackend2(Backend):
274 | def _backend_run(self) -> str:
275 | return "test result 2"
276 |
277 | name_backend_dict = {"backend1": MyBackend2, "backend2": MyBackend2}
278 | backend_names = ["backend1", "backend2"]
279 |
280 | model = RunnableModel(name="test name")
281 | register_backends(name_backend_dict)
282 | model.set_backend(backend_names)
283 | model.print_run_params = True
284 | model.run()
285 |
286 | model = RunnableModel(name="test name")
287 | model.default_run_params = {"para1": 1}
288 | model.use_default_run_params()
289 |
290 |
291 | if __name__ == "__main__":
292 | unittest.main()
293 |
--------------------------------------------------------------------------------
/sciunit/unit_test/observation_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for observations."""
2 |
3 | import unittest
4 |
5 | from sciunit.utils import NotebookTools
6 |
7 |
8 | class ObservationsTestCase(unittest.TestCase, NotebookTools):
9 | """Unit tests for the sciunit module"""
10 |
11 | path = "."
12 |
13 | def test_observation_validation(self):
14 | """Test validation of observations against the `observation_schema`."""
15 | self.do_notebook("validate_observation")
16 |
--------------------------------------------------------------------------------
/sciunit/unit_test/score_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for scores and score collections"""
2 |
3 | import unittest
4 |
5 | import numpy as np
6 | from IPython.display import display
7 | from pandas import DataFrame
8 | from pandas.core.frame import DataFrame
9 | from pandas.core.series import Series
10 | from quantities import Quantity
11 |
12 | from sciunit import Score, ScoreArray, ScoreMatrix
13 | from sciunit.errors import InvalidScoreError
14 | from sciunit.models import Model
15 | from sciunit.scores import (
16 | BooleanScore,
17 | CohenDScore,
18 | ErrorScore,
19 | FloatScore,
20 | InsufficientDataScore,
21 | NAScore,
22 | NoneScore,
23 | PercentScore,
24 | RandomScore,
25 | RatioScore,
26 | TBDScore,
27 | ZScore,
28 | )
29 | from sciunit.scores.collections_m2m import ScoreArrayM2M, ScoreMatrixM2M
30 | from sciunit.tests import RangeTest, Test
31 | from sciunit.unit_test.base import SuiteBase
32 | from sciunit.utils import NotebookTools
33 |
34 |
35 | class ScoresTestCase(SuiteBase, unittest.TestCase, NotebookTools):
36 |
37 | path = "."
38 |
39 | def test_score_matrix_constructor(self):
40 | tests = [Test([1, 2, 3])]
41 | models = [Model()]
42 | scores = np.array([ZScore(1.0)])
43 | ScoreArray(tests)
44 | scoreMatrix = ScoreMatrix(tests, models, scores)
45 | scoreMatrix = ScoreMatrix(tests, models, scores, transpose=True)
46 |
47 | tests = Test([1, 2, 3])
48 | models = Model()
49 | ScoreMatrix(tests, models, scores)
50 |
51 | def test_score_matrix(self):
52 | t, t1, t2, m1, m2 = self.prep_models_and_tests()
53 | sm = t.judge(m1)
54 |
55 | self.assertRaises(TypeError, sm.__getitem__, 0)
56 |
57 | self.assertEqual(str(sm.get_group((t1, m1))), "Pass")
58 | self.assertEqual(str(sm.get_group((m1, t1))), "Pass")
59 | self.assertEqual(str(sm.get_group((m1.name, t1.name))), "Pass")
60 | self.assertEqual(str(sm.get_group((t1.name, m1.name))), "Pass")
61 |
62 | self.assertRaises(TypeError, sm.get_group, (0, 0))
63 | self.assertRaises(KeyError, sm.get_by_name, "This name does not exist")
64 |
65 | self.assertIsInstance(sm.__getattr__("score"), DataFrame)
66 | self.assertIsInstance(sm.norm_scores, DataFrame)
67 | self.assertIsInstance(sm.T, ScoreMatrix)
68 | # self.assertIsInstance(sm.to_html(True, True, True), str)
69 | # self.assertIsInstance(sm.to_html(), str)
70 |
71 | self.assertTrue(type(sm) is ScoreMatrix)
72 | self.assertTrue(sm[t1][m1].score)
73 | self.assertTrue(sm["test1"][m1].score)
74 | self.assertTrue(sm[m1]["test1"].score)
75 | self.assertFalse(sm[t2][m1].score)
76 | self.assertEqual(sm[(m1, t1)].score, True)
77 | self.assertEqual(sm[(m1, t2)].score, False)
78 | sm = t.judge([m1, m2])
79 | self.assertEqual(sm.stature(t1, m1), 1)
80 | self.assertEqual(sm.stature(t1, m2), 2)
81 | display(sm)
82 |
83 | ######### m2m #################
84 | t1.observation = [2, 3]
85 | smm2m = ScoreMatrixM2M(
86 | test=t1, models=[m1], scores=[[Score(1), Score(1)], [Score(1), Score(1)]]
87 | )
88 |
89 | self.assertIsInstance(smm2m.__getattr__("score"), DataFrame)
90 | self.assertIsInstance(smm2m.__getattr__("norm_scores"), DataFrame)
91 | self.assertIsInstance(smm2m.__getattr__("related_data"), DataFrame)
92 | self.assertRaises(KeyError, smm2m.get_by_name, "Not Exist")
93 | self.assertIsInstance(smm2m.norm_scores, DataFrame)
94 | self.assertRaises(KeyError, smm2m.get_by_name, "Not Exist")
95 | self.assertRaises(TypeError, smm2m.get_group, [0])
96 | self.assertIsInstance(smm2m.get_group([m1.name, t1.name]), Score)
97 | self.assertEqual(smm2m.get_group([m1.name, t1.name]).score, 1)
98 | self.assertIsInstance(smm2m.get_group([m1, t1]), Score)
99 | self.assertEqual(smm2m.get_group([m1, t1]).score, 1)
100 |
101 | def test_score_arrays(self):
102 | t, t1, t2, m1, m2 = self.prep_models_and_tests()
103 | sm = t.judge(m1)
104 | sa = sm[m1]
105 | self.assertTrue(type(sa) is ScoreArray)
106 | self.assertIsInstance(sa.__getattr__("score"), Series)
107 | self.assertRaises(KeyError, sa.get_by_name, "This name does not exist")
108 | self.assertEqual(list(sa.norm_scores.values), [1.0, 0.0])
109 | self.assertEqual(sa.stature(t1), 1)
110 | self.assertEqual(sa.stature(t2), 2)
111 | self.assertEqual(sa.stature(t1), 1)
112 | display(sa)
113 |
114 | ######### m2m #################
115 | sam2m = ScoreArrayM2M(
116 | test=t1, models=[m1], scores=[[Score(1), Score(1)], [Score(1), Score(1)]]
117 | )
118 | self.assertRaises(KeyError, sam2m.get_by_name, "Not Exist")
119 |
120 | def test_regular_score_types_1(self):
121 | self.assertEqual(PercentScore(0).norm_score, 0)
122 | self.assertEqual(PercentScore(100).norm_score, 1)
123 |
124 | score = PercentScore(42)
125 | self.assertRaises(InvalidScoreError, PercentScore, 101)
126 | self.assertRaises(InvalidScoreError, PercentScore, -1)
127 | self.assertEqual(str(score), "42.0%")
128 | self.assertEqual(score.norm_score, 0.42)
129 |
130 | self.assertEqual(1, ZScore(0.0).norm_score)
131 | self.assertEqual(0, ZScore(1e12).norm_score)
132 | self.assertEqual(0, ZScore(-1e12).norm_score)
133 |
134 | ZScore(0.7)
135 | score = ZScore.compute({"mean": 3.0, "std": 1.0}, {"value": 2.0})
136 |
137 | self.assertIsInstance(
138 | ZScore.compute({"mean": 3.0}, {"value": 2.0}), InsufficientDataScore
139 | )
140 | self.assertIsInstance(
141 | ZScore.compute({"mean": 3.0, "std": -1.0}, {"value": 2.0}),
142 | InsufficientDataScore,
143 | )
144 | self.assertIsInstance(
145 | ZScore.compute({"mean": np.nan, "std": np.nan}, {"value": np.nan}),
146 | InsufficientDataScore,
147 | )
148 | self.assertEqual(score.score, -1.0)
149 |
150 | self.assertEqual(1, CohenDScore(0.0).norm_score)
151 | self.assertEqual(0, CohenDScore(1e12).norm_score)
152 | self.assertEqual(0, CohenDScore(-1e12).norm_score)
153 | CohenDScore(-0.3)
154 | score = CohenDScore.compute(
155 | {"mean": 3.0, "std": 1.0}, {"mean": 2.0, "std": 1.0}
156 | )
157 |
158 | self.assertAlmostEqual(-0.707, score.score, 3)
159 | self.assertEqual("D = -0.71", str(score))
160 |
161 | score = CohenDScore.compute(
162 | {"mean": 3.0, "std": 10.0, "n": 10}, {"mean": 2.5, "std": 10.0, "n": 10}
163 | )
164 | self.assertAlmostEqual(-0.05, score.score, 2)
165 |
166 | def test_regular_score_types_2(self):
167 | BooleanScore(True)
168 | BooleanScore(False)
169 | score = BooleanScore.compute(5, 5)
170 | self.assertEqual(score.norm_score, 1)
171 | score = BooleanScore.compute(4, 5)
172 | self.assertEqual(score.norm_score, 0)
173 |
174 | self.assertEqual(1, BooleanScore(True).norm_score)
175 | self.assertEqual(0, BooleanScore(False).norm_score)
176 |
177 | t = RangeTest([2, 3])
178 | score.test = t
179 | score.describe()
180 | score.description = "Lorem Ipsum"
181 | score.describe()
182 |
183 | score = FloatScore(3.14)
184 | self.assertRaises(
185 | InvalidScoreError, score.check_score, Quantity([1, 2, 3], "J")
186 | )
187 |
188 | obs = np.array([1.0, 2.0, 3.0])
189 | pred = np.array([1.0, 2.0, 4.0])
190 | score = FloatScore.compute_ssd(obs, pred)
191 | self.assertEqual(str(score), "1")
192 | self.assertEqual(score.score, 1.0)
193 |
194 | score = RatioScore(1.2)
195 | self.assertEqual(1, RatioScore(1.0).norm_score)
196 | self.assertEqual(0, RatioScore(1e12).norm_score)
197 | self.assertEqual(0, RatioScore(1e-12).norm_score)
198 |
199 | self.assertEqual(str(score), "Ratio = 1.20")
200 |
201 | self.assertRaises(InvalidScoreError, RatioScore, -1.0)
202 | score = RatioScore.compute({"mean": 4.0, "std": 1.0}, {"value": 2.0})
203 |
204 | self.assertEqual(score.score, 0.5)
205 |
206 | def test_irregular_score_types(self):
207 | e = Exception("This is an error")
208 | score = ErrorScore(e)
209 | score = NAScore(None)
210 | score = TBDScore(None)
211 | score = NoneScore(None)
212 | score = NoneScore("this is a string")
213 | self.assertIsInstance(str(score), str)
214 | self.assertRaises(InvalidScoreError, NoneScore, ["this is a string list"])
215 |
216 | score = InsufficientDataScore(None)
217 | self.assertEqual(score.norm_score, None)
218 |
219 | def test_only_lower_triangle(self):
220 | """Test validation of observations against the `observation_schema`."""
221 | self.do_notebook("test_only_lower_triangle")
222 |
223 | def test_RandomScore(self):
224 | """Note: RandomScore is only used for debugging purposes"""
225 | score = RandomScore(0.5)
226 | self.assertEqual("0.5", str(score))
227 |
228 | def test_Score(self):
229 | self.assertIsInstance(Score.compute({}, {}), NotImplementedError)
230 | score = Score(0.5)
231 | self.assertEqual(score.norm_score, 0.5)
232 | self.assertAlmostEqual(score.log_norm_score, -0.693, 2)
233 | self.assertAlmostEqual(score.log2_norm_score, -1.0, 1)
234 | self.assertAlmostEqual(score.log10_norm_score, -0.301, 1)
235 | self.assertIsInstance(score.raw, str)
236 | score._raw = "this is a string"
237 | self.assertIsNone(score.raw)
238 | self.assertIsInstance(score.__repr__(), str)
239 | self.assertIsInstance(score.__str__(), str)
240 |
241 | self.assertFalse(score.__ne__(score))
242 | self.assertTrue(score.__ne__(Score(998.0)))
243 | self.assertFalse(score.__ne__(0.5))
244 | self.assertTrue(score.__ne__(0.6))
245 |
246 | self.assertFalse(score.__gt__(score))
247 | self.assertTrue(score.__gt__(Score(0.2)))
248 | self.assertFalse(score.__gt__(0.5))
249 | self.assertTrue(score.__gt__(0.2))
250 |
251 | self.assertFalse(score.__lt__(score))
252 | self.assertTrue(score.__lt__(Score(0.9)))
253 | self.assertFalse(score.__lt__(0.5))
254 | self.assertTrue(score.__lt__(0.9))
255 |
256 | self.assertTrue(score.__le__(score))
257 | self.assertTrue(score.__le__(Score(0.5)))
258 | self.assertTrue(score.__le__(0.5))
259 | self.assertTrue(score.__le__(0.5))
260 | self.assertFalse(score.__le__(0.1))
261 | self.assertFalse(score.__le__(Score(0.1)))
262 |
263 | self.assertIsInstance(score.score_type, str)
264 |
265 | def test_ErrorScore(self):
266 | score = ErrorScore(0.5)
267 | self.assertEqual(0.0, score.norm_score)
268 | self.assertIsInstance(score.summary, str)
269 | self.assertIsInstance(score._describe(), str)
270 | self.assertIsInstance(str(score), str)
271 |
272 |
273 | if __name__ == "__main__":
274 | unittest.main()
275 |
--------------------------------------------------------------------------------
/sciunit/unit_test/test_only_lower_triangle.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "from sciunit.models.examples import ConstModel\n",
10 | "from sciunit.tests import TestM2M\n",
11 | "from sciunit.scores import RatioScore\n",
12 | "from sciunit.capabilities import ProducesNumber"
13 | ]
14 | },
15 | {
16 | "cell_type": "code",
17 | "execution_count": 2,
18 | "metadata": {},
19 | "outputs": [],
20 | "source": [
21 | "class MyTestM2M(TestM2M): \n",
22 | " required_capabilities = (ProducesNumber,)\n",
23 | " score_type = RatioScore \n",
24 | " \n",
25 | " def generate_prediction(self, model): \n",
26 | " return model.produce_number() "
27 | ]
28 | },
29 | {
30 | "cell_type": "code",
31 | "execution_count": 11,
32 | "metadata": {},
33 | "outputs": [],
34 | "source": [
35 | "models = [ConstModel(x, name='%s producer'%x) for x in range(1,4)]"
36 | ]
37 | },
38 | {
39 | "cell_type": "code",
40 | "execution_count": 12,
41 | "metadata": {},
42 | "outputs": [
43 | {
44 | "data": {
45 | "text/html": [
46 | "\n",
47 | "\n",
60 | "
\n",
61 | " \n",
62 | " \n",
63 | " | \n",
64 | " 1 producer | \n",
65 | " 2 producer | \n",
66 | " 3 producer | \n",
67 | "
\n",
68 | " \n",
69 | " \n",
70 | " \n",
71 | " 1 producer | \n",
72 | " Ratio = 1.00 | \n",
73 | " Ratio = 2.00 | \n",
74 | " Ratio = 3.00 | \n",
75 | "
\n",
76 | " \n",
77 | " 2 producer | \n",
78 | " Ratio = 0.50 | \n",
79 | " Ratio = 1.00 | \n",
80 | " Ratio = 1.50 | \n",
81 | "
\n",
82 | " \n",
83 | " 3 producer | \n",
84 | " Ratio = 0.33 | \n",
85 | " Ratio = 0.67 | \n",
86 | " Ratio = 1.00 | \n",
87 | "
\n",
88 | " \n",
89 | "
\n",
90 | "
"
91 | ],
92 | "text/plain": [
93 | " 1 producer 2 producer 3 producer\n",
94 | "1 producer Ratio = 1.00 Ratio = 2.00 Ratio = 3.00\n",
95 | "2 producer Ratio = 0.50 Ratio = 1.00 Ratio = 1.50\n",
96 | "3 producer Ratio = 0.33 Ratio = 0.67 Ratio = 1.00"
97 | ]
98 | },
99 | "execution_count": 12,
100 | "metadata": {},
101 | "output_type": "execute_result"
102 | }
103 | ],
104 | "source": [
105 | "test = MyTestM2M() \n",
106 | "test.judge(models) "
107 | ]
108 | },
109 | {
110 | "cell_type": "code",
111 | "execution_count": 13,
112 | "metadata": {},
113 | "outputs": [
114 | {
115 | "data": {
116 | "text/html": [
117 | "\n",
118 | "\n",
131 | "
\n",
132 | " \n",
133 | " \n",
134 | " | \n",
135 | " 1 producer | \n",
136 | " 2 producer | \n",
137 | " 3 producer | \n",
138 | "
\n",
139 | " \n",
140 | " \n",
141 | " \n",
142 | " 1 producer | \n",
143 | " Ratio = 1.00 | \n",
144 | " Ratio = 2.00 | \n",
145 | " Ratio = 3.00 | \n",
146 | "
\n",
147 | " \n",
148 | " 2 producer | \n",
149 | " Ratio = 2.00 | \n",
150 | " Ratio = 1.00 | \n",
151 | " Ratio = 1.50 | \n",
152 | "
\n",
153 | " \n",
154 | " 3 producer | \n",
155 | " Ratio = 3.00 | \n",
156 | " Ratio = 1.50 | \n",
157 | " Ratio = 1.00 | \n",
158 | "
\n",
159 | " \n",
160 | "
\n",
161 | "
"
162 | ],
163 | "text/plain": [
164 | " 1 producer 2 producer 3 producer\n",
165 | "1 producer Ratio = 1.00 Ratio = 2.00 Ratio = 3.00\n",
166 | "2 producer Ratio = 2.00 Ratio = 1.00 Ratio = 1.50\n",
167 | "3 producer Ratio = 3.00 Ratio = 1.50 Ratio = 1.00"
168 | ]
169 | },
170 | "execution_count": 13,
171 | "metadata": {},
172 | "output_type": "execute_result"
173 | }
174 | ],
175 | "source": [
176 | "test.judge(models, only_lower_triangle=True)"
177 | ]
178 | }
179 | ],
180 | "metadata": {
181 | "kernelspec": {
182 | "display_name": "Python 3",
183 | "language": "python",
184 | "name": "python3"
185 | },
186 | "language_info": {
187 | "codemirror_mode": {
188 | "name": "ipython",
189 | "version": 3
190 | },
191 | "file_extension": ".py",
192 | "mimetype": "text/x-python",
193 | "name": "python",
194 | "nbconvert_exporter": "python",
195 | "pygments_lexer": "ipython3",
196 | "version": "3.7.1"
197 | }
198 | },
199 | "nbformat": 4,
200 | "nbformat_minor": 2
201 | }
202 |
--------------------------------------------------------------------------------
/sciunit/unit_test/test_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for (sciunit) tests and test suites"""
2 |
3 | import unittest
4 |
5 | import quantities as pq
6 |
7 | from sciunit import Model, TestSuite, config
8 | from sciunit.capabilities import ProducesNumber
9 | from sciunit.errors import Error, InvalidScoreError, ObservationError, ParametersError
10 | from sciunit.models.examples import ConstModel, UniformModel
11 | from sciunit.scores import BooleanScore, FloatScore, ZScore
12 | from sciunit.scores.collections import ScoreMatrix
13 | from sciunit.tests import ProtocolToFeaturesTest, RangeTest, Test, TestM2M
14 |
15 | from .base import SuiteBase
16 |
17 |
18 | class TestsTestCase(unittest.TestCase):
19 | """Unit tests for the sciunit module"""
20 |
21 | def setUp(self):
22 | self.M = UniformModel
23 | self.T = RangeTest
24 |
25 | def test_get_test_description(self):
26 | t = self.T([2, 3])
27 | t.describe()
28 | t.description = "Lorem Ipsum"
29 | t.describe()
30 |
31 | class MyTest(self.T):
32 | """Lorem Ipsum"""
33 |
34 | t = MyTest([2, 3])
35 | t.description = None
36 | self.assertEqual(t.describe(), "Lorem Ipsum")
37 |
38 | def test_check_model_capabilities(self):
39 | t = self.T([2, 3])
40 | m = self.M(2, 3)
41 | t.check(m)
42 |
43 | def test_rangetest(self):
44 | from sciunit.converters import NoConversion
45 |
46 | range_2_3_test = RangeTest(observation=[2, 3])
47 | range_2_3_test.converter = NoConversion()
48 | one_model = ConstModel(2.5)
49 | self.assertTrue(range_2_3_test.check_capabilities(one_model))
50 | score = range_2_3_test.judge(one_model)
51 | self.assertTrue(isinstance(score, BooleanScore))
52 | self.assertEqual(score.score, True)
53 | self.assertTrue(score.test is range_2_3_test)
54 | self.assertTrue(score.model is one_model)
55 |
56 | def test_Test(self):
57 | pv = config["PREVALIDATE"]
58 | config["PREVALIDATE"] = 1
59 | with self.assertRaises(ObservationError):
60 | t = Test(None)
61 |
62 | with self.assertRaises(ObservationError):
63 |
64 | class Test2(Test):
65 | observation_schema = None
66 | score_type = ZScore
67 | units = pq.pA
68 |
69 | def generate_prediction(self):
70 | return 1
71 |
72 | t = Test2({"mean": 5 * pq.pA})
73 |
74 | t = Test({})
75 | self.assertRaises(
76 | ObservationError, t.validate_observation, "I am not an observation"
77 | )
78 | t.observation_schema = {}
79 | t.validate_observation({0: 0, 1: 1})
80 | Test.observation_schema = [{}, {}]
81 | self.assertListEqual(t.observation_schema_names(), ["Schema 1", "Schema 2"])
82 | config["PREVALIDATE"] = pv
83 |
84 | self.assertRaises(ParametersError, t.validate_params, None)
85 | self.assertRaises(ParametersError, t.validate_params, "I am not an observation")
86 | t.params_schema = {}
87 | t.validate_params({0: 1, 1: 2})
88 |
89 | self.assertRaises(Error, t.check_capabilities, "I am not a model")
90 | t.condition_model(Model())
91 | self.assertRaises(NotImplementedError, t.generate_prediction, Model())
92 | self.assertRaises(NotImplementedError, t.optimize, Model())
93 |
94 | self.assertTrue(t.compute_score({0: 2, 1: 2}, {0: 2, 1: 2}).score)
95 | self.assertFalse(t.compute_score({0: -2, 1: 2}, {0: 2, 1: -2}).score)
96 | t.score_type = None
97 | self.assertRaises(NotImplementedError, t.compute_score, {}, {})
98 |
99 | t.score_type = BooleanScore
100 | self.assertRaises(InvalidScoreError, t.check_score_type, FloatScore(0.5))
101 |
102 |
103 | class TestSuitesTestCase(SuiteBase, unittest.TestCase):
104 | """Unit tests for the sciunit module"""
105 |
106 | def test_testsuite(self):
107 | t1 = self.T([2, 3])
108 | t2 = self.T([5, 6])
109 | m1 = self.M(2, 3)
110 | m2 = self.M(5, 6)
111 | t = TestSuite([t1, t2])
112 | t.judge([m1, m2])
113 | self.assertIsInstance(t.check([m1, m2]), ScoreMatrix)
114 | capa_list = t.check_capabilities(m1)
115 | self.assertTrue(capa_list[0])
116 | self.assertTrue(capa_list[1])
117 |
118 | t = TestSuite(
119 | {"test 1": t1, "test 2": t2, "test 3 (non-Test)": "I am not a Test"}
120 | )
121 | self.assertRaises(TypeError, t.assert_tests, 0)
122 | self.assertRaises(TypeError, t.assert_tests, [0])
123 | self.assertRaises(TypeError, t.assert_models, 0)
124 | self.assertRaises(TypeError, t.assert_models, [0])
125 | self.assertRaises(NotImplementedError, t.optimize, m1)
126 | self.assertRaises(KeyError, t.__getitem__, "wrong name")
127 | self.assertIsInstance(t[0], RangeTest)
128 |
129 | t.judge([m1, m2])
130 | t = TestSuite([t1, t2], skip_models=[m1], include_models=[m2])
131 | t.judge([m1, m2])
132 |
133 | def test_testsuite_hooks(self):
134 | t1 = self.T([2, 3])
135 | t1.hook_called = False
136 | t2 = self.T([5, 6])
137 | m = self.M(2, 3)
138 |
139 | def f(test, tests, score, a=None):
140 | self.assertEqual(score, True)
141 | self.assertEqual(a, 1)
142 | t1.hook_called = True
143 |
144 | ts = TestSuite(
145 | [t1, t2], name="MySuite", hooks={t1: {"f": f, "kwargs": {"a": 1}}}
146 | )
147 | ts.judge(m)
148 | self.assertEqual(t1.hook_called, True)
149 |
150 | def test_testsuite_from_observations(self):
151 | m = self.M(2, 3)
152 | ts = TestSuite.from_observations(
153 | [(self.T, [2, 3]), (self.T, [5, 6])], name="MySuite"
154 | )
155 | ts.judge(m)
156 |
157 | def test_testsuite_set_verbose(self):
158 | t1 = self.T([2, 3])
159 | t2 = self.T([5, 6])
160 | t = TestSuite([t1, t2])
161 | t.set_verbose(True)
162 | self.assertEqual(t1.verbose, True)
163 | self.assertEqual(t2.verbose, True)
164 |
165 | def test_testsuite_serialize(self):
166 | tests = [RangeTest(observation=(x, x + 3)) for x in [1, 2, 3, 4]]
167 | ts = TestSuite(tests, name="RangeSuite")
168 | self.assertTrue(isinstance(ts.json(), str))
169 |
170 |
171 | class M2MsTestCase(unittest.TestCase):
172 | """Tests for the M2M flavor of tests and test suites"""
173 |
174 | def setUp(self):
175 | self.myModel1 = ConstModel(100.0, "Model1")
176 | self.myModel2 = ConstModel(110.0, "Model2")
177 |
178 | class NumberTest_M2M(TestM2M):
179 | """Dummy Test"""
180 |
181 | score_type = FloatScore
182 | description = "Tests the parameter 'value' between two models"
183 |
184 | def __init__(self, observation=None, name="ValueTest-M2M"):
185 | TestM2M.__init__(self, observation, name)
186 | self.required_capabilities += (ProducesNumber,)
187 |
188 | def generate_prediction(self, model, verbose=False):
189 | """Implementation of sciunit.Test.generate_prediction."""
190 | prediction = model.produce_number()
191 | return prediction
192 |
193 | def compute_score(self, prediction1, prediction2):
194 | """Implementation of sciunit.Test.score_prediction."""
195 | score = FloatScore(prediction1 - prediction2)
196 | score.description = "Difference between model predictions"
197 | return score
198 |
199 | self.NumberTest_M2M = NumberTest_M2M
200 |
201 | def test_testm2m_with_observation(self):
202 | myTest = self.NumberTest_M2M(observation=95.0)
203 | myScore = myTest.judge([self.myModel1, self.myModel2])
204 |
205 | # Test model vs observation
206 | self.assertEqual(myScore[myTest][self.myModel1], -5.0)
207 | self.assertEqual(myScore[self.myModel1][myTest], 5.0)
208 | self.assertEqual(myScore["observation"][self.myModel2], -15.0)
209 | self.assertEqual(myScore[self.myModel2]["observation"], 15.0)
210 |
211 | # Test model vs model
212 | self.assertEqual(myScore[self.myModel1][self.myModel2], -10.0)
213 | self.assertEqual(myScore[self.myModel2][self.myModel1], 10.0)
214 |
215 | def test_testm2m_without_observation(self):
216 | myTest = self.NumberTest_M2M(observation=None)
217 | myScore = myTest.judge([self.myModel1, self.myModel2])
218 |
219 | # Test model vs model; different ways of specifying individual scores
220 | self.assertEqual(myScore[self.myModel1][self.myModel2], -10.0)
221 | self.assertEqual(myScore[self.myModel2][self.myModel1], 10.0)
222 | self.assertEqual(myScore["Model1"][self.myModel2], -10.0)
223 | self.assertEqual(myScore["Model2"][self.myModel1], 10.0)
224 | self.assertEqual(myScore[self.myModel1][self.myModel1], 0.0)
225 | self.assertEqual(myScore["Model2"]["Model2"], 0.0)
226 |
227 | def test_testm2m(self):
228 | myTest = TestM2M(observation=95.0)
229 | myTest.validate_observation(None)
230 | myTest.score_type = None
231 | self.assertRaises(NotImplementedError, myTest.compute_score, {}, {})
232 | myTest.score_type = BooleanScore
233 | self.assertTrue(myTest.compute_score(95, 96))
234 | self.assertRaises(TypeError, myTest.judge, "str")
235 |
236 |
237 | class ProtocolToFeaturesTestCase(unittest.TestCase):
238 | def test_ProtocolToFeaturesTest(self):
239 | t = ProtocolToFeaturesTest([1, 2, 3])
240 | m = Model()
241 | m.run = lambda: 0
242 |
243 | self.assertIsInstance(t.generate_prediction(m), NotImplementedError)
244 | self.assertIsInstance(t.setup_protocol(m), NotImplementedError)
245 | self.assertIsInstance(t.get_result(m), NotImplementedError)
246 | self.assertIsInstance(t.extract_features(m, list()), NotImplementedError)
247 |
248 |
249 | if __name__ == "__main__":
250 | unittest.main()
251 |
--------------------------------------------------------------------------------
/sciunit/unit_test/utils_tests.py:
--------------------------------------------------------------------------------
1 | """Unit tests for sciunit utility functions and classes"""
2 |
3 | import tempfile
4 | import unittest
5 |
6 | import sciunit
7 | from sciunit.utils import use_backend_cache
8 | import numpy as np
9 |
10 |
11 | class CacheTestCase(unittest.TestCase):
12 |
13 | def test_basic_cache(self):
14 | class dummy_test(sciunit.Test):
15 |
16 | # The name of the cache key param determines the cache key location
17 | @use_backend_cache(cache_key_param='dummy_cache_key')
18 | def create_random_matrix(self, model):
19 | # Generate a random matrix that will land in cache
20 | return np.random.randint(0, 100, size=(5,5))
21 |
22 | class dummy_avg_test(dummy_test):
23 |
24 | default_params = {'dummy_cache_key': '1234'}
25 |
26 | @use_backend_cache
27 | def generate_prediction(self, model):
28 | return np.mean(self.create_random_matrix(model))
29 |
30 | class dummy_std_test(dummy_test):
31 |
32 | default_params = {'dummy_cache_key': '1234'}
33 |
34 | @use_backend_cache
35 | def generate_prediction(self, model):
36 | return np.std(self.create_random_matrix(model))
37 |
38 | @use_backend_cache
39 | def warning_function(self):
40 | return 'I am supposed to fail, because I have no model'
41 |
42 | class dummy_backend(sciunit.models.backends.Backend):
43 | pass
44 |
45 | class dummy_model(sciunit.models.RunnableModel):
46 | pass
47 |
48 |
49 | # Initialize dummy tests and models
50 | avg_corr_test1 = dummy_avg_test([], dummy_cache_key='1234')
51 | avg_corr_test2 = dummy_avg_test([], dummy_cache_key='5678')
52 | std_corr_test = dummy_std_test([])
53 | modelA = dummy_model(name='modelA', backend=dummy_backend)
54 | modelB = dummy_model(name='modelB', backend=dummy_backend)
55 |
56 | # Run predictions for the first time
57 | avg_corrsA1 = avg_corr_test1.generate_prediction(model=modelA)
58 | avg_corrsA2 = avg_corr_test2.generate_prediction(modelA)
59 | cached_predictionA1_avg = avg_corr_test1.get_backend_cache(model=modelA)
60 | cached_predictionA2_avg = avg_corr_test2.get_backend_cache(model=modelA)
61 | dummy_matrixA1 = avg_corr_test1.get_backend_cache(model=modelA,
62 | key='1234')
63 | dummy_matrixA2 = avg_corr_test2.get_backend_cache(model=modelA,
64 | key='5678')
65 | # dummy matrix is already generated
66 | # and cached specific for modelA with key '1234'
67 | std_corrsA = std_corr_test.generate_prediction(modelA)
68 | cached_predictionA_std = std_corr_test.get_backend_cache(model=modelA)
69 | dummy_matrixA_std = std_corr_test.get_backend_cache(model=modelA,
70 | key='1234')
71 |
72 | # Check if cached predictions are equal to original computations
73 | self.assertTrue(avg_corrsA1 == cached_predictionA1_avg)
74 | self.assertTrue(std_corrsA == cached_predictionA_std)
75 |
76 | # Check that different tests yield different predictions
77 | # These are floats, unlikely to ever be the same by chance
78 | self.assertTrue(cached_predictionA1_avg != cached_predictionA2_avg)
79 | self.assertTrue(cached_predictionA1_avg != cached_predictionA_std)
80 |
81 | # Check cached matrices are the same
82 | self.assertTrue(np.any(dummy_matrixA1 != dummy_matrixA2))
83 | self.assertTrue(np.all(dummy_matrixA1 == dummy_matrixA_std))
84 |
85 | """Check that a different model will have a different chache"""
86 | avg_corrsB = avg_corr_test1.generate_prediction(modelB)
87 | cached_predictionB1_avg = avg_corr_test1.get_backend_cache(model=modelB)
88 | dummy_matrixB1 = avg_corr_test1.get_backend_cache(model=modelB,
89 | key='1234')
90 | self.assertTrue(cached_predictionA1_avg != cached_predictionB1_avg)
91 | self.assertTrue(np.any(dummy_matrixA1 != dummy_matrixB1))
92 |
93 | """Test the failing cases of the decorator"""
94 | with self.assertWarns(Warning):
95 | std_corr_test.warning_function()
96 |
97 | with self.assertWarns(Warning):
98 | test = dummy_test([])
99 | test.create_random_matrix(modelA)
100 |
101 | class UtilsTestCase(unittest.TestCase):
102 | """Unit tests for sciunit.utils"""
103 |
104 | def test_warnings_traceback(self):
105 | from sciunit.utils import set_warnings_traceback, warn_with_traceback
106 |
107 | set_warnings_traceback(True)
108 | warn_with_traceback("This is a test warning", Warning, "utils_tests.py", 13)
109 |
110 | set_warnings_traceback(False)
111 | warn_with_traceback("This is a test warning", Warning, "utils_tests.py", 16)
112 |
113 | def test_notebook(self):
114 | from sciunit.utils import NotebookTools
115 |
116 | notebookObj = NotebookTools()
117 | notebookObj.execute_notebook("../docs/chapter1")
118 |
119 | def test_log(self):
120 | from sciunit.base import log, strip_html
121 | from sciunit.utils import html_log
122 |
123 | str1 = "test log 1"
124 | str1_stripped = "test log 1"
125 | str2 = "test log 2"
126 | self.assertEqual(strip_html(str1), str1_stripped)
127 | log(str1_stripped)
128 | html_log(str1, str2)
129 |
130 | def test_assert_dimensionless(self):
131 | import quantities as pq
132 |
133 | from sciunit.utils import assert_dimensionless
134 |
135 | assert_dimensionless(3 * pq.s * pq.Hz)
136 | try:
137 | assert_dimensionless(3 * pq.s)
138 | except TypeError:
139 | pass
140 | else:
141 | raise Exception("Should have produced a type error")
142 |
143 | def test_dict_hash(self):
144 | from sciunit.utils import dict_hash
145 |
146 | d1 = {"a": 1, "b": 2, "c": 3}
147 | d2 = {"c": 3, "a": 1, "b": 2}
148 | dh1 = dict_hash(d1)
149 | dh2 = dict_hash(d2)
150 | self.assertTrue(type(dh1) is str)
151 | self.assertTrue(type(dh2) is str)
152 | self.assertEqual(d1, d2)
153 |
154 | def test_import_module_from_path(self):
155 | from sciunit.utils import import_module_from_path
156 |
157 | temp_file = tempfile.mkstemp(suffix=".py")[1]
158 | with open(temp_file, "w") as f:
159 | f.write("value = 42")
160 | module = import_module_from_path(temp_file)
161 | self.assertEqual(module.value, 42)
162 |
163 | def test_versioned(self):
164 | from sciunit.models.examples import ConstModel
165 |
166 | m = ConstModel(37)
167 | print("Commit hash is %s" % m.version)
168 | print("Remote URL is %s" % m.remote_url)
169 | self.assertTrue("sciunit" in m.remote_url)
170 |
171 | def test_MockDevice(self):
172 | from io import StringIO
173 |
174 | from sciunit.utils import MockDevice
175 |
176 | s = StringIO()
177 | myMD = MockDevice(s)
178 | myMD.write("test mock device writing")
179 |
180 | def test_memoize(self):
181 | from random import randint
182 |
183 | from sciunit.utils import memoize
184 |
185 | @memoize
186 | def f(a):
187 | return a + randint(0, 1000000)
188 |
189 | # Should be equal despite the random integer
190 | # because of memoization
191 | self.assertEqual(f(3), f(3))
192 |
193 | def test_intern(self):
194 | from sciunit.utils import class_intern
195 |
196 | class N:
197 | def __init__(self, n):
198 | self.n = n
199 |
200 | five = N(5)
201 | five2 = N(5)
202 | self.assertNotEqual(five, five2)
203 |
204 | # Add the decorator to the class N.
205 | N = class_intern(N)
206 |
207 | five = N(5)
208 | five2 = N(5)
209 | self.assertEqual(five, five2)
210 |
211 |
212 | if __name__ == "__main__":
213 | unittest.main()
214 |
--------------------------------------------------------------------------------
/sciunit/unit_test/validate_observation.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "import quantities as pq\n",
10 | "import sciunit\n",
11 | "from sciunit.errors import ObservationError"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 2,
17 | "metadata": {},
18 | "outputs": [],
19 | "source": [
20 | "class MyTest(sciunit.Test):\n",
21 | " observation_schema = {'mean': {'type': 'float'}, \n",
22 | " 'std': {'type': 'float'}}"
23 | ]
24 | },
25 | {
26 | "cell_type": "code",
27 | "execution_count": 3,
28 | "metadata": {},
29 | "outputs": [],
30 | "source": [
31 | "test = MyTest({'mean': 3.1, 'std': 1.4}) # Correctly formatted observation"
32 | ]
33 | },
34 | {
35 | "cell_type": "code",
36 | "execution_count": 4,
37 | "metadata": {},
38 | "outputs": [
39 | {
40 | "name": "stdout",
41 | "output_type": "stream",
42 | "text": [
43 | "{'observation': [{'std': ['must be of float type']}]}\n"
44 | ]
45 | }
46 | ],
47 | "source": [
48 | "try:\n",
49 | " test = MyTest({'mean': 3.1, 'std': '1.4'}) # Incorrectly formatted; note string value for 'std' key\n",
50 | "except ObservationError as e:\n",
51 | " print(e)"
52 | ]
53 | },
54 | {
55 | "cell_type": "code",
56 | "execution_count": 5,
57 | "metadata": {},
58 | "outputs": [],
59 | "source": [
60 | "class QuantityTest(sciunit.Test):\n",
61 | " observation_schema = {'mean': {'type': 'quantity'},\n",
62 | " 'std': {'type': 'quantity'}}"
63 | ]
64 | },
65 | {
66 | "cell_type": "code",
67 | "execution_count": 6,
68 | "metadata": {},
69 | "outputs": [],
70 | "source": [
71 | "test = QuantityTest({'mean': 3.1*pq.V, 'std': 1.4*pq.V}) # Correctly formatted observation"
72 | ]
73 | },
74 | {
75 | "cell_type": "code",
76 | "execution_count": 7,
77 | "metadata": {},
78 | "outputs": [
79 | {
80 | "name": "stdout",
81 | "output_type": "stream",
82 | "text": [
83 | "{'observation': [{'mean': ['must be of quantity type']}]}\n"
84 | ]
85 | }
86 | ],
87 | "source": [
88 | "try:\n",
89 | " test = QuantityTest({'mean': 3.1, 'std': 1.4*pq.V}) # Incorrectly formatted; no quantity for 'mean' key\n",
90 | "except ObservationError as e:\n",
91 | " print(e)"
92 | ]
93 | },
94 | {
95 | "cell_type": "code",
96 | "execution_count": 8,
97 | "metadata": {},
98 | "outputs": [],
99 | "source": [
100 | "test = QuantityTest({'mean': 3.1*pq.mV, 'std': 1.4*pq.mV}) # Correctly formatted observation"
101 | ]
102 | },
103 | {
104 | "cell_type": "code",
105 | "execution_count": 9,
106 | "metadata": {},
107 | "outputs": [],
108 | "source": [
109 | "class QuantityTest2(sciunit.Test):\n",
110 | " observation_schema = {'mean': {'units': True, 'required': True},\n",
111 | " 'std': {'units': True, 'min':0, 'required': True},\n",
112 | " 'n': {'type': 'integer', 'min': 1}}\n",
113 | " units = pq.V"
114 | ]
115 | },
116 | {
117 | "cell_type": "code",
118 | "execution_count": 10,
119 | "metadata": {},
120 | "outputs": [],
121 | "source": [
122 | "test = QuantityTest2({'mean': 3.1*pq.mV, 'std': 1.4*pq.mV}) # Sould validate"
123 | ]
124 | },
125 | {
126 | "cell_type": "code",
127 | "execution_count": 11,
128 | "metadata": {},
129 | "outputs": [
130 | {
131 | "name": "stdout",
132 | "output_type": "stream",
133 | "text": [
134 | "{'observation': [{'sd': [\"Must have units of 'volt'\"]}]}\n"
135 | ]
136 | }
137 | ],
138 | "source": [
139 | "try:\n",
140 | " test = QuantityTest2({'mean': 3.1*pq.mV, 'std': 1.4*pq.ms}) # Incorrect units for 'std' key\n",
141 | "except ObservationError as e:\n",
142 | " print(e)"
143 | ]
144 | }
145 | ],
146 | "metadata": {
147 | "kernelspec": {
148 | "display_name": "Python 3",
149 | "language": "python",
150 | "name": "python3"
151 | },
152 | "language_info": {
153 | "codemirror_mode": {
154 | "name": "ipython",
155 | "version": 3
156 | },
157 | "file_extension": ".py",
158 | "mimetype": "text/x-python",
159 | "name": "python",
160 | "nbconvert_exporter": "python",
161 | "pygments_lexer": "ipython3",
162 | "version": "3.5.2"
163 | }
164 | },
165 | "nbformat": 4,
166 | "nbformat_minor": 2
167 | }
168 |
--------------------------------------------------------------------------------
/sciunit/unit_test/validator_tests.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | import quantities as pq
4 |
5 |
6 | class ValidatorTestCase(unittest.TestCase):
7 | def test_register(self):
8 | class TestClass:
9 | intValue = 0
10 |
11 | def getIntValue(self):
12 | return self.intValue
13 |
14 | from cerberus import TypeDefinition, Validator
15 |
16 | from sciunit.validators import register_quantity, register_type
17 |
18 | register_type(TestClass, "TestType1")
19 | q = pq.Quantity([1, 2, 3], "J")
20 | register_quantity(q, "TestType2")
21 | self.assertIsInstance(Validator.types_mapping["TestType1"], TypeDefinition)
22 | self.assertIsInstance(Validator.types_mapping["TestType2"], TypeDefinition)
23 |
24 | def test_ObservationValidator(self):
25 | import random
26 |
27 | from sciunit import Test
28 | from sciunit.validators import ObservationValidator
29 |
30 | long_test_list = [None] * 100
31 | for index in range(100):
32 | long_test_list[index] = random.randint(1, 1000)
33 |
34 | q = pq.Quantity([1, 2, 3], "ft")
35 | units = q.simplified.units
36 | units.name = "UnitName"
37 |
38 | testObj = Test({})
39 | testObj.units = units
40 | testObj.observation = long_test_list
41 | obsVal = ObservationValidator(test=testObj)
42 |
43 | # test constructor
44 | self.assertRaises(BaseException, ObservationValidator)
45 | self.assertIsInstance(obsVal, ObservationValidator)
46 |
47 | # test _validate_iterable
48 | obsVal._validate_iterable(True, "key", long_test_list)
49 | self.assertRaises(
50 | BaseException,
51 | obsVal._validate_iterable,
52 | is_iterable=True,
53 | key="Test key",
54 | value=0,
55 | )
56 |
57 | # test _validate_units
58 | self.assertRaises(
59 | BaseException,
60 | obsVal._validate_units,
61 | has_units=True,
62 | key="Test Key",
63 | value="I am not units",
64 | )
65 |
66 | # units in test object is q.simplified.units
67 | obsVal._validate_units(has_units=True, key="TestKey", value=q)
68 |
69 | # units in test object is a dict
70 | testObj.units = {"TestKey": units}
71 | obsVal._validate_units(has_units=True, key="TestKey", value=q)
72 |
73 | print(
74 | "debug here..............................................................................."
75 | )
76 | # Units dismatch
77 | q2 = pq.Quantity([1], "J")
78 | self.assertRaises(
79 | BaseException,
80 | obsVal._validate_units,
81 | has_units=True,
82 | key="TestKey",
83 | value=q2,
84 | )
85 |
86 | def test_ParametersValidator(self):
87 | from sciunit.validators import ParametersValidator
88 |
89 | paraVal = ParametersValidator()
90 |
91 | # test validate_quantity
92 | q = pq.Quantity([1, 2, 3], "A")
93 | paraVal.validate_quantity(q)
94 | self.assertRaises(
95 | BaseException, paraVal.validate_quantity, "I am not a quantity"
96 | )
97 |
98 | q = pq.Quantity([1, 2, 3], pq.s)
99 | self.assertTrue(paraVal._validate_type_time(q))
100 | self.assertRaises(BaseException, paraVal._validate_type_voltage, q)
101 | self.assertRaises(BaseException, paraVal._validate_type_current, q)
102 |
103 | q = pq.Quantity([1, 2, 3], pq.V)
104 | self.assertTrue(paraVal._validate_type_voltage(q))
105 | self.assertRaises(BaseException, paraVal._validate_type_time, q)
106 | self.assertRaises(BaseException, paraVal._validate_type_current, q)
107 |
108 | q = pq.Quantity([1, 2, 3], pq.A)
109 | self.assertTrue(paraVal._validate_type_current(q))
110 | self.assertRaises(BaseException, paraVal._validate_type_voltage, q)
111 | self.assertRaises(BaseException, paraVal._validate_type_time, q)
112 |
113 | self.assertRaises(
114 | BaseException, paraVal._validate_type_current, "I am not a quantity"
115 | )
116 | self.assertRaises(
117 | BaseException, paraVal._validate_type_voltage, "I am not a quantity"
118 | )
119 | self.assertRaises(
120 | BaseException, paraVal._validate_type_time, "I am not a quantity"
121 | )
122 |
123 |
124 | if __name__ == "__main__":
125 | unittest.main()
126 |
--------------------------------------------------------------------------------
/sciunit/validators.py:
--------------------------------------------------------------------------------
1 | """Cerberus validator classes for SciUnit."""
2 |
3 | import inspect
4 | from typing import Any
5 |
6 | import quantities as pq
7 | from cerberus import TypeDefinition, Validator
8 |
9 |
10 | def register_type(cls, name: str) -> None:
11 | """Register `name` as a type to validate as an instance of class `cls`.
12 |
13 | Args:
14 | cls: a class
15 | name (str): the name to be registered.
16 | """
17 | x = TypeDefinition(name, (cls,), ())
18 | Validator.types_mapping[name] = x
19 |
20 |
21 | def register_quantity(quantity: pq.Quantity, name: str) -> None:
22 | """Register `name` as a type to validate as an instance of the class of `quantity`.
23 |
24 | Args:
25 | quantity (pq.Quantity): a quantity.
26 | name (str): the name to be registered.
27 | """
28 |
29 | x = TypeDefinition(name, (quantity.__class__,), ())
30 | Validator.types_mapping[name] = x
31 |
32 |
33 | class ObservationValidator(Validator):
34 | """Cerberus validator class for observations.
35 |
36 | Attributes:
37 | test (Test): A sciunit test instance to be validated.
38 | _error (str, str): The error that happens during the validating process.
39 | """
40 |
41 | def __init__(self, *args, **kwargs):
42 | """Constructor of ObservationValidator.
43 |
44 | Must pass `test` as a keyword argument. Cannot be a positional argument without modifications to cerberus.
45 |
46 | Raises:
47 | Exception: "Observation validator constructor must have a `test` keyword argument."
48 | """
49 |
50 | try:
51 | self.test = kwargs["test"]
52 | except KeyError:
53 | raise Exception(
54 | (
55 | "Observation validator constructor must have "
56 | "a `test` keyword argument"
57 | )
58 | )
59 | super(ObservationValidator, self).__init__(*args, **kwargs)
60 | register_type(pq.quantity.Quantity, "quantity")
61 |
62 | def _validate_iterable(self, is_iterable: bool, key: str, value: Any) -> None:
63 | """Validate fields with `iterable` key in schema set to True
64 | The rule's arguments are validated against this schema:
65 | {'type': 'boolean'}
66 | """
67 | if is_iterable:
68 | try:
69 | iter(value)
70 | except TypeError:
71 | self._error(key, "Must be iterable (e.g. a list or array)")
72 |
73 | def _validate_units(self, has_units: bool, key: str, value: Any) -> None:
74 | """Validate fields with `units` key in schema set to True.
75 | The rule's arguments are validated against this schema:
76 | {'type': 'boolean'}
77 | """
78 | if has_units:
79 | if isinstance(self.test.units, dict):
80 | required_units = self.test.units[key]
81 | else:
82 | required_units = self.test.units
83 | if not isinstance(value, pq.quantity.Quantity):
84 | #self._error(key, "Must be a python quantity")
85 | value *= pq.dimensionless
86 | provided_units = value.simplified.units
87 | if not isinstance(required_units, pq.Dimensionless):
88 | required_units = required_units.simplified.units
89 | else:
90 | required_units = required_units.units
91 | if not required_units == provided_units:
92 | self._error(key, "Must have units of '%s'" % self.test.units.name)
93 |
94 |
95 | class ParametersValidator(Validator):
96 | """Cerberus validator class for observations.
97 |
98 | Attributes:
99 | units_type ([type]): The type of Python quantity's unit.
100 | _error (str, str): value is not a Python quantity instance.
101 | """
102 |
103 | # doc is needed here
104 | units_map = {"time": "s", "voltage": "V", "current": "A"}
105 |
106 | def validate_quantity(self, value: pq.quantity.Quantity) -> None:
107 | """Validate that the value is of the `Quantity` type.
108 |
109 | Args:
110 | value (pq.quantity.Quantity): The Quantity instance to be validated.
111 | """
112 | if not isinstance(value, pq.quantity.Quantity):
113 | self._error("%s" % value, "Must be a Python quantity.")
114 |
115 | def validate_units(self, value: pq.quantity.Quantity) -> bool:
116 | """Validate units, assuming that it was called by _validate_type_*.
117 |
118 | Args:
119 | value (pq.quantity.Quantity): A python quantity instance.
120 |
121 | Returns:
122 | bool: Whether it is valid.
123 | """
124 | self.validate_quantity(value)
125 | self.units_type = inspect.stack()[1][3].split("_")[-1]
126 | assert self.units_type, (
127 | "`validate_units` should not be called "
128 | "directly. It should be called by a "
129 | "_validate_type_* methods that sets "
130 | "`units_type`"
131 | )
132 | units = getattr(pq, self.units_map[self.units_type])
133 | if not value.simplified.units == units:
134 | self._error("%s" % value, "Must have dimensions of %s." % self.units_type)
135 | return True
136 |
137 | def _validate_type_time(self, value: pq.quantity.Quantity) -> bool:
138 | """Validate fields requiring `units` of seconds.
139 |
140 | Args:
141 | value (pq.quantity.Quantity): A python quantity instance.
142 |
143 | Returns:
144 | bool: Whether it is valid.
145 | """
146 | return self.validate_units(value)
147 |
148 | def _validate_type_voltage(self, value: pq.quantity.Quantity) -> bool:
149 | """Validate fields requiring `units` of volts.
150 |
151 | Args:
152 | value (pq.quantity.Quantity): A python quantity instance.
153 |
154 | Returns:
155 | bool: Whether it is valid.
156 | """
157 | return self.validate_units(value)
158 |
159 | def _validate_type_current(self, value: pq.quantity.Quantity) -> bool:
160 | """Validate fields requiring `units` of amps.
161 |
162 | Args:
163 | value (pq.quantity.Quantity): A python quantity instance.
164 |
165 | Returns:
166 | bool: Whether it is valid.
167 | """
168 | return self.validate_units(value)
169 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [metadata]
2 | name = sciunit
3 | version = 0.2.8
4 | description = A test-driven framework for formally validating scientific models against data.
5 | long_description = file: README.md
6 | long_description_content_type = text/markdown
7 | url = http://sciunit.scidash.org
8 |
9 | author = Rick Gerkin
10 | author_email = rgerkin@asu.edu
11 | license = MIT
12 | classifiers =
13 | License :: OSI Approved :: MIT License
14 | Programming Language :: Python :: 3
15 | Programming Language :: Python :: 3.7
16 | Programming Language :: Python :: 3.8
17 | Programming Language :: Python :: 3.9
18 |
19 |
20 | [options]
21 | zip_safe = False
22 | packages = find:
23 | python_requires = >=3.7
24 | install_requires =
25 | bs4
26 | cerberus>=1.2
27 | deepdiff
28 | gitpython
29 | importlib-metadata
30 | ipykernel
31 | ipython
32 | jsonpickle
33 | lxml
34 | matplotlib
35 | nbconvert
36 | nbformat
37 | pandas>=0.18
38 | quantities>=0.13.0
39 |
40 | [options.entry_points]
41 | console_scripts = sciunit=sciunit.__main__:main
42 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import setuptools
2 | import site
3 | import sys
4 |
5 | site.ENABLE_USER_SITE = "--user" in sys.argv[1:]
6 |
7 | setuptools.setup()
8 |
--------------------------------------------------------------------------------
/test.sh:
--------------------------------------------------------------------------------
1 | pip -q install coveralls
2 | if [ ! -d "../scidash" ]; then
3 | git clone -b cosmosuite http://github.com/scidash/scidash ../scidash
4 | fi
5 | # All tests listed in sciunit/unit_test/active.py will be run
6 | UNIT_TEST_SUITE="sciunit.unit_test buffer"
7 | coverage run -m --source=. --omit=*unit_test*,setup.py,.eggs $UNIT_TEST_SUITE
8 |
--------------------------------------------------------------------------------