├── .coveragerc ├── .docstr.yaml ├── .github ├── dependabot.yml └── workflows │ ├── examples.yml │ ├── lint.yml │ └── unit_tests.yml ├── .gitignore ├── CITATION.cff ├── LICENSE ├── README.md ├── docs ├── Makefile ├── complete_api.rst ├── conf.py ├── docs_requirements.txt ├── examples.md ├── index.rst ├── installation.rst ├── make.bat ├── paper.md ├── source │ ├── modules.rst │ ├── uncertainty_wizard.internal_utils.rst │ ├── uncertainty_wizard.models.ensemble_utils.rst │ ├── uncertainty_wizard.models.rst │ ├── uncertainty_wizard.models.stochastic_utils.rst │ ├── uncertainty_wizard.quantifiers.rst │ └── uncertainty_wizard.rst ├── user_guide_models.rst ├── user_guide_quantifiers.rst └── uwiz_logo.PNG ├── examples ├── LoadStochasticFromKeras.ipynb ├── MnistEnsemble.ipynb ├── MnistStochasticSequential.ipynb ├── __init.py └── multi_device.py ├── requirements.txt ├── scripts ├── docker_build.sh ├── format.sh ├── gen_dockerf.sh ├── pypi_deploy.sh ├── run_coverage.sh ├── run_docstring_coverage.sh ├── run_examples.sh └── run_unit_tests.sh ├── setup.py ├── test_requirements.txt ├── tests_unit ├── __init.py ├── internals_tests │ ├── __init__.py │ └── test_tf_version_resolver.py ├── keras_assumptions_tests │ ├── __init__.py │ ├── test_dropout_seed.py │ └── test_experimental_calls_exist.py ├── models_tests │ ├── __init__.py │ ├── test_context_manager.py │ ├── test_functional_stochastic.py │ ├── test_lazy_ensemble.py │ ├── test_sequential_stochastic.py │ ├── test_stochastic_layers.py │ └── test_uwiz_model.py └── quantifiers_tests │ ├── __init__.py │ ├── test_mean_softmax.py │ ├── test_mutual_information.py │ ├── test_one_shot_classifiers.py │ ├── test_predictive_entropy.py │ ├── test_quantifiers_registry.py │ ├── test_stddev.py │ └── test_variation_ratio.py ├── uncertainty_wizard ├── __init__.py ├── internal_utils │ ├── __init__.py │ ├── _uwiz_warning.py │ └── tf_version_resolver.py ├── models │ ├── __init__.py │ ├── _load_model.py │ ├── _stochastic │ │ ├── __init__.py │ │ ├── _abstract_stochastic.py │ │ ├── _from_keras.py │ │ ├── _functional_stochastic.py │ │ ├── _sequential_stochastic.py │ │ └── _stochastic_mode.py │ ├── _uwiz_model.py │ ├── ensemble_utils │ │ ├── __init__.py │ │ ├── _callables.py │ │ ├── _lazy_contexts.py │ │ ├── _lazy_ensemble.py │ │ └── _save_config.py │ └── stochastic_utils │ │ ├── __init__.py │ │ ├── broadcaster.py │ │ └── layers.py └── quantifiers │ ├── __init__.py │ ├── mean_softmax.py │ ├── mutual_information.py │ ├── one_shot_classifiers.py │ ├── predictive_entropy.py │ ├── quantifier.py │ ├── quantifier_registry.py │ ├── regression_quantifiers.py │ └── variation_ratio.py └── version.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | # Regexes for lines to exclude from consideration 3 | exclude_lines = 4 | # Have to re-enable the standard pragma 5 | pragma: no cover 6 | 7 | # Don't complain about missing debug-only code: 8 | def __repr__ 9 | if self\.debug 10 | 11 | # Don't complain if tests don't hit defensive assertion code: 12 | raise AssertionError 13 | raise NotImplementedError 14 | pass 15 | 16 | omit = 17 | # Test environment has no GPUs, so we cannot run GPU config on it 18 | uncertainty_wizard/models/ensemble_utils/_lazy_contexts.py -------------------------------------------------------------------------------- /.docstr.yaml: -------------------------------------------------------------------------------- 1 | paths: 2 | - uncertainty_wizard 3 | skip_magic: True 4 | skip_file_doc: True 5 | skip_init: True 6 | skip_private: True 7 | skip_class_def: True 8 | fail-under: 100 -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | ignore: 13 | - dependency-name: "*" 14 | update-types: ["version-update:semver-patch"] 15 | - dependency-name: "black" 16 | update-types: ["version-update:semver-minor"] 17 | - dependency-name: "tensorflow" # Reason: We want backwards compatibility 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /.github/workflows/examples.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | 3 | name: Examples 4 | 5 | on: 6 | push: 7 | branches: [ main ] 8 | pull_request: 9 | branches: [ main ] 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: [3.8] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - uses: actions/setup-node@v2-beta 21 | with: 22 | node-version: '12' 23 | check-latest: true 24 | 25 | - name: Set up Python ${{ matrix.python-version }} 26 | uses: actions/setup-python@v2 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install -r requirements.txt 34 | pip install -r test_requirements.txt 35 | 36 | - name: Run Tests 37 | run: bash scripts/run_examples.sh -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: actions/setup-node@v2-beta 11 | with: 12 | node-version: '12' 13 | check-latest: true 14 | 15 | - name: Set up Python 3.8 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: 3.8 19 | 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install -r test_requirements.txt 24 | 25 | - name: flake8 26 | run: | 27 | # Taken from https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python 28 | # stop the build if there are Python syntax errors or undefined names 29 | flake8 uncertainty_wizard tests_unit --count --select=E9,F63,F7,F82 --show-source --statistics 30 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 31 | flake8 uncertainty_wizard tests_unit --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 32 | 33 | - name: black 34 | run: | 35 | black uncertainty_wizard tests_unit --check --diff --color 36 | 37 | - name: isort 38 | run: | 39 | isort uncertainty_wizard tests_unit --check-only --profile black 40 | 41 | - name: Docstring Coverage 42 | run: | 43 | docstr-coverage uncertainty_wizard -------------------------------------------------------------------------------- /.github/workflows/unit_tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | 3 | name: Unit Tests 4 | 5 | on: [push, pull_request] 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: [3.8] 13 | tf-version: [2.7.0, 2.8.0, 2.9.0, 2.10.0, 2.11.0] 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: actions/setup-node@v2-beta 18 | with: 19 | node-version: '12' 20 | check-latest: true 21 | 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | 27 | - name: Install dependencies (tf ${{ matrix.tf-version }} ) 28 | run: | 29 | python -m pip install --upgrade pip 30 | pip install tensorflow==${{ matrix.tf-version }} 31 | pip install -r test_requirements.txt 32 | 33 | # Lazy fix for fact that some py/tf combinations don't like new versions of protobuf 34 | - name: Downgrade protobuf 35 | run: | 36 | pip install protobuf==3.20.0 37 | 38 | - name: Run Tests 39 | run: | 40 | bash scripts/run_coverage.sh 41 | bash <(curl -s https://codecov.io/bash) 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /virtualenv/ 2 | /.idea/ 3 | /venv/ 4 | /env/ 5 | /uwiz.egg-info/ 6 | /dist/ 7 | /build/ 8 | **/tmp/ 9 | /scripts/examples_build/ 10 | /examples_build/ 11 | /docs/_build/ 12 | /docker/ 13 | /htmlcov/ 14 | /coverage.xml 15 | /.coverage 16 | /uncertainty_wizard.egg-info/ 17 | /examples/temp-ensemble.txt 18 | **/__pycache__/ 19 | /lf_logs/* -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | 3 | message: "If you use this software, please cite it as below." 4 | type: software 5 | authors: 6 | - given-names: Michael 7 | family-names: Weiss 8 | email: michael.weiss@usi.ch 9 | affiliation: Università della Svizzera italiana 10 | orcid: 'https://orcid.org/0000-0002-8944-389X' 11 | - given-names: Paolo 12 | family-names: Tonella 13 | email: paolo.tonella@usi.ch 14 | affiliation: Università della Svizzera italiana 15 | orcid: 'https://orcid.org/0000-0003-3088-0339' 16 | identifiers: 17 | - type: doi 18 | value: 10.5281/zenodo.5121368 19 | description: The software archived on zenodo 20 | - type: doi 21 | value: 10.5281/zenodo.4651517 22 | description: The talk given when presenting the software 23 | - type: doi 24 | value: 10.1109/ICST49551.2021.00056 25 | description: The published paper 26 | title: >- 27 | Uncertainty-wizard: Fast and user-friendly neural 28 | network uncertainty quantification 29 | doi: 10.5281/zenodo.5121368 30 | date-released: 2020-12-18 31 | url: "https://github.com/testingautomated-usi/uncertainty-wizard" 32 | repository: 'https://zenodo.org/record/5121368' 33 | repository-artifact: 'https://pypi.org/project/uncertainty-wizard/' 34 | license: MIT 35 | preferred-citation: 36 | type: article 37 | authors: 38 | - given-names: Michael 39 | family-names: Weiss 40 | email: michael.weiss@usi.ch 41 | affiliation: Università della Svizzera italiana 42 | orcid: 'https://orcid.org/0000-0002-8944-389X' 43 | - given-names: Paolo 44 | family-names: Tonella 45 | email: paolo.tonella@usi.ch 46 | affiliation: Università della Svizzera italiana 47 | orcid: 'https://orcid.org/0000-0003-3088-0339' 48 | doi: "10.1109/ICST49551.2021.00056" 49 | journal: "2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)" 50 | month: 4 51 | start: 436 # First page number 52 | end: 441 # Last page number 53 | title: "Uncertainty-Wizard: Fast and User-Friendly Neural Network Uncertainty Quantification" 54 | year: 2021 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Michael Weiss and Paolo Tonella and the Università della Svizzera italiana 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![UNCERTAINTY WIZARD](https://github.com/testingautomated-usi/uncertainty-wizard/raw/main/docs/uwiz_logo.PNG) 2 | 3 |

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Documentation Status 13 | 14 | 15 | 16 | 17 | 18 | PyPI 19 | 20 | DOI 21 |

22 | 23 |

24 | 25 | 26 | Best Paper Award at ICST 2021 - Testing Tool Track 27 | 28 | 29 |

30 | 31 | Uncertainty wizard is a plugin on top of `tensorflow.keras`, 32 | allowing to easily and efficiently create uncertainty-aware deep neural networks: 33 | 34 | * Plain Keras Syntax: Use the layers and APIs you know and love. 35 | * Conversion from keras: Convert existing keras models into uncertainty aware models. 36 | * Smart Randomness: Use the same model for point predictions and sampling based inference. 37 | * Fast ensembles: Train and evaluate deep ensembles lazily loaded and using parallel processing - optionally on multiple GPUs. 38 | * Super easy setup: Pip installable. Only tensorflow as dependency. 39 | 40 | #### Installation 41 | 42 | It's as easy as `pip install uncertainty-wizard` 43 | 44 | #### Requirements 45 | `uncertainty-wizard` is tested on python 3.8 and recent tensorflow versions. 46 | Other versions (python 3.6+ and tensorflow 2.3+) should mostly work as well, but may require some mild tweaks. 47 | 48 | 49 | #### Documentation 50 | Our documentation is deployed to 51 | [uncertainty-wizard.readthedocs.io](https://uncertainty-wizard.readthedocs.io/). 52 | In addition, as uncertainty wizard has a 100% docstring coverage on public method and classes, 53 | your IDE will be able to provide you with a good amount of docs out of the box. 54 | 55 | You may also want to check out the technical tool paper [(preprint)](https://arxiv.org/abs/2101.00982), 56 | describing uncertainty wizard functionality and api as of version `v0.1.0`. 57 | 58 | #### Examples 59 | A set of small and easy examples, perfect to get started can be found in the 60 | [models user guide](https://uncertainty-wizard.readthedocs.io/en/latest/user_guide_models.html) 61 | and the [quantifiers user guide](https://uncertainty-wizard.readthedocs.io/en/latest/user_guide_quantifiers.html). 62 | Larger and examples are also provided - and you can run them in colab right away. 63 | You can find them here: [Jupyter examples](https://uncertainty-wizard.readthedocs.io/en/latest/examples.html). 64 | 65 | #### Authors and Papers 66 | 67 | Uncertainty wizard was developed by Michael Weiss and Paolo Tonella at USI (Lugano, Switzerland). 68 | If you use it for your research, please cite these papers: 69 | 70 | @inproceedings{Weiss2021FailSafe, 71 | title={Fail-safe execution of deep learning based systems through uncertainty monitoring}, 72 | author={Weiss, Michael and Tonella, Paolo}, 73 | booktitle={2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)}, 74 | pages={24--35}, 75 | year={2021}, 76 | organization={IEEE} 77 | } 78 | 79 | @inproceedings{Weiss2021UncertaintyWizard, 80 | title={Uncertainty-wizard: Fast and user-friendly neural network uncertainty quantification}, 81 | author={Weiss, Michael and Tonella, Paolo}, 82 | booktitle={2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)}, 83 | pages={436--441}, 84 | year={2021}, 85 | organization={IEEE} 86 | } 87 | 88 | The first paper [(preprint)](https://arxiv.org/abs/2102.00902) provides 89 | an empricial study comparing the approaches implemented in uncertainty wizard, 90 | and a list of lessons learned useful for reasearchers working with uncertainty wizard. 91 | The second paper [(preprint)](https://arxiv.org/abs/2101.00982) is a technical tool paper, 92 | providing a more detailed discussion of uncertainty wizards api and implementation. 93 | 94 | References to the original work introducing the techniques implemented 95 | in uncertainty wizard are provided in the papers listed above. 96 | 97 | #### Contributing 98 | Issues and PRs are welcome! Before investing a lot of time for a PR, please open an issue first, describing your contribution. 99 | This way, we can make sure that the contribution fits well into this repository. 100 | We also mark issues which are great to start contributing as as [good first issues](https://github.com/testingautomated-usi/uncertainty-wizard/contribute). 101 | If you want to implement an existing issue, don't forget to comment on it s.t. everyone knows that you are working on it. 102 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SPHINXPROJ = uncertainty_wizard 9 | SOURCEDIR = . 10 | BUILDDIR = _build 11 | 12 | # Put it first so that "make" without argument is like "make help". 13 | help: 14 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 15 | 16 | .PHONY: help Makefile 17 | 18 | # Catch-all target: route all unknown targets to Sphinx using the new 19 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 20 | %: Makefile 21 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | -------------------------------------------------------------------------------- /docs/complete_api.rst: -------------------------------------------------------------------------------- 1 | All Uncertainty Wizard Methods 2 | ****************************** 3 | 4 | Uncertainty Wizard aims for (and has, as far as we know) a 100% DocString coverage on public classes and methods. 5 | We recommend consulting the DocStrings in your IDE while using uncertainty wizard. 6 | Alternatively, the API can also be explored here: 7 | 8 | * :ref:`genindex` 9 | * :ref:`modindex` 10 | 11 | Note that these indexes are auto-generated by Sphinx and are not always very nice to look at. 12 | Apparently a standard in many python packages, some people seem to like them, though ;-) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('./../')) 16 | # Path to venv on windows machine 17 | sys.path.insert(0, os.path.abspath('./../virtualenv/Lib/site-packages')) 18 | 19 | import version as version_config 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = 'Uncertainty Wizard' 24 | copyright = 'Michael Weiss and Paolo Tonella at the Università della Svizzera Italiana. License: MIT' 25 | author = 'Michael Weiss' 26 | 27 | # The full version, including alpha/beta/rc tags 28 | # version = '0.14' 29 | version = version_config.VERSION 30 | release = version_config.RELEASE 31 | 32 | 33 | # -- General configuration --------------------------------------------------- 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be 36 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 37 | # ones. 38 | extensions = [ 39 | # Used to generate documentation from docstrings 40 | 'sphinx.ext.autodoc', 41 | # 'sphinx.ext.autosummary', 42 | # Check for Documentation Coverage 43 | 'sphinx.ext.coverage', 44 | # Enable understanding of various documentation styles 45 | 'sphinx.ext.napoleon', 46 | # Allow to use markdown files 47 | 'recommonmark', 48 | ] 49 | 50 | # Add any paths that contain templates here, relative to this directory. 51 | templates_path = ['_templates'] 52 | 53 | # List of patterns, relative to source directory, that match files and 54 | # directories to ignore when looking for source files. 55 | # This pattern also affects html_static_path and html_extra_path. 56 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 57 | 58 | 59 | # -- Options for HTML output ------------------------------------------------- 60 | 61 | # The theme to use for HTML and HTML Help pages. See the documentation for 62 | # a list of builtin themes. 63 | # 64 | html_theme = 'sphinx_rtd_theme' 65 | 66 | # Add any paths that contain custom static files (such as style sheets) here, 67 | # relative to this directory. They are copied after the builtin static files, 68 | # so a file named "default.css" will overwrite the builtin "default.css". 69 | html_static_path = ['_static'] 70 | 71 | # Disabled for Double Blindness 72 | html_context = { 73 | 'display_github': True, 74 | 'github_user': 'testingautomated-usi', 75 | 'github_repo': 'uncertainty_wizard', 76 | 'github_version': 'main/docs/', 77 | } 78 | -------------------------------------------------------------------------------- /docs/docs_requirements.txt: -------------------------------------------------------------------------------- 1 | recommonmark 2 | setuptools 3 | sphinx -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | ## Examples 2 | 3 | 4 | Besides the examples provided in the user guides for the usage of [models](./user_guide_models) 5 | and [quantifiers](./user_guide_quantifiers), the following Jupyter notebooks explain 6 | specific tasks: 7 | 8 | 9 | - **Creating a Stochastic Model using the Sequential API** 10 | This shows the simplest, and recommended, way to create an uncertainty aware DNN which 11 | is capable of calculating uncertainties and confidences based on point prediction approaches 12 | as well as on stochastic samples based approaches (e.g. MC-Dropout) 13 | 14 | Open In Colab 15 | View on Github 16 | 17 | - **Convert a traditional keras Model into an uncertainty-aware model** 18 | This shows how you can use any keras model you may have, which was not created through uncertainty wizard, 19 | into an uncertainty-aware DNN. 20 | 21 | Open In Colab 22 | View on Github 23 | 24 | 25 | - **Create a lazily loaded and highly parallelizable Deep Ensemble** 26 | This show the fastest way to create an even faster implementation of the powerful Deep Ensembles - 27 | in a way which respects the fact that your PC and your GPU are powerful and your time is costly. 28 | 29 | Open In Colab 30 | View on Github 31 | 32 | - **Multi-Device Ensemble** 33 | This shows an example on how to use uncertainty wizard lazy ensembles using multiple gpus in parallel. 34 | 35 | View on Github 36 | 37 | *Note: As this example is only applicable to machines with two gpus, we do not provide a colab link or jupyter notebook, 38 | but instead a classical python script to be run locally.* 39 | 40 | More examples will be added when we get feedback from our first users about the steps they found non-obvious. 41 | In the meantime, you may want to check out the [Complete API Documentation](./complete_api). -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Uncertainty Wizard 2 | ================== 3 | 4 | Uncertainty wizard is a plugin on top of ``tensorflow.keras``, 5 | allowing to easily and efficiently create uncertainty-aware deep neural networks: 6 | 7 | - **Plain Keras Syntax:** Use the layers and APIs you know and love. 8 | - **Conversion from keras:** Convert existing keras models into uncertainty aware models. 9 | - **Smart Randomness:** Use the same model for point predictions and sampling based inference. 10 | - **Fast ensembles:** Train and evaluate deep ensembles lazily loaded and using parallel processing. 11 | - **Super easy setup:** Pip installable. Only tensorflow as dependency. 12 | 13 | 14 | 15 | .. toctree:: 16 | :caption: Documentation 17 | :maxdepth: 1 18 | 19 | Installation 20 | User Guide: Models 21 | User Guide: Quantifiers 22 | Examples 23 | Complete API 24 | Paper 25 | Sources on Github 26 | 27 | 28 | .. _TensorflowGuide: https://www.tensorflow.org/guide 29 | 30 | Note that our documentation assumes basic knowledge of the tensorflow.keras API. 31 | If you do not know tensorflow.keras yet, check out the TensorflowGuide_. 32 | 33 | 34 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | #################### 3 | 4 | The installation is as simple as 5 | 6 | .. code-block:: console 7 | 8 | pip install uncertainty-wizard 9 | 10 | Then, in any file where you want to use uncertainty wizard, add the following import statement: 11 | 12 | .. code-block:: python 13 | 14 | import uncertainty_wizard as uwiz 15 | 16 | Dependencies 17 | ************ 18 | We acknowledge that a slim dependency tree is critical to many practical projects. 19 | Thus, the only dependency of uncertainty wizard is ``tensorflow>=2.3.0``. 20 | 21 | Note however that if you are still using python 3.6, you have to install 22 | the backport `dataclasses`. 23 | 24 | .. note:: 25 | Uncertainty Wizard is tested with tensorflow 2.3.0 and tensorflow is evolving quickly. 26 | Please do not hesitate to report an issue if you find broken functionality in more recent tensorflow versions. 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | export PYTHONPATH="$PWD" 4 | 5 | pushd %~dp0 6 | 7 | REM Command file for Sphinx documentation 8 | 9 | if "%SPHINXBUILD%" == "" ( 10 | set SPHINXBUILD=sphinx-build 11 | ) 12 | set SOURCEDIR=. 13 | set BUILDDIR=_build 14 | 15 | if "%1" == "" goto help 16 | 17 | %SPHINXBUILD% >NUL 2>NUL 18 | if errorlevel 9009 ( 19 | echo. 20 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 21 | echo.installed, then set the SPHINXBUILD environment variable to point 22 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 23 | echo.may add the Sphinx directory to PATH. 24 | echo. 25 | echo.If you don't have Sphinx installed, grab it from 26 | echo.http://sphinx-doc.org/ 27 | exit /b 1 28 | ) 29 | 30 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 31 | goto end 32 | 33 | :help 34 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 35 | 36 | :end 37 | popd 38 | -------------------------------------------------------------------------------- /docs/paper.md: -------------------------------------------------------------------------------- 1 | ## Paper 2 | 3 | Uncertainty wizard was developed by Michael Weiss and Paolo Tonella at USI (Lugano, Switzerland). 4 | If you use it for your research, please cite these papers: 5 | 6 | @inproceedings{Weiss2021FailSafe, 7 | title={Fail-safe execution of deep learning based systems through uncertainty monitoring}, 8 | author={Weiss, Michael and Tonella, Paolo}, 9 | booktitle={2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)}, 10 | pages={24--35}, 11 | year={2021}, 12 | organization={IEEE} 13 | } 14 | 15 | @inproceedings{Weiss2021UncertaintyWizard, 16 | title={Uncertainty-wizard: Fast and user-friendly neural network uncertainty quantification}, 17 | author={Weiss, Michael and Tonella, Paolo}, 18 | booktitle={2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST)}, 19 | pages={436--441}, 20 | year={2021}, 21 | organization={IEEE} 22 | } 23 | 24 | 25 | The first paper [(preprint)](https://arxiv.org/abs/2102.00902) provides 26 | an empricial study comparing the approaches implemented in uncertainty wizard, 27 | and a list of lessons learned useful for reasearchers working with uncertainty wizard. 28 | The second paper [(preprint)](https://arxiv.org/abs/2101.00982) is a technical tool paper, 29 | providing a more detailed discussion of uncertainty wizards api and implementation. 30 | 31 | References to the original work introducing the techniques implemented 32 | in uncertainty wizard are provided in the papers listed above. 33 | 34 | 35 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | uncertainty_wizard 2 | ================== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | uncertainty_wizard 8 | -------------------------------------------------------------------------------- /docs/source/uncertainty_wizard.internal_utils.rst: -------------------------------------------------------------------------------- 1 | uncertainty\_wizard.internal\_utils package 2 | =========================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: uncertainty_wizard.internal_utils 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/uncertainty_wizard.models.ensemble_utils.rst: -------------------------------------------------------------------------------- 1 | uncertainty\_wizard.models.ensemble\_utils package 2 | ================================================== 3 | 4 | Classes 5 | 6 | .. automodule:: uncertainty_wizard.models.ensemble_utils 7 | :members: 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/source/uncertainty_wizard.models.rst: -------------------------------------------------------------------------------- 1 | uncertainty\_wizard.models package 2 | ================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | uncertainty_wizard.models.ensemble_utils 11 | uncertainty_wizard.models.stochastic_utils 12 | 13 | Module contents 14 | --------------- 15 | 16 | .. automodule:: uncertainty_wizard.models 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /docs/source/uncertainty_wizard.models.stochastic_utils.rst: -------------------------------------------------------------------------------- 1 | uncertainty\_wizard.models.stochastic\_utils package 2 | ==================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | uncertainty\_wizard.models.stochastic\_utils.layers module 8 | ---------------------------------------------------------- 9 | 10 | .. automodule:: uncertainty_wizard.models.stochastic_utils.layers 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | -------------------------------------------------------------------------------- /docs/source/uncertainty_wizard.quantifiers.rst: -------------------------------------------------------------------------------- 1 | uncertainty\_wizard.quantifiers package 2 | ======================================= 3 | 4 | 5 | Module contents 6 | --------------- 7 | 8 | .. automodule:: uncertainty_wizard.quantifiers 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | -------------------------------------------------------------------------------- /docs/source/uncertainty_wizard.rst: -------------------------------------------------------------------------------- 1 | uncertainty\_wizard package 2 | =========================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | uncertainty_wizard.internal_utils 11 | uncertainty_wizard.models 12 | uncertainty_wizard.quantifiers 13 | 14 | Module contents 15 | --------------- 16 | 17 | .. automodule:: uncertainty_wizard 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | -------------------------------------------------------------------------------- /docs/user_guide_quantifiers.rst: -------------------------------------------------------------------------------- 1 | User Guide: Quantifiers 2 | ############################## 3 | 4 | Quantifiers are dependencies, injectable into prediction calls, 5 | which calculate predictions and uncertainties or confidences 6 | from DNN outputs: 7 | 8 | .. code-block:: python 9 | :caption: Use of quantifiers on uwiz models 10 | 11 | # Let's use a quantifier that calculates the entropy on a regression variable as uncertainty 12 | predictions, entropy = model.predict_quantified(x_test, quantifier='predictive_entropy') 13 | 14 | # Equivalently, we can pass the quantifier as object 15 | quantifier = uwiz.quantifiers.PredictiveEntropy() 16 | predictions, entropy = model.predict_quantified(x_test, quantifier=quantifer) 17 | 18 | # We can also pass multiple quantifiers. 19 | # In that case, `predict_quantified` returns a (prediction, confidence_or_uncertainty) tuple 20 | # for every passed quantifier. 21 | results = model.predict_quantified(x_test, quantifier=['predictive_entropy', 'mean_softmax') 22 | # results[0] is a tuple of predictions and entropies 23 | # results[1] is a tuple of predictions and mean softmax values 24 | 25 | Besides the prediction, quantifiers quantify either the networks confidence or its uncertainty. 26 | The difference between that two is as follows 27 | (assuming that the quantifier actually correctly captures the chance of misprediction): 28 | 29 | - In `uncertainty quantification`, the higher the value, the higher the chance of misprediction. 30 | - In `confidence quantification` the lower the value, the higher the chance of misprediction. 31 | 32 | For most applications where you use multiple quantifiers, you probably want to quantify 33 | either uncertainties or confidences to allow to use the quantifiers outputs interchangeable. 34 | Setting the param ``model.predict_quantified(..., as_confidence=True)`` 35 | convert uncertainties into confidences. ``as_confidence=False`` converts confidences into uncertainties. 36 | The default is 'None', in which case no conversions are made. 37 | 38 | .. note:: 39 | Independent on how many quantifiers you pass to the `predict_quantified` method, 40 | the outputs of the neural networks inference are re-used wherever possible for a more efficient execution. 41 | Thus, it is better to call `predict_quantified` with two quantifiers than 42 | to call `predict_quantified` twice, with one quantifier each. 43 | 44 | 45 | 46 | Quantifiers implemented in Uncertainty Wizard 47 | ********************************************* 48 | This Section provides an overview of the quantifiers provided in uncertainty wizard: 49 | For a precise discussion of the quantifiers listed here, please consult our paper 50 | and the docstrings of the quantifiers. 51 | 52 | Point Prediction Quantifiers 53 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 54 | 55 | +--------------------------------+------------------+------------------------------------------+ 56 | | | Class | | Problem Type | | Aliases | 57 | | | (uwiz.quantifiers.<...>) | | | | (besides class name) | 58 | +================================+==================+==========================================+ 59 | | | MaxSoftmax | | Classification | | SM, softmax, max_softmax, | 60 | +--------------------------------+------------------+------------------------------------------+ 61 | | | PredictionConfidenceScore | | Classification | | PCS, prediction_confidence_score | 62 | +--------------------------------+------------------+------------------------------------------+ 63 | | | SoftmaxEntropy | | Classification | | SE, softmax_entropy | 64 | +--------------------------------+------------------+------------------------------------------+ 65 | 66 | 67 | Monte Carlo Sampling Quantifiers 68 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 69 | 70 | 71 | +--------------------------------+------------------+------------------------------------------+ 72 | | | Class | | Problem Type | | Aliases | 73 | | | (uwiz.quantifiers.<...>) | | | | (besides class name) | 74 | +================================+==================+==========================================+ 75 | | | VariationRatio | | Classification | | VR, var_ratio, | 76 | | | | | | | variation_ratio | 77 | +--------------------------------+------------------+------------------------------------------+ 78 | | | PredictiveEntropy | | Classification | | PE, pred_entropy, | 79 | | | | | | | predictive_entropy | 80 | +--------------------------------+------------------+------------------------------------------+ 81 | | | MutualInformation | | Classification | | MI, mutu_info, | 82 | | | | | | | mutual_information | 83 | +--------------------------------+------------------+------------------------------------------+ 84 | | | MeanSoftmax | | Classification | | MS, mean_softmax, | 85 | | | | | | | ensembling | 86 | +--------------------------------+------------------+------------------------------------------+ 87 | | | StandardDeviation | | Regression | | STD, stddev, std_dev, | 88 | | | | | | | standard_deviation | 89 | +--------------------------------+------------------+------------------------------------------+ 90 | 91 | 92 | 93 | Custom Quantifers 94 | ***************** 95 | 96 | You can of course also use custom quantifiers with uncertainty wizard. 97 | It's as easy as extending ``uwiz.quantifiers.Quantifier`` and implement all abstract methods according 98 | to the description in the superclass method docstrings. 99 | 100 | Let's for example assume you want to create an **identity function** quantifier for a sampling based DNN 101 | (i.e., a stochastic DNN or a deep ensemble) for a classification problem, 102 | which does not actually calculate a prediction and uncertainty, but just returns the observed DNN outputs. 103 | This can be achieved using the following snippet: 104 | 105 | .. code-block:: python 106 | :caption: Custom quantifier definition: Identity Quantifier 107 | 108 | class IdentityQuantifer(uwiz.quantifiers.Quantifier): 109 | @classmethod 110 | def aliases(cls) -> List[str]: 111 | return ["custom::identity"] 112 | 113 | @classmethod 114 | def takes_samples(cls) -> bool: 115 | return True 116 | 117 | @classmethod 118 | def is_confidence(cls) -> bool: 119 | # Does not matter for the identity function 120 | return False 121 | 122 | @classmethod 123 | def calculate(cls, nn_outputs: np.ndarray): 124 | # Return None as prediction and all DNN outputs as 'quantification' 125 | return None, nn_outputs 126 | 127 | @classmethod 128 | def problem_type(cls) -> uwiz.ProblemType: 129 | return uwiz.ProblemType.CLASSIFICATION 130 | 131 | 132 | If you want to call your custom quantifier by its alias, you need to add it to the registry. 133 | To prevent name clashes in future uncertainty wizard versions, where more quantifiers might be registered by default, 134 | we recommend you to preprend "custom::" to any of your quantifiers aliases. 135 | 136 | .. code-block:: python 137 | :caption: Register a quantifier in the quantifier registry 138 | 139 | custom_instance = IdentityQuantifier() 140 | uwiz.quantifiers.QuantifierRegistry().register(custom_instance) 141 | 142 | model = # (...) uwiz model creation, compiling and fitting 143 | x_test = # (...) get the data for your predictions 144 | 145 | # Now this call, where we calculate the variation ratio, 146 | # and also return the observed DNN outputs... 147 | results = model.predict_quantified(x_test, num_samples=20, 148 | quantifier=["var_ratio", "custom::identity"]) 149 | # ... is equivalent to this call... 150 | results = model.predict_quantified(x_test, num_samples=20, 151 | quantifier=["var_ratio", IdentityQuantifier()]) 152 | 153 | 154 | .. warning:: 155 | Quantifiers added to the registry should be stateless and all their functions should be pure functions. 156 | Otherwise, reproduction of results might not be possible. -------------------------------------------------------------------------------- /docs/uwiz_logo.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/docs/uwiz_logo.PNG -------------------------------------------------------------------------------- /examples/LoadStochasticFromKeras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "## How to load a pre-trained keras model to be used for MC-Dropout and Point Predictions?" 7 | ], 8 | "metadata": { 9 | "collapsed": false, 10 | "pycharm": { 11 | "name": "#%% md\n" 12 | } 13 | } 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "outputs": [], 19 | "source": [ 20 | "import tensorflow as tf\n", 21 | "\n", 22 | "try:\n", 23 | " import uncertainty_wizard as uwiz\n", 24 | "except ModuleNotFoundError as e:\n", 25 | " # Uncertainty wizard was not installed. Install it now (we're probably on colab)\n", 26 | " !pip install uncertainty_wizard\n", 27 | " import uncertainty_wizard as uwiz" 28 | ], 29 | "metadata": { 30 | "collapsed": false, 31 | "pycharm": { 32 | "name": "#%%\n" 33 | } 34 | } 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "source": [ 39 | "**Step 1: Get the (plain) tf.keras model you want to cast to an uncertainty wizard model**\n", 40 | "\n", 41 | "In this example, we use a pre-trained efficientnet model, which we download through keras.\n", 42 | "You can of course also use one of your own pre-trained models." 43 | ], 44 | "metadata": { 45 | "collapsed": false 46 | } 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "outputs": [], 52 | "source": [ 53 | "# Let's load this big model. This will take a while\n", 54 | "keras_model = tf.keras.applications.EfficientNetB0(\n", 55 | " include_top=True, weights='imagenet', input_tensor=None, input_shape=None,\n", 56 | " pooling=None, classes=1000, classifier_activation='softmax')" 57 | ], 58 | "metadata": { 59 | "collapsed": false, 60 | "pycharm": { 61 | "name": "#%%\n" 62 | } 63 | } 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "source": [ 68 | "**Step 2: Cast to an uncertainty wizard model**" 69 | ], 70 | "metadata": { 71 | "collapsed": false 72 | } 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "outputs": [], 78 | "source": [ 79 | "# It's just one line.\n", 80 | "# However, given that our keras_model is fairly huge, processing will take a while.\n", 81 | "stochastic_model = uwiz.models.stochastic_from_keras(keras_model)\n", 82 | "\n", 83 | "print(stochastic_model.summary())\n", 84 | "\n", 85 | "print(\"Model successfully loaded - ready to make quantified predictions\")" 86 | ], 87 | "metadata": { 88 | "collapsed": false, 89 | "pycharm": { 90 | "name": "#%%\n" 91 | } 92 | } 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "source": [ 97 | "Are you unsure if your keras_model had any stochastic layers?\n", 98 | "**Don't worry** - uncertainty wizard has your back and will warn you if the resulting model is a deterministic one..." 99 | ], 100 | "metadata": { 101 | "collapsed": false, 102 | "pycharm": { 103 | "name": "#%% md\n" 104 | } 105 | } 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "source": [ 110 | "**Step 3: There is no step 3**\n", 111 | "\n", 112 | "You are already done converting and ready to now make quantified predictions:\n", 113 | "Use *stochastic_model.predict_quantified(...)* as shown in the example on how to use StochasticSequential models." 114 | ], 115 | "metadata": { 116 | "collapsed": false 117 | } 118 | } 119 | ], 120 | "metadata": { 121 | "kernelspec": { 122 | "display_name": "Python 3", 123 | "language": "python", 124 | "name": "python3" 125 | }, 126 | "language_info": { 127 | "codemirror_mode": { 128 | "name": "ipython", 129 | "version": 2 130 | }, 131 | "file_extension": ".py", 132 | "mimetype": "text/x-python", 133 | "name": "python", 134 | "nbconvert_exporter": "python", 135 | "pygments_lexer": "ipython2", 136 | "version": "2.7.6" 137 | } 138 | }, 139 | "nbformat": 4, 140 | "nbformat_minor": 0 141 | } -------------------------------------------------------------------------------- /examples/MnistEnsemble.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "## How to create a Deep Ensemble for MNIST" 7 | ], 8 | "metadata": { 9 | "collapsed": false 10 | } 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "outputs": [], 16 | "source": [ 17 | "import tensorflow as tf\n", 18 | "\n", 19 | "try:\n", 20 | " import uncertainty_wizard as uwiz\n", 21 | "except ModuleNotFoundError as e:\n", 22 | " # Uncertainty wizard was not installed. Install it now (we're probably on colab)\n", 23 | " !pip install uncertainty_wizard\n", 24 | " import uncertainty_wizard as uwiz" 25 | ], 26 | "metadata": { 27 | "collapsed": false, 28 | "pycharm": { 29 | "name": "#%%\n" 30 | } 31 | } 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "source": [ 36 | "**Step 1: Downloading Preprocessing the data**\n", 37 | "\n", 38 | "This is the same that we would do on any regular keras mnist classifier,\n", 39 | "except that we do not have to one-hot encode the test labels, as uncertainty wizards quantifiers\n", 40 | "will determine the winning class for us" 41 | ], 42 | "metadata": { 43 | "collapsed": false, 44 | "pycharm": { 45 | "name": "#%% md\n" 46 | } 47 | } 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "outputs": [], 53 | "source": [ 54 | "# Lets cache the train data on the file system,\n", 55 | "# and at the same time also prepare the test data for later\n", 56 | "_,(x_test, y_test) = tf.keras.datasets.mnist.load_data()\n", 57 | "x_test = (x_test.astype('float32') / 255).reshape(x_test.shape[0], 28, 28, 1)\n" 58 | ], 59 | "metadata": { 60 | "collapsed": false, 61 | "pycharm": { 62 | "name": "#%%\n" 63 | } 64 | } 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "source": [ 69 | "**Step 2: Define the model creation & training process in a picklable function**\n", 70 | "\n", 71 | "Just create a function at the root of your file, using plain tensorflow code. \n", 72 | "The function should return a newly created model and a second return value (typically the training history).\n", 73 | "\n", 74 | "This function will be called repetitively to create the atomic models in the ensemble.\n", 75 | "The optional return values will be collected and returned after the creation of the ensemble." 76 | ], 77 | "metadata": { 78 | "collapsed": false 79 | } 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "outputs": [], 85 | "source": [ 86 | "def model_creation_and_training(model_id: int):\n", 87 | " import tensorflow as tf\n", 88 | "\n", 89 | " model = tf.keras.models.Sequential()\n", 90 | " model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))\n", 91 | " model.add(tf.keras.layers.Conv2D(64, (3, 3), activation='relu'))\n", 92 | " model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\n", 93 | " model.add(tf.keras.layers.Flatten())\n", 94 | " model.add(tf.keras.layers.Dense(128, activation='relu'))\n", 95 | " model.add(tf.keras.layers.Dense(10, activation='softmax'))\n", 96 | " model.compile(loss=tf.keras.losses.categorical_crossentropy,\n", 97 | " optimizer=tf.keras.optimizers.Adadelta(),\n", 98 | " metrics=['accuracy'])\n", 99 | " (x_train, y_train), (_,_) = tf.keras.datasets.mnist.load_data()\n", 100 | " x_train = (x_train.astype('float32') / 255).reshape(x_train.shape[0], 28, 28, 1)\n", 101 | " y_train = tf.keras.utils.to_categorical(y_train, num_classes=10)\n", 102 | " # Note that we set the number of epochs to just 1, to be able to run this notebook quickly\n", 103 | " # Set the number of epochs higher if you want to optimally train the network\n", 104 | " fit_history = model.fit(x_train, y_train, validation_split=0.1, batch_size=32, epochs=1,\n", 105 | " verbose=1, callbacks=[tf.keras.callbacks.EarlyStopping(patience=2)])\n", 106 | " return model, fit_history.history\n" 107 | ], 108 | "metadata": { 109 | "collapsed": false, 110 | "pycharm": { 111 | "name": "#%%\n" 112 | } 113 | } 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "source": [ 118 | "**Step 3: Create Ensemble**\n", 119 | "\n", 120 | "Let's create a Lazy Ensemble instance, i.e., a definition of how many atomic models should be included in our ensemble,\n", 121 | "where they should be persisted, ... Note that this first call does not create or train any models and is thus super fast.\n", 122 | "\n", 123 | "After this definition, we can create the atomic models in the lazy ensemble using the function defined above." 124 | ], 125 | "metadata": { 126 | "collapsed": false 127 | } 128 | }, 129 | { 130 | "cell_type": "code", 131 | "source": [ 132 | "ensemble = uwiz.models.LazyEnsemble(num_models=2, # For the sake of this example. Use more in practice!\n", 133 | " model_save_path=\"/tmp/ensemble\",\n", 134 | " # Colab infrastructure is relatively weak.\n", 135 | " # Thus, lets disable multiprocessing and train on the main process.\n", 136 | " # Any argument >= 1 would result in (typically more efficient) multiprocessing\n", 137 | " # on a more powerful machine\n", 138 | " default_num_processes=0)\n", 139 | "# Creates, trains and persists atomic models using our function defined above\n", 140 | "training_histories = ensemble.create(create_function=model_creation_and_training)" 141 | ], 142 | "metadata": { 143 | "collapsed": false, 144 | "pycharm": { 145 | "name": "#%%\n" 146 | } 147 | }, 148 | "execution_count": null, 149 | "outputs": [] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "source": [ 154 | "**Step 4: Make predictions and get the uncertainties and confidences**\n", 155 | "\n", 156 | "If your test data is a numpy array, its as easy as shown in the code below.\n", 157 | "\n", 158 | "For customized prediction procedures, \n", 159 | "or a non-numpy test set, check out the documentation for ensemble.quantify_predictions where you\n", 160 | "can hook up an arbitrary prediction function - similar to the training function defined and used in step 2 and 3" 161 | ], 162 | "metadata": { 163 | "collapsed": false 164 | } 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "outputs": [], 170 | "source": [ 171 | "# Get two one-dimensional np arrays: One containing the predictions and one containing the confidences\n", 172 | "predictions, confidences = ensemble.predict_quantified(x_test,\n", 173 | " quantifier='mean_softmax')\n" 174 | ], 175 | "metadata": { 176 | "collapsed": false, 177 | "pycharm": { 178 | "name": "#%%\n" 179 | } 180 | } 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "source": [], 185 | "metadata": { 186 | "collapsed": false, 187 | "pycharm": { 188 | "name": "#%% md\n" 189 | } 190 | } 191 | } 192 | ], 193 | "metadata": { 194 | "kernelspec": { 195 | "display_name": "Python 3", 196 | "language": "python", 197 | "name": "python3" 198 | }, 199 | "language_info": { 200 | "codemirror_mode": { 201 | "name": "ipython", 202 | "version": 2 203 | }, 204 | "file_extension": ".py", 205 | "mimetype": "text/x-python", 206 | "name": "python", 207 | "nbconvert_exporter": "python", 208 | "pygments_lexer": "ipython2", 209 | "version": "2.7.6" 210 | }, 211 | "pycharm": { 212 | "stem_cell": { 213 | "cell_type": "raw", 214 | "source": [], 215 | "metadata": { 216 | "collapsed": false 217 | } 218 | } 219 | } 220 | }, 221 | "nbformat": 4, 222 | "nbformat_minor": 0 223 | } -------------------------------------------------------------------------------- /examples/MnistStochasticSequential.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "## How to create a stochastic model for MNIST? (I.e., a model which can be used for MC-Dropout and Point Predictions)" 7 | ], 8 | "metadata": { 9 | "collapsed": false, 10 | "pycharm": { 11 | "name": "#%% md\n" 12 | } 13 | } 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "outputs": [], 19 | "source": [ 20 | "import tensorflow as tf\n", 21 | "\n", 22 | "try:\n", 23 | " import uncertainty_wizard as uwiz\n", 24 | "except ModuleNotFoundError as e:\n", 25 | " # Uncertainty wizard was not installed. Install it now (we're probably on colab)\n", 26 | " !pip install uncertainty_wizard\n", 27 | " import uncertainty_wizard as uwiz" 28 | ], 29 | "metadata": { 30 | "collapsed": false, 31 | "pycharm": { 32 | "name": "#%%\n" 33 | } 34 | } 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "source": [ 39 | "**Step 1: Downloading Preprocessing the data**\n", 40 | "\n", 41 | "This is the same that we would do on any regular keras mnist classifier,\n", 42 | "except that we do not have to one-hot encode the test labels, as uncertainty wizards quantifiers\n", 43 | "will determine the winning class (one not its one-hot encoding) for us" 44 | ], 45 | "metadata": { 46 | "collapsed": false, 47 | "pycharm": { 48 | "name": "#%% md\n" 49 | } 50 | } 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "outputs": [], 56 | "source": [ 57 | "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n", 58 | "x_train = (x_train.astype('float32') / 255).reshape(x_train.shape[0], 28, 28, 1)\n", 59 | "x_test = (x_test.astype('float32') / 255).reshape(x_test.shape[0], 28, 28, 1)\n", 60 | "y_train = tf.keras.utils.to_categorical(y_train, num_classes=10)\n" 61 | ], 62 | "metadata": { 63 | "collapsed": false, 64 | "pycharm": { 65 | "name": "#%%\n" 66 | } 67 | } 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "source": [ 72 | "**Step 2: Creating a Stochastic Uncertainty-Wizard Model using the Sequential API**\n", 73 | "\n", 74 | "Note that only the first line is different from doing the same in plain keras, the rest is equivalent.\n", 75 | "We must however ensure to use at least one randomized layer (e.g. tf.keras.layers.Dropout)" 76 | ], 77 | "metadata": { 78 | "collapsed": false 79 | } 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "outputs": [], 85 | "source": [ 86 | "# Let's create an uncertainty wizard model!\n", 87 | "# We want to use a dropout-based stochastic model using the functional interface\n", 88 | "\n", 89 | "model = uwiz.models.StochasticSequential()" 90 | ], 91 | "metadata": { 92 | "collapsed": false, 93 | "pycharm": { 94 | "name": "#%%\n" 95 | } 96 | } 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "outputs": [], 102 | "source": [ 103 | "# In the functional interface, we can add the layer as if we were working\n", 104 | "# on a regular tf.keras.model.Sequential() using add()\n", 105 | "\n", 106 | "model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))\n", 107 | "model.add(tf.keras.layers.Conv2D(64, (3, 3), activation='relu'))\n", 108 | "model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\n", 109 | "\n", 110 | "# Because we are building a stochastic model, we have to include at last one of the following keras layers\n", 111 | "# tf.keras.layers.Dropout, tf.keras.layers.GaussionNoise, tf.keras.layers.GaussianDropout\n", 112 | "model.add(tf.keras.layers.Dropout(0.5))\n", 113 | "\n", 114 | "model.add(tf.keras.layers.Flatten())\n", 115 | "model.add(tf.keras.layers.Dense(128, activation='relu'))\n", 116 | "model.add(tf.keras.layers.Dense(10, activation='softmax'))" 117 | ], 118 | "metadata": { 119 | "collapsed": false, 120 | "pycharm": { 121 | "name": "#%%\n" 122 | } 123 | } 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "outputs": [], 129 | "source": [ 130 | "# Compiling and fitting is the same as in regular keras models as well:\n", 131 | "\n", 132 | "model.compile(loss=tf.keras.losses.categorical_crossentropy,\n", 133 | " optimizer=tf.keras.optimizers.Adadelta(),\n", 134 | " metrics=['accuracy'])\n", 135 | "\n", 136 | "# Note that we set the number of epochs to just 1, to be able to run this notebook quickly\n", 137 | "# Set the number of epochs higher if you want to optimally train the network.\n", 138 | "# Also note that obviously, with epochs=1, the currently passed callback does not do anything.\n", 139 | "model.fit(x_train, y_train, validation_split=0.1, batch_size=32, epochs=1,\n", 140 | " verbose=1, callbacks=[tf.keras.callbacks.EarlyStopping(patience=2)])\n" 141 | ], 142 | "metadata": { 143 | "collapsed": false, 144 | "pycharm": { 145 | "name": "#%%\n" 146 | } 147 | } 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "outputs": [], 153 | "source": [ 154 | "# Let's print the summary of the inner model that is wrapped by the stochastic model we just created.\n", 155 | "# Note that the Dropout-Layer was replaced by an UwizBernoulliDropout-Layer\n", 156 | "# This allows the uncertainty wizard to control randomness during inference.\n", 157 | "print(model.inner.summary())\n" 158 | ], 159 | "metadata": { 160 | "collapsed": false, 161 | "pycharm": { 162 | "name": "#%%\n" 163 | } 164 | } 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "source": [ 169 | "**Step 3: Make predictions and get the uncertainties and confidences**" 170 | ], 171 | "metadata": { 172 | "collapsed": false 173 | } 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "outputs": [], 179 | "source": [ 180 | "quantifiers = ['pcs', 'mean_softmax']\n", 181 | "results = model.predict_quantified(x_test,\n", 182 | " quantifier=quantifiers,\n", 183 | " batch_size=64,\n", 184 | " sample_size=32,\n", 185 | " verbose=1)\n", 186 | "\n", 187 | "# results[0][0] contains the point-predictions from the 'pcs' quantifier\n", 188 | "# (i.e., argmax of a single non-randomized network forward pass)\n", 189 | "# results[0][1] contains the prediction confidence scores for these predictions\n", 190 | "\n", 191 | "# results[1][0] contains the predictions from the 'mean_softmax' quantifier\n", 192 | "# (i.e., argmax of 32 averages forward pass samples on the randomized DNN)\n", 193 | "# results[1][1] contains the corresponding confidence\n", 194 | "# (i.e., the average softmax value of the class with the highest average softmax value)\n", 195 | "\n" 196 | ], 197 | "metadata": { 198 | "collapsed": false, 199 | "pycharm": { 200 | "name": "#%%\n" 201 | } 202 | } 203 | } 204 | ], 205 | "metadata": { 206 | "kernelspec": { 207 | "display_name": "Python 3", 208 | "language": "python", 209 | "name": "python3" 210 | }, 211 | "language_info": { 212 | "codemirror_mode": { 213 | "name": "ipython", 214 | "version": 2 215 | }, 216 | "file_extension": ".py", 217 | "mimetype": "text/x-python", 218 | "name": "python", 219 | "nbconvert_exporter": "python", 220 | "pygments_lexer": "ipython2", 221 | "version": "2.7.6" 222 | }, 223 | "pycharm": { 224 | "stem_cell": { 225 | "cell_type": "raw", 226 | "source": [], 227 | "metadata": { 228 | "collapsed": false 229 | } 230 | } 231 | } 232 | }, 233 | "nbformat": 4, 234 | "nbformat_minor": 0 235 | } -------------------------------------------------------------------------------- /examples/__init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/examples/__init.py -------------------------------------------------------------------------------- /examples/multi_device.py: -------------------------------------------------------------------------------- 1 | # ============================== 2 | # It does not make sense to run this from colab (as only one GPU) 3 | # Thus, this is not set up as a jupyter notebook official notebook 4 | # ============================== 5 | from typing import Dict 6 | 7 | import tensorflow 8 | 9 | import uncertainty_wizard as uwiz 10 | 11 | 12 | class MultiGpuContext(uwiz.models.ensemble_utils.DeviceAllocatorContextManagerV2): 13 | @classmethod 14 | def file_path(cls) -> str: 15 | return "temp-ensemble.txt" 16 | 17 | @classmethod 18 | def run_on_cpu(cls) -> bool: 19 | # Running on CPU is almost never a good idea. 20 | # The CPU should be available for data preprocessing. 21 | # Also, training on CPU is typically much slower than on a gpu. 22 | return False 23 | 24 | @classmethod 25 | def virtual_devices_per_gpu(cls) -> Dict[int, int]: 26 | # Here, we configure a setting with two gpus 27 | # On gpu 0, two atomic models will be executed at the same time 28 | # On gpu 1, three atomic models will be executed at the same time 29 | return {0: 2, 1: 3} 30 | 31 | 32 | def train_model(model_id): 33 | import tensorflow as tf 34 | 35 | model = tf.keras.models.Sequential() 36 | model.add( 37 | tf.keras.layers.Conv2D( 38 | 32, 39 | kernel_size=(3, 3), 40 | activation="relu", 41 | padding="same", 42 | input_shape=(32, 32, 3), 43 | ) 44 | ) 45 | model.add( 46 | tf.keras.layers.Conv2D( 47 | 32, kernel_size=(3, 3), activation="relu", padding="same" 48 | ) 49 | ) 50 | model.add(tf.keras.layers.MaxPooling2D((2, 2))) 51 | model.add(tf.keras.layers.Dropout(0.2)) 52 | model.add( 53 | tf.keras.layers.Conv2D( 54 | 64, kernel_size=(3, 3), activation="relu", padding="same" 55 | ) 56 | ) 57 | model.add( 58 | tf.keras.layers.Conv2D( 59 | 64, kernel_size=(3, 3), activation="relu", padding="same" 60 | ) 61 | ) 62 | model.add(tf.keras.layers.MaxPooling2D((2, 2))) 63 | model.add(tf.keras.layers.Dropout(0.2)) 64 | model.add( 65 | tf.keras.layers.Conv2D( 66 | 128, kernel_size=(3, 3), activation="relu", padding="same" 67 | ) 68 | ) 69 | model.add( 70 | tf.keras.layers.Conv2D( 71 | 128, kernel_size=(3, 3), activation="relu", padding="same" 72 | ) 73 | ) 74 | model.add(tf.keras.layers.MaxPooling2D((2, 2))) 75 | model.add(tf.keras.layers.Dropout(0.2)) 76 | model.add(tf.keras.layers.Flatten()) 77 | model.add(tf.keras.layers.Dense(128, activation="relu")) 78 | model.add(tf.keras.layers.Dropout(0.2)) 79 | model.add(tf.keras.layers.Dense(10, activation="softmax")) 80 | 81 | opt = tf.keras.optimizers.SGD(lr=0.001, momentum=0.9) 82 | model.compile(optimizer=opt, loss="categorical_crossentropy", metrics=["accuracy"]) 83 | 84 | (x_train, y_train), (_, _) = tf.keras.datasets.cifar10.load_data() 85 | x_train = x_train / 255.0 86 | y_train = tf.keras.utils.to_categorical(y_train, 10) 87 | 88 | # For the sake of this example, let's use just one epoch. 89 | # Of course, for higher accuracy, you should use more. 90 | model.fit(x_train, y_train, batch_size=32, epochs=1) 91 | 92 | return model, "history_not_returned" 93 | 94 | 95 | if __name__ == "__main__": 96 | # Make sure the training data is cached on the fs before the multiprocessing starts 97 | # Otherwise, all processes will simultaneously attempt to download and cache data, 98 | # which will fail as they break each others caches 99 | tensorflow.keras.datasets.cifar10.load_data() 100 | 101 | # set this path to where you want to save the ensemble 102 | temp_dir = "/tmp/ensemble" 103 | ensemble = uwiz.models.LazyEnsemble( 104 | num_models=20, 105 | model_save_path=temp_dir, 106 | delete_existing=True, 107 | default_num_processes=5, 108 | ) 109 | ensemble.create(train_model, context=MultiGpuContext) 110 | 111 | print("Ensemble was successfully trained") 112 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow>=2.7.0 2 | # The following is only required for python 3.6 3 | # dataclasses==0.6 4 | -------------------------------------------------------------------------------- /scripts/docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER_DIR="./docker/" 4 | 5 | # Parse flags 6 | 7 | FOR_GPU='false' 8 | FULL_BUILD='false' 9 | TAG="" 10 | 11 | # Process all options supplied on the command line 12 | while getopts ":t:gf" opt; do 13 | case $opt in 14 | 'g') 15 | # Update the value of the option x flag we defined above 16 | FOR_GPU='true' 17 | echo "Building image for GPU" 18 | ;; 19 | 'f') 20 | # Update the value of the option x flag we defined above 21 | FULL_BUILD='true' 22 | echo "Building full image (i.e., including source and test files)" 23 | ;; 24 | 't') 25 | TAG=$OPTARG 26 | echo "Building with tag '$TAG'" 27 | ;; 28 | '?') 29 | echo "INVALID OPTION - ${OPTARG}" >&2 30 | exit 1 31 | ;; 32 | ':') 33 | echo "MISSING ARGUMENT for option - ${OPTARG}" >&2 34 | exit 1 35 | ;; 36 | *) 37 | echo "UNIMPLEMENTED OPTION - ${OPTKEY}" >&2 38 | exit 1 39 | ;; 40 | esac 41 | done 42 | 43 | shift $((OPTIND - 1)) 44 | 45 | PROJECT_NAME="${1}" 46 | 47 | # Make sure we run in root folder of repository 48 | if ! [[ -d "$PROJECT_NAME" ]]; then 49 | printf "ERROR: Folder %s does not exist. " "${PROJECT_NAME}" 50 | printf "Make sure to run this script from the root folder of your repo and that you pass your " 51 | printf "project name (source folder name) as first argument to this script. " 52 | echo "(e.g., ./scripts/build_docker_image.sh -g -f my_project)" 53 | exit 1 54 | fi 55 | 56 | 57 | if [[ "" == "$TAG" ]] 58 | then 59 | TAG="${PROJECT_NAME}/${PROJECT_NAME}:snapshot" 60 | echo "Using default tag: '$TAG'" 61 | fi 62 | 63 | if [[ 'true' == "$FOR_GPU" ]] 64 | then 65 | PU_FOLDER="${DOCKER_DIR}gpu/" 66 | else 67 | PU_FOLDER="${DOCKER_DIR}cpu/" 68 | fi 69 | 70 | if [[ 'true' == "$FULL_BUILD" ]] 71 | then 72 | FULL_FOLDER="${PU_FOLDER}full/" 73 | else 74 | FULL_FOLDER="${PU_FOLDER}env/" 75 | fi 76 | 77 | DOCKERFILE="${FULL_FOLDER}Dockerfile" 78 | 79 | docker build -f "$DOCKERFILE" -t "$TAG" . -------------------------------------------------------------------------------- /scripts/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | set -x 3 | 4 | isort --profile black uncertainty_wizard tests_unit --profile black 5 | autoflake --remove-all-unused-imports --recursive --remove-unused-variables \ 6 | --in-place uncertainty_wizard tests_unit --exclude=__init__.py 7 | 8 | black uncertainty_wizard tests_unit 9 | isort --profile black uncertainty_wizard tests_unit --profile black 10 | -------------------------------------------------------------------------------- /scripts/gen_dockerf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER_DIR="./docker/" 4 | SOURCES_DIR="${1}" 5 | 6 | 7 | if ! [[ -d "./$SOURCES_DIR" ]]; then 8 | printf "ERROR: Folder %s does not exist. " "${SOURCES_DIR}" 9 | printf "Make sure to run this script from the root folder of your repo and that you pass your " 10 | echo "project name (source folder name) as first argument to this script." 11 | exit 1 12 | fi 13 | if [[ "" == "$SOURCES_DIR" ]]; then 14 | printf "ERROR: You did not specify your project name (source folder). " 15 | echo "Make sure to pass your project name (source folder name) as first positional argument to this script." 16 | exit 1 17 | fi 18 | 19 | # =============================================== 20 | # Iterate and parse all lines of requirements.txt 21 | # =============================================== 22 | NON_TF_LINES=() 23 | 24 | while IFS="" read -r line || [ -n "$line" ]; do 25 | 26 | # Check if line is tensorflow import with version specified. If so, read version number 27 | if [[ $line =~ tensorflow==* ]]; then 28 | TF_VERSION=$(echo $line | cut -d'=' -f 3) 29 | echo 'tensorflow dependency: ' "$TF_VERSION" 30 | 31 | # Check if line is tensorflow import without version specified or . If so, read version number 32 | elif [[ $line =~ "tensorflow>="* ]]; then 33 | echo 'tensorflow dependency: ' "latest" 34 | TF_VERSION="latest" 35 | 36 | # Check if tensorflow is in requirements.txt without version number 37 | elif [[ $line == 'tensorflow' ]]; then 38 | printf "WARNING: Please specify version for your tensorflow import (e.g., tensorflow==2.3.0)" 39 | echo "For now, we will just base the dockerfile on tensorflow/tensorflow:latest" 40 | TF_VERSION="latest" 41 | 42 | else 43 | if ! [[ $line = \#* || $line =~ ==* ]] 44 | then 45 | printf "WARNING: You did not specify an exact version of dependency '%s'. " "${line}" 46 | echo "Multiple builds of the generated dockerfile may lead to different images." 47 | fi 48 | NON_TF_LINES+=("$line") 49 | fi 50 | 51 | done > "${folder}Dockerfile" 83 | done 84 | 85 | echo "FROM tensorflow/tensorflow:${TF_VERSION}-gpu" >> "${DEP_GPU_DIR}Dockerfile" 86 | echo "FROM tensorflow/tensorflow:${TF_VERSION}" >> "${DEP_CPU_DIR}Dockerfile" 87 | echo "FROM tensorflow/tensorflow:${TF_VERSION}-gpu" >> "${FULL_GPU_DIR}Dockerfile" 88 | echo "FROM tensorflow/tensorflow:${TF_VERSION}" >> "${FULL_CPU_DIR}Dockerfile" 89 | 90 | 91 | # ============================================================== 92 | # Create the requirements file without the tensorflow dependency 93 | # ============================================================== 94 | 95 | # Header 96 | { 97 | echo "# =========================================================================================================" 98 | echo "# This is an automatically generated file. It has the same content as the root folder 'requirements.tex'," 99 | echo "# except for the tensorflow import (which is removed as it will be inherited from tensorflow docker image)" 100 | echo "# =========================================================================================================" 101 | echo "" 102 | } >> "${DOCKER_DIR}requirements.txt" 103 | 104 | # Print Lines 105 | for req_line in "${NON_TF_LINES[@]}" 106 | do 107 | echo "$req_line" >> "${DOCKER_DIR}requirements.txt" 108 | done 109 | 110 | 111 | # ============================================================== 112 | # Pip install requirements file (this is the same for all tags) 113 | # ============================================================== 114 | for folder in "${ALL_FOLDERS[@]}" 115 | do 116 | { 117 | echo "" 118 | echo "# Update pip and install all pip dependencies" 119 | echo "RUN /usr/bin/python3 -m pip install --upgrade pip" 120 | echo "COPY ${DOCKER_DIR}requirements.txt /opt/project/requirements.txt" 121 | echo "RUN pip install -r /opt/project/requirements.txt" 122 | } >> "${folder}Dockerfile" 123 | done 124 | 125 | 126 | # =============== 127 | # COPY Resources 128 | # =============== 129 | FULL_FOLDERS=("$FULL_GPU_DIR" "$FULL_CPU_DIR") 130 | for folder in "${FULL_FOLDERS[@]}" 131 | do 132 | { 133 | echo "" 134 | echo "# Copy the resources folder" 135 | echo "COPY ./resources /opt/project/resources" 136 | } >> "${folder}Dockerfile" 137 | done 138 | 139 | 140 | 141 | # ===================================== 142 | # COPY Project Sources, including tests 143 | # ===================================== 144 | for folder in "${FULL_FOLDERS[@]}" 145 | do 146 | { 147 | echo "" 148 | echo "# Copy full project (sources + tests). This does *NOT* include the mount folder." 149 | echo "COPY ./${SOURCES_DIR} /opt/project/${SOURCES_DIR}" 150 | echo "COPY ./tests /opt/project/tests" 151 | } >> "${folder}Dockerfile" 152 | done 153 | 154 | echo "Regenerated ./docker/ folder based on your requirements.txt" -------------------------------------------------------------------------------- /scripts/pypi_deploy.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | 4 | # Process all options supplied on the command line 5 | while getopts "p" opt; do 6 | case $opt in 7 | 'p') 8 | # Update the value of the option x flag we defined above 9 | PROD=true 10 | echo "Deploying to production pypi" 11 | ;; 12 | *) 13 | echo "UNIMPLEMENTED OPTION - ${OPTKEY}" >&2 14 | exit 1 15 | ;; 16 | esac 17 | done 18 | 19 | rm dist/* 20 | python3 setup.py sdist bdist_wheel 21 | 22 | # Make sure we run in root folder of repository 23 | if ! [ "$PROD" = true ]; then 24 | echo "Deploying to TEST (test.pypi). Use the -p flag to deploy to prod." 25 | twine upload --repository testpypi --skip-existing dist/* 26 | else 27 | twine upload --skip-existing dist/* 28 | fi 29 | -------------------------------------------------------------------------------- /scripts/run_coverage.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | set -x 3 | 4 | export PYTHONPATH="$PWD" 5 | coverage run -m --source=uncertainty_wizard unittest discover tests_unit 6 | coverage report -m 7 | coverage xml 8 | -------------------------------------------------------------------------------- /scripts/run_docstring_coverage.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | set -x 3 | 4 | export PYTHONPATH="$PWD" 5 | python -m docstr-coverage uncertainty_wizard --skipfiledoc --skip-private --failunder=100 -------------------------------------------------------------------------------- /scripts/run_examples.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | set -x 3 | 4 | export PYTHONPATH="$PWD" 5 | 6 | jupyter nbconvert --to script ./examples/*.ipynb --output-dir='./examples_build/' 7 | 8 | for f in 'examples_build/'*.py; do python "$f"; done -------------------------------------------------------------------------------- /scripts/run_unit_tests.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | set -x 3 | 4 | export PYTHONPATH="$PWD" 5 | python -m unittest discover tests_unit -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | import version 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | setuptools.setup( 9 | name="uncertainty-wizard", 10 | version=version.RELEASE, 11 | author="Michael Weiss", 12 | author_email="code@mweiss.ch", 13 | description="Quick access to uncertainty and confidence of Keras networks.", 14 | long_description=long_description, 15 | long_description_content_type="text/markdown", 16 | url="https://github.com/testingautomated-usi/uncertainty_wizard", 17 | packages=setuptools.find_packages(), 18 | classifiers=[ 19 | "Development Status :: 4 - Beta", 20 | "Programming Language :: Python :: 3", 21 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 22 | "License :: OSI Approved :: MIT License", 23 | "Operating System :: OS Independent", 24 | ], 25 | python_requires='>=3.6', 26 | ) 27 | -------------------------------------------------------------------------------- /test_requirements.txt: -------------------------------------------------------------------------------- 1 | jupyterlab==3.1.4 2 | coverage==6.0.2 3 | docstr-coverage==2.2.0 4 | isort==5.10.1 5 | black==22.3.0 6 | flake8==4.0.1 7 | autoflake==1.4 -------------------------------------------------------------------------------- /tests_unit/__init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/tests_unit/__init.py -------------------------------------------------------------------------------- /tests_unit/internals_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/tests_unit/internals_tests/__init__.py -------------------------------------------------------------------------------- /tests_unit/internals_tests/test_tf_version_resolver.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import tensorflow as tf 4 | 5 | from uncertainty_wizard.internal_utils import tf_version_resolver 6 | 7 | 8 | class EnsembleFunctionalTest(TestCase): 9 | def call_current_tf_is_older_than(self, version, expected_outcome): 10 | result = tf_version_resolver.current_tf_version_is_older_than(version=version) 11 | if expected_outcome: 12 | self.assertTrue(result) 13 | else: 14 | self.assertFalse(result) 15 | 16 | def test_current_tf_version_is_older_than_is_false(self): 17 | # Test regular case 18 | self.call_current_tf_is_older_than("1.2.3", False) 19 | self.call_current_tf_is_older_than("2.0.0", False) 20 | 21 | # Test release candidate 22 | self.call_current_tf_is_older_than("1.2.3rc2", False) 23 | 24 | # Call on same version 25 | self.call_current_tf_is_older_than(tf.__version__, False) 26 | 27 | def test_current_tf_version_is_older_than_is_true(self): 28 | # Test regular case 29 | self.call_current_tf_is_older_than("3.2.1", True) 30 | self.call_current_tf_is_older_than("2.36.0", True) 31 | 32 | # Test release candidate 33 | self.call_current_tf_is_older_than("3.2.1rc1", True) 34 | 35 | def test_tf_version_is_older_than_fallback(self): 36 | invalid = "1.2.invalid" 37 | 38 | # Test with fallback provided 39 | result = tf_version_resolver.current_tf_version_is_older_than( 40 | version=invalid, fallback=True 41 | ) 42 | self.assertTrue(result) 43 | result = tf_version_resolver.current_tf_version_is_older_than( 44 | version=invalid, fallback=False 45 | ) 46 | self.assertFalse(result) 47 | 48 | # Test without fallback provided: In this case it should return True 49 | self.call_current_tf_is_older_than(invalid, True) 50 | 51 | def test_tf_version_is_older_than_raises_warning(self): 52 | invalid = "1.2.invalid" 53 | with self.assertWarns(RuntimeWarning): 54 | self.call_current_tf_is_older_than(invalid, True) 55 | 56 | def call_current_tf_is_newer_than(self, version, expected_outcome): 57 | result = tf_version_resolver.current_tf_version_is_newer_than(version=version) 58 | if expected_outcome: 59 | self.assertTrue(result) 60 | else: 61 | self.assertFalse(result) 62 | 63 | def test_current_tf_version_is_newer_is_true(self): 64 | # Test regular case 65 | self.call_current_tf_is_newer_than("1.2.3", True) 66 | self.call_current_tf_is_newer_than("2.0.0", True) 67 | 68 | # Test release candidate 69 | self.call_current_tf_is_newer_than("1.2.3rc2", True) 70 | 71 | def test_current_tf_version_is_newer_is_false(self): 72 | # Call on same version 73 | self.call_current_tf_is_newer_than(tf.__version__, False) 74 | 75 | def test_tf_version_is_newer_than_fallback(self): 76 | invalid = "1.2.invalid" 77 | 78 | # Test with fallback provided 79 | result = tf_version_resolver.current_tf_version_is_newer_than( 80 | version=invalid, fallback=True 81 | ) 82 | self.assertTrue(result) 83 | result = tf_version_resolver.current_tf_version_is_newer_than( 84 | version=invalid, fallback=False 85 | ) 86 | self.assertFalse(result) 87 | 88 | # Test without fallback provided: In this case it should return False 89 | self.call_current_tf_is_newer_than(invalid, False) 90 | 91 | def test_tf_version_is_newer_than_raises_warning(self): 92 | invalid = "1.2.invalid" 93 | with self.assertWarns(RuntimeWarning): 94 | self.call_current_tf_is_newer_than(invalid, False) 95 | -------------------------------------------------------------------------------- /tests_unit/keras_assumptions_tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests in this package do *not* actually test uncertainty-wizard, 2 | but they make sure that assumptions about tensorflow which have been taken 3 | and tested during uncertainty-wizard, but which are not yet stable or well documented, 4 | are still true in any further tf version for which the tests are executed.""" 5 | -------------------------------------------------------------------------------- /tests_unit/keras_assumptions_tests/test_dropout_seed.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | 7 | class TestDropoutSeedAssumptions(TestCase): 8 | def test_dropout_is_random_with_no_tf_seed(self): 9 | dropout = tf.keras.layers.Dropout(rate=0.1) 10 | noseed_a = dropout.call(np.ones((10, 100)), training=True) 11 | noseed_b = dropout.call(np.ones((10, 100)), training=True) 12 | self.assertFalse(np.all(noseed_a == noseed_b)) 13 | 14 | def test_dropout_seed_is_not_sufficient_for_reproduction(self): 15 | dropout_a = tf.keras.layers.Dropout(rate=0.1, seed=1) 16 | noseed_a = dropout_a.call(np.ones((10, 100)), training=True) 17 | dropout_b = tf.keras.layers.Dropout(rate=0.1, seed=1) 18 | noseed_b = dropout_b.call(np.ones((10, 100)), training=True) 19 | self.assertFalse(np.all(noseed_a == noseed_b)) 20 | 21 | def test_dropout_is_deterministic_with_tf_seed(self): 22 | dropout = tf.keras.layers.Dropout(rate=0.1) 23 | tf.random.set_seed(123) 24 | seeded_a = dropout.call(np.ones((10, 100)), training=True) 25 | tf.random.set_seed(123) 26 | seeded_b = dropout.call(np.ones((10, 100)), training=True) 27 | self.assertTrue(np.all(seeded_a == seeded_b)) 28 | -------------------------------------------------------------------------------- /tests_unit/keras_assumptions_tests/test_experimental_calls_exist.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import unittest 3 | from unittest import TestCase 4 | 5 | import tensorflow as tf 6 | 7 | from uncertainty_wizard.internal_utils.tf_version_resolver import ( 8 | current_tf_version_is_older_than, 9 | ) 10 | 11 | 12 | class TestExperimentalAPIAreAvailable(TestCase): 13 | """ 14 | These tests are 'dependency-regression tests': 15 | They check that the experimental functions uwiz relies on 16 | are present with the expected signature in the used tf version. 17 | This way, we ensure to observe when the experimental methods 18 | become stable (and thus conditional imports have to be added) 19 | """ 20 | 21 | def test_list_physical_devices(self): 22 | self.assertTrue("list_physical_devices" in dir(tf.config.experimental)) 23 | parameters = inspect.signature( 24 | tf.config.experimental.list_physical_devices 25 | ).parameters 26 | self.assertTrue("device_type" in parameters) 27 | self.assertEqual(1, len(parameters)) 28 | 29 | @unittest.skipIf( 30 | not current_tf_version_is_older_than("2.10.0"), "Known to fail for tf >= 2.10.0" 31 | ) 32 | def test_virtual_device_configuration(self): 33 | self.assertTrue("VirtualDeviceConfiguration" in dir(tf.config.experimental)) 34 | parameters = inspect.signature( 35 | tf.config.experimental.VirtualDeviceConfiguration 36 | ).parameters 37 | self.assertTrue("memory_limit" in parameters) 38 | self.assertTrue("experimental_priority" in parameters) 39 | self.assertEqual(2, len(parameters)) 40 | 41 | def test_set_visible_devices(self): 42 | self.assertTrue("set_visible_devices" in dir(tf.config.experimental)) 43 | parameters = inspect.signature( 44 | tf.config.experimental.set_visible_devices 45 | ).parameters 46 | self.assertTrue("devices" in parameters) 47 | self.assertTrue("device_type" in parameters) 48 | self.assertEqual(2, len(parameters)) 49 | 50 | def test_set_virtual_device_configuration(self): 51 | self.assertTrue( 52 | "set_virtual_device_configuration" in dir(tf.config.experimental) 53 | ) 54 | parameters = inspect.signature( 55 | tf.config.experimental.set_virtual_device_configuration 56 | ).parameters 57 | self.assertTrue("device" in parameters) 58 | self.assertTrue("logical_devices" in parameters) 59 | self.assertEqual(2, len(parameters)) 60 | -------------------------------------------------------------------------------- /tests_unit/models_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/tests_unit/models_tests/__init__.py -------------------------------------------------------------------------------- /tests_unit/models_tests/test_context_manager.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import os 3 | import pathlib 4 | import pickle 5 | from typing import Dict 6 | from unittest import TestCase 7 | 8 | from uncertainty_wizard.models.ensemble_utils import DeviceAllocatorContextManager 9 | 10 | FILE_PATH = "temp_test_6543543168" 11 | 12 | 13 | class TestDeviceAllocator(DeviceAllocatorContextManager, abc.ABC): 14 | @classmethod 15 | def file_path(cls) -> str: 16 | return FILE_PATH 17 | 18 | @classmethod 19 | def gpu_memory_limit(cls) -> int: 20 | return 2000 21 | 22 | @classmethod 23 | def acquire_lock_timeout(cls) -> int: 24 | return 1 25 | 26 | @classmethod 27 | def virtual_devices_per_gpu(cls) -> Dict[int, int]: 28 | return {0: 1, 1: 2} 29 | 30 | 31 | class FunctionalStochasticTest(TestCase): 32 | def tearDown(cls) -> None: 33 | # Delete temp files 34 | TestDeviceAllocator.before_start() 35 | 36 | def test_cleans_files_before_start(self): 37 | pathlib.Path(FILE_PATH).touch() 38 | TestDeviceAllocator.before_start() 39 | self.assertFalse( 40 | os.path.isfile(FILE_PATH), 41 | "Specified temp file (allocation or lockfile) was not deleted", 42 | ) 43 | 44 | def test_acquire_and_release_lock(self): 45 | TestDeviceAllocator.before_start() 46 | # Test lock was acquired 47 | lockfile = TestDeviceAllocator._acquire_lock() 48 | self.assertTrue(os.path.isfile(FILE_PATH + ".lock"), "Lockfile was not created") 49 | # Test lock file can not be acquired twice 50 | with self.assertRaises(RuntimeError) as e_context: 51 | TestDeviceAllocator._acquire_lock() 52 | self.assertTrue( 53 | "Ensemble process was not capable of acquiring lock" 54 | in str(e_context.exception) 55 | ) 56 | # Test lock is released 57 | TestDeviceAllocator._release_lock(lockfile) 58 | self.assertFalse( 59 | os.path.isfile(FILE_PATH + ".lock"), "Lockfile was not deleted" 60 | ) 61 | 62 | def _assert_device_selection( 63 | self, chosen_device, expected_device, expected_availabilities 64 | ): 65 | self.assertEqual(chosen_device, expected_device) 66 | with open(FILE_PATH, "rb") as file: 67 | availabilities = pickle.load(file) 68 | self.assertEqual(availabilities, expected_availabilities) 69 | 70 | def test_get_availabilities_and_choose_device(self): 71 | TestDeviceAllocator.before_start() 72 | # Make sure as a first device, the once with highest availability is selected 73 | chosen_device = TestDeviceAllocator._get_availabilities_and_choose_device() 74 | self._assert_device_selection(chosen_device, 1, {-1: 0, 0: 1, 1: 1}) 75 | 76 | # Select most available device (with lowest id as tiebreaker) 77 | chosen_device = TestDeviceAllocator._get_availabilities_and_choose_device() 78 | self._assert_device_selection(chosen_device, 0, {-1: 0, 0: 0, 1: 1}) 79 | chosen_device = TestDeviceAllocator._get_availabilities_and_choose_device() 80 | self._assert_device_selection(chosen_device, 1, {-1: 0, 0: 0, 1: 0}) 81 | 82 | # Workaround to make CPU and GPU available. Also tests that device releasing works 83 | TestDeviceAllocator._release_device(-1) 84 | TestDeviceAllocator._release_device(0) 85 | with open(FILE_PATH, "rb") as file: 86 | availabilities = pickle.load(file) 87 | self.assertEqual(availabilities, {-1: 1, 0: 1, 1: 0}) 88 | 89 | # Make sure if an alternative is available, CPU is not selected 90 | chosen_device = TestDeviceAllocator._get_availabilities_and_choose_device() 91 | self._assert_device_selection(chosen_device, 0, {-1: 1, 0: 0, 1: 0}) 92 | 93 | # Make sure if the only thing available, CPU is selected 94 | chosen_device = TestDeviceAllocator._get_availabilities_and_choose_device() 95 | self._assert_device_selection(chosen_device, -1, {-1: 0, 0: 0, 1: 0}) 96 | 97 | # No capacity left, make sure error is thrown 98 | with self.assertRaises(ValueError) as e_context: 99 | TestDeviceAllocator._get_availabilities_and_choose_device() 100 | self.assertTrue("No available devices. " in str(e_context.exception)) 101 | -------------------------------------------------------------------------------- /tests_unit/models_tests/test_functional_stochastic.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | import uncertainty_wizard as uwiz 7 | from uncertainty_wizard.internal_utils import UncertaintyWizardWarning 8 | from uncertainty_wizard.models import StochasticFunctional 9 | from uncertainty_wizard.models.stochastic_utils.layers import UwizBernoulliDropout 10 | from uncertainty_wizard.quantifiers import StandardDeviation 11 | 12 | 13 | class FunctionalStochasticTest(TestCase): 14 | @staticmethod 15 | def _dummy_model(): 16 | stochastic_mode = uwiz.models.StochasticMode() 17 | x = tf.keras.layers.Input(shape=1000) 18 | output = UwizBernoulliDropout(rate=0.5, stochastic_mode=stochastic_mode)(x) 19 | return StochasticFunctional( 20 | inputs=x, outputs=output, stochastic_mode=stochastic_mode 21 | ) 22 | 23 | def test_predict_is_deterministic(self): 24 | model = self._dummy_model() 25 | y = model.predict(x=np.ones((10, 1000))) 26 | self.assertTrue(np.all(y == 1)) 27 | 28 | def test_sampled_predict_is_not_deterministic(self): 29 | model = self._dummy_model() 30 | self._assert_random_samples(model) 31 | 32 | def test_sampled_turning_sampling_on_and_off_iteratively(self): 33 | model = self._dummy_model() 34 | self._test_randomized_on_off(model) 35 | 36 | def _test_randomized_on_off(self, model): 37 | for _ in range(2): 38 | self._assert_random_samples(model) 39 | 40 | y = model.predict(x=np.ones((10, 1000))) 41 | self.assertTrue(np.all(y == 1)) 42 | 43 | def _assert_random_samples(self, model): 44 | y, std = model.predict_quantified( 45 | x=np.ones((10, 1000)), quantifier=StandardDeviation(), sample_size=20 46 | ) 47 | self.assertFalse(np.all(y == 1), y) 48 | self.assertFalse(np.all(std == 0), std) 49 | 50 | def test_warns_on_compile_if_not_stochastic(self): 51 | stochastic_mode = uwiz.models.StochasticMode() 52 | x = tf.keras.layers.Input(shape=1000) 53 | output = tf.keras.layers.Dropout(rate=0.5)(x) 54 | model = StochasticFunctional( 55 | inputs=x, outputs=output, stochastic_mode=stochastic_mode 56 | ) 57 | with self.assertWarns(UncertaintyWizardWarning): 58 | model.compile(loss="mse") 59 | 60 | def test_save_and_load_model(self): 61 | stochastic = self._dummy_model() 62 | # Model can currently (as of tf 2.1) only be saved if build, fit or predict was called 63 | stochastic.predict(np.ones((10, 1000))) 64 | stochastic.save("/tmp/test_save_and_load_model_stochastic") 65 | del stochastic 66 | stochastic_loaded = uwiz.models.load_model( 67 | "/tmp/test_save_and_load_model_stochastic" 68 | ) 69 | self._test_randomized_on_off(stochastic_loaded) 70 | 71 | def test_weights_and_stochastic_mode_on_clone_from_keras(self): 72 | # Prepare a model with dropout to be used to create a StochasticModel 73 | inputs = tf.keras.layers.Input(1000) 74 | x = tf.keras.layers.Dense( 75 | 1000, kernel_initializer="random_normal", bias_initializer="zeros" 76 | )(inputs) 77 | x = tf.keras.layers.Dropout(0.5)(x) 78 | x = tf.keras.layers.Dense( 79 | 10, kernel_initializer="random_normal", bias_initializer="zeros" 80 | )(x) 81 | x = tf.keras.layers.Dense(10, activation=tf.keras.activations.relu)(x) 82 | keras_model = tf.keras.Model(inputs=inputs, outputs=x) 83 | keras_model.compile(loss="mse", optimizer="adam", metrics=["mse"]) 84 | keras_model.fit( 85 | np.ones((20, 1000), dtype=float), np.zeros((20, 10)), batch_size=1, epochs=1 86 | ) 87 | 88 | # Call the model under test 89 | uwiz_model = uwiz.models.stochastic_from_keras(keras_model) 90 | 91 | # Demo input for tests 92 | inp_data = np.ones((10, 1000), dtype=float) 93 | 94 | # Assert that both models make the same predictions 95 | keras_prediction = keras_model.predict(inp_data) 96 | uwiz_prediction = uwiz_model.predict(inp_data) 97 | np.testing.assert_array_equal(keras_prediction, uwiz_prediction) 98 | 99 | # Test that stochastic mode is working on cloned model 100 | self._assert_random_samples(uwiz_model) 101 | 102 | def test_randomness_error_on_clone_from_keras(self): 103 | inputs = tf.keras.layers.Input(10) 104 | x = tf.keras.layers.Dense( 105 | 10, kernel_initializer="random_normal", bias_initializer="zeros" 106 | )(inputs) 107 | x = tf.keras.layers.Dense(10, activation=tf.keras.activations.relu)(x) 108 | keras_model = tf.keras.Model(inputs=inputs, outputs=x) 109 | keras_model.compile(loss="mse", optimizer="adam", metrics=["mse"]) 110 | keras_model.fit( 111 | np.ones((20, 10), dtype=float), np.zeros((20, 10)), batch_size=1, epochs=1 112 | ) 113 | 114 | # make sure no validation error is thrown when determinism is expected 115 | _ = uwiz.models.stochastic_from_keras(keras_model, expect_determinism=True) 116 | 117 | self.assertRaises( 118 | ValueError, lambda: uwiz.models.stochastic_from_keras(keras_model) 119 | ) 120 | -------------------------------------------------------------------------------- /tests_unit/models_tests/test_lazy_ensemble.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | import uncertainty_wizard as uwiz 7 | from uncertainty_wizard.quantifiers import StandardDeviation 8 | 9 | DUMMY_MODEL_PATH = "tmp/dummy_lazy_ensemble" 10 | 11 | 12 | def create_dummy_atomic_model(model_id: int): 13 | model = tf.keras.models.Sequential() 14 | model.add(tf.keras.layers.Input(shape=1000)) 15 | model.add(tf.keras.layers.Dropout(rate=0.5)) 16 | model.add(tf.keras.layers.Dense(1)) 17 | model.compile(loss="mse", optimizer="adam") 18 | return model, None 19 | 20 | 21 | def dummy_fit_process(model_id: int, model: tf.keras.Model): 22 | history = model.fit( 23 | x=np.ones((100, 1000)), y=np.ones((100, 1)), epochs=10, batch_size=100 24 | ) 25 | return model, history.history 26 | 27 | 28 | def dummy_predict_process(model_id: int, model: tf.keras.Model): 29 | return model.predict(np.ones((10, 1000))) 30 | 31 | 32 | def dummy_independent_task(model_id: int): 33 | return model_id 34 | 35 | 36 | # Note: So far we have mostly smoke tests. 37 | class LazyEnsembleTest(TestCase): 38 | def test_dummy_in_distinct_process(self): 39 | ensemble = uwiz.models.LazyEnsemble( 40 | num_models=2, model_save_path=DUMMY_MODEL_PATH 41 | ) 42 | ensemble.create(create_function=create_dummy_atomic_model) 43 | pred, std = ensemble.predict_quantified( 44 | x=np.ones((10, 1000)), quantifier="std", num_processes=1 45 | ) 46 | self.assertEqual(pred.shape, (10, 1)) 47 | self.assertEqual(std.shape, (10, 1)) 48 | 49 | def test_dummy_in_main_process(self): 50 | ensemble = uwiz.models.LazyEnsemble( 51 | num_models=2, model_save_path=DUMMY_MODEL_PATH, default_num_processes=0 52 | ) 53 | ensemble.create(create_function=create_dummy_atomic_model) 54 | pred, std = ensemble.predict_quantified( 55 | x=np.ones((10, 1000)), quantifier="std", num_processes=0 56 | ) 57 | self.assertEqual(pred.shape, (10, 1)) 58 | self.assertEqual(std.shape, (10, 1)) 59 | 60 | def test_result_as_dict(self): 61 | ensemble = uwiz.models.LazyEnsemble( 62 | num_models=2, model_save_path=DUMMY_MODEL_PATH, default_num_processes=0 63 | ) 64 | ensemble.create(create_function=create_dummy_atomic_model) 65 | res = ensemble.predict_quantified( 66 | x=np.ones((10, 1000)), 67 | quantifier="std", 68 | num_processes=0, 69 | return_alias_dict=True, 70 | ) 71 | assert isinstance(res, dict) 72 | for alias in StandardDeviation().aliases(): 73 | assert alias in res 74 | assert type(res[alias]) == tuple 75 | assert len(res[alias]) == 2 76 | assert res[alias][0].shape == (10, 1) 77 | assert res[alias][1].shape == (10, 1) 78 | 79 | def test_dummy_main_and_one_distinct_process_are_equivalent(self): 80 | ensemble = uwiz.models.LazyEnsemble( 81 | num_models=2, model_save_path=DUMMY_MODEL_PATH 82 | ) 83 | ensemble.create(create_function=create_dummy_atomic_model) 84 | pred_p, std_p = ensemble.predict_quantified( 85 | x=np.ones((10, 1000)), quantifier="std", num_processes=1 86 | ) 87 | 88 | pred_m, std_m = ensemble.predict_quantified( 89 | x=np.ones((10, 1000)), quantifier="std", num_processes=0 90 | ) 91 | self.assertTrue(np.all(std_m == std_p)) 92 | self.assertTrue(np.all(pred_m == pred_p)) 93 | 94 | def test_dummy_main_and_two_distinct_processes_are_equivalent(self): 95 | ensemble = uwiz.models.LazyEnsemble( 96 | num_models=2, model_save_path=DUMMY_MODEL_PATH 97 | ) 98 | ensemble.create(create_function=create_dummy_atomic_model) 99 | pred_p, std_p = ensemble.predict_quantified( 100 | x=np.ones((10, 1000)), quantifier="std", num_processes=2 101 | ) 102 | 103 | pred_m, std_m = ensemble.predict_quantified( 104 | x=np.ones((10, 1000)), quantifier="std", num_processes=0 105 | ) 106 | self.assertTrue(np.all(std_m == std_p)) 107 | self.assertTrue(np.all(pred_m == pred_p)) 108 | 109 | def test_save_and_load(self): 110 | ensemble = uwiz.models.LazyEnsemble( 111 | num_models=2, model_save_path=DUMMY_MODEL_PATH 112 | ) 113 | ensemble.create(create_function=create_dummy_atomic_model) 114 | pred_p, std_p = ensemble.predict_quantified( 115 | x=np.ones((10, 1000)), quantifier="std", num_processes=0 116 | ) 117 | # Saving not required, lazy ensembles are always saved on file 118 | del ensemble 119 | 120 | ensemble = uwiz.models.load_model(DUMMY_MODEL_PATH) 121 | pred_m, std_m = ensemble.predict_quantified( 122 | x=np.ones((10, 1000)), quantifier="std", num_processes=0 123 | ) 124 | self.assertTrue(np.all(std_m == std_p)) 125 | self.assertTrue(np.all(pred_m == pred_p)) 126 | 127 | def smoke_full(self, num_processes): 128 | ensemble = uwiz.models.LazyEnsemble( 129 | num_models=2, 130 | model_save_path=DUMMY_MODEL_PATH, 131 | default_num_processes=num_processes, 132 | ) 133 | ensemble.create(create_function=create_dummy_atomic_model) 134 | fit_history = ensemble.modify(map_function=dummy_fit_process) 135 | self.assertIsNotNone(fit_history) 136 | fit_history = ensemble.fit( 137 | x=np.ones((100, 1000)), y=np.ones((100, 1)), epochs=10, batch_size=100 138 | ) 139 | self.assertIsNotNone(fit_history) 140 | 141 | lazy_pred_p, lazy_pred_std = ensemble.quantify_predictions( 142 | quantifier="std", consume_function=dummy_predict_process 143 | ) 144 | np_based_pred_p, np_based_std = ensemble.predict_quantified( 145 | x=np.ones((10, 1000)), quantifier="std" 146 | ) 147 | self.assertTrue(np.all(lazy_pred_p == np_based_pred_p)) 148 | self.assertTrue(np.all(lazy_pred_std == np_based_std)) 149 | 150 | returned_ids = ensemble.run_model_free(task=dummy_independent_task) 151 | self.assertTrue(returned_ids == [0, 1], f"returned was: {returned_ids}") 152 | 153 | def test_main_smoke_full(self): 154 | self.smoke_full(0) 155 | 156 | def test_main_smoke_distinct_process(self): 157 | self.smoke_full(1) 158 | -------------------------------------------------------------------------------- /tests_unit/models_tests/test_sequential_stochastic.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | import uncertainty_wizard as uwiz 7 | from uncertainty_wizard.internal_utils import UncertaintyWizardWarning 8 | from uncertainty_wizard.models import StochasticSequential 9 | from uncertainty_wizard.quantifiers import MaxSoftmax, StandardDeviation, VariationRatio 10 | 11 | 12 | class SequentialStochasticTest(TestCase): 13 | @staticmethod 14 | def _dummy_model(): 15 | model = StochasticSequential() 16 | model.add(tf.keras.layers.Input(shape=1000)) 17 | model.add(tf.keras.layers.Dropout(rate=0.5)) 18 | return model 19 | 20 | @staticmethod 21 | def _dummy_classifier(): 22 | model = StochasticSequential() 23 | model.add(tf.keras.layers.Input(shape=1000)) 24 | model.add(tf.keras.layers.Dropout(rate=0.5)) 25 | model.add(tf.keras.layers.Dense(10, activation="softmax")) 26 | # compile the model 27 | model.compile( 28 | optimizer=tf.keras.optimizers.Adam(), 29 | loss=tf.keras.losses.CategoricalCrossentropy(), 30 | metrics=[tf.keras.metrics.CategoricalAccuracy()], 31 | ) 32 | return model 33 | 34 | def test_result_as_dict(self): 35 | model = self._dummy_classifier() 36 | x = np.ones((10, 1000)) 37 | res = model.predict_quantified( 38 | x=x, 39 | quantifier=[ 40 | "MaxSoftmax", 41 | VariationRatio(), 42 | ], 43 | return_alias_dict=True, 44 | ) 45 | 46 | self.assertTrue(isinstance(res, dict)) 47 | for key, values in res.items(): 48 | self.assertTrue(isinstance(key, str)) 49 | self.assertTrue(isinstance(values, tuple)) 50 | self.assertEqual(len(values), 2) 51 | self.assertEqual(values[0].shape, (10,)) 52 | self.assertEqual(values[1].shape, (10,)) 53 | 54 | for q in [MaxSoftmax(), VariationRatio()]: 55 | for a in q.aliases(): 56 | self.assertTrue(a in res.keys()) 57 | 58 | def test_return_type_default_multi_quant(self): 59 | model = self._dummy_classifier() 60 | x = np.ones((10, 1000)) 61 | res = model.predict_quantified(x=x, quantifier=["MaxSoftmax", VariationRatio()]) 62 | self.assertTrue(isinstance(res, list)) 63 | self.assertTrue(len(res), 2) 64 | 65 | def test_return_type_default_single_quant(self): 66 | model = self._dummy_classifier() 67 | x = np.ones((10, 1000)) 68 | res = model.predict_quantified(x=x, quantifier="MaxSoftmax") 69 | self.assertTrue(isinstance(res, tuple)) 70 | self.assertTrue(len(res), 2) 71 | 72 | def test_predict_is_deterministic(self): 73 | model = self._dummy_model() 74 | y = model.predict(x=np.ones((10, 1000))) 75 | self.assertTrue(np.all(y == 1)) 76 | 77 | def test_sampled_predict_is_not_deterministic(self): 78 | model = self._dummy_model() 79 | self._assert_random_samples(model) 80 | 81 | def test_sampled_turning_sampling_on_and_off_iteratively(self): 82 | model = self._dummy_model() 83 | self._test_randomized_on_off(model) 84 | 85 | def _test_randomized_on_off(self, model): 86 | for _ in range(2): 87 | self._assert_random_samples(model) 88 | 89 | y = model.predict(x=np.ones((10, 1000))) 90 | self.assertTrue(np.all(y == 1)) 91 | 92 | def _assert_random_samples(self, model): 93 | y, std = model.predict_quantified( 94 | x=np.ones((10, 1000)), quantifier=StandardDeviation(), sample_size=20 95 | ) 96 | self.assertFalse(np.all(y == 1), y) 97 | self.assertFalse(np.all(std == 0), std) 98 | 99 | def test_warns_on_compile_if_not_stochastic(self): 100 | model = StochasticSequential() 101 | model.add(tf.keras.layers.Input(shape=1000)) 102 | model.add(tf.keras.layers.Dense(1000)) 103 | with self.assertWarns(UncertaintyWizardWarning): 104 | model.compile() 105 | 106 | def test_save_and_load_model(self): 107 | stochastic = self._dummy_model() 108 | # Model can currently (as of tf 2.1) only be saved if build, fit or predict was called 109 | stochastic.predict(np.ones((10, 1000))) 110 | stochastic.save("/tmp/model") 111 | del stochastic 112 | stochastic_loaded = uwiz.models.load_model("/tmp/model") 113 | self._test_randomized_on_off(stochastic_loaded) 114 | 115 | def test_weights_and_stochasicmode_on_clone_from_keras(self): 116 | # Prepare a model with dropout to be used to create a StochasticModel 117 | keras_model = tf.keras.models.Sequential( 118 | [ 119 | tf.keras.layers.Dense( 120 | 1000, kernel_initializer="random_normal", bias_initializer="zeros" 121 | ), 122 | tf.keras.layers.Dropout(0.5), 123 | tf.keras.layers.Dense( 124 | 10, kernel_initializer="random_normal", bias_initializer="zeros" 125 | ), 126 | tf.keras.layers.Dense(10, activation=tf.keras.activations.relu), 127 | ] 128 | ) 129 | keras_model.compile(loss="mse", optimizer="adam", metrics=["mse"]) 130 | keras_model.fit( 131 | np.ones((20, 1000), dtype=float), np.zeros((20, 10)), batch_size=1, epochs=1 132 | ) 133 | 134 | # Call the model under test 135 | uwiz_model = uwiz.models.stochastic_from_keras(keras_model) 136 | 137 | # Demo input for tests 138 | input = np.ones((10, 1000), dtype=float) 139 | 140 | # Assert that both models make the same predictions 141 | keras_prediction = keras_model.predict(input) 142 | uwiz_prediction = uwiz_model.predict(input) 143 | np.testing.assert_array_equal(keras_prediction, uwiz_prediction) 144 | 145 | # Test that stochastic mode is working on cloned model 146 | self._assert_random_samples(uwiz_model) 147 | 148 | def test_randomness_error_on_clone_from_keras(self): 149 | keras_model = tf.keras.models.Sequential( 150 | [ 151 | tf.keras.layers.Dense( 152 | 10, kernel_initializer="random_normal", bias_initializer="zeros" 153 | ), 154 | tf.keras.layers.Dense(10, activation=tf.keras.activations.relu), 155 | ] 156 | ) 157 | keras_model.compile(loss="mse", optimizer="adam", metrics=["mse"]) 158 | keras_model.fit( 159 | np.ones((20, 10), dtype=float), np.zeros((20, 10)), batch_size=1, epochs=1 160 | ) 161 | 162 | # make sure no validation error is thrown when determinism is expected 163 | _ = uwiz.models.stochastic_from_keras(keras_model, expect_determinism=True) 164 | 165 | self.assertRaises( 166 | ValueError, lambda: uwiz.models.stochastic_from_keras(keras_model) 167 | ) 168 | -------------------------------------------------------------------------------- /tests_unit/models_tests/test_stochastic_layers.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | import uncertainty_wizard as uwiz 7 | from uncertainty_wizard.internal_utils import UncertaintyWizardWarning 8 | from uncertainty_wizard.models._stochastic._stochastic_mode import StochasticMode 9 | from uncertainty_wizard.models.stochastic_utils.layers import ( 10 | UwizBernoulliDropout, 11 | UwizGaussianDropout, 12 | UwizGaussianNoise, 13 | ) 14 | 15 | DUMMY_INPUT = np.ones((10, 1000), dtype=np.float32) 16 | 17 | 18 | class StochasticLayerTest(TestCase): 19 | def test_warns_custom_keras_subtype(self): 20 | class SubDropout(tf.keras.layers.Dropout): 21 | def __init__(self, rate, **kwargs): 22 | super().__init__(rate, **kwargs) 23 | 24 | subclass_instance = SubDropout(rate=0.5) 25 | uwiz_type = uwiz.models.stochastic_utils.layers.UwizBernoulliDropout 26 | with self.assertWarns(UncertaintyWizardWarning) as cm: 27 | uwiz.models.stochastic_utils.layers._has_casting_preventing_subtype( 28 | subclass_instance, 29 | expected_type=tf.keras.layers.Dropout, 30 | corresponding_uw_type=uwiz_type, 31 | ) 32 | the_warning = cm.warning 33 | self.assertTrue( 34 | "make sure the models stochastic mode tensor is respected" 35 | in the_warning.args[0] 36 | ) 37 | 38 | def test_warns_custom_uwiz_subtype(self): 39 | class SubUwizGaussianNoise(UwizGaussianNoise): 40 | def __init__(self, stddev, stochastic_mode, **kwargs): 41 | super().__init__(stddev, stochastic_mode, **kwargs) 42 | 43 | subclass_instance = SubUwizGaussianNoise( 44 | stddev=0.5, stochastic_mode=StochasticMode() 45 | ) 46 | uwiz_type = uwiz.models.stochastic_utils.layers.UwizGaussianNoise 47 | with self.assertWarns(UncertaintyWizardWarning) as cm: 48 | uwiz.models.stochastic_utils.layers._has_casting_preventing_subtype( 49 | subclass_instance, 50 | expected_type=tf.keras.layers.GaussianNoise, 51 | corresponding_uw_type=uwiz_type, 52 | ) 53 | the_warning = cm.warning 54 | self.assertTrue( 55 | "ou know what you did and set up the stochastic mode correctly." 56 | in the_warning.args[0] 57 | ) 58 | 59 | def assert_listens_to_forceful_enabled(self, layer_constructor): 60 | """ 61 | Setup to test that the stochastic mode is read correctly in a noise layer in eager mode. 62 | To be called directly by the test class for the corresponding layer 63 | 64 | Args: 65 | layer_constructor (): No-Args lambda that creates a corresponding layer. 66 | 67 | Returns: 68 | None 69 | """ 70 | stochastic_mode = StochasticMode() 71 | self.assertTrue( 72 | tf.executing_eagerly(), "This test is supposed to test eager execution" 73 | ) 74 | layer = layer_constructor(stochastic_mode) 75 | stochastic_mode.as_tensor().assign(tf.ones((), dtype=bool)) 76 | enabled_result = layer(DUMMY_INPUT) 77 | self.assertFalse( 78 | tf.reduce_all(tf.equal(enabled_result, DUMMY_INPUT).numpy()), 79 | "No values were dropped", 80 | ) 81 | stochastic_mode.as_tensor().assign(tf.zeros((), dtype=bool)) 82 | disabled_result = layer(DUMMY_INPUT) 83 | self.assertTrue( 84 | tf.reduce_all(tf.equal(disabled_result, DUMMY_INPUT)).numpy(), 85 | "Some values changed - which should not happen", 86 | ) 87 | 88 | def assert_listens_to_forceful_enabled_in_sequential_predict( 89 | self, layer_constructor 90 | ): 91 | """ 92 | Setup to test that the stochastic mode is read correctly in a noise layer in eager mode, 93 | when used in a tf.keras.Sequential model for prediction. 94 | To be called directly by the test class for the corresponding layer 95 | 96 | Args: 97 | layer_constructor (): No-Args lambda that creates a corresponding layer. 98 | 99 | Returns: 100 | None 101 | """ 102 | stochastic_mode = StochasticMode() 103 | self.assertTrue( 104 | tf.executing_eagerly(), "This test is supposed to test eager execution" 105 | ) 106 | stochastic_mode.as_tensor().assign(tf.ones((), dtype=bool)) 107 | model = tf.keras.models.Sequential() 108 | model.add(tf.keras.Input(shape=1000)) 109 | model.add(layer_constructor(stochastic_mode)) 110 | enabled_result = model.predict(DUMMY_INPUT) 111 | self.assertFalse( 112 | tf.reduce_all(tf.equal(enabled_result, DUMMY_INPUT).numpy()), 113 | "No values were dropped", 114 | ) 115 | 116 | stochastic_mode.as_tensor().assign(tf.zeros((), dtype=bool)) 117 | model = tf.keras.models.Sequential() 118 | model.add(tf.keras.Input(shape=1000)) 119 | model.add(layer_constructor(stochastic_mode)) 120 | disabled_result = model.predict(DUMMY_INPUT) 121 | self.assertTrue( 122 | tf.reduce_all(tf.equal(disabled_result, DUMMY_INPUT)).numpy(), 123 | "Some values changed - which should not happen", 124 | ) 125 | 126 | def assert_listens_to_forceful_enabled_graph_mode(self, layer_constructor): 127 | """ 128 | Setup to test that the stochastic mode is read correctly in a noise layer. 129 | To be called directly by the test class for the corresponding layer 130 | 131 | Args: 132 | layer_constructor (): function that creates a layer. Argument: the stochastic mode instance to use 133 | 134 | Returns: 135 | None 136 | """ 137 | stochastic_mode = StochasticMode() 138 | 139 | @tf.function 140 | def run_test(): 141 | self.assertFalse( 142 | tf.executing_eagerly(), 143 | "This test is supposed to test disabled eager execution", 144 | ) 145 | layer = layer_constructor(stochastic_mode) 146 | stochastic_mode.as_tensor().assign(tf.ones((), dtype=bool)) 147 | enabled_result = layer(DUMMY_INPUT) 148 | input_as_tensor = tf.constant(DUMMY_INPUT, dtype=tf.float32) 149 | test_enabled = tf.debugging.Assert( 150 | tf.math.logical_not( 151 | tf.reduce_all(tf.equal(enabled_result, input_as_tensor)) 152 | ), 153 | [tf.constant("No values were dropped"), enabled_result], 154 | ) 155 | stochastic_mode.as_tensor().assign(tf.zeros((), dtype=bool)) 156 | disabled_result = layer(DUMMY_INPUT) 157 | test_disabled = tf.debugging.Assert( 158 | tf.reduce_all(tf.equal(disabled_result, input_as_tensor)), 159 | [ 160 | tf.constant("Some values changed - which should not happen"), 161 | disabled_result, 162 | ], 163 | ) 164 | stochastic_mode.as_tensor().assign(tf.zeros((), dtype=bool)) 165 | 166 | with tf.control_dependencies([test_enabled, test_disabled]): 167 | pass 168 | 169 | run_test() 170 | 171 | 172 | class TestBernoulliDropout(StochasticLayerTest): 173 | def test_listens_to_forceful_enabled(self): 174 | self.assert_listens_to_forceful_enabled( 175 | lambda sm: UwizBernoulliDropout(0.5, stochastic_mode=sm) 176 | ) 177 | 178 | def test_listens_to_forceful_enabled_in_sequential(self): 179 | self.assert_listens_to_forceful_enabled_in_sequential_predict( 180 | lambda sm: UwizBernoulliDropout(0.5, stochastic_mode=sm) 181 | ) 182 | 183 | def test_listens_to_forceful_enabled_non_eager(self): 184 | self.assert_listens_to_forceful_enabled_graph_mode( 185 | lambda sm: UwizBernoulliDropout(0.5, stochastic_mode=sm) 186 | ) 187 | 188 | def test_cast_from_keras(self): 189 | plain_keras = tf.keras.layers.Dropout(rate=0.5) 190 | stochastic_mode = StochasticMode() 191 | output = UwizBernoulliDropout.from_keras_layer( 192 | plain_keras, stochastic_mode=stochastic_mode 193 | ) 194 | self.assertIsInstance(output, UwizBernoulliDropout) 195 | self.assertEqual(output.rate, 0.5) 196 | self.assertEqual( 197 | stochastic_mode.as_tensor(), output.stochastic_mode.as_tensor() 198 | ) 199 | 200 | 201 | class TestGaussianDropout(StochasticLayerTest): 202 | def test_listens_to_forceful_enabled(self): 203 | self.assert_listens_to_forceful_enabled( 204 | lambda sm: UwizGaussianDropout(0.5, stochastic_mode=sm) 205 | ) 206 | 207 | def test_listens_to_forceful_enabled_in_sequential(self): 208 | self.assert_listens_to_forceful_enabled_in_sequential_predict( 209 | lambda sm: UwizGaussianDropout(0.5, stochastic_mode=sm) 210 | ) 211 | 212 | def test_listens_to_forceful_enabled_non_eager(self): 213 | self.assert_listens_to_forceful_enabled_graph_mode( 214 | lambda sm: UwizGaussianDropout(0.5, stochastic_mode=sm) 215 | ) 216 | 217 | def test_cast_from_keras(self): 218 | plain_keras = tf.keras.layers.GaussianDropout(rate=0.5) 219 | stochastic_mode = StochasticMode() 220 | output = UwizGaussianDropout.from_keras_layer( 221 | plain_keras, stochastic_mode=stochastic_mode 222 | ) 223 | self.assertIsInstance(output, UwizGaussianDropout) 224 | self.assertEqual(output.rate, 0.5) 225 | self.assertEqual( 226 | stochastic_mode.as_tensor(), output.stochastic_mode.as_tensor() 227 | ) 228 | 229 | 230 | class TestGaussianNoise(StochasticLayerTest): 231 | def test_listens_to_forceful_enabled(self): 232 | self.assert_listens_to_forceful_enabled( 233 | lambda sm: UwizGaussianNoise(0.5, stochastic_mode=sm) 234 | ) 235 | 236 | def test_listens_to_forceful_enabled_in_sequential(self): 237 | self.assert_listens_to_forceful_enabled_in_sequential_predict( 238 | lambda sm: UwizGaussianNoise(0.5, stochastic_mode=sm) 239 | ) 240 | 241 | def test_listens_to_forceful_enabled_non_eager(self): 242 | self.assert_listens_to_forceful_enabled_graph_mode( 243 | lambda sm: UwizGaussianNoise(0.5, stochastic_mode=sm) 244 | ) 245 | 246 | def test_cast_from_keras(self): 247 | plain_keras = tf.keras.layers.GaussianNoise(stddev=0.5) 248 | stochastic_mode = StochasticMode() 249 | output = UwizGaussianNoise.from_keras_layer( 250 | plain_keras, stochastic_mode=stochastic_mode 251 | ) 252 | self.assertIsInstance(output, UwizGaussianNoise) 253 | self.assertEqual(output.stddev, 0.5) 254 | self.assertEqual( 255 | stochastic_mode.as_tensor(), output.stochastic_mode.as_tensor() 256 | ) 257 | -------------------------------------------------------------------------------- /tests_unit/models_tests/test_uwiz_model.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from unittest import TestCase 3 | 4 | import numpy as np 5 | import tensorflow as tf 6 | 7 | import uncertainty_wizard as uwiz 8 | from uncertainty_wizard.internal_utils import UncertaintyWizardWarning 9 | 10 | 11 | class EnsembleFunctionalTest(TestCase): 12 | @staticmethod 13 | def _dummy_stochastic_classifier(): 14 | model = uwiz.models.StochasticSequential( 15 | layers=[ 16 | tf.keras.layers.Input(shape=1000), 17 | tf.keras.layers.Dense(1000), 18 | tf.keras.layers.Dropout(0.3), 19 | tf.keras.layers.Dense(1000), 20 | tf.keras.layers.Dense(10), 21 | tf.keras.layers.Softmax(), 22 | ] 23 | ) 24 | model.compile(loss="mse") 25 | # The labels make no sense for a softmax output layer, but this does not matter 26 | model.fit(x=np.ones((10, 1000)), y=np.ones((10, 10)), epochs=2) 27 | return model 28 | 29 | def test_error_if_invalid_quantifier_type(self): 30 | model = self._dummy_stochastic_classifier() 31 | # Test that only string and objects are accepted as quantifiers 32 | with self.assertRaises(TypeError): 33 | model.predict_quantified(np.ones((10, 1000)), quantifier=5) 34 | 35 | def test_error_if_point_predictors(self): 36 | model = self._dummy_stochastic_classifier() 37 | # Test that only string and objects are accepted as quantifiers 38 | with self.assertRaises(ValueError): 39 | model.predict_quantified( 40 | np.ones((10, 1000)), quantifier=["PCS", "SoftmaxEntropy"] 41 | ) 42 | 43 | def test_error_if_point_predictors(self): 44 | model = self._dummy_stochastic_classifier() 45 | # Test that only string and objects are accepted as quantifiers 46 | with self.assertWarns(UncertaintyWizardWarning): 47 | model.predict_quantified( 48 | np.ones((10, 1000)), quantifier=["PCS", "SoftmaxEntropy"] 49 | ) 50 | 51 | # Test that no warning is printed if passed individually 52 | with warnings.catch_warnings(record=True) as w: 53 | model.predict_quantified(np.ones((10, 1000)), quantifier=["SoftmaxEntropy"]) 54 | model.predict_quantified(np.ones((10, 1000)), quantifier=["PCS"]) 55 | self.assertEqual(len(w), 0, w) 56 | -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/tests_unit/quantifiers_tests/__init__.py -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/test_mean_softmax.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard import ProblemType 6 | from uncertainty_wizard.quantifiers import MeanSoftmax, QuantifierRegistry 7 | 8 | 9 | class TestMeanSoftmax(TestCase): 10 | 11 | # ================= 12 | # Test Class Methods 13 | # ================= 14 | 15 | def test_string_representation(self): 16 | self.assertTrue( 17 | isinstance(QuantifierRegistry.find("mean_softmax"), MeanSoftmax) 18 | ) 19 | self.assertTrue(isinstance(QuantifierRegistry.find("ensembling"), MeanSoftmax)) 20 | self.assertTrue(isinstance(QuantifierRegistry.find("MS"), MeanSoftmax)) 21 | self.assertTrue(isinstance(QuantifierRegistry.find("MeanSoftmax"), MeanSoftmax)) 22 | 23 | def test_is_confidence(self): 24 | self.assertTrue(MeanSoftmax.is_confidence()) 25 | self.assertTrue(MeanSoftmax().is_confidence()) 26 | 27 | def test_samples_type_declaration(self): 28 | self.assertTrue(MeanSoftmax.takes_samples()) 29 | 30 | def test_problem_type(self): 31 | self.assertEqual(MeanSoftmax.problem_type(), ProblemType.CLASSIFICATION) 32 | 33 | # ================== 34 | # Test Logic 35 | # ================= 36 | 37 | def test_single_input_no_entropy(self): 38 | softmax_values = np.array([[[1, 0], [1, 0], [1, 0]]]) 39 | predictions, sm_value = MeanSoftmax.calculate(softmax_values) 40 | self.assertEqual(1, len(predictions.shape)) 41 | self.assertEqual(1, predictions.shape[0]) 42 | self.assertEqual(1, len(sm_value.shape)) 43 | self.assertEqual(1, sm_value.shape[0]) 44 | self.assertEqual(0, predictions[0]) 45 | self.assertAlmostEqual(1, sm_value[0], delta=0.0005) 46 | 47 | def test_two_inputs_high_pred_entropy(self): 48 | softmax_values = np.array( 49 | [ 50 | [[0.5, 0.5], [0.5, 0.5], [0.5, 0.5], [0.5, 0.5]], 51 | [[1, 0], [0, 1], [1, 0], [0, 1]], 52 | ] 53 | ) 54 | predictions, sm_value = MeanSoftmax.calculate(softmax_values) 55 | self.assertEqual(1, len(predictions.shape)) 56 | self.assertEqual(2, predictions.shape[0]) 57 | self.assertEqual(1, len(sm_value.shape)) 58 | self.assertEqual(2, sm_value.shape[0]) 59 | self.assertAlmostEqual(0.5, sm_value[0], delta=0.0005) 60 | self.assertAlmostEqual(0.5, sm_value[1], delta=0.0005) 61 | 62 | def test_as_confidence_flag(self): 63 | # Some hypothetical mean softmax values 64 | inputs = np.ones(10) 65 | 66 | # Cast to uncertainty 67 | as_uncertainty = MeanSoftmax.cast_conf_or_unc( 68 | as_confidence=False, superv_scores=inputs 69 | ) 70 | np.testing.assert_equal(as_uncertainty, inputs * (-1)) 71 | 72 | # Cast to confidence (which it already is, thus no change) 73 | as_confidence = MeanSoftmax.cast_conf_or_unc( 74 | as_confidence=True, superv_scores=inputs 75 | ) 76 | np.testing.assert_equal(as_confidence, inputs) 77 | 78 | # No casting whatsoever 79 | as_confidence_2 = MeanSoftmax.cast_conf_or_unc( 80 | as_confidence=None, superv_scores=inputs 81 | ) 82 | np.testing.assert_equal(as_confidence_2, inputs) 83 | -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/test_mutual_information.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard import ProblemType 6 | from uncertainty_wizard.quantifiers import MutualInformation, QuantifierRegistry 7 | 8 | 9 | class TestMutualInformation(TestCase): 10 | 11 | # ================= 12 | # Test Class Methods 13 | # ================= 14 | 15 | def test_string_representation(self): 16 | self.assertTrue( 17 | isinstance(QuantifierRegistry.find("mutual_information"), MutualInformation) 18 | ) 19 | self.assertTrue( 20 | isinstance(QuantifierRegistry.find("mutu_info"), MutualInformation) 21 | ) 22 | self.assertTrue(isinstance(QuantifierRegistry.find("MI"), MutualInformation)) 23 | self.assertTrue( 24 | isinstance(QuantifierRegistry.find("MutualInformation"), MutualInformation) 25 | ) 26 | 27 | def test_is_confidence(self): 28 | self.assertFalse(MutualInformation.is_confidence()) 29 | self.assertFalse(MutualInformation().is_confidence()) 30 | 31 | def test_samples_type_declaration(self): 32 | self.assertTrue(MutualInformation.takes_samples()) 33 | 34 | def test_problem_type(self): 35 | self.assertEqual(MutualInformation.problem_type(), ProblemType.CLASSIFICATION) 36 | 37 | # ================== 38 | # Test Logic 39 | # ================= 40 | 41 | def test_single_input_no_entropy(self): 42 | softmax_values = np.array([[[1, 0], [1, 0], [1, 0]]]) 43 | predictions, predictive_entropy = MutualInformation.calculate(softmax_values) 44 | self.assertEqual(1, len(predictions.shape)) 45 | self.assertEqual(1, predictions.shape[0]) 46 | self.assertEqual(1, len(predictive_entropy.shape)) 47 | self.assertEqual(1, predictive_entropy.shape[0]) 48 | self.assertEqual(0, predictions[0]) 49 | self.assertAlmostEqual(0, predictive_entropy[0], delta=0.0001) 50 | 51 | def test_two_inputs_high_pred_entropy(self): 52 | softmax_values = np.array( 53 | [ 54 | [[0.5, 0.5], [0.5, 0.5], [0.5, 0.5], [0.5, 0.5]], 55 | [[1, 0], [0, 1], [1, 0], [0, 1]], 56 | ] 57 | ) 58 | predictions, predictive_entropy = MutualInformation.calculate(softmax_values) 59 | self.assertEqual(1, len(predictions.shape)) 60 | self.assertEqual(2, predictions.shape[0]) 61 | self.assertEqual(1, len(predictive_entropy.shape)) 62 | self.assertEqual(2, predictive_entropy.shape[0]) 63 | self.assertAlmostEqual(0, predictive_entropy[0], delta=0.0001) 64 | self.assertAlmostEqual(1, predictive_entropy[1], delta=0.0001) 65 | 66 | def test_as_confidence_flag(self): 67 | # Some hypothetical entropies 68 | inputs = np.ones(10) 69 | 70 | # Cast to uncertainty (which it already is, thus no change) 71 | as_uncertainty = MutualInformation.cast_conf_or_unc( 72 | as_confidence=False, superv_scores=inputs 73 | ) 74 | np.testing.assert_equal(as_uncertainty, inputs) 75 | 76 | # Cast to confidence 77 | as_confidence = MutualInformation.cast_conf_or_unc( 78 | as_confidence=True, superv_scores=inputs 79 | ) 80 | np.testing.assert_equal(as_confidence, inputs * (-1)) 81 | 82 | # No casting whatsoever 83 | as_confidence_2 = MutualInformation.cast_conf_or_unc( 84 | as_confidence=None, superv_scores=inputs 85 | ) 86 | np.testing.assert_equal(as_confidence_2, inputs) 87 | -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/test_predictive_entropy.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard import ProblemType 6 | from uncertainty_wizard.quantifiers import PredictiveEntropy, QuantifierRegistry 7 | 8 | 9 | class TestPredictiveEntropy(TestCase): 10 | 11 | # ================= 12 | # Test Class Methods 13 | # ================= 14 | 15 | def test_string_representation(self): 16 | self.assertTrue( 17 | isinstance(QuantifierRegistry.find("predictive_entropy"), PredictiveEntropy) 18 | ) 19 | self.assertTrue( 20 | isinstance(QuantifierRegistry.find("pred_entropy"), PredictiveEntropy) 21 | ) 22 | self.assertTrue(isinstance(QuantifierRegistry.find("PE"), PredictiveEntropy)) 23 | self.assertTrue( 24 | isinstance(QuantifierRegistry.find("PredictiveEntropy"), PredictiveEntropy) 25 | ) 26 | 27 | def test_is_confidence(self): 28 | self.assertFalse(PredictiveEntropy.is_confidence()) 29 | self.assertFalse(PredictiveEntropy().is_confidence()) 30 | 31 | def test_samples_type_declaration(self): 32 | self.assertTrue(PredictiveEntropy.takes_samples()) 33 | 34 | def test_problem_type(self): 35 | self.assertEqual(PredictiveEntropy.problem_type(), ProblemType.CLASSIFICATION) 36 | 37 | # ================== 38 | # Test Logic 39 | # ================= 40 | 41 | def test_single_input_no_entropy(self): 42 | softmax_values = np.array([[[1, 0], [1, 0], [1, 0]]]) 43 | predictions, predictive_entropy = PredictiveEntropy.calculate(softmax_values) 44 | self.assertEqual(1, len(predictions.shape)) 45 | self.assertEqual(1, predictions.shape[0]) 46 | self.assertEqual(1, len(predictive_entropy.shape)) 47 | self.assertEqual(1, predictive_entropy.shape[0]) 48 | self.assertEqual(0, predictions[0]) 49 | self.assertAlmostEqual(0, predictive_entropy[0], delta=0.0001) 50 | 51 | def test_two_inputs_high_pred_entropy(self): 52 | softmax_values = np.array( 53 | [ 54 | [[0.5, 0.5], [0.5, 0.5], [0.5, 0.5], [0.5, 0.5]], 55 | [[1, 0], [0, 1], [1, 0], [0, 1]], 56 | ] 57 | ) 58 | predictions, predictive_entropy = PredictiveEntropy.calculate(softmax_values) 59 | self.assertEqual(1, len(predictions.shape)) 60 | self.assertEqual(2, predictions.shape[0]) 61 | self.assertEqual(1, len(predictive_entropy.shape)) 62 | self.assertEqual(2, predictive_entropy.shape[0]) 63 | self.assertAlmostEqual(1, predictive_entropy[0], delta=0.0001) 64 | self.assertAlmostEqual(1, predictive_entropy[1], delta=0.0001) 65 | -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/test_quantifiers_registry.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from uncertainty_wizard.quantifiers import QuantifierRegistry, VariationRatio 4 | 5 | 6 | class TestMutualInformation(TestCase): 7 | 8 | # Note that the correct registering of all default quantifiers is tested in the corresponding quantifiers tests 9 | 10 | def test_error_if_invalid_quantifier_alias(self): 11 | with self.assertRaises(ValueError): 12 | QuantifierRegistry.find("nonexistent_q_hi1ö2rn1ld") 13 | 14 | def test_error_if_alias_already_exists(self): 15 | with self.assertRaises(ValueError): 16 | QuantifierRegistry.register(VariationRatio()) 17 | -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/test_stddev.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard import ProblemType 6 | from uncertainty_wizard.quantifiers import QuantifierRegistry, StandardDeviation 7 | 8 | 9 | class TestStandardDeviation(TestCase): 10 | 11 | # ================= 12 | # Test Class Methods 13 | # ================= 14 | 15 | def test_string_representation(self): 16 | self.assertTrue( 17 | isinstance(QuantifierRegistry.find("standard_deviation"), StandardDeviation) 18 | ) 19 | self.assertTrue(isinstance(QuantifierRegistry.find("std"), StandardDeviation)) 20 | self.assertTrue( 21 | isinstance(QuantifierRegistry.find("StandardDeviation"), StandardDeviation) 22 | ) 23 | self.assertTrue( 24 | isinstance(QuantifierRegistry.find("stddev"), StandardDeviation) 25 | ) 26 | 27 | def test_is_confidence(self): 28 | self.assertFalse(StandardDeviation.is_confidence()) 29 | self.assertFalse(StandardDeviation().is_confidence()) 30 | 31 | def test_samples_type_declaration(self): 32 | self.assertTrue(StandardDeviation.takes_samples()) 33 | 34 | def test_problem_type(self): 35 | self.assertEqual(StandardDeviation.problem_type(), ProblemType.REGRESSION) 36 | 37 | # ================== 38 | # Test Logic 39 | # ================= 40 | 41 | def test_happy_path_single(self): 42 | values = np.array( 43 | [[[1.1, 0.8, 0.08, 0.02], [0.2, 0.7, 0.08, 0.02], [0.5, 0.4, 0.08, 0.02]]] 44 | ) 45 | predictions, std = StandardDeviation.calculate(values) 46 | self.assertEqual(2, len(predictions.shape)) 47 | self.assertEqual(1, predictions.shape[0]) 48 | self.assertEqual(4, predictions.shape[1]) 49 | self.assertEqual(2, len(std.shape)) 50 | self.assertEqual(4, std.shape[1]) 51 | self.assertEqual(1, std.shape[0]) 52 | self.assertAlmostEqual(1.8 / 3, predictions[0][0], delta=0.0001) 53 | self.assertAlmostEqual(1.9 / 3, predictions[0][1], delta=0.0001) 54 | self.assertAlmostEqual(0.24 / 3, predictions[0][2], delta=0.0001) 55 | self.assertAlmostEqual(0.06 / 3, predictions[0][3], delta=0.0001) 56 | self.assertAlmostEqual( 57 | np.std(np.array([1.1, 0.2, 0.5])), std[0][0], delta=0.0001 58 | ) 59 | self.assertAlmostEqual( 60 | np.std(np.array([0.8, 0.7, 0.4])), std[0][1], delta=0.0001 61 | ) 62 | self.assertAlmostEqual(0, std[0][2], delta=0.0001) 63 | self.assertAlmostEqual(0, std[0][3], delta=0.0001) 64 | -------------------------------------------------------------------------------- /tests_unit/quantifiers_tests/test_variation_ratio.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard import ProblemType 6 | from uncertainty_wizard.quantifiers import QuantifierRegistry, VariationRatio 7 | 8 | 9 | class TestVariationRatio(TestCase): 10 | 11 | # ================= 12 | # Test Class Methods 13 | # ================= 14 | 15 | def test_string_representation(self): 16 | self.assertTrue( 17 | isinstance(QuantifierRegistry.find("variation_ratio"), VariationRatio) 18 | ) 19 | self.assertTrue( 20 | isinstance(QuantifierRegistry.find("var_ratio"), VariationRatio) 21 | ) 22 | self.assertTrue(isinstance(QuantifierRegistry.find("VR"), VariationRatio)) 23 | self.assertTrue( 24 | isinstance(QuantifierRegistry.find("VariationRatio"), VariationRatio) 25 | ) 26 | 27 | def test_is_confidence(self): 28 | self.assertFalse(VariationRatio.is_confidence()) 29 | self.assertFalse(VariationRatio().is_confidence()) 30 | 31 | def test_samples_type_declaration(self): 32 | self.assertTrue(VariationRatio.takes_samples()) 33 | 34 | def test_problem_type(self): 35 | self.assertEqual(VariationRatio.problem_type(), ProblemType.CLASSIFICATION) 36 | 37 | # ================== 38 | # Test Logic 39 | # ================= 40 | 41 | def test_happy_path_single(self): 42 | softmax_values = np.array( 43 | [[[0.1, 0.8, 0.08, 0.02], [0.2, 0.7, 0.08, 0.02], [0.5, 0.4, 0.08, 0.02]]] 44 | ) 45 | predictions, vr = VariationRatio.calculate(softmax_values) 46 | self.assertEqual(1, len(predictions.shape)) 47 | self.assertEqual(1, predictions.shape[0]) 48 | self.assertEqual(1, len(vr.shape)) 49 | self.assertEqual(1, vr.shape[0]) 50 | self.assertEqual(1, predictions[0]) 51 | self.assertAlmostEqual(0.33333, vr[0], delta=0.0001) 52 | 53 | def test_happy_path_batch(self): 54 | softmax_values = np.array( 55 | [ 56 | [ 57 | [0.1, 0.8, 0.08, 0.02], 58 | [0.2, 0.7, 0.08, 0.02], 59 | [0.5, 0.4, 0.08, 0.02], 60 | ], 61 | [ 62 | [0.1, 0.08, 0.8, 0.02], 63 | [0.02, 0.16, 0.8, 0.02], 64 | [0.01, 0.17, 0.8, 0.02], 65 | ], 66 | ] 67 | ) 68 | predictions, vr = VariationRatio.calculate(softmax_values) 69 | self.assertEqual(1, len(predictions.shape)) 70 | self.assertEqual(2, predictions.shape[0]) 71 | self.assertEqual(1, len(vr.shape)) 72 | self.assertEqual(2, vr.shape[0]) 73 | self.assertEqual(1, predictions[0]) 74 | self.assertAlmostEqual(0.33333, vr[0], delta=0.0001) 75 | self.assertEqual(2, predictions[1]) 76 | self.assertAlmostEqual(0, vr[1], delta=0.0001) 77 | -------------------------------------------------------------------------------- /uncertainty_wizard/__init__.py: -------------------------------------------------------------------------------- 1 | from . import models, quantifiers 2 | from .quantifiers.quantifier import ProblemType 3 | -------------------------------------------------------------------------------- /uncertainty_wizard/internal_utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ Utils """ 2 | 3 | __all__ = ["UncertaintyWizardWarning", "tf_version_resolver"] 4 | 5 | from . import tf_version_resolver 6 | from ._uwiz_warning import UncertaintyWizardWarning 7 | -------------------------------------------------------------------------------- /uncertainty_wizard/internal_utils/_uwiz_warning.py: -------------------------------------------------------------------------------- 1 | class UncertaintyWizardWarning(Warning): 2 | """ 3 | Class for warnings generated by user uncertainty wizard. 4 | """ 5 | -------------------------------------------------------------------------------- /uncertainty_wizard/internal_utils/tf_version_resolver.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from typing import Union 3 | 4 | import tensorflow as tf 5 | 6 | 7 | def _compare_expected_to_current_tf_version(expected_version) -> Union[None, int]: 8 | """ 9 | Compares the 'x.y.z' version parts of a passed expected version and the actual tensorflow version. 10 | The result is negative if the expected version is newer, positive if the expected version is older 11 | and 0 if they are the same. 12 | If one of the versions cannot be parsed, a warning is triggered and 'None' is returned. 13 | :param expected_version: 14 | :return: an int if a comparison was made and None if parsing was impossible 15 | """ 16 | actual_version = tf.version.VERSION 17 | 18 | # replaces rc, dev and b with dots to make the version strings comparable 19 | dotted_actual_version = actual_version.replace("rc", ".-1.") 20 | dotted_expected_version = expected_version.replace("rc", ".-1.") 21 | 22 | # Inspection disabling reason: We really want to catch all exceptions. 23 | # noinspection PyBroadException 24 | try: 25 | expected_v_splits = [int(v) for v in dotted_expected_version.split(".")] 26 | actual_v_splits = [int(v) for v in dotted_actual_version.split(".")] 27 | except Exception: 28 | warnings.warn( 29 | f"One of the version strings '{expected_version}' (requested) " 30 | f"or '{actual_version}' was not parsed: " 31 | f"We are trying to use a suitable guess about your tf compatibility and thus," 32 | f"you may not actually note any problems." 33 | f"However, to be safe, please report this issue (with this warning) " 34 | f"to the uncertainty wizard maintainers.", 35 | RuntimeWarning, 36 | ) 37 | return None 38 | 39 | if len(expected_v_splits) > len(actual_v_splits): 40 | actual_v_splits += [1000] * (len(expected_v_splits) - len(actual_v_splits)) 41 | elif len(expected_v_splits) < len(actual_v_splits): 42 | expected_v_splits += [1000] * (len(actual_v_splits) - len(expected_v_splits)) 43 | 44 | for act, expected in zip(actual_v_splits, expected_v_splits): 45 | if expected > act: 46 | return 1 47 | elif expected < act: 48 | return -1 49 | # Version equality 50 | return 0 51 | 52 | 53 | def current_tf_version_is_older_than(version: str, fallback: Union[bool, None] = True): 54 | """ 55 | A method to check whether the loaded tensorflow version is older than a passed version. 56 | :param version: A tensorflow version string, e.g. '2.3.0' 57 | :param fallback: If a problem occurs during parsing, the value of fallback will be returned 58 | :return: True if the used tensorflow version is older than the version specified in the passed string 59 | """ 60 | comp = _compare_expected_to_current_tf_version(version) 61 | if comp is None: 62 | return fallback 63 | elif comp > 0: 64 | return True 65 | else: 66 | return False 67 | 68 | 69 | def current_tf_version_is_newer_than(version: str, fallback: Union[bool, None] = False): 70 | """ 71 | A method to check whether the loaded tensorflow version is younger than a passed version. 72 | :param version: A tensorflow version string, e.g. '2.3.0' 73 | :param fallback: If a problem occurs during parsing, the value of fallback will be returned 74 | :return: True if the used tensorflow version is newer than the version specified in the passed string 75 | """ 76 | comp = _compare_expected_to_current_tf_version(version) 77 | if comp is None: 78 | return fallback 79 | elif comp >= 0: 80 | return False 81 | else: 82 | return True 83 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Uncertainty wizard models and corresponding utilities 3 | """ 4 | 5 | __all__ = [ 6 | # Models 7 | "Stochastic", 8 | "StochasticSequential", 9 | "StochasticFunctional", 10 | "LazyEnsemble", 11 | # Helper Objects 12 | "StochasticMode", 13 | # Model factories 14 | "stochastic_from_keras", 15 | "load_model", 16 | ] 17 | from ._load_model import load_model 18 | from ._stochastic._abstract_stochastic import Stochastic 19 | from ._stochastic._from_keras import stochastic_from_keras 20 | from ._stochastic._functional_stochastic import StochasticFunctional 21 | from ._stochastic._sequential_stochastic import StochasticSequential 22 | from ._stochastic._stochastic_mode import StochasticMode 23 | from .ensemble_utils._lazy_ensemble import LazyEnsemble 24 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_load_model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import warnings 5 | 6 | import tensorflow as tf 7 | 8 | import uncertainty_wizard as uwiz 9 | from uncertainty_wizard.internal_utils import ( 10 | UncertaintyWizardWarning, 11 | tf_version_resolver, 12 | ) 13 | 14 | from ._stochastic._functional_stochastic import StochasticFunctional 15 | from ._stochastic._sequential_stochastic import StochasticSequential 16 | 17 | 18 | def load_model(path, custom_objects: dict = None, compile=None, options=None): 19 | """ 20 | Loads an uncertainty wizard model that was saved using model.save(...). 21 | See the documentation of `tf.keras.models.load_model` for further information about the method params. 22 | 23 | For lazy ensembles: As they are lazy, only the folder path and the number of models are interpreted 24 | by this model loading - no keras models are actually loaded yet. 25 | Thus, custom_objects, compile and options must not be specified. 26 | 27 | :param path: The path of the folder where the ensemble was saved. 28 | :param custom_objects: Dict containing methods for custom deserialization of objects. 29 | :param compile: Whether to compile the models. 30 | :param options: Load options, check tf.keras documentation for precise information. 31 | 32 | :return: An uwiz model. 33 | """ 34 | # Note: a path without file extension is not necessarily an ensemble in a folder 35 | # It could be a user who stored a stochastic model with default (non specified) file ending 36 | # Thus, we have to check if the folder exists and contains an ensemble config file 37 | ensemble_config_path = uwiz.models.ensemble_utils._lazy_ensemble.config_file_path( 38 | path 39 | ) 40 | if ( 41 | os.path.isdir(path) 42 | and os.path.exists(path) 43 | and os.path.exists(ensemble_config_path) 44 | ): 45 | return _load_ensemble( 46 | path=path, custom_objects=custom_objects, compile=compile, options=options 47 | ) 48 | else: 49 | return _load_stochastic( 50 | path=path, custom_objects=custom_objects, compile=compile, options=options 51 | ) 52 | 53 | 54 | def _load_stochastic(path, custom_objects: dict = None, compile=None, options=None): 55 | """Attempts to load the model at the provided path as a stochastic model""" 56 | 57 | # Note: We currently intentionally don't define stochastic layers as custom_objects 58 | # as they have no methods other than call that we rely on, and thus the (robust and easy to maintain) 59 | # tf deserialization is sufficient 60 | if tf_version_resolver.current_tf_version_is_older_than("2.3.0", fallback=True): 61 | if options is not None: 62 | raise ValueError( 63 | "Load-Options are not supported by tensorflow<2.3.0." 64 | "Please do not specify any options when you call 'uwiz.models.load_model'" 65 | "or upgrade to a tensorflow version >= 2.3.0" 66 | ) 67 | inner = tf.keras.models.load_model( 68 | path, custom_objects=custom_objects, compile=compile 69 | ) 70 | else: 71 | inner = tf.keras.models.load_model( 72 | path, custom_objects=custom_objects, compile=compile, options=options 73 | ) 74 | assert hasattr( 75 | inner, "_stochastic_mode_tensor" 76 | ), "Looks like the model which is being deserialized is not an uwiz stochastic model" 77 | 78 | if isinstance(inner, tf.keras.models.Sequential): 79 | return StochasticSequential._wrap(inner) 80 | else: 81 | return StochasticFunctional._wrap(inner) 82 | 83 | 84 | def _load_ensemble(path, custom_objects: dict = None, compile=None, options=None): 85 | """Creates a lazy ensemble with the provided path as root dir. No models are acutally loaded yet (as in 'lazy').""" 86 | 87 | if compile is not None or options is not None or custom_objects is not None: 88 | warnings.warn( 89 | "Parameters compile, custom_objects and options are still ignored in lazy ensembles." 90 | "Support may be added in the future.", 91 | UncertaintyWizardWarning, 92 | ) 93 | 94 | with open( 95 | uwiz.models.ensemble_utils._lazy_ensemble.config_file_path(path=path), "r" 96 | ) as f: 97 | config = json.load(f) 98 | 99 | num_models = config["num_models"] 100 | ensemble = uwiz.models.LazyEnsemble( 101 | num_models=num_models, 102 | model_save_path=path, 103 | expect_model=True, 104 | delete_existing=False, 105 | ) 106 | logging.info( 107 | "Loaded ensemble. You may want to override the default_num_processes 'model.default_num_processes'" 108 | ) 109 | return ensemble 110 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_stochastic/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_stochastic/_from_keras.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | import tensorflow as tf 4 | 5 | from uncertainty_wizard.models._stochastic._stochastic_mode import StochasticMode 6 | 7 | from ._abstract_stochastic import Stochastic 8 | from ._functional_stochastic import StochasticFunctional 9 | from ._sequential_stochastic import StochasticSequential 10 | 11 | 12 | def stochastic_from_keras( 13 | model: tf.keras.models.Model, 14 | input_tensors=None, 15 | clone_function=None, 16 | expect_determinism=False, 17 | temp_weights_path="tmp/weights", 18 | ): 19 | """ 20 | Creates a stochastic instance from a given `tf.keras.models.Sequential` model: 21 | The new model will have the same structure (layers) and weights as the passed model. 22 | 23 | All stochastic layers (e.g. tf.keras.layers.Dropout) will be used for randomization during randomized predictions. 24 | If no stochastic layers are present, a ValueError is thrown. 25 | The raising of the error can be suppressed by setting `expect_determinism` to true. 26 | 27 | If your model contains custom layers, you can pass a function to `clone_function` to clone your custom layers, 28 | or place the annotation `@tf.keras.utils.register_keras_serializable()` on your custom layers, 29 | and make sure the `get_config` and `from_config` methods are implemented. 30 | (uncertainty wizard will serialize and deserialize all layers). 31 | 32 | :param model: The model to copy. Remains unchanged. 33 | :param input_tensors: Optional tensors to use as input_tensors for new model. See the corresponding parameter in `tf.keras.models.clone_model` for details. 34 | :param _clone_function: Optional function to use to clone layers. Will be applied to all layers except input layers and stochastic layers. See the corresponding parameter in `tf.keras.models.clone_model` for more details. 35 | :param expect_determinism: If True, deterministic models (e.g. models without stochastic layers) are accepted and no ValueError is thrown. 36 | :param temp_weights_path: The model weights are temporarily saved to the disk at this path. Folder is deleted after successful completion. 37 | :return: A newly created stochastic model 38 | """ 39 | # _clone_function is some layer cloning behavior that can be specified by the user 40 | # If none is specified, we use keras default (see `tf.keras.models.clone_model`) 41 | if clone_function is None: 42 | 43 | def _clone_function(layer): 44 | return layer.__class__.from_config(layer.get_config()) 45 | 46 | # We wrap the users (or default) clone function in a clone function 47 | # that replaces stochastic layers with uncertainty wizard stochastic layers 48 | stochastic_mode = StochasticMode() 49 | is_stochastic_layer = [] 50 | 51 | def _uncertainty_wizard_aware_clone_function(layer): 52 | new_layer = Stochastic._replace_layer_if_possible( 53 | layer, stochastic_mode=stochastic_mode 54 | ) 55 | if new_layer == layer: 56 | # Layer was not mapped to an uncertainty wizard layer, thus the default clone function is applied 57 | new_layer = _clone_function(layer) 58 | is_stochastic_layer.append(False) 59 | else: 60 | is_stochastic_layer.append(True) 61 | return new_layer 62 | 63 | # Clone the keras model to become the new inner model 64 | new_inner = tf.keras.models.clone_model( 65 | model=model, 66 | input_tensors=input_tensors, 67 | clone_function=_uncertainty_wizard_aware_clone_function, 68 | ) 69 | new_inner.stochastic_mode_tensor = stochastic_mode.as_tensor() 70 | 71 | if not expect_determinism and not any(is_stochastic_layer): 72 | raise ValueError( 73 | "The passed model had no stochastic layers." 74 | "If that is intended (and you do not plan to use any sampling based quantifiers)" 75 | "you can set the flag `expect_determinism = True`, i.e., " 76 | "calling `SequentialStochastic.clone_from_keras(keras_model,expect_determinism = True)`" 77 | ) 78 | 79 | # Restore the Weights 80 | model.save_weights(temp_weights_path) 81 | new_inner.load_weights(temp_weights_path) 82 | # Remove temporarily stored weights 83 | shutil.rmtree(temp_weights_path, ignore_errors=True) 84 | 85 | # Put the wrapper around the new model 86 | if isinstance(model, tf.keras.models.Sequential): 87 | target_class = StochasticSequential 88 | else: 89 | target_class = StochasticFunctional 90 | 91 | # Consenting Adults: The _wrap method is intended to be used here 92 | # but declared private as it is not intended to be used by the uwiz user 93 | return target_class._wrap( 94 | inner=new_inner, stochastic_mode_tensor=stochastic_mode.as_tensor() 95 | ) 96 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_stochastic/_functional_stochastic.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from uncertainty_wizard.models._stochastic._abstract_stochastic import Stochastic 4 | from uncertainty_wizard.models._stochastic._stochastic_mode import StochasticMode 5 | 6 | 7 | class StochasticFunctional(Stochastic): 8 | """ 9 | A stochastic wrapper of a `tf.keras.Model`, allowing to build models using the functional interface. 10 | Note that when using the functional interface, you need to use `uwiz.models.stochastic.layers` 11 | or build your own Stochastic-Mode dependent stochastic layers. See the online user guide for more info. 12 | 13 | """ 14 | 15 | # Include the superclass documentation 16 | __doc__ += Stochastic.__doc__ 17 | 18 | def __init__( 19 | self, inputs, outputs, stochastic_mode: StochasticMode, name: str = None 20 | ): 21 | """ 22 | Create a new functional model, equivalent to calling tf.keras.Model(...). 23 | 24 | In addition, a stochastic mode instance has to be passed. 25 | The same instance also has to be passed to any randomized uwiz.layers instances 26 | which are part of this model. 27 | This allows to dynamically enable and disable randomness in the predictions. 28 | :param inputs: See the corresponding tf.keras.Model(...) docs 29 | :param outputs: See the corresponding tf.keras.Model(...) docs 30 | :param stochastic_mode: A stochastic mode instance 31 | :param name: See the corresponding tf.keras.Model(...) docs 32 | """ 33 | super().__init__() 34 | self._inner_model = tf.keras.Model(inputs=inputs, outputs=outputs, name=name) 35 | self._inner_model._stochastic_mode_tensor = stochastic_mode.as_tensor() 36 | 37 | # docstr-coverage:inherited 38 | @property 39 | def inner(self): 40 | return self._inner_model 41 | 42 | # docstr-coverage:inherited 43 | @property 44 | def stochastic_mode_tensor(self): 45 | return self._inner_model._stochastic_mode_tensor 46 | 47 | @classmethod 48 | def _wrap(cls, inner: tf.keras.Model, stochastic_mode_tensor=None): 49 | if stochastic_mode_tensor is None: 50 | assert inner._stochastic_mode_tensor is not None, ( 51 | "Uncertainty Wizard internal error. " 52 | "Trying to wrap a model that has no stochastic_mode_tensor, " 53 | "and no external stochastic_mode_tensor is passed to attach" 54 | ) 55 | stochastic_mode_tensor = inner._stochastic_mode_tensor 56 | stochastic_mode = StochasticMode(stochastic_mode_tensor) 57 | return StochasticFunctional( 58 | inner.inputs, inner.outputs, stochastic_mode=stochastic_mode 59 | ) 60 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_stochastic/_sequential_stochastic.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import tensorflow as tf 4 | 5 | from uncertainty_wizard.models._stochastic._abstract_stochastic import Stochastic 6 | from uncertainty_wizard.models._stochastic._stochastic_mode import StochasticMode 7 | 8 | 9 | class StochasticSequential(Stochastic): 10 | """ 11 | A stochastic wrapper of `tf.keras.models.Sequential` models, suitable for MC Dropout 12 | and similar sampling based approaches on randomized models. 13 | """ 14 | 15 | # Include the superclass documentation 16 | __doc__ += Stochastic.__doc__ 17 | 18 | # docstr-coverage:inherited 19 | def __init__(self, layers=None, name=None): 20 | super().__init__() 21 | # Create an empty sequential model with a stochastic mode 22 | self._inner_sequential = tf.keras.models.Sequential(name=name) 23 | self._inner_sequential._stochastic_mode_tensor = StochasticMode().as_tensor() 24 | # Append all the passed layers 25 | if layers is not None: 26 | for layer in layers: 27 | self.add(layer) 28 | 29 | @classmethod 30 | def _wrap(cls, inner, stochastic_mode_tensor=None): 31 | model = StochasticSequential() 32 | model._inner_sequential = inner 33 | if stochastic_mode_tensor is None: 34 | assert model._inner_sequential._stochastic_mode_tensor is not None, ( 35 | "Uncertainty Wizard internal error. " 36 | "Trying to wrap a model that has no stochastic_mode_tensor, " 37 | "and no external stochastic_mode_tensor is passed to attach" 38 | ) 39 | else: 40 | model._inner_sequential._stochastic_mode_tensor = stochastic_mode_tensor 41 | return model 42 | 43 | # docstr-coverage:inherited 44 | @property 45 | def inner(self): 46 | return self._inner_sequential 47 | 48 | # docstr-coverage:inherited 49 | @property 50 | def stochastic_mode_tensor(self): 51 | return self._inner_sequential._stochastic_mode_tensor 52 | 53 | def add(self, layer, prevent_use_for_sampling=False): 54 | """ 55 | Adds the layer to the model. See docs of `tf.keras.model.Sequential.add(layer)` for details. 56 | 57 | In addition, layers of type 58 | `tf.keras.layers.Dropout`, 59 | `tf.keras.layers.GaussianNoise` and 60 | `tf.keras.layers.GaussianDropout` 61 | are overridden by equivalent layers which allow to be enabled during inference for randomized predictions. 62 | 63 | Arguments: 64 | layer: layer instance to be added to the model. 65 | prevent_use_for_sampling: Do not use the layer for randomization during inference. Has only effect on layers of type `Dropout`, `GaussianNoise` or `GaussianDropout` 66 | """ 67 | # Add noise layer, if applicable 68 | if not prevent_use_for_sampling: 69 | layer = self._replace_layer_if_possible( 70 | layer, stochastic_mode=self._get_stochastic_mode() 71 | ) 72 | # Add normal as defined by user 73 | self.inner.add(layer) 74 | 75 | def get_config(self): 76 | """ 77 | Not supported 78 | :return: An empty config 79 | """ 80 | logging.warning( 81 | """ 82 | It looks like you are trying to serialize a StochasticSequential model. 83 | Please note that to save an StochasticSequential model, you have to call `model.save(...)` 84 | and to load it, you have to use `StochasticSequential.load_model(...)` 85 | """ 86 | ) 87 | return [] 88 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_stochastic/_stochastic_mode.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | batch_size = 128 4 | num_classes = 10 5 | 6 | 7 | class StochasticMode: 8 | """ 9 | Stochastic mode is a wrapper for a bool tensor which serves as flag during inference in an uwiz stochastic model: 10 | If the flag is True, the inference in randomized. Otherwise, randomization is disabled. 11 | 12 | When creating a StochasticFunctional model, you need to create a new StochasticMode(), 13 | use it for any of your (custom?) layers that should have a behavior in a stochastic environment 14 | than in a detererministic environment (for example your own randomization layer). 15 | """ 16 | 17 | def __init__(self, tensor=None): 18 | """ 19 | Create a new stochastic mode. If not provided, a new flag tensor will be created. 20 | :param tensor: Pass your own boolean tensorflow Variable. Use is not recommended. 21 | """ 22 | if tensor is not None: 23 | self._enabled = tensor 24 | else: 25 | self._enabled = tf.Variable( 26 | initial_value=False, trainable=False, dtype=bool 27 | ) 28 | 29 | def as_tensor(self): 30 | """ 31 | Get the tensor wrapped by this stochastic mode 32 | :return: A boolean tensor 33 | """ 34 | return self._enabled 35 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/_uwiz_model.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from typing import Iterable, Union 3 | 4 | from uncertainty_wizard.internal_utils import UncertaintyWizardWarning 5 | from uncertainty_wizard.quantifiers import Quantifier, QuantifierRegistry 6 | 7 | 8 | class _UwizModel: 9 | @staticmethod 10 | def _quantifiers_as_list(quantifier): 11 | is_single_quantifier = False 12 | if isinstance(quantifier, list): 13 | quantifiers = quantifier 14 | else: 15 | quantifiers = [quantifier] 16 | is_single_quantifier = True 17 | 18 | quantifier_objects = [] 19 | for quantifier in quantifiers: 20 | if isinstance(quantifier, str): 21 | quantifier_objects.append(QuantifierRegistry.find(quantifier)) 22 | elif isinstance(quantifier, Quantifier): 23 | quantifier_objects.append(quantifier) 24 | else: 25 | raise TypeError( 26 | "The passed quantifier is neither a quantifier instance nor a quantifier alias (str)." 27 | f"Type of the passed object {str(type(quantifier))}" 28 | ) 29 | 30 | point_prediction_quantifiers = [ 31 | q for q in quantifier_objects if q.takes_samples() is False 32 | ] 33 | samples_based_quantifiers = [ 34 | q for q in quantifier_objects if q.takes_samples() is True 35 | ] 36 | return ( 37 | quantifier_objects, 38 | point_prediction_quantifiers, 39 | samples_based_quantifiers, 40 | is_single_quantifier, 41 | ) 42 | 43 | @staticmethod 44 | def _check_quantifier_heterogenity( 45 | as_confidence: Union[None, bool], quantifiers: Iterable[Quantifier] 46 | ) -> None: 47 | if as_confidence is None: 48 | num_uncertainties = sum(q.is_confidence() is False for q in quantifiers) 49 | num_confidences = sum(q.is_confidence() is True for q in quantifiers) 50 | if num_confidences > 0 and num_uncertainties > 0: 51 | warnings.warn( 52 | """ 53 | You are predicting both confidences and uncertainties. 54 | When comparing the two, keep in mind that confidence is expected to 55 | correlate positively with the probability of a correct prediction, 56 | while uncertainty is expected to correlate negatively. 57 | 58 | You may want to use the `as_confidence` flag when calling `predict_quantified`: 59 | If set to false, it multiplies confidences by (-1). 60 | If set to true, it multiplies uncertainties by (-1). 61 | 62 | See uwiz.Quantifier.cast_conf_or_unc for more details. 63 | """, 64 | category=UncertaintyWizardWarning, 65 | stacklevel=3, 66 | ) 67 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/ensemble_utils/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "EnsembleContextManager", 3 | "DynamicGpuGrowthContextManager", 4 | "NoneContextManager", 5 | "DeviceAllocatorContextManager", 6 | "DeviceAllocatorContextManagerV2", 7 | "CpuOnlyContextManager", 8 | "SaveConfig", 9 | ] 10 | 11 | from ._lazy_contexts import ( 12 | CpuOnlyContextManager, 13 | DeviceAllocatorContextManager, 14 | DeviceAllocatorContextManagerV2, 15 | DynamicGpuGrowthContextManager, 16 | EnsembleContextManager, 17 | NoneContextManager, 18 | ) 19 | from ._save_config import SaveConfig 20 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/ensemble_utils/_callables.py: -------------------------------------------------------------------------------- 1 | import gc 2 | from dataclasses import dataclass 3 | from typing import Dict, Tuple, Union 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | 9 | @dataclass 10 | class DataLoadedPredictor: 11 | """ 12 | The default task to be executed for predictions where the input data is a numpy array. 13 | Leaves the serialization and deserialization of the array to the python multiprocessing library, 14 | and does thus not explicitly implement it here. 15 | """ 16 | 17 | x_test: np.ndarray 18 | batch_size: int 19 | steps: int = None 20 | 21 | def __call__(self, model_id: int, model: tf.keras.Model): 22 | """Simple call to keras predict, formulated as __call__ to allow for constructor params.""" 23 | return model.predict( 24 | x=self.x_test, batch_size=self.batch_size, steps=self.steps, verbose=1 25 | ) 26 | 27 | 28 | @dataclass 29 | class NumpyFitProcess: 30 | """ 31 | This is a class used as callable for the serialization and deserialization of numpy arrays 32 | which are then used in the keras fit process. 33 | """ 34 | 35 | x: Union[str, np.ndarray] = None 36 | y: Union[str, np.ndarray] = None 37 | batch_size: int = None 38 | epochs: int = 1 39 | verbose: int = 1 40 | # Callbacks not supported in this default process (as type does not guarantee picklability) 41 | # callbacks = None, 42 | validation_split: float = 0.0 43 | validation_data: Union[Tuple[str, str], Tuple[np.ndarray, np.ndarray]] = None 44 | shuffle: bool = True 45 | class_weight: Dict[int, float] = None 46 | sample_weight: np.ndarray = None 47 | initial_epoch: int = 0 48 | steps_per_epoch: int = None 49 | validation_steps: int = None 50 | validation_freq: int = 1 51 | 52 | # Max_queue_size, workers and use_multiprocessing not supported as we force input to be numpy array 53 | # max_queue_size = 10, 54 | # workers = 1, 55 | # use_multiprocessing = False 56 | 57 | def __call__( 58 | self, model_id: int, model: tf.keras.Model 59 | ) -> Tuple[tf.keras.Model, tf.keras.callbacks.History]: 60 | """Simple call to keras fit, formulated as __call__ to allow for constructor params.""" 61 | x = np.load(self.x, allow_pickle=True) if isinstance(self.x, str) else self.x 62 | y = np.load(self.y, allow_pickle=True) if isinstance(self.y, str) else self.y 63 | if self.validation_data is not None and isinstance( 64 | self.validation_data[0], str 65 | ): 66 | val_x = np.load(self.validation_data[0], allow_pickle=True) 67 | val_y = np.load(self.validation_data[1], allow_pickle=True) 68 | val_data = (val_x, val_y) 69 | else: 70 | val_data = self.validation_data 71 | history = model.fit( 72 | x=x, 73 | y=y, 74 | batch_size=self.batch_size, 75 | epochs=self.epochs, 76 | verbose=self.verbose, 77 | validation_split=self.validation_split, 78 | validation_data=val_data, 79 | shuffle=self.shuffle, 80 | class_weight=self.class_weight, 81 | sample_weight=self.sample_weight, 82 | initial_epoch=self.initial_epoch, 83 | steps_per_epoch=self.steps_per_epoch, 84 | validation_steps=self.validation_steps, 85 | validation_freq=self.validation_freq, 86 | ) 87 | del x 88 | del y 89 | if val_data: 90 | del val_data 91 | gc.collect() 92 | return model, history.history 93 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/ensemble_utils/_save_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shutil 4 | 5 | 6 | def _preprocess_path(path: str) -> str: 7 | """ 8 | Preprocesses the path, i.e., 9 | it makes sure there's no tailing '/' or '\' 10 | :param path: The path as passed by the user 11 | :return: The path of the folder on which the model will be stored 12 | """ 13 | if path.endswith("/") or path.endswith("\\"): 14 | path = path[: len(path) - 1] 15 | return path 16 | 17 | 18 | class SaveConfig: 19 | """ 20 | This is a class containing information and utility methods about the saving of the ensemble. 21 | Currently, it only contains a few static fields. This may change in the future. 22 | 23 | Instances of the SaveConfig can be used in the save_single_model and load_single_model 24 | methods of EnsembleContextManagers. However, consider SaveConfigs as read-only classes. 25 | In addition, instances should only be created internally by uncertainty wizard 26 | and not in custom user code. 27 | 28 | """ 29 | 30 | def __init__( 31 | self, ensemble_save_base_path: str, delete_existing: bool, expect_model: bool 32 | ): 33 | """ 34 | ** WARNING ** Typically, there's no need for uwiz users to call this contructor: 35 | They are created by uncertainty wizard and passed to (potentially custom made) context managers. 36 | 37 | Creates a new save config. 38 | Note that this automatically triggers the preparation of the specified save_path 39 | (e.g. the deletion of existing 40 | :param ensemble_save_base_path: Where to store the ensemble 41 | :param delete_existing: If the folder is non empty and delete_existing is True, 42 | the folder will be cleared. If this is false and the folder is non-empty, 43 | a warning will be logged. 44 | :param expect_model: If this is True, we expect that there is already an 45 | ensemble file in the folder and the warning described for 'delete_existing' 46 | will not be logged. 47 | """ 48 | self._ensemble_save_base_path = _preprocess_path(ensemble_save_base_path) 49 | self._create_or_empty_folder( 50 | path=ensemble_save_base_path, 51 | overwrite=delete_existing, 52 | expect_model=expect_model, 53 | ) 54 | 55 | def filepath(self, model_id: int): 56 | """ 57 | This methods builds the path on which a particular atomic model should be saved / found 58 | :param model_id: the id of the atomic model 59 | :return: A path, where a folder named after the model id is appended to self.ensemble_save_base_path 60 | """ 61 | return f"{self._ensemble_save_base_path}/{model_id}" 62 | 63 | @property 64 | def ensemble_save_base_path(self) -> str: 65 | """ 66 | Returns the path (as string) where this ensemble is stored. 67 | This path is always a folder and after successful creation of the ensemble, 68 | it will contain the ensemble config file and a subfolder for every atomic model. 69 | :return: The save path of this ensemble as string 70 | """ 71 | return self._ensemble_save_base_path 72 | 73 | @staticmethod 74 | def _create_or_empty_folder(path: str, overwrite: bool, expect_model=False) -> None: 75 | if os.path.exists(path=path): 76 | if overwrite: 77 | shutil.rmtree(path, ignore_errors=False, onerror=None) 78 | os.mkdir(path) 79 | elif not expect_model: 80 | logging.info( 81 | f"A folder {path} already exists. " 82 | f"We will assume it contains a lazy ensemble. " 83 | f"Specify `delete_existing=True` to empty the folder when creating the LazyEnsemble." 84 | ) 85 | else: 86 | os.makedirs(path) 87 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/stochastic_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testingautomated-usi/uncertainty-wizard/ec6600d293326b859271bc8375125cd8832768ac/uncertainty_wizard/models/stochastic_utils/__init__.py -------------------------------------------------------------------------------- /uncertainty_wizard/models/stochastic_utils/broadcaster.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | from typing import Any 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | 9 | class Broadcaster(abc.ABC): 10 | """Abstract class to inject sampling-related logic""" 11 | 12 | def __init__(self, batch_size: int, verbose, steps, sample_size, **kwargs): 13 | self.batch_size = batch_size 14 | self.verbose = verbose 15 | self.steps = steps 16 | self.sample_size = sample_size 17 | 18 | @abc.abstractmethod 19 | def broadcast_inputs(self, x, **kwargs) -> Any: 20 | """Replicates every input in x `num_samples` times. 21 | 22 | Replication should happen in place. For example, 23 | inputs [a,b,c] with sample size 3 should lead to output 24 | [a,a,a,b,b,b,c,c,c] and not [a,b,c,a,b,c,a,b,c]. 25 | 26 | The return type is arbitrary, but typically a `tf.data.Dataset`. 27 | It will be used as `inputs` to `self.predict`. 28 | """ 29 | 30 | @abc.abstractmethod 31 | def predict(self, model: tf.keras.Model, inputs: Any) -> Any: 32 | """Returns predictions for the given inputs on the passed model""" 33 | 34 | @abc.abstractmethod 35 | def reshape_outputs(self, outputs: np.ndarray, **kwargs) -> Any: 36 | """Reshape predictions to be used by sampling based quantifiers. 37 | 38 | For the default sampling-based quantifiers shipped with uwiz, such as 39 | `uwiz.quantifiers.VariationRatio`, predictions are expected to have the shape 40 | (num_inputs, num_samples, ...). 41 | The outputs of `self.predict` typically have shape (num_inputs * num_samples, ...). 42 | It's this methods responsibility to bring the inputs to the right shape. 43 | """ 44 | 45 | 46 | class DefaultBroadcaster(Broadcaster): 47 | """Implements a Default Broadcaster, supporting the most typical usecases.""" 48 | 49 | # docstr-coverage:inherited 50 | def predict(self, model: tf.keras.Model, inputs: Any) -> Any: 51 | if self.steps is None: 52 | steps = None 53 | else: 54 | steps = self.steps * self.sample_size 55 | return model.predict(inputs, verbose=self.verbose, steps=steps) 56 | 57 | # docstr-coverage:inherited 58 | def broadcast_inputs(self, x, **kwargs) -> tf.data.Dataset: 59 | if isinstance(x, tf.data.Dataset): 60 | logging.debug( 61 | "You passed a tf.data.Dataset to predict_quantified in a stochastic model" 62 | "using the default broadcaster." 63 | "tf.data.Datasets passed to this method must not be batched. We take care of the batching." 64 | "Please make sure that your dataset is not batched (we can not check that)." 65 | ) 66 | x_as_ds = x 67 | elif isinstance(x, np.ndarray): 68 | x_as_ds = tf.data.Dataset.from_tensor_slices(x) 69 | else: 70 | raise ValueError( 71 | "At the moment, uwiz stochastic models support only (unbatched)" 72 | "numpy arrays and tf.data.Datasets as inputs. " 73 | "Please transform your input in one of these forms or inject a custom broadcaster." 74 | ) 75 | 76 | # Repeat every input `sample_size` many times in-place 77 | num_samples_tensor = tf.reshape(tf.constant(self.sample_size), [1]) 78 | 79 | @tf.function 80 | @tf.autograph.experimental.do_not_convert 81 | def _expand_to_sample_size(inp): 82 | shape = tf.concat((num_samples_tensor, tf.shape(inp)), axis=0) 83 | return tf.broadcast_to(input=inp, shape=shape) 84 | 85 | inputs = x_as_ds.map(_expand_to_sample_size).unbatch() 86 | 87 | # Batch the resulting dataset 88 | return inputs.batch(batch_size=self.batch_size) 89 | 90 | # docstr-coverage:inherited 91 | def reshape_outputs(self, outputs: np.ndarray, **kwargs) -> np.ndarray: 92 | output_shape = list(outputs.shape) 93 | output_shape.insert(0, -1) 94 | output_shape[1] = self.sample_size 95 | return outputs.reshape(output_shape) 96 | -------------------------------------------------------------------------------- /uncertainty_wizard/models/stochastic_utils/layers.py: -------------------------------------------------------------------------------- 1 | """ 2 | The layers in this file are extensions of the randomized keras layers, 3 | which are modified in a way to take the stochastic mode into account. 4 | """ 5 | import warnings 6 | 7 | import tensorflow as tf 8 | 9 | from uncertainty_wizard.internal_utils import UncertaintyWizardWarning 10 | from uncertainty_wizard.models._stochastic._stochastic_mode import StochasticMode 11 | 12 | _MISSING_STOCHASTIC_MODE_ERROR = ( 13 | "No stochastic mode instance was provided when creating the randomized layer. " 14 | "A stochastic mode is required to use the randomization for predictions" 15 | ) 16 | 17 | 18 | def _has_casting_preventing_subtype( 19 | instance, expected_type, corresponding_uw_type 20 | ) -> bool: 21 | i_type = type(instance) 22 | if issubclass(i_type, expected_type) and not issubclass(expected_type, i_type): 23 | # The instance is from a (strict) subclass of the expected type 24 | if isinstance(instance, corresponding_uw_type): 25 | warnings.warn( 26 | f"Looks like you are passing an {corresponding_uw_type} layer." 27 | f"For SequentialStochastic layers, it is sufficient to pass a layer of" 28 | f"the corresponding keras layer {expected_type}." 29 | f"We trust you that you know what you did and set up the stochastic mode correctly." 30 | f"Your layer will thus not be replaced, but added to the model as you provided it.", 31 | UncertaintyWizardWarning, 32 | ) 33 | else: 34 | warnings.warn( 35 | f"Looks like you are passing an instance of a custom subtype of {expected_type}." 36 | f"We typically replace {expected_type} instances with our own custom subtype." 37 | f"We will not do this with your custom subtype instance. " 38 | f"If you want to use it for randomness during inference, " 39 | f"make sure the models stochastic mode tensor is respected in your layer.", 40 | UncertaintyWizardWarning, 41 | ) 42 | return True 43 | return False 44 | 45 | 46 | class UwizBernoulliDropout(tf.keras.layers.Dropout): 47 | """ 48 | The extension of tf.keras.layers.Dropout to be used in uncertainty wizard stochastic models 49 | """ 50 | 51 | def __init__( 52 | self, 53 | rate, 54 | stochastic_mode: StochasticMode, 55 | noise_shape=None, 56 | seed=None, 57 | **kwargs, 58 | ): 59 | """ 60 | Create a new layer instance. This is essentially the same as creating a tf.keras.layers.Dropout layer, 61 | but in addition, a stochastic mode is expected, which will allow to dynamically toggle randomness at runtime. 62 | :param rate: see the corresponding keras docs. 63 | :param stochastic_mode: A stochastic mode instance. Must be the same thats used in the functional model this layer will be added to. 64 | :param noise_shape: see the corresponding keras docs. 65 | :param seed: see the corresponding keras docs. 66 | :param kwargs: see the corresponding keras docs. 67 | """ 68 | super().__init__(rate, noise_shape, seed, **kwargs) 69 | assert stochastic_mode is not None, _MISSING_STOCHASTIC_MODE_ERROR 70 | self.stochastic_mode = stochastic_mode 71 | # We must not call super from within tf.cond 72 | self._super = super() 73 | 74 | # docstr-coverage:inherited 75 | def call(self, inputs, training=None): 76 | return tf.cond( 77 | self.stochastic_mode.as_tensor(), 78 | lambda: self._super.call(inputs=inputs, training=True), 79 | lambda: self._super.call(inputs=inputs, training=training), 80 | ) 81 | 82 | @classmethod 83 | def from_keras_layer( 84 | cls, layer: tf.keras.layers.Dropout, stochastic_mode: StochasticMode 85 | ): 86 | """ 87 | Attempts to create a new UwizBernoulliDropout instance based on the configuration (i.e. dropout rate) 88 | of a passed Dropout layer 89 | :param layer: The layer from which to read the dropout layer 90 | :param stochastic_mode: The stochastic mode which allows to toggle randomness. 91 | :return: A UwizBernoulliDropout, if casting was successful. Otherwise (i.e., if the passed layer was a casting preventing subtype of Dropout), the passed layer is returned and a warning is printed to the console. 92 | """ 93 | if _has_casting_preventing_subtype( 94 | layer, tf.keras.layers.Dropout, UwizBernoulliDropout 95 | ): 96 | return layer 97 | else: 98 | rate = layer.rate 99 | noise_shape = layer.noise_shape 100 | seed = layer.seed 101 | return UwizBernoulliDropout( 102 | rate=rate, 103 | noise_shape=noise_shape, 104 | seed=seed, 105 | stochastic_mode=stochastic_mode, 106 | ) 107 | 108 | # docstr-coverage:inherited 109 | def get_config(self): 110 | config = super(UwizBernoulliDropout, self).get_config() 111 | config["name"] = "UwBernoulliDropout" 112 | return config 113 | 114 | 115 | class UwizGaussianDropout(tf.keras.layers.GaussianDropout): 116 | """ 117 | The extension of tf.keras.layers.GaussianDropout to be used in uncertainty wizard stochastic models 118 | """ 119 | 120 | def __init__(self, rate, stochastic_mode: StochasticMode, **kwargs): 121 | """ 122 | Create a new layer instance. This is essentially the same as creating a tf.keras.layers.Dropout layer, 123 | but in addition, a stochastic mode is expected, which will allow to dynamically toggle randomness at runtime. 124 | :param rate: see the corresponding keras docs. 125 | :param stochastic_mode: A stochastic mode instance. Must be the same which is going to be used in the functional model this layer will be added to. 126 | :param kwargs: see the corresponding keras docs. 127 | """ 128 | super().__init__(rate, **kwargs) 129 | assert stochastic_mode is not None, _MISSING_STOCHASTIC_MODE_ERROR 130 | self.stochastic_mode = stochastic_mode 131 | # We must not call super from within tf.cond 132 | self._super = super() 133 | 134 | # docstr-coverage:inherited 135 | def call(self, inputs, training=None): 136 | return tf.cond( 137 | self.stochastic_mode.as_tensor(), 138 | lambda: self._super.call(inputs=inputs, training=True), 139 | lambda: self._super.call(inputs=inputs, training=training), 140 | ) 141 | 142 | @classmethod 143 | def from_keras_layer( 144 | cls, layer: tf.keras.layers.GaussianDropout, stochastic_mode: StochasticMode 145 | ): 146 | """ 147 | Attempts to create a new UwizGaussianDropout instance based on the configuration (i.e. dropout rate) 148 | of a passed GaussianDropout layer 149 | :param layer: The layer from which to read the dropout layer 150 | :param stochastic_mode: The stochastic mode which allows to toggle randomness. 151 | :return: A UwizGaussianDropout, if casting was successful. Otherwise (i.e., if the passed layer was a casting preventing subtype of GaussianDropout), the passed layer is returned and a warning is printed to the console. 152 | """ 153 | if _has_casting_preventing_subtype( 154 | layer, tf.keras.layers.GaussianDropout, UwizGaussianDropout 155 | ): 156 | return layer 157 | else: 158 | rate = layer.rate 159 | return UwizGaussianDropout(rate=rate, stochastic_mode=stochastic_mode) 160 | 161 | # docstr-coverage:inherited 162 | def get_config(self): 163 | config = super(UwizGaussianDropout, self).get_config() 164 | config["name"] = "UwGaussianDropout" 165 | # No custom config yet. 166 | return config 167 | 168 | 169 | class UwizGaussianNoise(tf.keras.layers.GaussianNoise): 170 | """ 171 | The extension of tf.keras.layers.GaussianNoise to be used in uncertainty wizard stochastic models 172 | """ 173 | 174 | def __init__(self, stddev, stochastic_mode: StochasticMode, **kwargs): 175 | """ 176 | Create a new layer instance. This is essentially the same as creating a tf.keras.layers.Dropout layer, 177 | but in addition, a stochastic mode is expected, which will allow to dynamically toggle randomness at runtime. 178 | :param stddev: see the corresponding keras docs. 179 | :param stochastic_mode: A stochastic mode instance. Must be the same thats used in the functional model this layer will be added to. 180 | :param kwargs: see the corresponding keras docs. 181 | """ 182 | super().__init__(stddev, **kwargs) 183 | assert stochastic_mode is not None, _MISSING_STOCHASTIC_MODE_ERROR 184 | self.stochastic_mode = stochastic_mode 185 | # We must not call super from within tf.cond 186 | self._super = super() 187 | 188 | # docstr-coverage:inherited 189 | def call(self, inputs, training=None): 190 | return tf.cond( 191 | self.stochastic_mode.as_tensor(), 192 | lambda: self._super.call(inputs=inputs, training=True), 193 | lambda: self._super.call(inputs=inputs, training=training), 194 | ) 195 | 196 | @classmethod 197 | def from_keras_layer( 198 | cls, layer: tf.keras.layers.GaussianNoise, stochastic_mode: StochasticMode 199 | ): 200 | """ 201 | Attempts to create a new UwizGaussianNoise instance based on the configuration (i.e. the standard deviation) 202 | of a passed GaussianNoise layer 203 | :param layer: The layer from which to read the dropout layer 204 | :param stochastic_mode: The stochastic mode which allows to toggle randomness. 205 | :return: A UwizGaussianNoise, if casting was successful. Otherwise (i.e., if the passed layer was a casting preventing subtype of GaussianNoise), the passed layer is returned and a warning is printed to the console. 206 | """ 207 | if _has_casting_preventing_subtype( 208 | layer, tf.keras.layers.GaussianNoise, UwizGaussianNoise 209 | ): 210 | return layer 211 | else: 212 | stddev = layer.stddev 213 | return UwizGaussianNoise(stddev=stddev, stochastic_mode=stochastic_mode) 214 | 215 | # docstr-coverage:inherited 216 | def get_config(self): 217 | config = super(tf.keras.layers.GaussianNoise, self).get_config() 218 | config["name"] = "UwGaussianNoise" 219 | # No custom config yet. 220 | return config 221 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/__init__.py: -------------------------------------------------------------------------------- 1 | """This module contains all quantifiers used to infer prediction and confidence (or uncertainty) 2 | from neural network outputs. It also contains the QuantifierRegistry which allows to refer to 3 | quantifiers by alias.""" 4 | 5 | __all__ = [ 6 | "Quantifier", 7 | "MutualInformation", 8 | "PredictiveEntropy", 9 | "VariationRatio", 10 | "MeanSoftmax", 11 | "PredictionConfidenceScore", 12 | "MaxSoftmax", 13 | "SoftmaxEntropy", 14 | "StandardDeviation", 15 | "QuantifierRegistry", 16 | ] 17 | 18 | 19 | # Base class 20 | from .mean_softmax import MeanSoftmax 21 | 22 | # Sampling Based Classification Quantifiers 23 | from .mutual_information import MutualInformation 24 | 25 | # Point Predictor Classification Quantifiers 26 | from .one_shot_classifiers import MaxSoftmax, PredictionConfidenceScore, SoftmaxEntropy 27 | from .predictive_entropy import PredictiveEntropy 28 | from .quantifier import Quantifier 29 | 30 | # Registry 31 | from .quantifier_registry import QuantifierRegistry 32 | 33 | # Regression 34 | from .regression_quantifiers import StandardDeviation 35 | from .variation_ratio import VariationRatio 36 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/mean_softmax.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard.quantifiers.one_shot_classifiers import MaxSoftmax 6 | from uncertainty_wizard.quantifiers.quantifier import ConfidenceQuantifier, ProblemType 7 | 8 | 9 | class MeanSoftmax(ConfidenceQuantifier): 10 | """ 11 | A predictor & uncertainty quantifier, based on multiple samples (e.g. nn outputs) in a classification problem. 12 | 13 | Both the prediction and the uncertainty score are calculated using the average softmax values over all samples. 14 | This is sometimes also called 'ensembling', as it is often used in deep ensembles. 15 | """ 16 | 17 | # docstr-coverage:inherited 18 | @classmethod 19 | def aliases(cls) -> List[str]: 20 | return ["mean_softmax", "ensembling", "ms", "MeanSoftmax"] 21 | 22 | # docstr-coverage:inherited 23 | @classmethod 24 | def takes_samples(cls) -> bool: 25 | return True 26 | 27 | # docstr-coverage:inherited 28 | @classmethod 29 | def problem_type(cls) -> ProblemType: 30 | return ProblemType.CLASSIFICATION 31 | 32 | # docstr-coverage:inherited 33 | @classmethod 34 | def calculate(cls, nn_outputs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: 35 | 36 | # For simplicity, we let the predictions be calculated by the Variation Ratio Code 37 | # accepting a slight overhead from also calculating the actual variation ratio 38 | assert len(nn_outputs.shape) == 3, ( 39 | "nn_outputs for this quantifier must have shape " 40 | "(num_inputs, num_samples, num_classes)" 41 | ) 42 | 43 | # Take means over the samples 44 | means = np.mean(nn_outputs, axis=1) 45 | 46 | # Calculate argmax as prediction and max as confidence 47 | return MaxSoftmax.calculate(means) 48 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/mutual_information.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard.quantifiers.predictive_entropy import PredictiveEntropy, entropy 6 | from uncertainty_wizard.quantifiers.quantifier import ProblemType, UncertaintyQuantifier 7 | 8 | 9 | class MutualInformation(UncertaintyQuantifier): 10 | """ 11 | A predictor & uncertainty quantifier, based on multiple samples (e.g. nn outputs) in a classification problem 12 | 13 | The prediction is made using a plurality vote, i.e., the class with the highest value in most samples is selected. 14 | In the case of a tie, the class with the lowest index is selected. 15 | 16 | The uncertainty is quantified using the mutual information. 17 | See the docs for a precise explanation of mutual information. 18 | """ 19 | 20 | # docstr-coverage:inherited 21 | @classmethod 22 | def takes_samples(cls) -> bool: 23 | return True 24 | 25 | # docstr-coverage:inherited 26 | @classmethod 27 | def problem_type(cls) -> ProblemType: 28 | return ProblemType.CLASSIFICATION 29 | 30 | # docstr-coverage:inherited 31 | @classmethod 32 | def aliases(cls) -> List[str]: 33 | return ["mutu_info", "mutual_information", "mi", "MutualInformation"] 34 | 35 | # docstr-coverage:inherited 36 | @classmethod 37 | def calculate(cls, nn_outputs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: 38 | predictions, pred_entropy = PredictiveEntropy.calculate(nn_outputs) 39 | 40 | entropies = entropy(nn_outputs, axis=2) 41 | entropy_means = np.mean(entropies, axis=1) 42 | 43 | # The mutual information is the predictive entropy minus the mean of the entropies 44 | mutual_information = pred_entropy - entropy_means 45 | 46 | return predictions, mutual_information 47 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/one_shot_classifiers.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | 5 | import uncertainty_wizard as uwiz 6 | 7 | from .quantifier import ConfidenceQuantifier, ProblemType, UncertaintyQuantifier 8 | 9 | 10 | def _check_inputs_array(inputs, quantifier_name): 11 | """ 12 | Checks if the input array has the right shape and properties. 13 | Returns the array as well as a flag indicating whether the input was batched or a single sample 14 | """ 15 | assert inputs.ndim == 2, ( 16 | "The input to calculate {0} must be two" 17 | "dimensional (num_inputs x num_classes)".format(quantifier_name) 18 | ) 19 | 20 | # Check that all values are between 0 and 1 21 | assert np.all(inputs >= 0) and np.all(inputs <= 1), ( 22 | "{0} is built on softmax outputs, but the input array does not represent softmax outputs: " 23 | "There are entries which are not in the interval [0,1]".format(quantifier_name) 24 | ) 25 | 26 | # Check that all softmax values sum up to one 27 | assert np.all( 28 | np.isclose(np.sum(inputs, axis=1), np.ones(shape=inputs.shape[0]), 0.00001) 29 | ), ( 30 | "{0} is built on softmax outputs, but the input array does not represent softmax outputs: " 31 | "The per-sample values do not sum up to 1".format(quantifier_name) 32 | ) 33 | 34 | 35 | class PredictionConfidenceScore(ConfidenceQuantifier): 36 | """ 37 | The Prediction Confidence Score is a confidence metric in one-shot classification. 38 | Inputs/activations have to be normalized using the softmax function over all classes. 39 | The class with the highest activation is chosen as prediction, 40 | the difference between the two highest activations is used as confidence quantification. 41 | """ 42 | 43 | # docstr-coverage:inherited 44 | @classmethod 45 | def takes_samples(cls) -> bool: 46 | return False 47 | 48 | # docstr-coverage:inherited 49 | @classmethod 50 | def problem_type(cls) -> ProblemType: 51 | return ProblemType.CLASSIFICATION 52 | 53 | # docstr-coverage:inherited 54 | @classmethod 55 | def aliases(cls) -> List[str]: 56 | return ["pcs", "prediction_confidence_score", "PredictionConfidenceScore"] 57 | 58 | # docstr-coverage:inherited 59 | @classmethod 60 | def calculate(cls, nn_outputs: np.ndarray): 61 | _check_inputs_array(nn_outputs, quantifier_name="prediction_confidence_score") 62 | 63 | num_samples = nn_outputs.shape[0] 64 | calculated_predictions = np.argmax(nn_outputs, axis=1) 65 | max_values = nn_outputs[np.arange(num_samples), calculated_predictions] 66 | values_copy = nn_outputs.copy() 67 | values_copy[np.arange(num_samples), calculated_predictions] = -np.inf 68 | second_highest_values = np.max(values_copy, axis=1) 69 | 70 | pcs = max_values - second_highest_values 71 | return calculated_predictions, pcs 72 | 73 | 74 | class MaxSoftmax(ConfidenceQuantifier): 75 | """ 76 | The MaxSoftmax is a confidence metric in one-shot classification. 77 | It is the defaults in most simple use cases and sometimes also referred to 78 | as 'Vanilla Confidence Metric'. 79 | 80 | Inputs/activations have to be normalized using the softmax function over all classes. 81 | The class with the highest activation is chosen as prediction, 82 | the activation of this highest activation is used as confidence quantification. 83 | """ 84 | 85 | # docstr-coverage:inherited 86 | @classmethod 87 | def aliases(cls) -> List[str]: 88 | return ["softmax", "MaxSoftmax", "max_softmax", "sm"] 89 | 90 | # docstr-coverage:inherited 91 | @classmethod 92 | def takes_samples(cls) -> bool: 93 | return False 94 | 95 | # docstr-coverage:inherited 96 | @classmethod 97 | def problem_type(cls) -> ProblemType: 98 | return ProblemType.CLASSIFICATION 99 | 100 | # docstr-coverage:inherited 101 | @classmethod 102 | def calculate(cls, nn_outputs: np.ndarray): 103 | _check_inputs_array(nn_outputs, quantifier_name="softmax") 104 | 105 | num_samples = nn_outputs.shape[0] 106 | calculated_predictions = np.argmax(nn_outputs, axis=1) 107 | max_values = nn_outputs[np.arange(num_samples), calculated_predictions] 108 | 109 | return calculated_predictions, max_values 110 | 111 | 112 | class SoftmaxEntropy(UncertaintyQuantifier): 113 | """ 114 | The SoftmaxEntropy is a confidence metric in one-shot classification. 115 | 116 | Inputs/activations have to be normalized using the softmax function over all classes. 117 | The class with the highest activation is chosen as prediction, 118 | the entropy over all activations is used as uncertainty quantification. 119 | """ 120 | 121 | # docstr-coverage:inherited 122 | @classmethod 123 | def aliases(cls) -> List[str]: 124 | return ["softmax_entropy", "SoftmaxEntropy", "se"] 125 | 126 | # docstr-coverage:inherited 127 | @classmethod 128 | def takes_samples(cls) -> bool: 129 | return False 130 | 131 | # docstr-coverage:inherited 132 | @classmethod 133 | def problem_type(cls) -> ProblemType: 134 | return ProblemType.CLASSIFICATION 135 | 136 | # docstr-coverage:inherited 137 | @classmethod 138 | def calculate(cls, nn_outputs: np.ndarray): 139 | _check_inputs_array(nn_outputs, quantifier_name="softmax-entropy") 140 | 141 | calculated_predictions = np.argmax(nn_outputs, axis=1) 142 | entropies = uwiz.quantifiers.predictive_entropy.entropy(nn_outputs, axis=1) 143 | 144 | return calculated_predictions, entropies 145 | 146 | 147 | class DeepGini(UncertaintyQuantifier): 148 | """DeepGini - Uncertainty (1 minus sum of squared softmax outputs). 149 | 150 | 151 | See Feng. et. al., "Deepgini: prioritizing massive tests to enhance 152 | the robustness of deep neural networks" for more information. ISSTA 2020. 153 | 154 | The implementation is part of our paper: 155 | Michael Weiss and Paolo Tonella, Simple Techniques Work Surprisingly Well 156 | for Neural Network Test Prioritization and Active Learning (Replication Paper), 157 | ISSTA 2021. (forthcoming)""" 158 | 159 | # docstr-coverage:inherited 160 | @classmethod 161 | def aliases(cls) -> List[str]: 162 | return ["deep_gini", "DeepGini"] 163 | 164 | # docstr-coverage:inherited 165 | @classmethod 166 | def takes_samples(cls) -> bool: 167 | return False 168 | 169 | # docstr-coverage:inherited 170 | @classmethod 171 | def is_confidence(cls) -> bool: 172 | return False 173 | 174 | # docstr-coverage:inherited 175 | @classmethod 176 | def calculate(cls, nn_outputs: np.ndarray): 177 | predictions, _ = MaxSoftmax.calculate(nn_outputs) 178 | gini = 1 - np.sum(nn_outputs * nn_outputs, axis=1) 179 | return predictions, gini 180 | 181 | # docstr-coverage:inherited 182 | @classmethod 183 | def problem_type(cls) -> ProblemType: 184 | return ProblemType.CLASSIFICATION 185 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/predictive_entropy.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple 2 | 3 | import numpy as np 4 | 5 | from .quantifier import ProblemType, UncertaintyQuantifier 6 | from .variation_ratio import VariationRatio 7 | 8 | 9 | def entropy(data: np.ndarray, axis: int) -> np.ndarray: 10 | """ 11 | A utility method to compute the entropy. May also be used by other quantifiers which internally rely on entropy. 12 | 13 | Following standard convention, the logarithm used in the entropy calculation is on base 2. 14 | 15 | :param data: The values on which the entropy should be computed. 16 | :param axis: Entropy will be taken along this axis. 17 | :return: An array containing the entropies. It is one dimension smaller than the passed data (the specified axis was removed). 18 | """ 19 | # Remove zeros from nn_outputs (to allow to take logs) 20 | # Note that the actual increment (1e-20) does not matter, as it is multiplied by 0 below 21 | increments = np.zeros(shape=data.shape, dtype=np.float32) 22 | indexes_of_zeros = data == 0 23 | increments[indexes_of_zeros] = 1e-20 24 | nonzero_data = data + increments 25 | 26 | # These arrays can be quite large and are not used anymore - we free the space for the operations below 27 | del increments, indexes_of_zeros 28 | 29 | # Calculate and return the entropy 30 | return -np.sum(data * np.log2(nonzero_data), axis=axis) 31 | 32 | 33 | class PredictiveEntropy(UncertaintyQuantifier): 34 | """ 35 | A predictor & uncertainty quantifier, based on multiple samples (e.g. nn outputs) in a classification problem 36 | 37 | The prediction is made using a plurality vote, i.e., the class with the highest value in most samples is selected. 38 | In the case of a tie, the class with the lowest index is selected. 39 | 40 | The uncertainty is quantified using the predictive entropy; 41 | the entropy (base 2) of the per-class means of the sampled predictions. 42 | """ 43 | 44 | # docstr-coverage:inherited 45 | @classmethod 46 | def aliases(cls) -> List[str]: 47 | return ["predictive_entropy", "pred_entropy", "PE", "PredictiveEntropy"] 48 | 49 | # docstr-coverage:inherited 50 | @classmethod 51 | def takes_samples(cls) -> bool: 52 | return True 53 | 54 | # docstr-coverage:inherited 55 | @classmethod 56 | def problem_type(cls) -> ProblemType: 57 | return ProblemType.CLASSIFICATION 58 | 59 | # docstr-coverage:inherited 60 | @classmethod 61 | def calculate(cls, nn_outputs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: 62 | # For simplicity, we let the predictions be calculated by the Variation Ratio Code 63 | # accepting a slight overhead from also calculating the actual variation ratio 64 | predictions, _ = VariationRatio.calculate(nn_outputs) 65 | 66 | # Take means over the samples 67 | means = np.mean(nn_outputs, axis=1) 68 | 69 | # The predictive entropy is the entropy of the means 70 | predictive_entropy = entropy(means, axis=1) 71 | 72 | return predictions, predictive_entropy 73 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/quantifier.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import enum 3 | from typing import List, Union 4 | 5 | import numpy as np 6 | 7 | 8 | class ProblemType(enum.Enum): 9 | """ 10 | An enum used to distinguish between regression and classification problems. 11 | Might be extended later, to support also other or more specific ProblemTypes 12 | (e.g. one-class classification) 13 | """ 14 | 15 | REGRESSION = enum.auto() 16 | CLASSIFICATION = enum.auto() 17 | 18 | 19 | class Quantifier(abc.ABC): 20 | """ 21 | Quantifiers are dependencies, injectable into prediction calls, 22 | which calculate predictions and uncertainties or confidences 23 | from DNN outputs. 24 | 25 | The quantifier class is abstract and should not be directly implemented. 26 | Instead, new quantifiers should extend uwiz.quantifiers.ConfidenceQuantifier 27 | or uwiz.quantifiers.UncertaintyQuantifier instead. 28 | """ 29 | 30 | @classmethod 31 | @abc.abstractmethod 32 | def aliases(cls) -> List[str]: 33 | """ 34 | Aliases are string identifiers of this quantifier. 35 | They are used to select quantifiers by string in predict methods (need to be registered in quantifier_registry). 36 | 37 | Additionally, the first identifier in the list is used for logging purpose. 38 | Thus, the returned list have at least length 1. 39 | 40 | :return: list of quantifier identifiers 41 | """ 42 | pass # pragma: no cover 43 | 44 | @classmethod 45 | @abc.abstractmethod 46 | def is_confidence(cls) -> bool: 47 | """ 48 | Boolean flag indicating whether this quantifier quantifies uncertainty or confidence. 49 | They are different as follows (assuming that the quantifier actually correctly captures the chance of misprediction): 50 | 51 | - In `uncertainty quantification`, the higher the quantification, the higher the chance of misprediction. 52 | - in `confidence quantification` the lower the quantification, the higher the change of misprediction. 53 | 54 | :return: True iff this is a confidence quantifier, False if this is an uncertainty quantifier 55 | 56 | """ 57 | pass # pragma: no cover 58 | 59 | @classmethod 60 | @abc.abstractmethod 61 | def takes_samples(cls) -> bool: 62 | """ 63 | A flag indicating whether this quantifier relies on monte carlo samples 64 | (in which case the method returns True) 65 | or on a single neural network output 66 | (in which case the method return False) 67 | 68 | :return: True if this quantifier expects monte carlo samples for quantification. False otherwise. 69 | """ 70 | pass # pragma: no cover 71 | 72 | @classmethod 73 | @abc.abstractmethod 74 | def problem_type(cls) -> ProblemType: 75 | """ 76 | Specifies whether this quantifier is applicable to classification or regression problems 77 | :return: One of the two enum values REGRESSION or CLASSIFICATION 78 | """ 79 | pass # pragma: no cover 80 | 81 | @classmethod 82 | @abc.abstractmethod 83 | def calculate(cls, nn_outputs: np.ndarray): 84 | """ 85 | Calculates the predictions and uncertainties. 86 | 87 | 88 | Note this this assumes *batches* of neural network outputs. 89 | When using this method for a single nn output, make sure to reshape the passed array, 90 | e.g. using `x = np.expand_dims(x, axis=0)` 91 | 92 | The method returns a tuple of 93 | 94 | - A prediction (int or float) or array of predictions 95 | - A uncertainty or confidence quantification (float) or array of uncertainties 96 | 97 | :param nn_outputs: The NN outputs to be considered when determining prediction and uncertainty quantification 98 | :return: A tuple of prediction(s) and uncertainty(-ies). 99 | """ 100 | pass # pragma: no cover 101 | 102 | @classmethod 103 | def cast_conf_or_unc( 104 | cls, as_confidence: Union[None, bool], superv_scores: np.ndarray 105 | ) -> np.ndarray: 106 | """ 107 | Utility method to convert confidence metrics into uncertainty and vice versa. 108 | Call `is_confidence()` to find out if this is a uncertainty or a confidence metric. 109 | 110 | The supervisors scores are converted as follows: 111 | 112 | - Confidences are multiplied by (-1) iff `as_confidence` is False 113 | - Uncertainties are multiplied by (-1) iff `as_confidence` is True 114 | - Otherwise, the passed supervisor scores are returned unchanged. 115 | 116 | 117 | :param as_confidence: : A boolean indicating if the scores should be converted to confidences (True) or uncertainties (False) 118 | :param superv_scores: : The scores that are to be converted, provided a conversion is needed. 119 | :return: The converted scores or the unchanged `superv_scores` (if `as_confidence` is None or no conversion is needed) 120 | 121 | """ 122 | if as_confidence is not None and cls.is_confidence() != as_confidence: 123 | return superv_scores * -1 124 | return superv_scores 125 | 126 | 127 | class ConfidenceQuantifier(Quantifier, abc.ABC): 128 | """ 129 | An abstract Quantifier subclass, serving as superclass for all confidence quantifying quantifiers: 130 | In `confidence quantification` the lower the value, the higher the chance of misprediction. 131 | """ 132 | 133 | # docstr-coverage:inherited 134 | @classmethod 135 | def is_confidence(cls) -> bool: 136 | return True 137 | 138 | 139 | class UncertaintyQuantifier(Quantifier, abc.ABC): 140 | """ 141 | An abstract Quantifier subclass, serving as superclass for all uncertainty quantifying quantifiers: 142 | In `uncertainty quantification` the lower the value, the lower the chance of misprediction. 143 | """ 144 | 145 | # docstr-coverage:inherited 146 | @classmethod 147 | def is_confidence(cls) -> bool: 148 | return False 149 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/quantifier_registry.py: -------------------------------------------------------------------------------- 1 | from .mean_softmax import MeanSoftmax 2 | from .mutual_information import MutualInformation 3 | from .one_shot_classifiers import ( 4 | DeepGini, 5 | MaxSoftmax, 6 | PredictionConfidenceScore, 7 | SoftmaxEntropy, 8 | ) 9 | from .predictive_entropy import PredictiveEntropy 10 | from .quantifier import Quantifier 11 | from .regression_quantifiers import StandardDeviation 12 | from .variation_ratio import VariationRatio 13 | 14 | 15 | class QuantifierRegistry: 16 | """ 17 | The quantifier registry keeps track of all quantifiers and their string aliases. 18 | This is primarily used to allow to pass string representations of quantifiers in predict_quantified 19 | method calls, but may also be used for other purposes where dynamic quantifier selection is desired. 20 | """ 21 | 22 | _registries = dict() 23 | 24 | @classmethod 25 | def register(cls, quantifier: Quantifier) -> None: 26 | """ 27 | Use this method to add a new quantifier to the registry. 28 | :param quantifier: The quantifier instance to be added. 29 | :return: None 30 | """ 31 | for alias in quantifier.aliases(): 32 | if alias.lower() in cls._registries: 33 | raise ValueError( 34 | f"A quantifier with alias '{alias}' is already registered." 35 | ) 36 | cls._registries[alias.lower()] = quantifier 37 | 38 | @classmethod 39 | def find(cls, alias: str) -> Quantifier: 40 | """ 41 | Find quantifiers by their id. 42 | :param alias: A string representation of the quantifier, as defined in the quantifiers aliases method 43 | :return: A quantifier instance 44 | """ 45 | record = cls._registries.get(alias.lower()) 46 | if record is None: 47 | raise ValueError( 48 | f"No quantifier with alias '{alias}' was found. Check if you made any typos." 49 | f"If you use the alias of a custom quantifier (i.e., not an uwiz default quantifier)," 50 | f"make sure to register it through `uwiz.QuantifierRegistry.register(...)`" 51 | ) 52 | return record 53 | 54 | 55 | # Register uwiz classification quantifiers 56 | QuantifierRegistry.register(MaxSoftmax()) 57 | QuantifierRegistry.register(PredictionConfidenceScore()) 58 | QuantifierRegistry.register(SoftmaxEntropy()) 59 | QuantifierRegistry.register(DeepGini()) 60 | QuantifierRegistry.register(VariationRatio()) 61 | QuantifierRegistry.register(PredictiveEntropy()) 62 | QuantifierRegistry.register(MutualInformation()) 63 | QuantifierRegistry.register(MeanSoftmax()) 64 | 65 | # Register uwiz classification quantifiers 66 | QuantifierRegistry.register(StandardDeviation()) 67 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/regression_quantifiers.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | 5 | from .quantifier import ProblemType, UncertaintyQuantifier 6 | 7 | 8 | def validate_shape(nn_outputs): 9 | """Makes sure the nn outputs is a valid input for the regression classifiers in this file.""" 10 | assert len(nn_outputs.shape) >= 2, ( 11 | "nn_outputs for Average Standard Deviation must have shape " 12 | "(number_of_inputs, num_samples, pred_dim_1, pred_dim_2, ...)" 13 | ) 14 | num_inputs = nn_outputs.shape[0] 15 | num_samples = nn_outputs.shape[1] 16 | return num_inputs, num_samples 17 | 18 | 19 | class StandardDeviation(UncertaintyQuantifier): 20 | """ 21 | Measures the standard deviation over different samples of a regression problem, i.e., an arbitrary problem, 22 | which is used as Uncertainty and the mean of the samples as prediction 23 | 24 | This implementation can handle both regression prediction consisting of a single scalar dnn output 25 | as well as larger-shaped dnn outputs. In the latter case, entropy is calculated and returned 26 | for every position in the dnn output shape. 27 | """ 28 | 29 | # docstr-coverage:inherited 30 | @classmethod 31 | def takes_samples(cls) -> bool: 32 | return True 33 | 34 | # docstr-coverage:inherited 35 | @classmethod 36 | def problem_type(cls) -> ProblemType: 37 | return ProblemType.REGRESSION 38 | 39 | # docstr-coverage:inherited 40 | @classmethod 41 | def aliases(cls) -> List[str]: 42 | return ["standard_deviation", "std_dev", "std", "stddev", "StandardDeviation"] 43 | 44 | # docstr-coverage:inherited 45 | @classmethod 46 | def calculate(cls, nn_outputs: np.ndarray): 47 | _, _ = validate_shape(nn_outputs) 48 | predictions = np.mean(nn_outputs, axis=1) 49 | uncertainties = np.std(nn_outputs, axis=1) 50 | return predictions, uncertainties 51 | -------------------------------------------------------------------------------- /uncertainty_wizard/quantifiers/variation_ratio.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple 2 | 3 | import numpy as np 4 | 5 | from uncertainty_wizard.quantifiers.quantifier import ProblemType, UncertaintyQuantifier 6 | 7 | 8 | class VariationRatio(UncertaintyQuantifier): 9 | """ 10 | A predictor & uncertainty quantifier, based on multiple samples (e.g. nn outputs) in a classification problem 11 | 12 | The prediction is made using a plurality vote, i.e., the class with the highest value in most samples is selected. 13 | In the case of a tie, the class with the lowest index is selected. 14 | 15 | The uncertainty is quantified using the variation ratio `1 - w / S`, 16 | where w is the number of samples where the overall prediction equals the prediction of the sample 17 | and S is the total number of samples. 18 | """ 19 | 20 | # docstr-coverage:inherited 21 | @classmethod 22 | def takes_samples(cls) -> bool: 23 | return True 24 | 25 | # docstr-coverage:inherited 26 | @classmethod 27 | def problem_type(cls) -> ProblemType: 28 | return ProblemType.CLASSIFICATION 29 | 30 | # docstr-coverage:inherited 31 | @classmethod 32 | def aliases(cls) -> List[str]: 33 | return ["variation_ratio", "vr", "var_ratio", "VariationRatio"] 34 | 35 | # docstr-coverage:inherited 36 | @classmethod 37 | def calculate(cls, nn_outputs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: 38 | assert len(nn_outputs.shape) == 3, ( 39 | "nn_outputs for this quantifier must have shape " 40 | "(num_inputs, num_samples, num_classes)" 41 | ) 42 | num_inputs = nn_outputs.shape[0] 43 | num_samples = nn_outputs.shape[1] 44 | num_classes = nn_outputs.shape[2] 45 | 46 | sofmax_table = np.reshape(nn_outputs, (num_inputs * num_samples, num_classes)) 47 | per_sample_argmax = np.argmax(sofmax_table, axis=1) 48 | 49 | is_max_array = np.zeros( 50 | shape=(num_inputs * num_samples, num_classes), dtype=bool 51 | ) 52 | is_max_array[np.arange(num_inputs * num_samples), per_sample_argmax] = True 53 | 54 | per_input_is_max_array = np.reshape( 55 | is_max_array, (num_inputs, num_samples, num_classes) 56 | ) 57 | 58 | sum_array_dtype = cls._sum_array_dtype(num_samples) 59 | votes_counts = np.sum(per_input_is_max_array, axis=1, dtype=sum_array_dtype) 60 | 61 | predictions = cls._prediction_array_with_appropriate_dtype( 62 | num_classes, num_inputs 63 | ) 64 | np.argmax(votes_counts, axis=1, out=predictions) 65 | max_counts = votes_counts[np.arange(num_inputs), predictions] 66 | 67 | vr = 1 - max_counts / num_samples 68 | return predictions, vr 69 | 70 | @classmethod 71 | def _sum_array_dtype(cls, num_samples): 72 | """Selects an appropriate dtype (np.uint16 or np.uint8) based on the number of samples""" 73 | # uint16 allows up to 65535 samples (which is way above reasonable for any problem) 74 | # If there are up to 255 samples per input, we can save half the memory using uint8 75 | sum_array_dtype = np.uint16 76 | if num_samples < 256: 77 | sum_array_dtype = np.uint8 78 | return sum_array_dtype 79 | 80 | @classmethod 81 | def _prediction_array_with_appropriate_dtype(cls, num_classes, num_inputs): 82 | """Creates an empty one-dimensional array with the number of inputs as length 83 | and an appropriate dtype (np.uint16 or np.uint8). 84 | This can then be used to store the predictions""" 85 | # uint16 allows up to 65535 samples (which is way above reasonable for any problem) 86 | # If there are up to 255 samples per input, we can save half the memory using uint8 87 | predictions = np.empty(shape=num_inputs, dtype=np.uint16) 88 | if num_classes < 256: 89 | predictions = np.empty(shape=num_inputs, dtype=np.uint8) 90 | return predictions 91 | -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | VERSION = "0.4.0" 2 | # RELEASE = VERSION + "-alpha2" 3 | # RELEASE = VERSION + "-beta1" 4 | # RELEASE = VERSION + "-rc1" 5 | RELEASE = VERSION 6 | --------------------------------------------------------------------------------