├── .bumpversion.cfg ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ ├── documentation.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── publish.yml │ └── test.yml ├── .gitignore ├── .readthedocs.yml ├── AUTHORS.rst ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── Makefile ├── README.rst ├── RELEASE_CHECKLIST.md ├── doc ├── Makefile ├── __init__.py ├── _static │ ├── css │ │ └── project-template.css │ └── js │ │ └── copybutton.js ├── _templates │ ├── __init__.py │ ├── class.rst │ ├── function.rst │ └── numpydoc_docstring.py ├── api.rst ├── choosing_the_right_algorithm.rst ├── choosing_the_right_algorithm_classification.rst ├── choosing_the_right_algorithm_regression.rst ├── conf.py ├── images │ ├── 1d_data.png │ ├── calibration_basic.png │ ├── classification_methods.png │ ├── comp-methods.csv │ ├── comp-methods.png │ ├── cp_cross.png │ ├── cp_prefit.png │ ├── cp_split.png │ ├── decision_tree.png │ ├── educational_visual.png │ ├── jackknife.png │ ├── jackknife_cut.jpg │ ├── jackknife_cut.png │ ├── jackknife_cv.png │ ├── jackknife_jackknife.png │ ├── jackknife_naive.png │ ├── list_methods.png │ ├── mapie_internals_classification.png │ ├── mapie_internals_regression.png │ ├── mapie_logo_nobg_cut.png │ ├── mapie_logo_white_cut.png │ ├── mapie_predictions.png │ ├── mapie_roadmap.png │ ├── mondrian.png │ ├── prefit.png │ ├── r_hat_plus.png │ ├── uncertainty_example_blog.png │ └── v1_release_notes_gitdiff.png ├── index.rst ├── index_binary_classification.rst ├── index_classification.rst ├── index_mondrian.rst ├── index_regression.rst ├── make.bat ├── notebooks_calibration.rst ├── notebooks_risk_control.rst ├── quick_start.rst ├── split_cross_conformal.rst ├── theoretical_description_binary_classification.rst ├── theoretical_description_calibration.rst ├── theoretical_description_classification.rst ├── theoretical_description_conformity_scores.rst ├── theoretical_description_metrics.rst ├── theoretical_description_mondrian.rst ├── theoretical_description_regression.rst ├── theoretical_description_risk_control.rst ├── tutorial_classification_files │ ├── tutorial_classification_12_0.png │ ├── tutorial_classification_15_0.png │ ├── tutorial_classification_20_0.png │ ├── tutorial_classification_25_0.png │ ├── tutorial_classification_28_0.png │ └── tutorial_classification_6_0.png ├── tutorial_mondrian_regression_files │ ├── tutorial_mondrian_regression_13_0.png │ ├── tutorial_mondrian_regression_15_2.png │ ├── tutorial_mondrian_regression_2_0.png │ ├── tutorial_mondrian_regression_5_0.png │ └── tutorial_mondrian_regression_8_1.png ├── tutorial_multilabel_classification_files │ ├── tutorial_multilabel_classification_10_0.png │ ├── tutorial_multilabel_classification_11_0.png │ └── tutorial_multilabel_classification_4_1.png ├── tutorial_regression_files │ ├── tutorial_regression_10_0.png │ ├── tutorial_regression_16_0.png │ ├── tutorial_regression_17_0.png │ ├── tutorial_regression_18_0.png │ ├── tutorial_regression_19_0.png │ ├── tutorial_regression_28_0.png │ ├── tutorial_regression_29_0.png │ ├── tutorial_regression_30_0.png │ ├── tutorial_regression_31_0.png │ ├── tutorial_regression_32_0.png │ ├── tutorial_regression_34_0.png │ ├── tutorial_regression_35_0.png │ ├── tutorial_regression_36_0.png │ ├── tutorial_regression_37_0.png │ ├── tutorial_regression_38_0.png │ ├── tutorial_regression_39_0.png │ ├── tutorial_regression_40_0.png │ ├── tutorial_regression_41_0.png │ ├── tutorial_regression_42_0.png │ ├── tutorial_regression_42_2.png │ ├── tutorial_regression_44_2.png │ ├── tutorial_regression_48_0.png │ ├── tutorial_regression_49_0.png │ ├── tutorial_regression_50_0.png │ ├── tutorial_regression_52_0.png │ ├── tutorial_regression_53_0.png │ ├── tutorial_regression_55_0.png │ ├── tutorial_regression_56_0.png │ ├── tutorial_regression_58_0.png │ ├── tutorial_regression_59_0.png │ ├── tutorial_regression_62_0.png │ ├── tutorial_regression_65_0.png │ ├── tutorial_regression_66_0.png │ ├── tutorial_regression_69_0.png │ ├── tutorial_regression_70_0.png │ ├── tutorial_regression_72_0.png │ ├── tutorial_regression_73_0.png │ ├── tutorial_regression_74_0.png │ └── tutorial_regression_9_0.png └── v1_release_notes.rst ├── environment.ci.yml ├── environment.dev.yml ├── environment.doc.yml ├── environment.notebooks.yml ├── examples ├── README.rst ├── calibration │ ├── 1-quickstart │ │ ├── README.rst │ │ └── plot_calibration_hypothesis_testing.py │ ├── 2-advanced-analysis │ │ ├── README.rst │ │ └── plot_asymptotic_convergence_of_p_values.py │ └── README.rst ├── classification │ ├── 1-quickstart │ │ ├── README.rst │ │ └── plot_quickstart_classification.py │ ├── 2-advanced-analysis │ │ ├── README.rst │ │ ├── plot_comp_methods_on_2d_dataset.py │ │ ├── plot_crossconformal.py │ │ └── plot_main-tutorial-binary-classification.py │ ├── 3-scientific-articles │ │ ├── README.rst │ │ └── plot_sadinle2019_example.py │ ├── 4-other-notebooks │ │ └── README.rst │ └── README.rst ├── data │ └── demand_temperature.csv ├── mondrian │ ├── 1-quickstart │ │ ├── README.rst │ │ └── plot_main-tutorial-mondrian-regression.py │ └── README.rst ├── multilabel_classification │ ├── 1-quickstart │ │ ├── README.rst │ │ └── plot_tutorial_risk_control.py │ └── README.rst └── regression │ ├── 1-quickstart │ ├── README.rst │ ├── plot_compare_conformity_scores.py │ ├── plot_heteroscedastic_1d_data.py │ ├── plot_homoscedastic_1d_data.py │ ├── plot_prefit.py │ ├── plot_toy_model.py │ └── plot_ts-tutorial.py │ ├── 2-advanced-analysis │ ├── README.rst │ ├── plot-coverage-width-based-criterion.py │ ├── plot_ResidualNormalisedScore_tutorial.py │ ├── plot_both_uncertainties.py │ ├── plot_conditional_coverage.py │ ├── plot_conformal_predictive_distribution.py │ ├── plot_coverage_validity.py │ ├── plot_cqr_symmetry_difference.py │ ├── plot_cqr_tutorial.py │ ├── plot_main-tutorial-regression.py │ ├── plot_nested-cv.py │ └── plot_timeseries_enbpi.py │ ├── 3-scientific-articles │ ├── README.rst │ ├── plot_barber2020_simulations.py │ ├── plot_kim2020_simulations.py │ └── plot_zaffran2022_comparison.py │ ├── 4-other-notebooks │ └── README.rst │ └── README.rst ├── mapie ├── __init__.py ├── _machine_precision.py ├── _version.py ├── aggregation_functions.py ├── calibration.py ├── classification.py ├── conformity_scores │ ├── __init__.py │ ├── bounds │ │ ├── __init__.py │ │ ├── absolute.py │ │ ├── gamma.py │ │ └── residuals.py │ ├── classification.py │ ├── interface.py │ ├── regression.py │ ├── sets │ │ ├── __init__.py │ │ ├── aps.py │ │ ├── lac.py │ │ ├── naive.py │ │ ├── raps.py │ │ ├── topk.py │ │ └── utils.py │ └── utils.py ├── control_risk │ ├── __init__.py │ ├── crc_rcps.py │ ├── ltt.py │ ├── p_values.py │ └── risks.py ├── estimator │ ├── __init__.py │ ├── classifier.py │ └── regressor.py ├── metrics │ ├── __init__.py │ ├── calibration.py │ ├── classification.py │ └── regression.py ├── regression │ ├── __init__.py │ ├── quantile_regression.py │ ├── regression.py │ └── time_series_regression.py ├── risk_control.py ├── subsample.py ├── tests │ ├── __init__.py │ ├── test_aggregations_functions.py │ ├── test_calibration.py │ ├── test_classification.py │ ├── test_common.py │ ├── test_conformity_scores_bounds.py │ ├── test_conformity_scores_sets.py │ ├── test_conformity_scores_utils.py │ ├── test_control_risk.py │ ├── test_metrics.py │ ├── test_quantile_regression.py │ ├── test_regression.py │ ├── test_risk_control.py │ ├── test_subsample.py │ ├── test_time_series_regression.py │ └── test_utils.py └── utils.py ├── mypy.ini ├── notebooks ├── Makefile ├── README.md ├── calibration │ └── top_label_calibration.ipynb ├── classification │ ├── Cifar10.ipynb │ ├── tutorial_risk_control_precision.ipynb │ └── tutorial_risk_control_recall.ipynb └── regression │ ├── exoplanets.ipynb │ ├── exoplanets_mass.csv │ ├── ts-changepoint.ipynb │ └── tutorial_regression.ipynb ├── pyproject.toml ├── requirements.dev.txt ├── requirements.doc.txt └── tests_v1 ├── test_functional ├── README.md ├── test_functional.py ├── test_non_regression_classification.py ├── test_non_regression_regression.py └── utils.py └── test_unit ├── README.md ├── test_conformity_scores_utils.py ├── test_regression.py └── test_utils.py /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.0.1 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:pyproject.toml] 7 | search = version = "{current_version}" 8 | replace = version = "{new_version}" 9 | 10 | [bumpversion:file:mapie/_version.py] 11 | search = __version__ = "{current_version}" 12 | replace = __version__ = "{new_version}" 13 | 14 | [bumpversion:file:doc/conf.py] 15 | search = version = "{current_version}" 16 | replace = version = "{new_version}" 17 | 18 | [bumpversion:file:CITATION.cff] 19 | search = version: {current_version} 20 | replace = version: {new_version} 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "" 5 | labels: bug 6 | assignees: "" 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to "..." 16 | 2. Click on "..." 17 | 3. Scroll down to "..." 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - MAPIE Version [e.g. 0.3.2] 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation 3 | about: Add or improve MAPIE documentation 4 | title: "" 5 | labels: documentation 6 | assignees: "" 7 | 8 | --- 9 | 10 | **Is your documentation request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I don't understand why [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to be documented. 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "" 5 | labels: "enhancement" 6 | assignees: "" 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 4 | 5 | Fixes #(issue) 6 | 7 | ## Type of change 8 | 9 | Please remove options that are irrelevant. 10 | 11 | - Bug fix (non-breaking change which fixes an issue) 12 | - New feature (non-breaking change which adds functionality) 13 | - Breaking change (fix or feature that would cause existing functionality to not work as expected) 14 | - This change requires a documentation update 15 | 16 | # How Has This Been Tested? 17 | 18 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration 19 | 20 | - [ ] Test A 21 | - [ ] Test B 22 | 23 | # Checklist 24 | 25 | - [ ] I have read the [contributing guidelines](https://github.com/scikit-learn-contrib/MAPIE/blob/master/CONTRIBUTING.rst) 26 | - [ ] I have updated the [HISTORY.rst](https://github.com/scikit-learn-contrib/MAPIE/blob/master/HISTORY.rst) and [AUTHORS.rst](https://github.com/scikit-learn-contrib/MAPIE/blob/master/AUTHORS.rst) files 27 | - [ ] Linting passes successfully: `make lint` 28 | - [ ] Typing passes successfully: `make type-check` 29 | - [ ] Unit tests pass successfully: `make tests` 30 | - [ ] Coverage is 100%: `make coverage` 31 | - [ ] When updating documentation: doc builds successfully and without warnings: `make doc` 32 | - [ ] When updating documentation: code examples in doc run successfully: `make doctest` -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPi 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set up Python 13 | uses: actions/setup-python@v2 14 | with: 15 | python-version: "3.10" 16 | - name: Install build dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install setuptools wheel twine build 20 | - name: Build package 21 | run: python -m build 22 | - name: Publish package 23 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 24 | with: 25 | user: __token__ 26 | password: ${{ secrets.PYPI_API_TOKEN_VBL }} 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | build: 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | matrix: 14 | include: 15 | # minimum supported dependencies 16 | - os: ubuntu-latest 17 | python-version: "3.9" 18 | numpy-version: "=1.23" 19 | sklearn-version: "=1.4" 20 | # latest stable dependencies 21 | - os: ubuntu-latest 22 | python-version: "3.11" 23 | numpy-version: "" 24 | sklearn-version: "" 25 | - os: windows-latest 26 | python-version: "3.11" 27 | numpy-version: "" 28 | sklearn-version: "" 29 | - os: macos-latest 30 | python-version: "3.11" 31 | numpy-version: "" 32 | sklearn-version: "" 33 | defaults: 34 | run: 35 | shell: bash -l {0} 36 | 37 | steps: 38 | - name: Git clone 39 | uses: actions/checkout@v2 40 | - name: Set up virtual environment 41 | uses: conda-incubator/setup-miniconda@v2 42 | with: 43 | python-version: ${{ matrix.python-version }} 44 | environment-file: environment.ci.yml 45 | channels: defaults, conda-forge 46 | miniconda-version: "latest" 47 | - name: Install numpy 48 | run: conda install numpy${{ matrix.numpy-version }} 49 | - name: Install sklearn 50 | run: conda install scikit-learn${{ matrix.sklearn-version }} 51 | - name: Check linting 52 | run: make lint 53 | - name: Check static typing 54 | run: make type-check 55 | - name: Test and coverage with pytest 56 | run: make coverage 57 | - name: Code coverage 58 | run: codecov 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .DS_Store 6 | .mypy* 7 | 8 | # C extensions 9 | *.so 10 | 11 | # documentation specific 12 | doc/_build/ 13 | doc/examples_classification/ 14 | doc/examples_regression/ 15 | doc/examples_calibration/ 16 | doc/examples_multilabel_classification/ 17 | doc/examples_mondrian/ 18 | doc/auto_examples/ 19 | doc/modules/generated/ 20 | doc/datasets/generated/ 21 | doc/generated/ 22 | 23 | # Distribution / packaging 24 | 25 | .Python 26 | env/ 27 | build/ 28 | develop-eggs/ 29 | dist/ 30 | downloads/ 31 | eggs/ 32 | .eggs/ 33 | lib/ 34 | lib64/ 35 | parts/ 36 | sdist/ 37 | var/ 38 | *.egg-info/ 39 | .installed.cfg 40 | *.egg 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Tests / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .coverage 56 | .coverage.* 57 | .cache 58 | nosetests.xml 59 | coverage.xml 60 | *,cover 61 | .hypothesis/ 62 | 63 | # Translations 64 | *.mo 65 | *.pot 66 | 67 | # Django stuff: 68 | *.log 69 | 70 | # Sphinx documentation 71 | doc/_build/ 72 | doc/generated/ 73 | 74 | # Notebooks checkpoints 75 | **.ipynb_checkpoints 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # VSCode 81 | .vscode 82 | .idea/ 83 | 84 | # Images 85 | *.jpeg 86 | 87 | # ZIP files 88 | *.zip 89 | 90 | # Pyenv 91 | .python-version -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "mambaforge-22.9" 7 | jobs: 8 | post_build: 9 | - cd doc && make doctest 10 | 11 | python: 12 | install: 13 | - method: pip 14 | path: . 15 | 16 | conda: 17 | environment: environment.doc.yml 18 | 19 | sphinx: 20 | builder: html 21 | configuration: doc/conf.py 22 | fail_on_warning: false 23 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Thibault Cordier 9 | * Vincent Blot 10 | * Louis Lacombe 11 | * Valentin Laurent 12 | * Hussein Jawad 13 | 14 | Emeritus Core Developers 15 | ------------------------ 16 | 17 | * Grégoire Martinon 18 | * Vianney Taquet 19 | 20 | Contributors 21 | ------------ 22 | 23 | * Nicolas Brunel 24 | * Issam Ibnouhsein 25 | * François Deheeger 26 | * Rémi Adon 27 | * Alizé Papp 28 | * Abdou Akim Goumbala 29 | * Adirtha Borgohain 30 | * Thomas Morzadec 31 | * Julien Roussel 32 | * Arnaud Capitaine 33 | * Tarik Tazi 34 | * Daniel Herbst 35 | * Candice Moyet 36 | * Sofiane Ziane 37 | * Remi Colliaux 38 | * Arthur Phan 39 | * Rafael Saraiva 40 | * Mehdi Elion 41 | * Sami Kaddani 42 | * Pierre de Fréminville 43 | * Ambros Marzetta 44 | * Carl McBride Ellis 45 | * Baptiste Calot 46 | * Leonardo Garma 47 | * Mohammed Jawhar 48 | * Syed Affan 49 | * Cyprien Bertran 50 | * Brijesh Thummar 51 | * Faustin Pulvéric 52 | * Chaoqi Zhang 53 | To be continued ... 54 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: "Cordier" 5 | given-names: "Thibault" 6 | orcid: "https://orcid.org/0000-0000-0000-0000" 7 | title: "MAPIE - Model Agnostic Prediction Interval Estimator" 8 | version: 1.0.1 9 | date-released: 2019-04-30 10 | url: "https://github.com/scikit-learn-contrib/MAPIE" 11 | preferred-citation: 12 | type: conference-paper 13 | title: "Flexible and Systematic Uncertainty Estimation with Conformal Prediction via the MAPIE library" 14 | authors: 15 | - family-names: "Cordier" 16 | given-names: "Thibault" 17 | - family-names: "Blot" 18 | given-names: "Vincent" 19 | - family-names: "Lacombe" 20 | given-names: "Louis" 21 | - family-names: "Morzadec" 22 | given-names: "Thomas" 23 | - family-names: "Capitaine" 24 | given-names: "Arnaud" 25 | - family-names: "Brunel" 26 | given-names: "Nicolas" 27 | collection-title: "Conformal and Probabilistic Prediction with Applications" 28 | collection-type: proceedings 29 | pages: "549--581" 30 | year: 2023 31 | organization: "PMLR" 32 | old-citation: 33 | type: article 34 | authors: 35 | - family-names: "Taquet" 36 | given-names: "Vianney" 37 | - family-names: "Blot" 38 | given-names: "Vincent" 39 | - family-names: "Morzadec" 40 | given-names: "Thomas" 41 | - family-names: "Lacombe" 42 | given-names: "Louis" 43 | - family-names: "Brunel" 44 | given-names: "Nicolas" 45 | doi: "10.48550/arXiv.2207.12274" 46 | journal: "arXiv preprint arXiv:2207.12274" 47 | title: "MAPIE: an open-source library for distribution-free uncertainty quantification" 48 | year: 2021 49 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | thibault.a.cordier@capgemini.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Contribution guidelines 3 | ======================= 4 | 5 | What to work on? 6 | ---------------- 7 | 8 | You are welcome to propose and contribute new ideas. 9 | We encourage you to `open an issue `_ so that we can align on the work to be done. 10 | It is generally a good idea to have a quick discussion before opening a pull request that is potentially out-of-scope. 11 | 12 | Fork/clone/pull 13 | --------------- 14 | 15 | The typical workflow for contributing to `mapie` is: 16 | 17 | 1. Fork the ``master`` branch from the `GitHub repository `_. 18 | 2. Clone your fork locally. 19 | 3. Commit changes. 20 | 4. Push the changes to your fork. 21 | 5. Send a pull request from your fork back to the original ``master`` branch. 22 | 23 | Local setup 24 | ----------- 25 | 26 | We encourage you to use a virtual environment, with Python `3.11`. You'll want to activate it every time you want to work on `mapie`. 27 | 28 | You can create a virtual environment via ``conda``: 29 | 30 | .. code-block:: sh 31 | 32 | $ conda env create -f environment.dev.yml 33 | $ conda activate mapie 34 | 35 | Alternatively, using ``pip``, create a virtual environment and install dependencies with the following command: 36 | 37 | .. code-block:: sh 38 | 39 | $ pip install -r requirements.dev.txt 40 | 41 | If you work on Mac, you may have to install libomp manually in order to install LightGBM: 42 | 43 | .. code-block:: sh 44 | 45 | $ brew install libomp 46 | 47 | Finally, install ``mapie`` in development mode: 48 | 49 | .. code-block:: sh 50 | 51 | $ pip install -e . 52 | 53 | 54 | Documenting your change 55 | ----------------------- 56 | 57 | If you're adding a public class or function, then you'll need to add a docstring with a doctest. We follow the `numpy docstring convention `_, so please do too. 58 | Any estimator should follow the `scikit-learn API `_, so please follow these guidelines. 59 | 60 | In order to build the documentation locally, you first need to install some dependencies: 61 | 62 | Create a dedicated virtual environment via ``conda``: 63 | 64 | .. code-block:: sh 65 | 66 | $ conda env create -f environment.doc.yml 67 | $ conda activate mapie-doc 68 | 69 | Alternatively, using ``pip``, create a different virtual environment than the one used for development, and install the dependencies: 70 | 71 | .. code-block:: sh 72 | 73 | $ pip install -r requirements.doc.txt 74 | $ pip install -e . 75 | 76 | Finally, once dependencies are installed, you can build the documentation locally by running: 77 | 78 | .. code-block:: sh 79 | 80 | $ make clean-doc 81 | $ make doc 82 | 83 | 84 | Updating changelog 85 | ------------------ 86 | 87 | You can make your contribution visible by: 88 | 89 | 1. Adding your name to the Contributors section of `AUTHORS.rst `_ 90 | 2. Adding a line describing your change into `HISTORY.rst `_ 91 | 92 | Testing 93 | ------- 94 | 95 | Linting 96 | ^^^^^^^ 97 | 98 | These tests absolutely have to pass. 99 | 100 | .. code-block:: sh 101 | 102 | $ make lint 103 | 104 | 105 | Static typing 106 | ^^^^^^^^^^^^^ 107 | 108 | These tests absolutely have to pass. 109 | 110 | .. code-block:: sh 111 | 112 | $ make type-check 113 | 114 | 115 | Unit tests 116 | ^^^^^^^^^^ 117 | 118 | These tests absolutely have to pass. 119 | 120 | .. code-block:: sh 121 | 122 | $ make tests 123 | 124 | Coverage 125 | ^^^^^^^^ 126 | 127 | The coverage should absolutely be 100%. 128 | 129 | .. code-block:: sh 130 | 131 | $ make coverage 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021, Vianney TAQUET, Grégoire MARTINON, Nicolas BRUNEL, Issam IBNOUHSEIN, François DEHEEGER and MAPIE contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of MAPIE nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ### Config ### 2 | .PHONY: tests doc build 3 | 4 | 5 | ### Checks that are run in GitHub CI ### 6 | lint: 7 | flake8 examples mapie notebooks tests_v1 --max-line-length=88 8 | 9 | type-check: 10 | mypy mapie 11 | 12 | coverage: 13 | pytest -vsx \ 14 | --cov-branch \ 15 | --cov=mapie \ 16 | --cov-report term-missing \ 17 | --pyargs mapie tests_v1 \ 18 | --cov-fail-under=100 \ 19 | --no-cov-on-fail \ 20 | --doctest-modules 21 | 22 | 23 | ### Checks that are run in ReadTheDocs CI ### 24 | doc: 25 | $(MAKE) html -C doc 26 | 27 | doctest: 28 | # Tests .. testcode:: blocks in documentation, among other things 29 | $(MAKE) doctest -C doc 30 | 31 | 32 | ### Other utilities (for local use) ### 33 | all-checks: 34 | $(MAKE) lint 35 | $(MAKE) type-check 36 | $(MAKE) coverage 37 | 38 | tests: 39 | pytest -vs --doctest-modules mapie 40 | python -m pytest -vs tests_v1 41 | 42 | clean-doc: 43 | $(MAKE) clean -C doc 44 | 45 | build: 46 | python -m build 47 | 48 | clean-build: 49 | rm -rf build dist MAPIE.egg-info 50 | 51 | clean: 52 | rm -rf .mypy_cache .pytest_cache .coverage* 53 | rm -rf **__pycache__ 54 | $(MAKE) clean-build 55 | $(MAKE) clean-doc 56 | -------------------------------------------------------------------------------- /RELEASE_CHECKLIST.md: -------------------------------------------------------------------------------- 1 | # Release checklist 2 | 3 | - [ ] Edit HISTORY.rst and AUTHORS.rst to make sure it’s up-to-date and add release date 4 | - [ ] Make sure tests run, pass and cover 100% of the package: 5 | * `make lint` 6 | * `make type-check` 7 | * `make tests` 8 | * `make coverage` 9 | - [ ] Make sure documentation builds without warnings and shows nicely: 10 | * `make doc` 11 | - [ ] Commit every change from the steps below 12 | - [ ] Update the version number with `bump2version major|minor|patch` 13 | - [ ] Build source distribution: 14 | * `make clean-build` 15 | * `make build` 16 | - [ ] Check that your package is ready for publication: `twine check dist/*` 17 | - [ ] Push the commit created by bump2version: `git push origin master` 18 | - [ ] Push the tag created by bump2version:: `git push --tags` 19 | - [ ] Upload it to TestPyPi: 20 | * you need to create an account on test.pypi.org first if you don't have one, then an API key, and ask one the existing MAPIE maintainer to add you as a maintainer 21 | * `twine upload --repository-url https://test.pypi.org/legacy/ dist/*` (use `__token__` as username and your api token as password) 22 | - [ ] Test upload on TestPyPi: 23 | * create a new empty virtual environment 24 | * `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ mapie` 25 | - [ ] Create new release on GitHub for this tag. 26 | - [ ] Merge the automatically created pull request on https://github.com/conda-forge/mapie-feedstock. You need to be added as a maintainer on this repo first. To create the pull request 27 | manually to avoid waiting for automation, create an issue with the name `@conda-forge-admin, please update version` 28 | -------------------------------------------------------------------------------- /doc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/__init__.py -------------------------------------------------------------------------------- /doc/_static/css/project-template.css: -------------------------------------------------------------------------------- 1 | @import url("theme.css"); 2 | 3 | .highlight a { 4 | text-decoration: underline; 5 | } 6 | 7 | .deprecated p { 8 | padding: 10px 7px 10px 10px; 9 | color: #b94a48; 10 | background-color: #F3E5E5; 11 | border: 1px solid #eed3d7; 12 | } 13 | 14 | .deprecated p span.versionmodified { 15 | font-weight: bold; 16 | } 17 | -------------------------------------------------------------------------------- /doc/_static/js/copybutton.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | /* Add a [>>>] button on the top-right corner of code samples to hide 3 | * the >>> and ... prompts and the output and thus make the code 4 | * copyable. */ 5 | var div = $('.highlight-python .highlight,' + 6 | '.highlight-python3 .highlight,' + 7 | '.highlight-pycon .highlight,' + 8 | '.highlight-default .highlight') 9 | var pre = div.find('pre'); 10 | 11 | // get the styles from the current theme 12 | pre.parent().parent().css('position', 'relative'); 13 | var hide_text = 'Hide the prompts and output'; 14 | var show_text = 'Show the prompts and output'; 15 | var border_width = pre.css('border-top-width'); 16 | var border_style = pre.css('border-top-style'); 17 | var border_color = pre.css('border-top-color'); 18 | var button_styles = { 19 | 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', 20 | 'border-color': border_color, 'border-style': border_style, 21 | 'border-width': border_width, 'color': border_color, 'text-size': '75%', 22 | 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', 23 | 'border-radius': '0 3px 0 0' 24 | } 25 | 26 | // create and add the button to all the code blocks that contain >>> 27 | div.each(function(index) { 28 | var jthis = $(this); 29 | if (jthis.find('.gp').length > 0) { 30 | var button = $('>>>'); 31 | button.css(button_styles) 32 | button.attr('title', hide_text); 33 | button.data('hidden', 'false'); 34 | jthis.prepend(button); 35 | } 36 | // tracebacks (.gt) contain bare text elements that need to be 37 | // wrapped in a span to work with .nextUntil() (see later) 38 | jthis.find('pre:has(.gt)').contents().filter(function() { 39 | return ((this.nodeType == 3) && (this.data.trim().length > 0)); 40 | }).wrap(''); 41 | }); 42 | 43 | // define the behavior of the button when it's clicked 44 | $('.copybutton').click(function(e){ 45 | e.preventDefault(); 46 | var button = $(this); 47 | if (button.data('hidden') === 'false') { 48 | // hide the code output 49 | button.parent().find('.go, .gp, .gt').hide(); 50 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); 51 | button.css('text-decoration', 'line-through'); 52 | button.attr('title', show_text); 53 | button.data('hidden', 'true'); 54 | } else { 55 | // show the code output 56 | button.parent().find('.go, .gp, .gt').show(); 57 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); 58 | button.css('text-decoration', 'none'); 59 | button.attr('title', hide_text); 60 | button.data('hidden', 'false'); 61 | } 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /doc/_templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/_templates/__init__.py -------------------------------------------------------------------------------- /doc/_templates/class.rst: -------------------------------------------------------------------------------- 1 | :mod:`{{module}}`.{{objname}} 2 | {{ underline }}============== 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autoclass:: {{ objname }} 7 | :members: 8 | 9 | {% block methods %} 10 | .. automethod:: __init__ 11 | {% endblock %} 12 | 13 | .. include:: {{module}}.{{objname}}.examples 14 | 15 | .. raw:: html 16 | 17 |
18 | -------------------------------------------------------------------------------- /doc/_templates/function.rst: -------------------------------------------------------------------------------- 1 | :mod:`{{module}}`.{{objname}} 2 | {{ underline }}==================== 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autofunction:: {{ objname }} 7 | 8 | .. include:: {{module}}.{{objname}}.examples 9 | 10 | .. raw:: html 11 | 12 |
13 | -------------------------------------------------------------------------------- /doc/_templates/numpydoc_docstring.py: -------------------------------------------------------------------------------- 1 | {{index}} 2 | {{summary}} 3 | {{extended_summary}} 4 | {{parameters}} 5 | {{returns}} 6 | {{yields}} 7 | {{other_parameters}} 8 | {{attributes}} 9 | {{raises}} 10 | {{warns}} 11 | {{warnings}} 12 | {{see_also}} 13 | {{notes}} 14 | {{references}} 15 | {{examples}} 16 | {{methods}} 17 | -------------------------------------------------------------------------------- /doc/api.rst: -------------------------------------------------------------------------------- 1 | ######### 2 | MAPIE API 3 | ######### 4 | 5 | Regression 6 | ============================= 7 | 8 | Conformalizers 9 | --------------- 10 | 11 | .. autosummary:: 12 | :toctree: generated/ 13 | :template: class.rst 14 | 15 | mapie.regression.SplitConformalRegressor 16 | mapie.regression.CrossConformalRegressor 17 | mapie.regression.JackknifeAfterBootstrapRegressor 18 | mapie.regression.ConformalizedQuantileRegressor 19 | mapie.regression.TimeSeriesRegressor 20 | 21 | Metrics 22 | -------- 23 | 24 | .. autosummary:: 25 | :toctree: generated/ 26 | :template: function.rst 27 | 28 | mapie.metrics.regression.regression_coverage_score 29 | mapie.metrics.regression.regression_mean_width_score 30 | mapie.metrics.regression.regression_ssc 31 | mapie.metrics.regression.regression_ssc_score 32 | mapie.metrics.regression.hsic 33 | mapie.metrics.regression.coverage_width_based 34 | mapie.metrics.regression.regression_mwi_score 35 | 36 | Conformity Scores 37 | ------------------- 38 | 39 | .. autosummary:: 40 | :toctree: generated/ 41 | :template: class.rst 42 | 43 | mapie.conformity_scores.BaseRegressionScore 44 | mapie.conformity_scores.AbsoluteConformityScore 45 | mapie.conformity_scores.GammaConformityScore 46 | mapie.conformity_scores.ResidualNormalisedScore 47 | 48 | Resampling 49 | ----------- 50 | 51 | .. autosummary:: 52 | :toctree: generated/ 53 | :template: class.rst 54 | 55 | mapie.subsample.BlockBootstrap 56 | mapie.subsample.Subsample 57 | 58 | Classification 59 | ============== 60 | 61 | Conformalizers 62 | --------------- 63 | 64 | .. autosummary:: 65 | :toctree: generated/ 66 | :template: class.rst 67 | 68 | mapie.classification.SplitConformalClassifier 69 | mapie.classification.CrossConformalClassifier 70 | 71 | Metrics 72 | -------- 73 | 74 | .. autosummary:: 75 | :toctree: generated/ 76 | :template: function.rst 77 | 78 | mapie.metrics.classification.classification_coverage_score 79 | mapie.metrics.classification.classification_mean_width_score 80 | mapie.metrics.classification.classification_ssc 81 | mapie.metrics.classification.classification_ssc_score 82 | 83 | Conformity Scores 84 | ------------------ 85 | 86 | .. autosummary:: 87 | :toctree: generated/ 88 | :template: class.rst 89 | 90 | mapie.conformity_scores.BaseClassificationScore 91 | mapie.conformity_scores.NaiveConformityScore 92 | mapie.conformity_scores.LACConformityScore 93 | mapie.conformity_scores.APSConformityScore 94 | mapie.conformity_scores.RAPSConformityScore 95 | mapie.conformity_scores.TopKConformityScore 96 | 97 | Risk Control 98 | ========================== 99 | 100 | .. autosummary:: 101 | :toctree: generated/ 102 | :template: class.rst 103 | 104 | mapie.risk_control.PrecisionRecallController 105 | 106 | Calibration 107 | =========== 108 | 109 | Conformalizer 110 | -------------- 111 | 112 | .. autosummary:: 113 | :toctree: generated/ 114 | :template: class.rst 115 | 116 | mapie.calibration.TopLabelCalibrator 117 | 118 | Metrics 119 | -------- 120 | 121 | .. autosummary:: 122 | :toctree: generated/ 123 | :template: function.rst 124 | 125 | mapie.metrics.calibration.expected_calibration_error 126 | mapie.metrics.calibration.top_label_ece 127 | mapie.metrics.calibration.cumulative_differences 128 | mapie.metrics.calibration.kolmogorov_smirnov_cdf 129 | mapie.metrics.calibration.kolmogorov_smirnov_p_value 130 | mapie.metrics.calibration.kolmogorov_smirnov_statistic 131 | mapie.metrics.calibration.kuiper_cdf 132 | mapie.metrics.calibration.kuiper_p_value 133 | mapie.metrics.calibration.kuiper_statistic 134 | mapie.metrics.calibration.length_scale 135 | mapie.metrics.calibration.spiegelhalter_p_value 136 | mapie.metrics.calibration.spiegelhalter_statistic 137 | 138 | Utils 139 | ============================== 140 | 141 | .. autosummary:: 142 | :toctree: generated/ 143 | :template: function.rst 144 | 145 | mapie.utils.train_conformalize_test_split 146 | -------------------------------------------------------------------------------- /doc/choosing_the_right_algorithm.rst: -------------------------------------------------------------------------------- 1 | ############################################ 2 | Choosing the right algorithm 3 | ############################################ 4 | 5 | Following is a simple decision tree to help you getting started quickly with MAPIE. 6 | 7 | Reality is of course a bit more complex, so feel free to browse the documentation for nuanced explanations. 8 | 9 | .. image:: images/decision_tree.png 10 | :align: center 11 | -------------------------------------------------------------------------------- /doc/choosing_the_right_algorithm_classification.rst: -------------------------------------------------------------------------------- 1 | ############################################ 2 | Choosing the right algorithm 3 | ############################################ 4 | 5 | Following is a simple decision tree to help you you getting started quickly with MAPIE. 6 | 7 | Reality is of course a bit more complex, so feel free to browse the documentation for nuanced explanations. 8 | 9 | .. image:: images/decision_tree.png 10 | :align: center 11 | -------------------------------------------------------------------------------- /doc/choosing_the_right_algorithm_regression.rst: -------------------------------------------------------------------------------- 1 | ############################################ 2 | Choose the right algorithm 3 | ############################################ 4 | 5 | Following is a simple decision tree to help you you getting started quickly with MAPIE. 6 | 7 | Reality is of course a bit more complex, so feel free to browse the documentation for nuanced explanations. 8 | 9 | .. image:: images/decision_tree.png 10 | :align: center 11 | -------------------------------------------------------------------------------- /doc/images/1d_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/1d_data.png -------------------------------------------------------------------------------- /doc/images/calibration_basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/calibration_basic.png -------------------------------------------------------------------------------- /doc/images/classification_methods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/classification_methods.png -------------------------------------------------------------------------------- /doc/images/comp-methods.csv: -------------------------------------------------------------------------------- 1 | **Method**,**Theoretical coverage**,**Typical coverage**,**Training cost**,**Evaluation cost** 2 | **Naïve**,No guarantee,:math:`< 1-\alpha`,1,:math:`n_{test}` 3 | **Split**,:math:`\geq 1-\alpha`,:math:`\simeq 1-\alpha`,1,:math:`n_{test}` 4 | **Jackknife**,No guarantee,:math:`\simeq 1-\alpha`,:math:`n`,:math:`n_{test}` 5 | ,, or :math:`< 1-\alpha` if :math:`\hat{\mu}` unstable,, 6 | **Jackknife+**,:math:`\geq 1-2\alpha`,:math:`\simeq 1-\alpha`,:math:`n`,:math:`n \times n_{test}` 7 | **Jackknife-minmax**,:math:`\geq 1-\alpha`,:math:`> 1-\alpha`,:math:`n`,:math:`n \times n_{test}` 8 | **CV**,No guarantee,:math:`\simeq 1-\alpha`,:math:`K`,:math:`n_{test}` 9 | ,, or :math:`< 1-\alpha` if :math:`\hat{\mu}` unstable,, 10 | **CV+**,:math:`\geq 1-2\alpha`,:math:`\gtrsim 1-\alpha`,:math:`K`,:math:`K \times n_{test}` 11 | **CV-minmax**,:math:`\geq 1-\alpha`,:math:`> 1-\alpha`,:math:`K`,:math:`K \times n_{test}` 12 | **Jackknife-aB+**,:math:`\geq 1-2\alpha`,:math:`\gtrsim 1-\alpha`,:math:`K`,:math:`K \times n_{test}` 13 | **Jackknife-aB-minmax**,:math:`\geq 1-\alpha`,:math:`> 1-\alpha`,:math:`K`,:math:`K \times n_{test}` 14 | **Conformalized quantile regressor**,:math:`\geq 1-\alpha`,:math:`\gtrsim 1-\alpha`,:math:`3`,:math:`3 \times n_{test}` 15 | **EnbPI**,:math:`\geq 1-\alpha` (asymptotic),:math:`\gtrsim 1-\alpha`,:math:`K`,:math:`K \times n_{test}` -------------------------------------------------------------------------------- /doc/images/comp-methods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/comp-methods.png -------------------------------------------------------------------------------- /doc/images/cp_cross.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/cp_cross.png -------------------------------------------------------------------------------- /doc/images/cp_prefit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/cp_prefit.png -------------------------------------------------------------------------------- /doc/images/cp_split.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/cp_split.png -------------------------------------------------------------------------------- /doc/images/decision_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/decision_tree.png -------------------------------------------------------------------------------- /doc/images/educational_visual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/educational_visual.png -------------------------------------------------------------------------------- /doc/images/jackknife.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/jackknife.png -------------------------------------------------------------------------------- /doc/images/jackknife_cut.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/jackknife_cut.jpg -------------------------------------------------------------------------------- /doc/images/jackknife_cut.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/jackknife_cut.png -------------------------------------------------------------------------------- /doc/images/jackknife_cv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/jackknife_cv.png -------------------------------------------------------------------------------- /doc/images/jackknife_jackknife.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/jackknife_jackknife.png -------------------------------------------------------------------------------- /doc/images/jackknife_naive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/jackknife_naive.png -------------------------------------------------------------------------------- /doc/images/list_methods.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/list_methods.png -------------------------------------------------------------------------------- /doc/images/mapie_internals_classification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mapie_internals_classification.png -------------------------------------------------------------------------------- /doc/images/mapie_internals_regression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mapie_internals_regression.png -------------------------------------------------------------------------------- /doc/images/mapie_logo_nobg_cut.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mapie_logo_nobg_cut.png -------------------------------------------------------------------------------- /doc/images/mapie_logo_white_cut.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mapie_logo_white_cut.png -------------------------------------------------------------------------------- /doc/images/mapie_predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mapie_predictions.png -------------------------------------------------------------------------------- /doc/images/mapie_roadmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mapie_roadmap.png -------------------------------------------------------------------------------- /doc/images/mondrian.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/mondrian.png -------------------------------------------------------------------------------- /doc/images/prefit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/prefit.png -------------------------------------------------------------------------------- /doc/images/r_hat_plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/r_hat_plus.png -------------------------------------------------------------------------------- /doc/images/uncertainty_example_blog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/uncertainty_example_blog.png -------------------------------------------------------------------------------- /doc/images/v1_release_notes_gitdiff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/images/v1_release_notes_gitdiff.png -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | .. toctree:: 4 | :maxdepth: 2 5 | :hidden: 6 | :caption: Getting Started 7 | 8 | quick_start 9 | split_cross_conformal 10 | choosing_the_right_algorithm 11 | v1_release_notes 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :hidden: 16 | :caption: Measure predictions uncertainty 17 | 18 | index_regression 19 | index_classification 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | :hidden: 24 | :caption: Control prediction errors 25 | 26 | theoretical_description_risk_control 27 | examples_multilabel_classification/1-quickstart/plot_tutorial_risk_control 28 | notebooks_risk_control 29 | 30 | .. toctree:: 31 | :maxdepth: 2 32 | :hidden: 33 | :caption: Calibrate multi-class predictions 34 | 35 | theoretical_description_calibration 36 | examples_calibration/index 37 | notebooks_calibration 38 | 39 | .. toctree:: 40 | :maxdepth: 2 41 | :hidden: 42 | :caption: Question & Answers 43 | 44 | Metrics: how to measure conformal prediction performance? 45 | index_mondrian 46 | 47 | .. toctree:: 48 | :maxdepth: 2 49 | :hidden: 50 | :caption: API 51 | 52 | api 53 | -------------------------------------------------------------------------------- /doc/index_binary_classification.rst: -------------------------------------------------------------------------------- 1 | The binary classification case 2 | ================================ 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | examples_classification/2-advanced-analysis/plot_main-tutorial-binary-classification 8 | theoretical_description_binary_classification -------------------------------------------------------------------------------- /doc/index_classification.rst: -------------------------------------------------------------------------------- 1 | Prediction sets (classification) 2 | ================================ 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | choosing_the_right_algorithm_classification 8 | examples_classification/1-quickstart/plot_quickstart_classification 9 | examples_classification/index 10 | theoretical_description_classification 11 | index_binary_classification -------------------------------------------------------------------------------- /doc/index_mondrian.rst: -------------------------------------------------------------------------------- 1 | Mondrian: how to use prior knowledge on groups when measuring uncertainty? 2 | ============================================================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | examples_mondrian/1-quickstart/plot_main-tutorial-mondrian-regression 8 | theoretical_description_mondrian 9 | -------------------------------------------------------------------------------- /doc/index_regression.rst: -------------------------------------------------------------------------------- 1 | Prediction intervals (regression) 2 | ================================= 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | choosing_the_right_algorithm_regression 8 | examples_regression/1-quickstart/plot_toy_model 9 | examples_regression/1-quickstart/plot_prefit 10 | examples_regression/index 11 | theoretical_description_regression 12 | theoretical_description_conformity_scores -------------------------------------------------------------------------------- /doc/notebooks_calibration.rst: -------------------------------------------------------------------------------- 1 | Calibration notebooks 2 | ===================== 3 | 4 | The following examples present advanced analyses on multi-class calibration. 5 | 6 | 7 | 1. Top-label calibration for outputs of ML models : `notebook `_ 8 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 9 | -------------------------------------------------------------------------------- /doc/notebooks_risk_control.rst: -------------------------------------------------------------------------------- 1 | Risk control notebooks 2 | ==================================== 3 | 4 | The following examples present advanced analyses on risk control problems with different methods proposed in MAPIE. 5 | 6 | 1. Overview of Recall Control for Multi-Label Classification : `recall_notebook `_ 7 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 8 | 9 | 2. Overview of Precision Control for Multi-Label Classification : `precision_notebook `_ 10 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------- /doc/quick_start.rst: -------------------------------------------------------------------------------- 1 | ###################### 2 | Quick Start with MAPIE 3 | ###################### 4 | 5 | This package allows you to easily estimate uncertainties in both regression and classification settings. 6 | In regression settings, **MAPIE** provides prediction intervals on single-output data. 7 | In classification settings, **MAPIE** provides prediction sets on multi-class data. 8 | In any case, **MAPIE** is compatible with any scikit-learn-compatible estimator. 9 | 10 | 11 | 1. Download and install the module 12 | ================================== 13 | 14 | Install via ``pip``: 15 | 16 | .. code:: python 17 | 18 | pip install mapie 19 | 20 | or via `conda`: 21 | 22 | .. code:: sh 23 | 24 | $ conda install -c conda-forge mapie 25 | 26 | To install directly from the github repository : 27 | 28 | .. code:: python 29 | 30 | pip install git+https://github.com/scikit-learn-contrib/MAPIE 31 | 32 | 33 | 2. Regression 34 | ===================== 35 | 36 | Let us start with a basic regression problem. 37 | Here, we generate one-dimensional noisy data that we fit with a MLPRegressor: `Use MAPIE to plot prediction intervals `_ 38 | 39 | 40 | 3. Classification 41 | ======================= 42 | 43 | Similarly, it's possible to do the same for a basic classification problem: `Use MAPIE to plot prediction sets `_ -------------------------------------------------------------------------------- /doc/split_cross_conformal.rst: -------------------------------------------------------------------------------- 1 | ################################################################ 2 | The conformalization (or "calibration") set 3 | ################################################################ 4 | 5 | **MAPIE** is based on two types of techniques for measuring uncertainty in regression and classification: 6 | 7 | - the split-conformal predictions, 8 | - the cross-conformal predictions. 9 | 10 | In all cases, the training/conformalization process can be broken down as follows: 11 | 12 | - Train a model using the training set (or full dataset if cross-conformal). 13 | - Estimate conformity scores using the conformalization set (or full dataset if cross-conformal). 14 | - Predict target on test data to obtain prediction intervals/sets based on these conformity scores. 15 | 16 | 17 | 1. Split conformal predictions 18 | ============================== 19 | 20 | - Compute conformity scores ("conformalization") on a conformalization set not seen by the model during training. 21 | (Use :func:`~mapie.utils.train_conformalize_test_split` to obtain the different sets.) 22 | 23 | **MAPIE** then uses the conformity scores to estimate sets associated with the desired coverage on new data with strong theoretical guarantees. 24 | 25 | Split conformal predictions with a pre-trained model 26 | ------------------------------------------------------------------------------------ 27 | 28 | .. image:: images/cp_prefit.png 29 | :width: 600 30 | :align: center 31 | 32 | 33 | Split conformal predictions with an untrained model 34 | ------------------------------------------------------------------------------------ 35 | 36 | .. image:: images/cp_split.png 37 | :width: 600 38 | :align: center 39 | 40 | 41 | 2. Cross conformal predictions 42 | ============================== 43 | 44 | - Conformity scores on the whole dataset obtained by cross-validation, 45 | - Perturbed models generated during the cross-validation. 46 | 47 | **MAPIE** then combines all these elements in a way that provides prediction intervals on new data with strong theoretical guarantees. 48 | 49 | .. image:: images/cp_cross.png 50 | :width: 600 51 | :align: center 52 | -------------------------------------------------------------------------------- /doc/theoretical_description_binary_classification.rst: -------------------------------------------------------------------------------- 1 | .. title:: Theoretical Description Binary Classification : contents 2 | 3 | .. _theoretical_description_binay_classification: 4 | 5 | ####################### 6 | Theoretical Description 7 | ####################### 8 | 9 | Note: in theoretical parts of the documentation, we use the following terms employed in the scientific literature: 10 | 11 | - `alpha` is equivalent to `1 - confidence_level`. It can be seen as a *risk level* 12 | - *calibrate* and *calibration*, are equivalent to *conformalize* and *conformalization*. 13 | 14 | — 15 | 16 | There are mainly three different ways to handle uncertainty quantification in binary classification: 17 | calibration (see :doc:`theoretical_description_calibration`), confidence interval (CI) for the probability 18 | :math:`P(Y \vert \hat{\mu}(X))` and prediction sets (see :doc:`theoretical_description_classification`). 19 | These 3 notions are tightly related for score-based classifier, as it is shown in [1]. 20 | 21 | Prediction sets can be computed in the same way for multiclass and binary classification with 22 | :class:`~mapie.classification.SplitConformalClassifier` or :class:`~mapie.classification.CrossConformalClassifier`, 23 | and there are the same theoretical guarantees. Nevertheless, prediction sets are often much less informative in the 24 | binary case than in the multiclass case. 25 | 26 | From Gupta et al [1]: 27 | 28 | PSs and CIs are only ‘informative’ if the sets or intervals produced by them are small. To quantify 29 | this, we measure CIs using their width (denoted as :math:`|C(.)|)`, and PSs using their diameter (defined as 30 | the width of the convex hull of the PS). For example, in the case of binary classification, the diameter 31 | of a PS is :math:`1` if the prediction set is :math:`\{0,1\}`, and :math:`0` otherwise (since :math:`Y\in\{0,1\}` 32 | always holds, the set :math:`\{0,1\}` is ‘uninformative’). A short CI such as :math:`[0.39, 0.41]` 33 | is more informative than a wider one such as :math:`[0.3, 0.5]`. 34 | 35 | In a few words, what you need to remember about these concepts : 36 | 37 | * *Calibration* is useful for transforming a score (typically given by an ML model) 38 | into the probability of making a good prediction. 39 | * *Set Prediction* gives the set of likely predictions with a probabilistic guarantee that the true label is in this set. 40 | * *Probabilistic Prediction* gives a confidence interval for the predictive distribution. 41 | 42 | 43 | 1. Set Prediction 44 | ----------------- 45 | 46 | Definition 1 (Prediction Set (PS) w.r.t :math:`f`) [1]. 47 | Fix a predictor :math:`\hat{\mu}:\mathcal{X} \to [0, 1]` and let :math:`(\mathcal{X}, \mathcal{Y}) \sim P`. 48 | Define the set of all subsets of :math:`\mathcal{Y}`, :math:`L = \{\{0\}, \{1\}, \{0, 1\}, \emptyset\}`. 49 | A function :math:`S:[0,1]\to\mathcal{L}` is said to be :math:`(1-\alpha)`-PS with respect to :math:`\hat{\mu}` if: 50 | 51 | .. math:: 52 | P(Y\in S(\hat{\mu}(X))) \geq 1 - \alpha 53 | 54 | PSs are typically studied for larger output sets, such as :math:`\mathcal{Y}_{regression}=\mathbb{R}` or 55 | :math:`\mathcal{Y}_{multiclass}=\{1, 2, ..., L > 2\}`. 56 | 57 | See :class:`~mapie.classification.SplitConformalClassifier` and :class:`~mapie.classification.CrossConformalClassifier` 58 | to use a set predictor. 59 | 60 | 61 | 2. Probabilistic Prediction 62 | --------------------------- 63 | 64 | Definition 2 (Confidence Interval (CI) w.r.t :math:`\hat{\mu}`) [1]. 65 | Fix a predictor :math:`\hat{\mu}:\mathcal{X} \to [0, 1]` and let :math:`(\mathcal{X}, \mathcal{Y}) \sim P`. 66 | Let :math:`I` denote the set of all subintervals of :math:`[0,1]`. 67 | A function :math:`C:[0,1]\to\mathcal{I}` is said to be :math:`(1-\alpha)`-CI with respect to :math:`\hat{\mu}` if: 68 | 69 | .. math:: 70 | P(\mathbb{E}[Y|\hat{\mu}(X)]\in C(\hat{\mu}(X))) \geq 1 - \alpha 71 | 72 | 73 | 3. Calibration 74 | -------------- 75 | 76 | Usually, calibration is understood as perfect calibration meaning (see :doc:`theoretical_description_calibration`). 77 | In practice, it is more reasonable to consider approximate calibration. 78 | 79 | Definition 3 (Approximate calibration) [1]. 80 | Fix a predictor :math:`\hat{\mu}:\mathcal{X} \to [0, 1]` and let :math:`(\mathcal{X}, \mathcal{Y}) \sim P`. 81 | The predictor :math:`\hat{\mu}:\mathcal{X} \to [0, 1]` is :math:`(\epsilon,\alpha)`-calibrated 82 | for some :math:`\epsilon,\alpha\in[0, 1]` if with probability at least :math:`1-\alpha`: 83 | 84 | .. math:: 85 | |\mathbb{E}[Y|\hat{\mu}(X)] - \hat{\mu}(X)| \leq \epsilon 86 | 87 | See :class:`~sklearn.calibration.CalibratedClassifierCV` or :class:`~mapie.calibration.TopLabelCalibrator` 88 | to use a calibrator. 89 | 90 | In the CP framework, it is worth noting that Venn predictors produce probability-type predictions 91 | for the labels of test objects which are guaranteed to be well-calibrated under the standard assumption 92 | that the observations are generated independently from the same distribution [2]. 93 | 94 | 95 | References 96 | ---------- 97 | 98 | [1] Gupta, Chirag, Aleksandr Podkopaev, and Aaditya Ramdas. 99 | "Distribution-free binary classification: prediction sets, confidence intervals, and calibration." 100 | Advances in Neural Information Processing Systems 33 (2020): 3711-3723. 101 | 102 | [2] Vovk, Vladimir, Alexander Gammerman, and Glenn Shafer. 103 | "Algorithmic Learning in a Random World." 104 | Springer Nature, 2022. -------------------------------------------------------------------------------- /doc/theoretical_description_calibration.rst: -------------------------------------------------------------------------------- 1 | .. title:: Theoretical Description : contents 2 | 3 | .. _theoretical_description_calibration: 4 | 5 | ####################### 6 | Theoretical Description 7 | ####################### 8 | 9 | Note: in theoretical parts of the documentation, we use the following terms employed in the scientific literature: 10 | 11 | - `alpha` is equivalent to `1 - confidence_level`. It can be seen as a *risk level* 12 | - *calibrate* and *calibration*, are equivalent to *conformalize* and *conformalization*. 13 | 14 | — 15 | 16 | One method for multi-class calibration has been implemented in MAPIE so far : 17 | Top-Label Calibration [1]. 18 | 19 | The goal of binary calibration is to transform a score (typically given by an ML model) that is not a probability into a 20 | probability. The algorithms that are used for calibration can be interpreted as estimators of the confidence level. Hence, 21 | they need independent and dependent variables to be fitted. 22 | 23 | The figure below illustrates what we would expect as a result of a calibration, with the scores predicted being closer to the 24 | true probability compared to the original output. 25 | 26 | .. image:: images/calibration_basic.png 27 | :width: 300 28 | :align: center 29 | 30 | 31 | Firstly, we introduce binary calibration, we denote the :math:`(h(X), y)` pair as the score and ground truth for the object. Hence, :math:`y` 32 | values are in :math:`{0, 1}`. The model is calibrated if for every output :math:`q \in [0, 1]`, we have: 33 | 34 | .. math:: 35 | Pr(Y = 1 \mid h(X) = q) = q 36 | 37 | where :math:`h()` is the score predictor. 38 | 39 | To apply calibration directly to a multi-class context, Gupta et al. propose a framework, multiclass-to-binary, in order to reduce 40 | a multi-class calibration to multiple binary calibrations (M2B). 41 | 42 | 43 | Top-Label 44 | --------- 45 | 46 | Top-Label calibration is a calibration technique introduced by Gupta et al. to calibrate the model according to the highest score and 47 | the corresponding class (see [1] Section 2). This framework offers to apply binary calibration techniques to multi-class calibration. 48 | 49 | More intuitively, top-label calibration simply performs a binary calibration (such as Platt scaling or isotonic regression) on the 50 | highest score and the corresponding class, whereas confidence calibration only calibrates on the highest score (see [1] Section 2). 51 | 52 | Let :math:`c` be the classifier and :math:`h` be the maximum score from the classifier. The couple :math:`(c, h)` is calibrated 53 | according to Top-Label calibration if: 54 | 55 | .. math:: 56 | Pr(Y = c(X) \mid h(X), c(X)) = h(X) 57 | 58 | 59 | References 60 | ---------- 61 | 62 | [1] Gupta, Chirag, and Aaditya K. Ramdas. 63 | "Top-label calibration and multiclass-to-binary reductions." 64 | arXiv preprint arXiv:2107.08353 (2021). 65 | 66 | [2] Arrieta-Ibarra I, Gujral P, Tannen J, Tygert M, Xu C. 67 | Metrics of calibration for probabilistic predictions. 68 | The Journal of Machine Learning Research. 69 | 2022 Jan 1;23(1):15886-940. 70 | 71 | [3] Tygert M. 72 | Calibration of P-values for calibration and for deviation 73 | of a subpopulation from the full population. 74 | arXiv preprint arXiv:2202.00100. 75 | 2022 Jan 31. 76 | 77 | [4] D. A. Darling. A. J. F. Siegert. 78 | The First Passage Problem for a Continuous Markov Process. 79 | Ann. Math. Statist. 24 (4) 624 - 639, December, 1953. 80 | 81 | [5] William Feller. 82 | The Asymptotic Distribution of the Range of Sums of 83 | Independent Random Variables. 84 | Ann. Math. Statist. 22 (3) 427 - 432 85 | September, 1951. 86 | 87 | [6] Spiegelhalter DJ. 88 | Probabilistic prediction in patient management and clinical trials. 89 | Statistics in medicine. 90 | 1986 Sep;5(5):421-33. -------------------------------------------------------------------------------- /doc/theoretical_description_conformity_scores.rst: -------------------------------------------------------------------------------- 1 | .. title:: Theoretical Description Conformity Scores : contents 2 | 3 | .. _theoretical_description_conformity_scores: 4 | 5 | ############################################# 6 | Theoretical Description for Conformity Scores 7 | ############################################# 8 | 9 | Note: in theoretical parts of the documentation, we use the following terms employed in the scientific literature: 10 | 11 | - `alpha` is equivalent to `1 - confidence_level`. It can be seen as a *risk level* 12 | - *calibrate* and *calibration*, are equivalent to *conformalize* and *conformalization*. 13 | 14 | — 15 | 16 | The :class:`mapie.conformity_scores.BaseRegressionScore` class implements various 17 | methods to compute conformity scores for regression. 18 | We give here a brief theoretical description of the scores included in the module. 19 | Note that it is possible for the user to create any conformal scores that are not 20 | already included in MAPIE by inheriting this class. 21 | 22 | Before describing the methods, let's briefly present the mathematical setting. 23 | With conformal predictions, we want to transform a heuristic notion of uncertainty 24 | from a model into a rigorous one, and the first step to do it is to choose a conformal score. 25 | The only requirement for the score function :math:`s(X, Y) \in \mathbb{R}` is 26 | that larger scores should encode worse agreement between :math:`X` and :math:`Y`. [1] 27 | 28 | There are two types of scores : the symmetric and asymmetric ones. 29 | The symmetric property defines the way of computing the quantile of the conformity 30 | scores when calculating the interval's bounds. If a score is symmetrical two 31 | quantiles will be computed : one on the right side of the distribution 32 | and the other on the left side. 33 | 34 | 1. The absolute residual score 35 | ------------------------------ 36 | 37 | The absolute residual score (:class:`mapie.conformity_scores.AbsoluteConformityScore`) 38 | is the simplest and most commonly used conformal score, it translates the error 39 | of the model : in regression, it is called the residual. 40 | 41 | .. math:: |Y-\hat{\mu}(X)| 42 | 43 | The intervals of prediction's bounds are then computed from the following formula : 44 | 45 | .. math:: [\hat{\mu}(X) - q(s), \hat{\mu}(X) + q(s)] 46 | 47 | Where :math:`q(s)` is the :math:`(1-\alpha)` quantile of the conformity scores. 48 | (see :doc:`theoretical_description_regression` for more details). 49 | 50 | With this score, the intervals of predictions will be constant over the whole dataset. 51 | This score is by default symmetric (*see above for definition*). 52 | 53 | 2. The gamma score 54 | ------------------ 55 | 56 | The gamma score [2] (:class:`mapie.conformity_scores.GammaConformityScore`) adds a 57 | notion of adaptivity with the normalization of the residuals by the predictions. 58 | 59 | .. math:: \frac{|Y-\hat{\mu}(X)|}{\hat{\mu}(X)} 60 | 61 | It computes adaptive intervals : intervals of different size on each example, with 62 | the following formula : 63 | 64 | .. math:: [\hat{\mu}(X) * (1 - q(s)), \hat{\mu}(X) * (1 + q(s))] 65 | 66 | Where :math:`q(s)` is the :math:`(1-\alpha)` quantile of the conformity scores. 67 | (see :doc:`theoretical_description_regression` for more details). 68 | 69 | This score is by default asymmetric (*see definition above*). 70 | 71 | Compared to the absolute residual score, it allows us to see regions with smaller intervals 72 | than others which are interpreted as regions with more certainty than others. 73 | It is important to note that, this conformity score is inversely proportional to the 74 | order of magnitude of the predictions. Therefore, the uncertainty is proportional to 75 | the order of magnitude of the predictions, implying that this score should be used 76 | in use cases where we want greater uncertainty when the prediction is high. 77 | 78 | 3. The residual normalized score 79 | -------------------------------- 80 | 81 | The residual normalized score [1] (:class:`mapie.conformity_scores.ResidualNormalisedScore`) 82 | is slightly more complex than the previous scores. 83 | The normalization of the residual is now done by the predictions of an additional model 84 | :math:`\hat\sigma` which learns to predict the base model residuals from :math:`X`. 85 | :math:`\hat\sigma` is trained on :math:`(X, |Y-\hat{\mu}(X)|)` and the formula of the score is: 86 | 87 | .. math:: \frac{|Y-\hat{\mu}(X)|}{\hat{\sigma}(X)} 88 | 89 | This score provides adaptive intervals : intervals of different sizes in each point 90 | with the following formula : 91 | 92 | .. math:: [\hat{\mu}(X) - q(s) * \hat{\sigma}(X), \hat{\mu}(X) + q(s) * \hat{\sigma}(X)] 93 | 94 | Where :math:`q(s)` is the :math:`(1-\alpha)` quantile of the conformity scores. 95 | (see :doc:`theoretical_description_regression` for more details). 96 | 97 | This score is by default symmetric (*see definition above*). Unlike the scores above, 98 | and due to the additional model required this score can only be used with split methods. 99 | 100 | Normalization by the learned residuals from :math:`X` adds to the score a knowledge of 101 | :math:`X` and its similarity to the other examples in the dataset. 102 | Compared to the gamma score, the other adaptive score implemented in MAPIE, 103 | it is not proportional to the uncertainty. 104 | 105 | 106 | Key takeaways 107 | ------------- 108 | 109 | - The absolute residual score is the basic conformity score and gives constant intervals. It is the one used by default by regression methods 110 | such as :class:`mapie.regression.SplitConformalRegressor`. 111 | - The gamma conformity score adds a notion of adaptivity by giving intervals of different sizes 112 | and is proportional to the uncertainty. 113 | - The residual normalized score is a conformity score that requires an additional model 114 | to learn the residuals of the model from :math:`X`. It gives very adaptive intervals 115 | without specific assumptions on the data. 116 | 117 | References 118 | ---------- 119 | 120 | [1] Lei, J., G'Sell, M., Rinaldo, A., Tibshirani, R. J., & Wasserman, L. (2018). Distribution-Free 121 | Predictive Inference for Regression. Journal of the American Statistical Association, 113(523), 1094–1111. 122 | Available from https://doi.org/10.1080/01621459.2017.1307116 123 | 124 | [2] Cordier, T., Blot, V., Lacombe, L., Morzadec, T., Capitaine, A. & Brunel, N.. (2023). 125 | Flexible and Systematic Uncertainty Estimation with Conformal Prediction via the MAPIE library. 126 | Available from https://proceedings.mlr.press/v204/cordier23a.html. 127 | -------------------------------------------------------------------------------- /doc/theoretical_description_mondrian.rst: -------------------------------------------------------------------------------- 1 | .. title:: Theoretical Description Mondrian : contents 2 | 3 | .. _theoretical_description_mondrian: 4 | 5 | ####################### 6 | Theoretical Description 7 | ####################### 8 | 9 | Note: in theoretical parts of the documentation, we use the following terms employed in the scientific literature: 10 | 11 | - `alpha` is equivalent to `1 - confidence_level`. It can be seen as a *risk level* 12 | - *calibrate* and *calibration*, are equivalent to *conformalize* and *conformalization*. 13 | 14 | — 15 | 16 | Mondrian conformal prediction (MCP) [1] is a method that allows to build prediction sets with a group-conditional 17 | coverage guarantee. The coverage guarantee is given by: 18 | 19 | .. math:: 20 | P \{Y_{n+1} \in \hat{C}_{n, \alpha}(X_{n+1}) | G_{n+1} = g\} \geq 1 - \alpha 21 | 22 | where :math:`G_{n+1}` is the group of the new test point :math:`X_{n+1}` and :math:`g` 23 | is a group in the set of groups :math:`\mathcal{G}`. 24 | 25 | MCP can be used with any split conformal predictor and can be particularly useful when one have a prior 26 | knowledge about existing groups whether the information is directly included in the features of the data or not. 27 | In a classification setting, the groups can be defined as the predicted classes of the data. Doing so, 28 | one can ensure that, for each predicted class, the coverage guarantee is satisfied. 29 | 30 | In order to achieve the group-conditional coverage guarantee, MCP simply classifies the data 31 | according to the groups and then applies the split conformal predictor to each group separately. 32 | 33 | The quantile of each group is defined as: 34 | 35 | .. math:: 36 | \widehat{q}^g =Quantile\left(s_1, ..., s_{n^g} ,\frac{\lceil (n^{(g)} + 1)(1-\alpha)\rceil}{n^{(g)}} \right) 37 | 38 | Where :math:`s_1, ..., s_{n^g}` are the conformity scores of the training points in group :math:`g` and :math:`n^{(g)}` 39 | is the number of training points in group :math:`g`. 40 | 41 | The following figure (from [1]) explains the process of Mondrian conformal prediction: 42 | 43 | .. image:: images/mondrian.png 44 | :width: 600 45 | :align: center 46 | 47 | References 48 | ---------- 49 | 50 | [1] Vladimir Vovk, David Lindsay, Ilia Nouretdinov, and Alex Gammerman. 51 | Mondrian confidence machine. 52 | Technical report, Royal Holloway University of London, 2003 53 | -------------------------------------------------------------------------------- /doc/tutorial_classification_files/tutorial_classification_12_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_classification_files/tutorial_classification_12_0.png -------------------------------------------------------------------------------- /doc/tutorial_classification_files/tutorial_classification_15_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_classification_files/tutorial_classification_15_0.png -------------------------------------------------------------------------------- /doc/tutorial_classification_files/tutorial_classification_20_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_classification_files/tutorial_classification_20_0.png -------------------------------------------------------------------------------- /doc/tutorial_classification_files/tutorial_classification_25_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_classification_files/tutorial_classification_25_0.png -------------------------------------------------------------------------------- /doc/tutorial_classification_files/tutorial_classification_28_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_classification_files/tutorial_classification_28_0.png -------------------------------------------------------------------------------- /doc/tutorial_classification_files/tutorial_classification_6_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_classification_files/tutorial_classification_6_0.png -------------------------------------------------------------------------------- /doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_13_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_13_0.png -------------------------------------------------------------------------------- /doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_15_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_15_2.png -------------------------------------------------------------------------------- /doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_2_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_2_0.png -------------------------------------------------------------------------------- /doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_5_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_5_0.png -------------------------------------------------------------------------------- /doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_8_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_mondrian_regression_files/tutorial_mondrian_regression_8_1.png -------------------------------------------------------------------------------- /doc/tutorial_multilabel_classification_files/tutorial_multilabel_classification_10_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_multilabel_classification_files/tutorial_multilabel_classification_10_0.png -------------------------------------------------------------------------------- /doc/tutorial_multilabel_classification_files/tutorial_multilabel_classification_11_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_multilabel_classification_files/tutorial_multilabel_classification_11_0.png -------------------------------------------------------------------------------- /doc/tutorial_multilabel_classification_files/tutorial_multilabel_classification_4_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_multilabel_classification_files/tutorial_multilabel_classification_4_1.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_10_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_10_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_16_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_16_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_17_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_17_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_18_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_18_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_19_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_19_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_28_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_28_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_29_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_29_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_30_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_30_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_31_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_31_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_32_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_32_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_34_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_34_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_35_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_35_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_36_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_36_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_37_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_37_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_38_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_38_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_39_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_39_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_40_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_40_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_41_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_41_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_42_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_42_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_42_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_42_2.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_44_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_44_2.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_48_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_48_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_49_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_49_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_50_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_50_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_52_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_52_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_53_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_53_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_55_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_55_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_56_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_56_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_58_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_58_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_59_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_59_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_62_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_62_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_65_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_65_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_66_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_66_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_69_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_69_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_70_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_70_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_72_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_72_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_73_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_73_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_74_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_74_0.png -------------------------------------------------------------------------------- /doc/tutorial_regression_files/tutorial_regression_9_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/doc/tutorial_regression_files/tutorial_regression_9_0.png -------------------------------------------------------------------------------- /environment.ci.yml: -------------------------------------------------------------------------------- 1 | name: mapie-ci 2 | channels: 3 | - defaults 4 | - conda-forge 5 | dependencies: 6 | - codecov 7 | - flake8 8 | - mypy<1.15 9 | - pandas 10 | - pytest-cov 11 | - typed-ast 12 | -------------------------------------------------------------------------------- /environment.dev.yml: -------------------------------------------------------------------------------- 1 | name: mapie-dev 2 | channels: 3 | - defaults 4 | - conda-forge 5 | dependencies: 6 | - bump2version=1.0.1 7 | - build 8 | - flake8=4.0.1 9 | - ipykernel=6.9.0 10 | - jupyter=1.0.0 11 | - mypy=1.7.1 12 | - numpydoc=1.1.0 13 | - numpy=1.22.3 14 | - pandas=1.3.5 15 | - pytest=6.2.5 16 | - pytest-cov=3.0.0 17 | - python=3.10 18 | - scikit-learn<1.6.0 19 | - sphinx=4.3.2 20 | - sphinx-gallery=0.10.1 21 | - sphinx_rtd_theme=1.0.0 22 | - twine 23 | - wheel 24 | -------------------------------------------------------------------------------- /environment.doc.yml: -------------------------------------------------------------------------------- 1 | name: mapie-doc 2 | channels: 3 | - defaults 4 | - conda-forge 5 | dependencies: 6 | - lightgbm=3.2.1 7 | - numpydoc=1.1.0 8 | - pandas=1.3.5 9 | - python=3.10 10 | - scikit-learn=1.5.2 11 | - sphinx=5.3.0 12 | - sphinx-gallery=0.10.1 13 | - sphinx_rtd_theme=1.0.0 14 | - typing_extensions=4.0.1 -------------------------------------------------------------------------------- /environment.notebooks.yml: -------------------------------------------------------------------------------- 1 | name: mapie-notebooks 2 | channels: 3 | - defaults 4 | - conda-forge 5 | dependencies: 6 | - ipykernel=6.9.0 7 | - jupyter=1.0.0 8 | - jupyterlab=3.2.9 9 | - jupytext=1.13.6 10 | - matplotlib=3.5.1 11 | - nbconvert=6.4.0 12 | - numpy=1.22.3 13 | - pandas=1.3.5 14 | - pip=22.0.3 15 | - pip: 16 | - scikeras==0.4.1 17 | - python=3.10 18 | - scikit-learn=1.0.1 19 | - seaborn=0.11.2 20 | - tensorflow=2.7.0 21 | - xgboost=1.5.1 22 | -------------------------------------------------------------------------------- /examples/README.rst: -------------------------------------------------------------------------------- 1 | .. _general_examples: 2 | 3 | General examples 4 | ================ -------------------------------------------------------------------------------- /examples/calibration/1-quickstart/README.rst: -------------------------------------------------------------------------------- 1 | .. _calibration_examples_1: 2 | 3 | 1. Quickstart examples 4 | ---------------------- 5 | 6 | The following examples present the main functionalities of MAPIE through basic quickstart calibration problems. -------------------------------------------------------------------------------- /examples/calibration/1-quickstart/plot_calibration_hypothesis_testing.py: -------------------------------------------------------------------------------- 1 | """ 2 | ========================================================= 3 | Testing for calibration in binary classification settings 4 | ========================================================= 5 | This example uses :func:`~mapie.metrics.kolmogorov_smirnov_pvalue` 6 | to test for calibration of scores output by binary classifiers. 7 | Other alternatives are :func:`~mapie.metrics.kuiper_pvalue` and 8 | :func:`~mapie.metrics.spieglehalter_pvalue`. 9 | 10 | These statistical tests are based on the following references: 11 | 12 | [1] Arrieta-Ibarra I, Gujral P, Tannen J, Tygert M, Xu C. 13 | Metrics of calibration for probabilistic predictions. 14 | The Journal of Machine Learning Research. 15 | 2022 Jan 1;23(1):15886-940. 16 | 17 | [2] Tygert M. 18 | Calibration of P-values for calibration and for deviation 19 | of a subpopulation from the full population. 20 | arXiv preprint arXiv:2202.00100. 21 | 2022 Jan 31. 22 | 23 | [3] D. A. Darling. A. J. F. Siegert. 24 | The First Passage Problem for a Continuous Markov Process. 25 | Ann. Math. Statist. 24 (4) 624 - 639, December, 26 | 1953. 27 | """ 28 | import numpy as np 29 | from matplotlib import pyplot as plt 30 | from sklearn.utils import check_random_state 31 | 32 | from numpy.typing import NDArray 33 | from mapie.metrics.calibration import ( 34 | cumulative_differences, kolmogorov_smirnov_p_value, 35 | length_scale, 36 | ) 37 | 38 | #################################################################### 39 | # 1. Create 1-dimensional dataset and scores to test for calibration 40 | # ------------------------------------------------------------------ 41 | # 42 | # We start by simulating a 1-dimensional binary classification problem. 43 | # We assume that the ground truth probability is driven by a sigmoid function, 44 | # and we generate label according to this probability distribution. 45 | 46 | 47 | def sigmoid(x: NDArray): 48 | y = 1 / (1 + np.exp(-x)) 49 | return y 50 | 51 | 52 | def generate_y_true_calibrated( 53 | y_prob: NDArray, 54 | random_state: int = 1 55 | ) -> NDArray: 56 | generator = check_random_state(random_state) 57 | uniform = generator.uniform(size=len(y_prob)) 58 | y_true = (uniform <= y_prob).astype(float) 59 | return y_true 60 | 61 | 62 | X = np.linspace(-5, 5, 2000) 63 | y_prob = sigmoid(X) 64 | y_true = generate_y_true_calibrated(y_prob) 65 | 66 | #################################################################### 67 | # Next we provide two additional miscalibrated scores (on purpose). 68 | 69 | 70 | y = { 71 | "y_prob": y_prob, 72 | "y_pred_1": sigmoid(1.3*X), 73 | "y_pred_2": sigmoid(0.7*X) 74 | } 75 | 76 | #################################################################### 77 | # This is how the two miscalibration curves stands next to the 78 | # ground truth. 79 | 80 | 81 | for name, y_score in y.items(): 82 | plt.plot(X, y_score, label=name) 83 | plt.title("Probability curves") 84 | plt.xlabel("x") 85 | plt.ylabel("y") 86 | plt.grid() 87 | plt.legend() 88 | plt.show() 89 | 90 | #################################################################### 91 | # Alternatively, you can readily see how much there is miscalibration 92 | # in this view where we plot scores against the ground truth probability. 93 | 94 | for name, y_score in y.items(): 95 | plt.plot(y_prob, y_score, label=name) 96 | plt.title("Probability curves") 97 | plt.xlabel("True probability") 98 | plt.ylabel("Estimated probability") 99 | plt.grid() 100 | plt.legend() 101 | plt.show() 102 | 103 | #################################################################### 104 | # 2. Visualizing and testing for miscalibration 105 | # ------------------------------------------------------------------ 106 | # 107 | # We leverage the Kolomogorov-Smirnov statistical test 108 | # :func:`~mapie.metrics.kolmogorov_smirnov_pvalue`. It is based 109 | # on the cumulative difference between sorted scores and labels. 110 | # If the null hypothesis holds (i.e., the scores are well calibrated), 111 | # the curve of the cumulative differences share some nice properties 112 | # with the standard Brownian motion, in particular its range and 113 | # maximum absolute value [1, 2]. 114 | # 115 | # Let's have a look. 116 | # 117 | # First we compute the cumulative differences. 118 | 119 | 120 | cum_diffs = { 121 | name: cumulative_differences(y_true, y_score) 122 | for name, y_score in y.items() 123 | } 124 | 125 | #################################################################### 126 | # We want to plot is along the proportion of scores taken into account. 127 | 128 | 129 | k = np.arange(len(y_true))/len(y_true) 130 | 131 | #################################################################### 132 | # We also want to compare the extension of the curve to that of a typical 133 | # Brownian motion. 134 | 135 | 136 | sigma = length_scale(y_prob) 137 | 138 | #################################################################### 139 | # Finally, we compute the p-value according to Kolmogorov-Smirnov test [2, 3]. 140 | 141 | 142 | p_values = { 143 | name: kolmogorov_smirnov_p_value(y_true, y_score) 144 | for name, y_score in y.items() 145 | } 146 | 147 | #################################################################### 148 | # The graph hereafter shows cumulative differences of each series of scores. 149 | # The horizontal bars are typical length scales expected if the null 150 | # hypothesis holds (standard Brownian motion). You can see that our two 151 | # miscalibrated scores overshoot these limits, and that their p-values 152 | # are accordingly very small. On the contrary, you can see that the 153 | # well calibrated ground truth perfectly lies within the expected bounds 154 | # with a p-value close to 1. 155 | # 156 | # So we conclude by both visual and statistical 157 | # arguments that we reject the null hypothesis for the two 158 | # miscalibrated scores ! 159 | 160 | 161 | for name, cum_diff in cum_diffs.items(): 162 | plt.plot(k, cum_diff, label=f"name (p-value = {p_values[name]:.5f})") 163 | plt.axhline(y=2*sigma, color="r", linestyle="--") 164 | plt.axhline(y=-2*sigma, color="r", linestyle="--") 165 | plt.title("Probability curves") 166 | plt.xlabel("Proportion of scores considered") 167 | plt.ylabel("Cumulative differences with the ground truth") 168 | plt.grid() 169 | plt.legend() 170 | plt.show() 171 | -------------------------------------------------------------------------------- /examples/calibration/2-advanced-analysis/README.rst: -------------------------------------------------------------------------------- 1 | .. _calibration_examples_2: 2 | 3 | 2. Advanced analysis 4 | -------------------- 5 | 6 | The following examples use MAPIE for discussing more complex calibration problems. -------------------------------------------------------------------------------- /examples/calibration/2-advanced-analysis/plot_asymptotic_convergence_of_p_values.py: -------------------------------------------------------------------------------- 1 | """ 2 | ================================================= 3 | Evaluating the asymptotic convergence of p-values 4 | ================================================= 5 | This example uses :func:`~mapie.metrics.kolmogorov_smirnov_pvalue`, 6 | :func:`~mapie.metrics.kuiper_pvalue` and 7 | :func:`~mapie.metrics.spieglehalter_pvalue`. We investigate 8 | the asymptotic convergence of these functions toward real p-values. 9 | Indeed, these quantities are only asymptotic p-values, i.e. when 10 | the number of observations is infinite. However, they can be safely 11 | used as real p-values even with moderate dataset sizes. This is what we 12 | are going to illustrate in this exampple. 13 | 14 | To this end, we generate many datasets that are calibrated by nature, 15 | and plot the distribution of the p-values. A p-value must follow 16 | a uniform distribution when computed on statistical samples following 17 | the null hypothesis. 18 | 19 | The argument can be retrieved from the following references: 20 | 21 | [1] Arrieta-Ibarra I, Gujral P, Tannen J, Tygert M, Xu C. 22 | Metrics of calibration for probabilistic predictions. 23 | The Journal of Machine Learning Research. 24 | 2022 Jan 1;23(1):15886-940. 25 | 26 | [2] Tygert M. 27 | Calibration of P-values for calibration and for deviation 28 | of a subpopulation from the full population. 29 | arXiv preprint arXiv:2202.00100. 30 | 2022 Jan 31. 31 | 32 | [3] D. A. Darling. A. J. F. Siegert. 33 | The First Passage Problem for a Continuous Markov Process. 34 | Ann. Math. Statist. 24 (4) 624 - 639, December, 35 | 1953. 36 | """ 37 | 38 | import numpy as np 39 | from matplotlib import pyplot as plt 40 | from sklearn.utils import check_random_state 41 | 42 | from numpy.typing import NDArray 43 | from mapie.metrics.calibration import (spiegelhalter_p_value) 44 | from mapie.metrics.calibration import kolmogorov_smirnov_p_value, kuiper_p_value 45 | 46 | ############################################################################## 47 | # First we need to generate scores that are perfecty calibrated. To do so, 48 | # we simply start from a given array of probabilities between 0 and 1, 49 | # and draw random labels 0 or 1 according to these probabilities. 50 | 51 | 52 | def generate_y_true_calibrated( 53 | y_prob: NDArray, 54 | random_state: int = 1 55 | ) -> NDArray: 56 | generator = check_random_state(random_state) 57 | uniform = generator.uniform(size=len(y_prob)) 58 | y_true = (uniform <= y_prob).astype(float) 59 | return y_true 60 | 61 | ############################################################################## 62 | # Then, we draw many different calibrated datasets, each with a fixed 63 | # dataset size. For each of these datasets, we compute the available p-values 64 | # implemented in MAPIE. 65 | 66 | 67 | n_sets = 10000 68 | n_points = 500 69 | ks_p_values = [] 70 | ku_p_values = [] 71 | sp_p_values = [] 72 | for i in range(n_sets): 73 | y_score = np.linspace(0, 1, n_points) 74 | y_true = generate_y_true_calibrated(y_score, random_state=i) 75 | ks_p_value = kolmogorov_smirnov_p_value(y_true, y_score) 76 | ku_p_value = kuiper_p_value(y_true, y_score) 77 | sp_p_value = spiegelhalter_p_value(y_true, y_score) 78 | ks_p_values.append(ks_p_value) 79 | ku_p_values.append(ku_p_value) 80 | sp_p_values.append(sp_p_value) 81 | ks_p_values = np.sort(ks_p_values) 82 | ku_p_values = np.sort(ku_p_values) 83 | sp_p_values = np.sort(sp_p_values) 84 | 85 | ############################################################################## 86 | # Finally, we plot the empirical cumulative distribution function of 87 | # the p-values computed on these many datasets. We see that even for 88 | # moderately sized datasets, the p-values computed closely follow the 89 | # expected uniform distribution under the null hypothesis. It appears that 90 | # Kuiper p-value is the slowest to converge compared to Spiegelhalter 91 | # and Kolmogorov-Smirnov. 92 | 93 | plt.hist( 94 | ks_p_values, 100, 95 | cumulative=True, density=True, histtype="step", label="Kolmogorov-Smirnov" 96 | ) 97 | plt.hist( 98 | ku_p_values, 100, 99 | cumulative=True, density=True, histtype="step", label="Kuiper" 100 | ) 101 | plt.hist( 102 | sp_p_values, 100, 103 | cumulative=True, density=True, histtype="step", label="Spiegelhalter" 104 | ) 105 | plt.plot([0, 1], [0, 1], "--", color="black") 106 | plt.title("Distribution of p-values for calibrated datasets") 107 | plt.xlabel("p-values") 108 | plt.ylabel("Cumulative count (%)") 109 | plt.grid() 110 | plt.legend() 111 | plt.show() 112 | -------------------------------------------------------------------------------- /examples/calibration/README.rst: -------------------------------------------------------------------------------- 1 | .. _calibration_examples: 2 | 3 | Calibration examples 4 | ======================= -------------------------------------------------------------------------------- /examples/classification/1-quickstart/README.rst: -------------------------------------------------------------------------------- 1 | .. _classification_examples_1: 2 | 3 | 1. Quickstart examples 4 | ---------------------- 5 | 6 | The following examples present the main functionalities of MAPIE through basic quickstart classification problems. -------------------------------------------------------------------------------- /examples/classification/1-quickstart/plot_quickstart_classification.py: -------------------------------------------------------------------------------- 1 | """ 2 | ====================================================== 3 | Use MAPIE to plot prediction sets 4 | ====================================================== 5 | 6 | In this example, we explain how to use MAPIE on a basic classification setting. 7 | """ 8 | 9 | ################################################################################## 10 | # We will use MAPIE to estimate prediction sets on a two-dimensional dataset with 11 | # three labels. 12 | 13 | import numpy as np 14 | from sklearn.neighbors import KNeighborsClassifier 15 | from sklearn.datasets import make_blobs 16 | from matplotlib import pyplot as plt 17 | from matplotlib.colors import ListedColormap 18 | from mapie.utils import train_conformalize_test_split 19 | from mapie.classification import SplitConformalClassifier 20 | from mapie.metrics.classification import classification_coverage_score 21 | 22 | np.random.seed(42) 23 | 24 | ############################################################################## 25 | # Firstly, let us create our dataset: 26 | 27 | X, y = make_blobs(n_samples=500, n_features=2, centers=3, cluster_std=3.4) 28 | 29 | (X_train, X_conformalize, X_test, 30 | y_train, y_conformalize, y_test) = train_conformalize_test_split( 31 | X, y, train_size=0.4, conformalize_size=0.4, test_size=0.2 32 | ) 33 | 34 | ############################################################################## 35 | # We fit our training data with a KNN estimator. 36 | # Then, we initialize a :class:`~mapie.classification.SplitConformalClassifier` 37 | # using our estimator, indicating that it has already been fitted with 38 | # `prefit=True`. 39 | # Lastly, we compute the prediction sets with the desired confidence level using the 40 | # ``conformalize`` and ``predict_set`` methods. 41 | 42 | classifier = KNeighborsClassifier(n_neighbors=10) 43 | classifier.fit(X_train, y_train) 44 | 45 | confidence_level = 0.95 46 | mapie_classifier = SplitConformalClassifier( 47 | estimator=classifier, confidence_level=confidence_level, prefit=True 48 | ) 49 | mapie_classifier.conformalize(X_conformalize, y_conformalize) 50 | y_pred, y_pred_set = mapie_classifier.predict_set(X_test) 51 | 52 | ############################################################################## 53 | # ``y_pred`` represents the point predictions as a ``np.ndarray`` of shape 54 | # ``(n_samples)``. 55 | # ``y_pred_set`` corresponds to the prediction sets as a ``np.ndarray`` of shape 56 | # ``(n_samples, 3, 1)``. This array contains only boolean values: ``True`` if the label 57 | # is included in the prediction set, and ``False`` if not. 58 | 59 | ############################################################################## 60 | # Finally, we can easily compute the coverage score (i.e., the proportion of times the 61 | # true labels fall within the predicted sets). 62 | 63 | coverage_score = classification_coverage_score(y_test, y_pred_set) 64 | print(f"For a confidence level of {confidence_level:.2f}, " 65 | f"the target coverage is {confidence_level:.3f}, " 66 | f"and the effective coverage is {coverage_score[0]:.3f}.") 67 | 68 | ############################################################################## 69 | # In this example, the effective coverage is slightly above the target coverage 70 | # (i.e., 0.95), indicating that the confidence level we set has been reached. 71 | # Therefore, we can confirm that the prediction sets effectively contain the 72 | # true label more than 95% of the time. 73 | 74 | ############################################################################## 75 | # Now, let us plot the confidence regions across the plane. 76 | # This plot will give us insights about what the prediction set looks like for each 77 | # point. 78 | 79 | x_min, x_max = np.min(X[:, 0]), np.max(X[:, 0]) 80 | y_min, y_max = np.min(X[:, 1]), np.max(X[:, 1]) 81 | step = 0.1 82 | 83 | xx, yy = np.meshgrid(np.arange(x_min, x_max, step), np.arange(y_min, y_max, step)) 84 | X_test_mesh = np.stack([xx.ravel(), yy.ravel()], axis=1) 85 | 86 | y_pred_set = mapie_classifier.predict_set(X_test_mesh)[1][:, :, 0] 87 | 88 | cmap_back = ListedColormap( 89 | [(0.7803921568627451, 0.9137254901960784, 0.7529411764705882), 90 | (0.9921568627450981, 0.8156862745098039, 0.6352941176470588), 91 | (0.6196078431372549, 0.6039215686274509, 0.7843137254901961), 92 | (0.7764705882352941, 0.8588235294117647, 0.9372549019607843), 93 | (0.6196078431372549, 0.6039215686274509, 0.7843137254901961), 94 | (0.6196078431372549, 0.6039215686274509, 0.7843137254901961)] 95 | ) 96 | cmap_dots = ListedColormap( 97 | [(0.19215686274509805, 0.5098039215686274, 0.7411764705882353), 98 | (0.9019607843137255, 0.3333333333333333, 0.050980392156862744), 99 | (0.19215686274509805, 0.6392156862745098, 0.32941176470588235)] 100 | ) 101 | 102 | plt.scatter( 103 | X_test_mesh[:, 0], X_test_mesh[:, 1], 104 | c=np.ravel_multi_index(y_pred_set.T, (2, 2, 2)), 105 | cmap=cmap_back, marker='.', s=10 106 | ) 107 | plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_dots) 108 | plt.xlabel("x1") 109 | plt.ylabel("x2") 110 | plt.title("Confidence regions with KNN") 111 | plt.show() 112 | 113 | ############################################################################## 114 | # On the plot above, the dots represent the samples from our dataset, with their 115 | # color indicating their respective label. 116 | # The blue, orange and green zones correspond to prediction sets 117 | # containing only the blue label, orange label and green label respectively. 118 | # The purple zone represents areas where the prediction sets contain more than one 119 | # label, indicating that the model is uncertain. 120 | -------------------------------------------------------------------------------- /examples/classification/2-advanced-analysis/README.rst: -------------------------------------------------------------------------------- 1 | .. _classification_examples_2: 2 | 3 | 2. Advanced analysis 4 | -------------------- 5 | 6 | The following examples use MAPIE for discussing more complex MAPIE classification problems. -------------------------------------------------------------------------------- /examples/classification/3-scientific-articles/README.rst: -------------------------------------------------------------------------------- 1 | .. _classification_examples_3: 2 | 3 | 3. Simulations from scientific articles 4 | --------------------------------------- 5 | 6 | The following examples reproduce the simulations from the scientific 7 | articles that present the methods implemented 8 | in MAPIE for classification settings. -------------------------------------------------------------------------------- /examples/classification/3-scientific-articles/plot_sadinle2019_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | ======================================================================================= 3 | Least Ambiguous Set-Valued Classifiers with Bounded Error Levels, Sadinle et al. (2019) 4 | ======================================================================================= 5 | 6 | We use :class:`~mapie_v1.classification.SplitConformalClassifier` to reproduce 7 | Example 7 from Sadinle et al. (2019). 8 | 9 | We consider a two-dimensional dataset with three labels. The distribution 10 | of the data is a bivariate normal with diagonal covariance matrices for 11 | each label. 12 | We model the data with Gaussian Naive Bayes classifier 13 | :class:`~sklearn.naive_bayes.GaussianNB` as a base model. 14 | 15 | Prediction sets are estimated by 16 | :class:`~mapie_v1.classification.SplitConformalClassifier` from the distribution of the 17 | softmax scores of the true labels for three confidence level values (0.8, 0.9, and 0.95) 18 | giving different class coverage levels. 19 | 20 | When the class coverage level is not large enough, the prediction sets can be 21 | empty. 22 | This happens because the model is uncertain at the border between two labels. 23 | These so-called null regions disappear for larger coverage levels. 24 | """ 25 | import matplotlib.pyplot as plt 26 | import numpy as np 27 | from sklearn.naive_bayes import GaussianNB 28 | 29 | from mapie.classification import SplitConformalClassifier 30 | 31 | # Create training set from multivariate normal distribution 32 | centers = [(0, 3.5), (-2, 0), (2, 0)] 33 | # covs = [[[1, 0], [0, 1]], [[2, 0], [0, 2]], [[5, 0], [0, 1]]] 34 | covs = [np.eye(2), np.eye(2) * 2, np.diag([5, 1])] 35 | x_min, x_max, y_min, y_max, step = -6, 8, -6, 8, 0.1 36 | n_samples = 500 37 | n_classes = 3 38 | confidence_level = [0.8, 0.9, 0.95] 39 | np.random.seed(42) 40 | X_train = np.vstack( 41 | [ 42 | np.random.multivariate_normal(center, cov, n_samples) 43 | for center, cov in zip(centers, covs) 44 | ] 45 | ) 46 | y_train = np.hstack([np.full(n_samples, i) for i in range(n_classes)]) 47 | 48 | 49 | # Create test from (x, y) coordinates 50 | xx, yy = np.meshgrid( 51 | np.arange(x_min, x_max, step), np.arange(x_min, x_max, step) 52 | ) 53 | X_test = np.stack([xx.ravel(), yy.ravel()], axis=1) 54 | 55 | # Apply SplitConformalClassifier on the dataset to get prediction sets 56 | clf = GaussianNB() 57 | clf.fit(X_train, y_train) 58 | y_pred = clf.predict(X_test) 59 | y_pred_proba = clf.predict_proba(X_test) 60 | y_pred_proba_max = np.max(y_pred_proba, axis=1) 61 | mapie = SplitConformalClassifier( 62 | estimator=clf, 63 | confidence_level=confidence_level, 64 | prefit=True, 65 | conformity_score="lac" 66 | ) 67 | mapie.conformalize(X_train, y_train) 68 | y_pred_mapie, y_ps_mapie = mapie.predict_set(X_test) 69 | 70 | # Plot the results 71 | tab10 = plt.cm.get_cmap("Purples", 4) 72 | colors = {0: "#1f77b4", 1: "#ff7f0e", 2: "#2ca02c", 3: "#d62728"} 73 | y_pred_col = list(map(colors.get, y_pred_mapie)) 74 | y_train_col = list(map(colors.get, y_train)) 75 | y_train_col = [colors[int(i)] for _, i in enumerate(y_train)] 76 | fig, axs = plt.subplots(1, 4, figsize=(20, 4)) 77 | axs[0].scatter( 78 | X_test[:, 0], X_test[:, 1], color=y_pred_col, marker=".", s=10, alpha=0.4 79 | ) 80 | axs[0].scatter( 81 | X_train[:, 0], 82 | X_train[:, 1], 83 | color=y_train_col, 84 | marker="o", 85 | s=10, 86 | edgecolor="k", 87 | ) 88 | axs[0].set_title("Predicted labels") 89 | for i, confidence_level_ in enumerate(confidence_level): 90 | y_ps_sums = y_ps_mapie[:, :, i].sum(axis=1) 91 | num_labels = axs[i + 1].scatter( 92 | X_test[:, 0], 93 | X_test[:, 1], 94 | c=y_ps_sums, 95 | marker=".", 96 | s=10, 97 | alpha=1, 98 | cmap=tab10, 99 | vmin=0, 100 | vmax=3, 101 | ) 102 | cbar = plt.colorbar(num_labels, ax=axs[i + 1]) 103 | axs[i + 1].set_title(f"Number of labels for confidence_level={confidence_level_}") 104 | plt.show() 105 | -------------------------------------------------------------------------------- /examples/classification/4-other-notebooks/README.rst: -------------------------------------------------------------------------------- 1 | .. _classification_examples_4: 2 | 3 | 4. Other notebooks 4 | -------------------- 5 | 6 | This section lists a series of Jupyter notebooks hosted on the MAPIE Github repository that can be run on Google Colab: 7 | 8 | - `Deep learning with MAPIE on the Cifar10 dataset `_ -------------------------------------------------------------------------------- /examples/classification/README.rst: -------------------------------------------------------------------------------- 1 | .. _classification_examples: 2 | 3 | All classification examples 4 | ============================ 5 | 6 | Following is a collection of notebooks demonstrating how to use MAPIE in classification. -------------------------------------------------------------------------------- /examples/mondrian/1-quickstart/README.rst: -------------------------------------------------------------------------------- 1 | .. _mondrian_examples_1: 2 | 3 | 1. Quickstart examples 4 | ---------------------- 5 | 6 | The following examples present the main functionalities of MAPIE through basic quickstart regression problems. -------------------------------------------------------------------------------- /examples/mondrian/README.rst: -------------------------------------------------------------------------------- 1 | .. _mondrian_examples: 2 | 3 | Mondrian examples 4 | ======================= -------------------------------------------------------------------------------- /examples/multilabel_classification/1-quickstart/README.rst: -------------------------------------------------------------------------------- 1 | .. _multilabel_classification_examples_1: 2 | 3 | 1. Quickstart examples 4 | ---------------------- 5 | 6 | The following examples present the main functionalities of MAPIE through basic quickstart regression problems. -------------------------------------------------------------------------------- /examples/multilabel_classification/README.rst: -------------------------------------------------------------------------------- 1 | .. _general_examples: 2 | 3 | General examples 4 | ================ -------------------------------------------------------------------------------- /examples/regression/1-quickstart/README.rst: -------------------------------------------------------------------------------- 1 | .. _regression_examples_1: 2 | 3 | 1. Quickstart 4 | ---------------------- 5 | 6 | The following examples present the main functionalities of MAPIE through basic quickstart regression problems. -------------------------------------------------------------------------------- /examples/regression/1-quickstart/plot_toy_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | ===================================================================================== 3 | Use MAPIE to plot prediction intervals 4 | ===================================================================================== 5 | An example plot of :class:`~mapie.regression.SplitConformalRegressor` used 6 | in the Quickstart. 7 | """ 8 | 9 | ################################################################################## 10 | # We will use MAPIE to estimate prediction intervals on a one-dimensional, 11 | # non-linear regression problem. 12 | 13 | import numpy as np 14 | from matplotlib import pyplot as plt 15 | from numpy.typing import NDArray 16 | from sklearn.neural_network import MLPRegressor 17 | from mapie.metrics.regression import regression_coverage_score 18 | from mapie.regression import SplitConformalRegressor 19 | from mapie.utils import train_conformalize_test_split 20 | 21 | RANDOM_STATE = 1 22 | 23 | ############################################################################## 24 | # Firstly, let us create our dataset: 25 | 26 | 27 | def f(x: NDArray) -> NDArray: 28 | """Polynomial function used to generate one-dimensional data.""" 29 | return np.array(5 * x + 5 * x**4 - 9 * x**2) 30 | 31 | 32 | rng = np.random.default_rng(1) 33 | sigma = 0.1 34 | n_samples = 10000 35 | X = np.linspace(0, 1, n_samples) 36 | y = f(X) + rng.normal(0, sigma, n_samples) 37 | X = X.reshape(-1, 1) 38 | (X_train, X_conformalize, X_test, 39 | y_train, y_conformalize, y_test) = train_conformalize_test_split( 40 | X, y, 41 | train_size=0.8, conformalize_size=0.1, test_size=0.1, 42 | random_state=RANDOM_STATE 43 | ) 44 | 45 | ############################################################################## 46 | # We fit our training data with a MLPRegressor. 47 | # Then, we initialize a :class:`~mapie.regression.SplitConformalRegressor` 48 | # using our estimator, indicating that it has already been fitted with 49 | # `prefit=True`. 50 | # Lastly, we compute the prediction intervals with the desired confidence level using 51 | # the ``conformalize`` and ``predict_interval`` methods. 52 | 53 | regressor = MLPRegressor(activation="relu", random_state=RANDOM_STATE) 54 | regressor.fit(X_train, y_train) 55 | 56 | confidence_level = 0.95 57 | mapie_regressor = SplitConformalRegressor( 58 | estimator=regressor, confidence_level=confidence_level, prefit=True 59 | ) 60 | mapie_regressor.conformalize(X_conformalize, y_conformalize) 61 | y_pred, y_pred_interval = mapie_regressor.predict_interval(X_test) 62 | 63 | ############################################################################## 64 | # ``y_pred`` represents the point predictions as a ``np.ndarray`` of shape 65 | # ``(n_samples)``. 66 | # ``y_pred_interval`` corresponds to the prediction intervals as a ``np.ndarray`` of 67 | # shape ``(n_samples, 2, 1)``, giving the lower and upper bounds of the intervals. 68 | 69 | ############################################################################## 70 | # Finally, we can easily compute the coverage score (i.e., the proportion of times the 71 | # true labels fall within the predicted intervals). 72 | 73 | coverage_score = regression_coverage_score(y_test, y_pred_interval) 74 | print(f"For a confidence level of {confidence_level:.2f}, " 75 | f"the target coverage is {confidence_level:.3f}, " 76 | f"and the effective coverage is {coverage_score[0]:.3f}.") 77 | 78 | ############################################################################## 79 | # In this example, the effective coverage is slightly above the target coverage 80 | # (i.e., 0.95), indicating that the confidence level we set has been reached. 81 | # Therefore, we can confirm that the prediction intervals effectively contain the 82 | # true label more than 95% of the time. 83 | 84 | ############################################################################## 85 | # Now, let us plot the estimated prediction intervals. 86 | 87 | plt.xlabel("x") 88 | plt.ylabel("y") 89 | plt.scatter(X_test, y_test, alpha=0.3) 90 | X_test = X_test.ravel() 91 | order = np.argsort(X_test) 92 | plt.plot(X_test[order], y_pred[order], color="C1") 93 | plt.plot(X_test[order], y_pred_interval[order][:, 0, 0], color="C1", ls="--") 94 | plt.plot(X_test[order], y_pred_interval[order][:, 1, 0], color="C1", ls="--") 95 | plt.fill_between( 96 | X_test[order], 97 | y_pred_interval[:, 0, 0][order].ravel(), 98 | y_pred_interval[:, 1, 0][order].ravel(), 99 | alpha=0.2, 100 | ) 101 | plt.title("Estimated prediction intervals with MLPRegressor") 102 | plt.show() 103 | 104 | ############################################################################## 105 | # On the plot above, the dots represent the samples from our dataset, while the 106 | # orange area corresponds to the estimated prediction intervals for each ``x`` value. 107 | -------------------------------------------------------------------------------- /examples/regression/2-advanced-analysis/README.rst: -------------------------------------------------------------------------------- 1 | .. _regression_examples_2: 2 | 3 | ----- 4 | 5 | 2. Advanced analysis 6 | -------------------- 7 | 8 | The following examples use MAPIE for discussing more complex MAPIE problems. -------------------------------------------------------------------------------- /examples/regression/2-advanced-analysis/plot_conformal_predictive_distribution.py: -------------------------------------------------------------------------------- 1 | """ 2 | ================================================================================ 3 | Conformal Predictive Distribution with MAPIE 4 | ================================================================================ 5 | 6 | """ 7 | 8 | ############################################################################## 9 | # In this advanced analysis, we propose to use MAPIE for Conformal Predictive 10 | # Distribution (CPD) in few steps. Here are some reference papers for more 11 | # information about CPD: 12 | # 13 | # [1] Schweder, T., & Hjort, N. L. (2016). Confidence, likelihood, probability 14 | # (Vol. 41). Cambridge University Press. 15 | # 16 | # [2] Vovk, V., Shen, J., Manokhin, V., & Xie, M. G. (2017, May). Nonparametric 17 | # predictive distributions based on conformal prediction. In Conformal and 18 | # probabilistic prediction and applications (pp. 82-102). PMLR. 19 | 20 | import warnings 21 | 22 | import numpy as np 23 | from matplotlib import pyplot as plt 24 | from sklearn.datasets import make_regression 25 | from sklearn.linear_model import LinearRegression 26 | 27 | from mapie.conformity_scores import (AbsoluteConformityScore, 28 | ResidualNormalisedScore) 29 | from mapie.regression import SplitConformalRegressor 30 | from mapie.utils import train_conformalize_test_split 31 | 32 | warnings.filterwarnings('ignore') 33 | 34 | RANDOM_STATE = 15 35 | 36 | 37 | ############################################################################## 38 | # 1. Generating toy dataset 39 | # ------------------------- 40 | # 41 | # Here, we propose just to generate data for regression task, then split it. 42 | 43 | X, y = make_regression( 44 | n_samples=1000, n_features=1, noise=20, random_state=RANDOM_STATE 45 | ) 46 | 47 | ( 48 | X_train, X_conformalize, X_test, y_train, y_conformalize, y_test 49 | ) = train_conformalize_test_split( 50 | X, y, 51 | train_size=0.6, conformalize_size=0.2, test_size=0.2, 52 | random_state=RANDOM_STATE 53 | ) 54 | 55 | 56 | plt.xlabel("x") 57 | plt.ylabel("y") 58 | plt.scatter(X_train, y_train, alpha=0.3) 59 | plt.show() 60 | 61 | 62 | ############################################################################## 63 | # 2. Defining a Conformal Predictive Distribution class with MAPIE 64 | # ------------------------------------------------------------------ 65 | # 66 | # To be able to obtain the cumulative distribution function of 67 | # a prediction with MAPIE, we propose here to wrap the 68 | # :class:`~mapie.regression.SplitConformalRegressor` to add a new method named 69 | # `get_cumulative_distribution_function`. 70 | 71 | class MapieConformalPredictiveDistribution(SplitConformalRegressor): 72 | 73 | def __init__(self, **kwargs) -> None: 74 | super().__init__(**kwargs) 75 | 76 | def get_cumulative_distribution_function(self, X): 77 | y_pred, _ = self.predict_interval(X) 78 | cs = self._mapie_regressor.conformity_scores_[ 79 | ~np.isnan(self._mapie_regressor.conformity_scores_)] 80 | res = self._conformity_score.get_estimation_distribution( 81 | y_pred.reshape((-1, 1)), cs, X=X 82 | ) 83 | return res 84 | 85 | 86 | ############################################################################## 87 | # Now, we propose to use it with two different conformity scores - 88 | # :class:`~mapie.conformity_scores.AbsoluteConformityScore` and 89 | # :class:`~mapie.conformity_scores.ResidualNormalisedScore` - 90 | # in split-conformal inference. 91 | 92 | mapie_regressor_1 = MapieConformalPredictiveDistribution( 93 | estimator=LinearRegression(), 94 | conformity_score=AbsoluteConformityScore(sym=False), 95 | prefit=False 96 | ) 97 | 98 | mapie_regressor_1.fit(X_train, y_train) 99 | mapie_regressor_1.conformalize(X_conformalize, y_conformalize) 100 | y_pred_1, _ = mapie_regressor_1.predict_interval(X_test) 101 | y_cdf_1 = mapie_regressor_1.get_cumulative_distribution_function(X_test) 102 | 103 | mapie_regressor_2 = MapieConformalPredictiveDistribution( 104 | estimator=LinearRegression(), 105 | conformity_score=ResidualNormalisedScore(sym=False, random_state=RANDOM_STATE), 106 | prefit=False 107 | ) 108 | 109 | mapie_regressor_2.fit(X_train, y_train) 110 | mapie_regressor_2.conformalize(X_conformalize, y_conformalize) 111 | y_pred_2, _ = mapie_regressor_2.predict_interval(X_test) 112 | y_cdf_2 = mapie_regressor_2.get_cumulative_distribution_function(X_test) 113 | 114 | plt.xlabel("x") 115 | plt.ylabel("y") 116 | plt.scatter(X_test, y_test, alpha=0.3) 117 | plt.plot(X_test, y_pred_1, color="C1") 118 | plt.show() 119 | 120 | 121 | ############################################################################## 122 | # 3. Visualizing the cumulative distribution function 123 | # --------------------------------------------------- 124 | # 125 | # We now propose to visualize the cumulative distribution functions of 126 | # the predictive distribution in a graph in order to compare the two methods. 127 | 128 | 129 | nb_bins = 100 130 | 131 | 132 | def plot_cdf(data, bins, **kwargs): 133 | counts, bins = np.histogram(data, bins=bins) 134 | cdf = np.cumsum(counts)/np.sum(counts) 135 | 136 | plt.plot( 137 | np.vstack((bins, np.roll(bins, -1))).T.flatten()[:-2], 138 | np.vstack((cdf, cdf)).T.flatten(), 139 | **kwargs 140 | ) 141 | 142 | 143 | plot_cdf( 144 | y_cdf_1[0], bins=nb_bins, label='Absolute Residual Score', alpha=0.8 145 | ) 146 | plot_cdf( 147 | y_cdf_2[0], bins=nb_bins, label='Normalized Residual Score', alpha=0.8 148 | ) 149 | plt.vlines( 150 | y_pred_1[0], 0, 1, label='Prediction', color="C2", linestyles='dashed' 151 | ) 152 | plt.legend(loc=2) 153 | plt.show() 154 | -------------------------------------------------------------------------------- /examples/regression/2-advanced-analysis/plot_cqr_symmetry_difference.py: -------------------------------------------------------------------------------- 1 | """ 2 | ========================================================================================================== 3 | The symmetric_correction parameter of ConformalizedQuantileRegressor 4 | ========================================================================================================== 5 | 6 | 7 | An example plot of :class:`~mapie.regression.ConformalizedQuantileRegressor` 8 | illustrating the impact of the ``symmetric_correction`` parameter. 9 | """ 10 | import numpy as np 11 | from matplotlib import pyplot as plt 12 | from sklearn.datasets import make_regression 13 | from sklearn.ensemble import GradientBoostingRegressor 14 | 15 | from mapie.metrics.regression import regression_coverage_score 16 | from mapie.regression import ConformalizedQuantileRegressor 17 | from mapie.utils import train_conformalize_test_split 18 | 19 | RANDOM_STATE = 1 20 | 21 | ############################################################################## 22 | # We generate a synthetic data. 23 | 24 | X, y = make_regression( 25 | n_samples=1000, n_features=1, noise=20, random_state=RANDOM_STATE 26 | ) 27 | 28 | ( 29 | X_train, X_conformalize, X_test, y_train, y_conformalize, y_test 30 | ) = train_conformalize_test_split( 31 | X, y, 32 | train_size=0.6, conformalize_size=0.2, test_size=0.2, 33 | random_state=RANDOM_STATE 34 | ) 35 | 36 | 37 | # Define confidence level 38 | confidence_level = 0.8 39 | 40 | # Initialize a Gradient Boosting Regressor for quantile regression 41 | gb_reg = GradientBoostingRegressor( 42 | loss="quantile", alpha=0.5, random_state=RANDOM_STATE 43 | ) 44 | 45 | # Using ConformalizedQuantileRegressor 46 | mapie_qr = ConformalizedQuantileRegressor( 47 | estimator=gb_reg, confidence_level=confidence_level) 48 | mapie_qr.fit(X_train, y_train) 49 | mapie_qr.conformalize(X_conformalize, y_conformalize) 50 | y_pred_sym, y_pis_sym = mapie_qr.predict_interval(X_test, symmetric_correction=True) 51 | y_pred_asym, y_pis_asym = mapie_qr.predict_interval(X_test, symmetric_correction=False) 52 | y_qlow = mapie_qr._mapie_quantile_regressor.estimators_[0].predict(X_test) 53 | y_qup = mapie_qr._mapie_quantile_regressor.estimators_[1].predict(X_test) 54 | 55 | print(f"y.shape: {y.shape}") 56 | print(f"y_pis_sym[:, 0].shape: {y_pis_sym[:, 0].shape}") 57 | print(f"y_pis_sym[:, 1].shape: {y_pis_sym[:, 1].shape}") 58 | # Calculate coverage scores 59 | coverage_score_sym = regression_coverage_score( 60 | y_test, y_pis_sym 61 | )[0] 62 | coverage_score_asym = regression_coverage_score( 63 | y_test, y_pis_asym 64 | )[0] 65 | 66 | # Sort the values for plotting 67 | order = np.argsort(X_test[:, 0]) 68 | X_test_sorted = X_test[order] 69 | y_pred_sym_sorted = y_pred_sym[order] 70 | y_pis_sym_sorted = y_pis_sym[order] 71 | y_pred_asym_sorted = y_pred_asym[order] 72 | y_pis_asym_sorted = y_pis_asym[order] 73 | y_qlow = y_qlow[order] 74 | y_qup = y_qup[order] 75 | 76 | ############################################################################## 77 | # We will plot the predictions and prediction intervals for both symmetric 78 | # and asymmetric intervals. The line represents the predicted values, the 79 | # dashed lines represent the prediction intervals, and the shaded area 80 | # represents the symmetric and asymmetric prediction intervals. 81 | 82 | plt.figure(figsize=(14, 7)) 83 | 84 | plt.subplot(1, 2, 1) 85 | plt.xlabel("x") 86 | plt.ylabel("y") 87 | plt.scatter(X_test, y_test, alpha=0.3) 88 | plt.plot(X_test_sorted, y_qlow, color="C1") 89 | plt.plot(X_test_sorted, y_qup, color="C1") 90 | plt.plot(X_test_sorted, y_pis_sym_sorted[:, 0], color="C1", ls="--") 91 | plt.plot(X_test_sorted, y_pis_sym_sorted[:, 1], color="C1", ls="--") 92 | plt.fill_between( 93 | X_test_sorted.ravel(), 94 | y_pis_sym_sorted[:, 0].ravel(), 95 | y_pis_sym_sorted[:, 1].ravel(), 96 | alpha=0.2, 97 | ) 98 | plt.title( 99 | f"Symmetric Intervals\n" 100 | f"Target and effective coverages for " 101 | f"confidence_level={confidence_level:.2f}; coverage={coverage_score_sym:.3f})" 102 | ) 103 | 104 | # Plot asymmetric prediction intervals 105 | plt.subplot(1, 2, 2) 106 | plt.xlabel("x") 107 | plt.ylabel("y") 108 | plt.scatter(X_test, y_test, alpha=0.3) 109 | plt.plot(X_test_sorted, y_qlow, color="C2") 110 | plt.plot(X_test_sorted, y_qup, color="C2") 111 | plt.plot(X_test_sorted, y_pis_asym_sorted[:, 0], color="C2", ls="--") 112 | plt.plot(X_test_sorted, y_pis_asym_sorted[:, 1], color="C2", ls="--") 113 | plt.fill_between( 114 | X_test_sorted.ravel(), 115 | y_pis_asym_sorted[:, 0].ravel(), 116 | y_pis_asym_sorted[:, 1].ravel(), 117 | alpha=0.2, 118 | ) 119 | plt.title( 120 | f"Asymmetric Intervals\n" 121 | f"Target and effective coverages for " 122 | f"confidence_level={confidence_level:.2f}; coverage={coverage_score_sym:.3f})" 123 | ) 124 | plt.tight_layout() 125 | plt.show() 126 | 127 | ############################################################################## 128 | # The symmetric intervals (``symmetric_correction=True``) use a combined set of 129 | # residuals for both bounds, while the asymmetric intervals 130 | # (``symmetric_correction=False``) use distinct residuals for each bound, 131 | # allowing for more flexible and accurate intervals that reflect the 132 | # heteroscedastic nature of the data. The resulting effective coverages 133 | # demonstrate the theoretical guarantee of the target coverage level 134 | # ``confidence_level``. 135 | -------------------------------------------------------------------------------- /examples/regression/2-advanced-analysis/plot_nested-cv.py: -------------------------------------------------------------------------------- 1 | """ 2 | ========================================================================================== 3 | Hyperparameters tuning with CrossConformalRegressor 4 | ========================================================================================== 5 | 6 | 7 | This example compares non-nested and nested cross-validation strategies 8 | when using 9 | :class:`~mapie.regression.CrossConformalRegressor`. 10 | 11 | In the regular sequential method, a cross-validation parameter search is performed 12 | on the entire training set. The best model is then used in MAPIE to estimate 13 | prediction intervals. However, as MAPIE computes residuals on 14 | the validation dataset used during hyperparameter tuning, it can lead to 15 | overfitting. This fools MAPIE into being slightly too optimistic with confidence 16 | intervals. 17 | 18 | To solve this problem, an alternative option is to perform a nested 19 | cross-validation parameter search directly within the MAPIE estimator on each 20 | *out-of-fold* dataset. 21 | This ensures that residuals seen by MAPIE are never seen by the algorithm 22 | beforehand. However, this method is much heavier computationally since 23 | it results in ``N * P`` calculations, where *N* is the number of 24 | *out-of-fold* models and *P* the number of parameter search cross-validations, 25 | versus ``N + P`` for the non-nested approach. 26 | 27 | Here, we compare the two strategies on a toy dataset. 28 | 29 | The two approaches give slightly different predictions with the nested CV 30 | approach estimating larger prediction interval in average. 31 | 32 | For this example, the two approaches result in identical scores and identical 33 | effective coverages. 34 | 35 | In the general case, the recommended approach is to use nested 36 | cross-validation, since it does not underestimate conformity scores and hence 37 | prediction intervals. However, in this particular example, effective 38 | coverages of both nested and non-nested methods are the same. 39 | """ 40 | import matplotlib.pyplot as plt 41 | import numpy as np 42 | from scipy.stats import randint 43 | from sklearn.ensemble import RandomForestRegressor 44 | from sklearn.metrics import root_mean_squared_error 45 | from sklearn.model_selection import RandomizedSearchCV, train_test_split 46 | from sklearn.datasets import make_sparse_uncorrelated 47 | 48 | from mapie.metrics.regression import regression_coverage_score 49 | from mapie.regression import CrossConformalRegressor 50 | 51 | 52 | RANDOM_STATE = 42 53 | 54 | # Load the toy data 55 | X, y = make_sparse_uncorrelated(500, random_state=RANDOM_STATE) 56 | 57 | # Split the data into training and test sets. 58 | X_train, X_test, y_train, y_test = train_test_split( 59 | X, y, test_size=0.2, random_state=RANDOM_STATE 60 | ) 61 | 62 | # Define the Random Forest model as base regressor with parameter ranges. 63 | rf_model = RandomForestRegressor(random_state=RANDOM_STATE, verbose=0) 64 | rf_params = {"max_depth": randint(2, 10), "n_estimators": randint(10, 100)} 65 | 66 | # Cross-validation and prediction-interval parameters. 67 | cv = 10 68 | n_iter = 5 69 | confidence_level = 0.95 70 | 71 | # Non-nested approach with the CV+ strategy using the Random Forest model. 72 | cv_obj = RandomizedSearchCV( 73 | rf_model, 74 | param_distributions=rf_params, 75 | n_iter=n_iter, 76 | cv=cv, 77 | scoring="neg_root_mean_squared_error", 78 | return_train_score=True, 79 | verbose=0, 80 | random_state=RANDOM_STATE, 81 | n_jobs=-1, 82 | ) 83 | cv_obj.fit(X_train, y_train) 84 | best_est = cv_obj.best_estimator_ 85 | mapie_non_nested = CrossConformalRegressor( 86 | estimator=best_est, method="plus", cv=cv, n_jobs=-1, 87 | confidence_level=confidence_level, random_state=RANDOM_STATE 88 | ) 89 | mapie_non_nested.fit_conformalize(X_train, y_train) 90 | y_pred_non_nested, y_pis_non_nested = mapie_non_nested.predict_interval( 91 | X_test, aggregate_predictions='median' 92 | ) 93 | widths_non_nested = y_pis_non_nested[:, 1, 0] - y_pis_non_nested[:, 0, 0] 94 | coverage_non_nested = regression_coverage_score( 95 | y_test, y_pis_non_nested 96 | )[0] 97 | score_non_nested = root_mean_squared_error(y_test, y_pred_non_nested) 98 | 99 | # Nested approach with the CV+ strategy using the Random Forest model. 100 | cv_obj = RandomizedSearchCV( 101 | rf_model, 102 | param_distributions=rf_params, 103 | n_iter=n_iter, 104 | cv=cv, 105 | scoring="neg_root_mean_squared_error", 106 | return_train_score=True, 107 | verbose=0, 108 | random_state=RANDOM_STATE, 109 | n_jobs=-1, 110 | ) 111 | mapie_nested = CrossConformalRegressor( 112 | estimator=cv_obj, method="plus", cv=cv, n_jobs=-1, 113 | confidence_level=confidence_level, random_state=RANDOM_STATE 114 | ) 115 | mapie_nested.fit_conformalize(X_train, y_train) 116 | y_pred_nested, y_pis_nested = mapie_nested.predict_interval( 117 | X_test, aggregate_predictions='median' 118 | ) 119 | widths_nested = y_pis_nested[:, 1, 0] - y_pis_nested[:, 0, 0] 120 | coverage_nested = regression_coverage_score( 121 | y_test, y_pis_nested 122 | )[0] 123 | score_nested = root_mean_squared_error(y_test, y_pred_nested) 124 | 125 | # Print scores and effective coverages. 126 | print( 127 | "Scores and effective coverages for the CV+ strategy using the " 128 | "Random Forest model." 129 | ) 130 | print( 131 | "Score on the test set for the non-nested and nested CV approaches: ", 132 | f"{score_non_nested: .3f}, {score_nested: .3f}", 133 | ) 134 | print( 135 | "Effective coverage on the test set for the non-nested " 136 | "and nested CV approaches: ", 137 | f"{coverage_non_nested: .3f}, {coverage_nested: .3f}", 138 | ) 139 | 140 | # Compare prediction interval widths. 141 | fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 6)) 142 | min_x = np.min([np.min(widths_nested), np.min(widths_non_nested)]) 143 | max_x = np.max([np.max(widths_nested), np.max(widths_non_nested)]) 144 | ax1.set_xlabel("Prediction interval width using the nested CV approach") 145 | ax1.set_ylabel("Prediction interval width using the non-nested CV approach") 146 | ax1.scatter(widths_nested, widths_non_nested) 147 | ax1.plot([min_x, max_x], [min_x, max_x], ls="--", color="k") 148 | ax2.axvline(x=0, color="r", lw=2) 149 | ax2.set_xlabel( 150 | "[width(non-nested CV) - width(nested CV)] / width(non-nested CV)" 151 | ) 152 | ax2.set_ylabel("Counts") 153 | ax2.hist( 154 | (widths_non_nested - widths_nested) / widths_non_nested, 155 | bins=15, 156 | edgecolor="black", 157 | ) 158 | plt.show() 159 | -------------------------------------------------------------------------------- /examples/regression/3-scientific-articles/README.rst: -------------------------------------------------------------------------------- 1 | .. _regression_examples_3: 2 | 3 | ----- 4 | 5 | 3. Simulations from scientific articles 6 | --------------------------------------- 7 | 8 | The following examples reproduce the simulations from the scientific 9 | articles that introduce the methods implemented 10 | in MAPIE for regression settings. -------------------------------------------------------------------------------- /examples/regression/4-other-notebooks/README.rst: -------------------------------------------------------------------------------- 1 | .. _regression_examples_5: 2 | 3 | ----- 4 | 5 | 4. Other notebooks 6 | -------------------------------------------- 7 | 8 | This section lists a series of Jupyter notebooks hosted on the MAPIE Github repository that can be run on Google Colab: 9 | 10 | - `Estimating prediction intervals for time series forecast with EnbPI and ACI `_ 11 | 12 | -------------------------------------------------------------------------------- /examples/regression/README.rst: -------------------------------------------------------------------------------- 1 | .. _regression_examples: 2 | 3 | All regression examples 4 | ======================== 5 | 6 | Following is a collection of notebooks demonstrating how to use MAPIE. -------------------------------------------------------------------------------- /mapie/__init__.py: -------------------------------------------------------------------------------- 1 | from . import ( 2 | classification, 3 | metrics, 4 | regression, 5 | utils, 6 | risk_control, 7 | calibration, 8 | subsample, 9 | ) 10 | from ._version import __version__ 11 | 12 | __all__ = [ 13 | "regression", 14 | "classification", 15 | "risk_control", 16 | "calibration", 17 | "metrics", 18 | "utils", 19 | "subsample", 20 | "__version__" 21 | ] 22 | -------------------------------------------------------------------------------- /mapie/_machine_precision.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | EPSILON = np.float64(1e-8) 4 | 5 | __all__ = ["EPSILON"] 6 | -------------------------------------------------------------------------------- /mapie/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.0.1" 2 | -------------------------------------------------------------------------------- /mapie/aggregation_functions.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional 2 | 3 | import numpy as np 4 | 5 | from numpy.typing import NDArray 6 | 7 | 8 | def phi1D( 9 | x: NDArray, 10 | B: NDArray, 11 | fun: Callable[[NDArray], NDArray], 12 | ) -> NDArray: 13 | """ 14 | The function phi1D is called by phi2D. 15 | It aims at applying a function ``fun`` after multiplying each row 16 | of B by x. 17 | 18 | Parameters 19 | ---------- 20 | x : NDArray of shape (n, ) 21 | 1D vector. 22 | B : NDArray of shape (k, n) 23 | 2D vector whose number of columns is the number of rows of x. 24 | fun : function 25 | Vectorized function applying to NDArray. 26 | 27 | Returns 28 | ------- 29 | NDArray 30 | The function fun is applied to the product of ``x`` and ``B``. 31 | Typically, ``fun`` is a numpy function, ignoring nan, 32 | with argument ``axis=1``. 33 | 34 | Examples 35 | -------- 36 | >>> import numpy as np 37 | >>> x = np.array([1, 2, 3, 4, 5]) 38 | >>> B = np.array([[1, 1, 1, np.nan, np.nan], 39 | ... [np.nan, np.nan, 1, 1, 1]]) 40 | >>> fun = lambda x: np.nanmean(x, axis=1) 41 | >>> res = phi1D(x, B, fun) 42 | >>> print(res) 43 | [2. 4.] 44 | """ 45 | return fun(x * B) 46 | 47 | 48 | def phi2D( 49 | A: NDArray, 50 | B: NDArray, 51 | fun: Callable[[NDArray], NDArray], 52 | ) -> NDArray: 53 | """ 54 | The function phi2D is a loop applying phi1D on each row of A. 55 | 56 | Parameters 57 | ---------- 58 | A : NDArray of shape (n_rowsA, n_columns) 59 | B : NDArray of shape (n_rowsB, n_columns) 60 | A and B must have the same number of columns. 61 | 62 | fun : function 63 | Vectorized function applying to NDArray, and that should ignore nan. 64 | 65 | Returns 66 | ------- 67 | NDArray of shape (n_rowsA, n_rowsB) 68 | Applies phi1D(x, B, fun) to each row x of A. 69 | 70 | Examples 71 | -------- 72 | >>> import numpy as np 73 | >>> A = np.array([[1, 2, 3, 4, 5],[6, 7, 8, 9, 10],[11, 12, 13, 14, 15]]) 74 | >>> B = np.array([[1, 1, 1, np.nan, np.nan], 75 | ... [np.nan, np.nan, 1, 1, 1]]) 76 | >>> fun = lambda x: np.nanmean(x, axis=1) 77 | >>> res = phi2D(A, B, fun) 78 | >>> print(res.ravel()) 79 | [ 2. 4. 7. 9. 12. 14.] 80 | """ 81 | return np.apply_along_axis(phi1D, axis=1, arr=A, B=B, fun=fun) 82 | 83 | 84 | def aggregate_all(agg_function: Optional[str], X: NDArray) -> NDArray: 85 | """ 86 | Applies np.nanmean(, axis=1) or np.nanmedian(, axis=1) according 87 | to the string ``agg_function``. 88 | 89 | Parameters 90 | ----------- 91 | X : NDArray of shape (n, p) 92 | Array of floats and nans 93 | 94 | Returns 95 | -------- 96 | NDArray of shape (n, 1): 97 | Array of the means or medians of each row of X 98 | 99 | Raises 100 | ------ 101 | ValueError 102 | If agg_function is ``None`` 103 | 104 | Examples 105 | -------- 106 | >>> import numpy as np 107 | >>> from mapie.aggregation_functions import aggregate_all 108 | >>> agg_function = "mean" 109 | >>> aggregate_all(agg_function, 110 | ... np.array([list(range(30)), 111 | ... list(range(30))])) 112 | array([14.5, 14.5]) 113 | 114 | """ 115 | if agg_function == "median": 116 | return np.nanmedian(X, axis=1) 117 | elif agg_function == "mean": 118 | return np.nanmean(X, axis=1) 119 | raise ValueError("Aggregation function called but not defined.") 120 | -------------------------------------------------------------------------------- /mapie/conformity_scores/__init__.py: -------------------------------------------------------------------------------- 1 | from .regression import BaseRegressionScore 2 | from .classification import BaseClassificationScore 3 | from .bounds import ( 4 | AbsoluteConformityScore, GammaConformityScore, ResidualNormalisedScore 5 | ) 6 | from .sets import ( 7 | APSConformityScore, LACConformityScore, NaiveConformityScore, 8 | RAPSConformityScore, TopKConformityScore 9 | ) 10 | 11 | 12 | __all__ = [ 13 | "BaseRegressionScore", 14 | "BaseClassificationScore", 15 | "AbsoluteConformityScore", 16 | "GammaConformityScore", 17 | "ResidualNormalisedScore", 18 | "NaiveConformityScore", 19 | "LACConformityScore", 20 | "APSConformityScore", 21 | "RAPSConformityScore", 22 | "TopKConformityScore" 23 | ] 24 | -------------------------------------------------------------------------------- /mapie/conformity_scores/bounds/__init__.py: -------------------------------------------------------------------------------- 1 | from .absolute import AbsoluteConformityScore 2 | from .gamma import GammaConformityScore 3 | from .residuals import ResidualNormalisedScore 4 | 5 | 6 | __all__ = [ 7 | "AbsoluteConformityScore", 8 | "GammaConformityScore", 9 | "ResidualNormalisedScore", 10 | ] 11 | -------------------------------------------------------------------------------- /mapie/conformity_scores/bounds/absolute.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from numpy.typing import ArrayLike, NDArray 4 | from mapie.conformity_scores import BaseRegressionScore 5 | 6 | 7 | class AbsoluteConformityScore(BaseRegressionScore): 8 | """ 9 | Absolute conformity score. 10 | 11 | The signed conformity score = y - y_pred. 12 | The conformity score is symmetrical. 13 | 14 | This is appropriate when the confidence interval is symmetrical and 15 | its range is approximatively the same over the range of predicted values. 16 | """ 17 | 18 | def __init__( 19 | self, 20 | sym: bool = True, 21 | ) -> None: 22 | super().__init__(sym=sym, consistency_check=True) 23 | 24 | def get_signed_conformity_scores( 25 | self, 26 | y: ArrayLike, 27 | y_pred: ArrayLike, 28 | **kwargs 29 | ) -> NDArray: 30 | """ 31 | Compute the signed conformity scores from the predicted values 32 | and the observed ones, from the following formula: 33 | signed conformity score = y - y_pred 34 | """ 35 | return np.subtract(y, y_pred) 36 | 37 | def get_estimation_distribution( 38 | self, 39 | y_pred: ArrayLike, 40 | conformity_scores: ArrayLike, 41 | **kwargs 42 | ) -> NDArray: 43 | """ 44 | Compute samples of the estimation distribution from the predicted 45 | values and the conformity scores, from the following formula: 46 | signed conformity score = y - y_pred 47 | <=> y = y_pred + signed conformity score 48 | 49 | ``conformity_scores`` can be either the conformity scores or 50 | the quantile of the conformity scores. 51 | """ 52 | return np.add(y_pred, conformity_scores) 53 | -------------------------------------------------------------------------------- /mapie/conformity_scores/bounds/gamma.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from numpy.typing import ArrayLike, NDArray 4 | from mapie.conformity_scores import BaseRegressionScore 5 | 6 | 7 | class GammaConformityScore(BaseRegressionScore): 8 | """ 9 | Gamma conformity score. 10 | 11 | The signed conformity score = (y - y_pred) / y_pred. 12 | The conformity score is not symmetrical. 13 | 14 | This is appropriate when the confidence interval is not symmetrical and 15 | its range depends on the predicted values. Like the Gamma distribution, 16 | its support is limited to strictly positive reals. 17 | """ 18 | 19 | def __init__( 20 | self, 21 | sym: bool = False, 22 | ) -> None: 23 | super().__init__(sym=sym, consistency_check=False) 24 | 25 | def _check_observed_data( 26 | self, 27 | y: ArrayLike, 28 | ) -> None: 29 | if not self._all_strictly_positive(y): 30 | raise ValueError( 31 | f"At least one of the observed target is negative " 32 | f"which is incompatible with {self.__class__.__name__}. " 33 | "All values must be strictly positive, " 34 | "in conformity with the Gamma distribution support." 35 | ) 36 | 37 | def _check_predicted_data( 38 | self, 39 | y_pred: ArrayLike, 40 | ) -> None: 41 | if not self._all_strictly_positive(y_pred): 42 | raise ValueError( 43 | f"At least one of the predicted target is negative " 44 | f"which is incompatible with {self.__class__.__name__}. " 45 | "All values must be strictly positive, " 46 | "in conformity with the Gamma distribution support." 47 | ) 48 | 49 | @staticmethod 50 | def _all_strictly_positive( 51 | y: ArrayLike, 52 | ) -> bool: 53 | return not np.any(np.less_equal(y, 0)) 54 | 55 | def get_signed_conformity_scores( 56 | self, 57 | y: ArrayLike, 58 | y_pred: ArrayLike, 59 | **kwargs 60 | ) -> NDArray: 61 | """ 62 | Compute the signed conformity scores from the observed values 63 | and the predicted ones, from the following formula: 64 | signed conformity score = (y - y_pred) / y_pred 65 | """ 66 | self._check_observed_data(y) 67 | self._check_predicted_data(y_pred) 68 | return np.divide(np.subtract(y, y_pred), y_pred) 69 | 70 | def get_estimation_distribution( 71 | self, 72 | y_pred: ArrayLike, 73 | conformity_scores: ArrayLike, 74 | **kwargs 75 | ) -> NDArray: 76 | """ 77 | Compute samples of the estimation distribution from the predicted 78 | values and the conformity scores, from the following formula: 79 | signed conformity score = (y - y_pred) / y_pred 80 | <=> y = y_pred * (1 + signed conformity score) 81 | 82 | ``conformity_scores`` can be either the conformity scores or 83 | the quantile of the conformity scores. 84 | """ 85 | self._check_predicted_data(y_pred) 86 | return np.multiply(y_pred, np.add(1, conformity_scores)) 87 | -------------------------------------------------------------------------------- /mapie/conformity_scores/interface.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from typing import Optional 3 | 4 | import numpy as np 5 | from sklearn.base import BaseEstimator 6 | 7 | from numpy.typing import NDArray 8 | 9 | 10 | class BaseConformityScore(metaclass=ABCMeta): 11 | """ 12 | Base class for conformity scores. 13 | 14 | This class should not be used directly. Use derived classes instead. 15 | """ 16 | 17 | def __init__(self) -> None: 18 | pass 19 | 20 | def set_external_attributes( 21 | self, 22 | **kwargs 23 | ) -> None: 24 | """ 25 | Set attributes that are not provided by the user. 26 | 27 | Must be overloaded by subclasses if necessary to add more attributes, 28 | particularly when the attributes are known after the object has been 29 | instantiated. 30 | """ 31 | 32 | def set_ref_predictor( 33 | self, 34 | predictor: BaseEstimator 35 | ): 36 | """ 37 | Set the reference predictor. 38 | 39 | Parameters 40 | ---------- 41 | predictor: BaseEstimator 42 | Reference predictor. 43 | """ 44 | self.predictor = predictor 45 | 46 | def split_data( 47 | self, 48 | X: NDArray, 49 | y: NDArray, 50 | y_enc: NDArray, 51 | sample_weight: Optional[NDArray] = None, 52 | groups: Optional[NDArray] = None, 53 | ): 54 | """ 55 | Split data. Keeps part of the data for the calibration estimator 56 | (separate from the calibration data). 57 | 58 | Parameters 59 | ---------- 60 | *args: Tuple of NDArray 61 | 62 | Returns 63 | ------- 64 | Tuple of NDArray 65 | Split data for training and calibration. 66 | """ 67 | self.n_samples_ = len(X) 68 | return X, y, y_enc, sample_weight, groups 69 | 70 | @abstractmethod 71 | def get_conformity_scores( 72 | self, 73 | y: NDArray, 74 | y_pred: NDArray, 75 | **kwargs 76 | ) -> NDArray: 77 | """ 78 | Placeholder for ``get_conformity_scores``. 79 | Subclasses should implement this method! 80 | 81 | Compute the sample conformity scores given the predicted and 82 | observed targets. 83 | 84 | Parameters 85 | ---------- 86 | y: NDArray of shape (n_samples,) 87 | Observed target values. 88 | 89 | y_pred: NDArray of shape (n_samples,) 90 | Predicted target values. 91 | 92 | Returns 93 | ------- 94 | NDArray of shape (n_samples,) 95 | Conformity scores. 96 | """ 97 | 98 | @staticmethod 99 | def get_quantile( 100 | conformity_scores: NDArray, 101 | alpha_np: NDArray, 102 | axis: int = 0, 103 | reversed: bool = False, 104 | unbounded: bool = False 105 | ) -> NDArray: 106 | """ 107 | Compute the alpha quantile of the conformity scores. 108 | 109 | Parameters 110 | ---------- 111 | conformity_scores: NDArray of shape (n_samples,) 112 | Values from which the quantile is computed. 113 | 114 | alpha_np: NDArray of shape (n_alpha,) 115 | NDArray of floats between ``0`` and ``1``, represents the 116 | uncertainty of the confidence set. 117 | 118 | axis: int 119 | The axis from which to compute the quantile. 120 | 121 | By default ``0``. 122 | 123 | reversed: bool 124 | Boolean specifying whether we take the upper or lower quantile, 125 | if False, the alpha quantile, otherwise the (1-alpha) quantile. 126 | 127 | By default ``False``. 128 | 129 | unbounded: bool 130 | Boolean specifying whether infinite prediction sets 131 | could be produced (when alpha_np is greater than or equal to 1.). 132 | 133 | By default ``False``. 134 | 135 | Returns 136 | ------- 137 | NDArray of shape (1, n_alpha) or (n_samples, n_alpha) 138 | The quantiles of the conformity scores. 139 | """ 140 | n_ref = conformity_scores.shape[1-axis] 141 | n_calib: int = np.min(np.sum(~np.isnan(conformity_scores), axis=axis)) 142 | signed = 1-2*reversed 143 | 144 | # Adapt alpha w.r.t upper/lower : alpha vs. 1-alpha 145 | alpha_ref = (1-2*alpha_np)*reversed + alpha_np 146 | 147 | # Adjust alpha w.r.t quantile correction 148 | alpha_cor = np.ceil(alpha_ref*(n_calib+1))/n_calib 149 | alpha_cor = np.clip(alpha_cor, a_min=0, a_max=1) 150 | 151 | # Compute the target quantiles: 152 | # If unbounded is True and alpha is greater than or equal to 1, 153 | # the quantile is set to infinity. 154 | # Otherwise, the quantile is calculated as the corrected lower quantile 155 | # of the signed conformity scores. 156 | quantile = signed * np.column_stack([ 157 | np.nanquantile( 158 | signed * conformity_scores, _alpha_cor, 159 | axis=axis, method="lower" 160 | ) if not (unbounded and _alpha >= 1) else np.inf * np.ones(n_ref) 161 | for _alpha, _alpha_cor in zip(alpha_ref, alpha_cor) 162 | ]) 163 | return quantile 164 | 165 | @abstractmethod 166 | def predict_set( 167 | self, 168 | X: NDArray, 169 | alpha_np: NDArray, 170 | **kwargs 171 | ): 172 | """ 173 | Compute the prediction sets on new samples based on the uncertainty of 174 | the target confidence set. 175 | 176 | Parameters: 177 | ----------- 178 | X: NDArray of shape (n_samples,) 179 | The input data or samples for prediction. 180 | 181 | alpha_np: NDArray of shape (n_alpha, ) 182 | Represents the uncertainty of the confidence set to produce. 183 | 184 | **kwargs: dict 185 | Additional keyword arguments. 186 | 187 | Returns: 188 | -------- 189 | The output structure depend on the subclass. 190 | The prediction sets for each sample and each alpha level. 191 | """ 192 | -------------------------------------------------------------------------------- /mapie/conformity_scores/sets/__init__.py: -------------------------------------------------------------------------------- 1 | from .naive import NaiveConformityScore 2 | from .lac import LACConformityScore 3 | from .aps import APSConformityScore 4 | from .raps import RAPSConformityScore 5 | from .topk import TopKConformityScore 6 | 7 | 8 | __all__ = [ 9 | "NaiveConformityScore", 10 | "LACConformityScore", 11 | "APSConformityScore", 12 | "RAPSConformityScore", 13 | "TopKConformityScore", 14 | ] 15 | -------------------------------------------------------------------------------- /mapie/conformity_scores/sets/topk.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, cast 2 | 3 | import numpy as np 4 | 5 | from mapie.conformity_scores.classification import BaseClassificationScore 6 | from mapie.conformity_scores.sets.utils import ( 7 | check_proba_normalized, get_true_label_position 8 | ) 9 | from mapie.estimator.classifier import EnsembleClassifier 10 | 11 | from mapie._machine_precision import EPSILON 12 | from numpy.typing import NDArray 13 | from mapie.utils import _compute_quantiles 14 | 15 | 16 | class TopKConformityScore(BaseClassificationScore): 17 | """ 18 | Top-K method-based non-conformity score. 19 | 20 | It is based on the sorted index of the probability of the true label in the 21 | softmax outputs, on the conformalization set. In case two probabilities are 22 | equal, both are taken, thus, the size of some prediction sets may be 23 | different from the others. 24 | 25 | References 26 | ---------- 27 | [1] Anastasios Nikolas Angelopoulos, Stephen Bates, Michael Jordan 28 | and Jitendra Malik. 29 | "Uncertainty Sets for Image Classifiers using Conformal Prediction." 30 | International Conference on Learning Representations 2021. 31 | 32 | Attributes 33 | ---------- 34 | classes: Optional[ArrayLike] 35 | Names of the classes. 36 | 37 | random_state: Optional[Union[int, RandomState]] 38 | Pseudo random number generator state. 39 | 40 | quantiles_: ArrayLike of shape (n_alpha) 41 | The quantiles estimated from ``get_sets`` method. 42 | """ 43 | 44 | def __init__(self) -> None: 45 | super().__init__() 46 | 47 | def get_conformity_scores( 48 | self, 49 | y: NDArray, 50 | y_pred: NDArray, 51 | y_enc: Optional[NDArray] = None, 52 | **kwargs 53 | ) -> NDArray: 54 | """ 55 | Get the conformity score. 56 | 57 | Parameters 58 | ---------- 59 | y: NDArray of shape (n_samples,) 60 | Observed target values. 61 | 62 | y_pred: NDArray of shape (n_samples,) 63 | Predicted target values. 64 | 65 | y_enc: NDArray of shape (n_samples,) 66 | Target values as normalized encodings. 67 | 68 | Returns 69 | ------- 70 | NDArray of shape (n_samples,) 71 | Conformity scores. 72 | """ 73 | # Casting 74 | y_enc = cast(NDArray, y_enc) 75 | 76 | # Conformity scores 77 | # Here we reorder the labels by decreasing probability and get the 78 | # position of each label from decreasing probability 79 | conformity_scores = get_true_label_position(y_pred, y_enc) 80 | 81 | return conformity_scores 82 | 83 | def get_predictions( 84 | self, 85 | X: NDArray, 86 | alpha_np: NDArray, 87 | estimator: EnsembleClassifier, 88 | **kwargs 89 | ) -> NDArray: 90 | """ 91 | Get predictions from an EnsembleClassifier. 92 | 93 | This method should be implemented by any subclass of the current class. 94 | 95 | Parameters 96 | ----------- 97 | X: NDArray of shape (n_samples, n_features) 98 | Observed feature values. 99 | 100 | alpha_np: NDArray of shape (n_alpha,) 101 | NDArray of floats between ``0`` and ``1``, represents the 102 | uncertainty of the confidence interval. 103 | 104 | estimator: EnsembleClassifier 105 | Estimator that is fitted to predict y from X. 106 | 107 | Returns 108 | -------- 109 | NDArray 110 | Array of predictions. 111 | """ 112 | y_pred_proba = estimator.predict(X, agg_scores="mean") 113 | y_pred_proba = check_proba_normalized(y_pred_proba, axis=1) 114 | y_pred_proba = np.repeat( 115 | y_pred_proba[:, :, np.newaxis], len(alpha_np), axis=2 116 | ) 117 | return y_pred_proba 118 | 119 | def get_conformity_score_quantiles( 120 | self, 121 | conformity_scores: NDArray, 122 | alpha_np: NDArray, 123 | estimator: EnsembleClassifier, 124 | **kwargs 125 | ) -> NDArray: 126 | """ 127 | Get the quantiles of the conformity scores for each uncertainty level. 128 | 129 | Parameters 130 | ----------- 131 | conformity_scores: NDArray of shape (n_samples,) 132 | Conformity scores for each sample. 133 | 134 | alpha_np: NDArray of shape (n_alpha,) 135 | NDArray of floats between 0 and 1, representing the uncertainty 136 | of the confidence interval. 137 | 138 | estimator: EnsembleClassifier 139 | Estimator that is fitted to predict y from X. 140 | 141 | Returns 142 | -------- 143 | NDArray 144 | Array of quantiles with respect to alpha_np. 145 | """ 146 | return _compute_quantiles(conformity_scores, alpha_np) 147 | 148 | def get_prediction_sets( 149 | self, 150 | y_pred_proba: NDArray, 151 | conformity_scores: NDArray, 152 | alpha_np: NDArray, 153 | estimator: EnsembleClassifier, 154 | **kwargs 155 | ) -> NDArray: 156 | """ 157 | Generate prediction sets based on the probability predictions, 158 | the conformity scores and the uncertainty level. 159 | 160 | Parameters 161 | ----------- 162 | y_pred_proba: NDArray of shape (n_samples, n_classes) 163 | Target prediction. 164 | 165 | conformity_scores: NDArray of shape (n_samples,) 166 | Conformity scores for each sample. 167 | 168 | alpha_np: NDArray of shape (n_alpha,) 169 | NDArray of floats between 0 and 1, representing the uncertainty 170 | of the confidence interval. 171 | 172 | estimator: EnsembleClassifier 173 | Estimator that is fitted to predict y from X. 174 | 175 | Returns 176 | -------- 177 | NDArray 178 | Array of quantiles with respect to alpha_np. 179 | """ 180 | y_pred_proba = y_pred_proba[:, :, 0] 181 | index_sorted = np.fliplr(np.argsort(y_pred_proba, axis=1)) 182 | y_pred_index_last = np.stack( 183 | [ 184 | index_sorted[:, quantile] 185 | for quantile in self.quantiles_ 186 | ], axis=1 187 | ) 188 | y_pred_proba_last = np.stack( 189 | [ 190 | np.take_along_axis( 191 | y_pred_proba, 192 | y_pred_index_last[:, iq].reshape(-1, 1), 193 | axis=1 194 | ) 195 | for iq, _ in enumerate(self.quantiles_) 196 | ], axis=2 197 | ) 198 | prediction_sets = np.greater_equal( 199 | y_pred_proba[:, :, np.newaxis] 200 | - y_pred_proba_last, 201 | -EPSILON 202 | ) 203 | 204 | return prediction_sets 205 | -------------------------------------------------------------------------------- /mapie/conformity_scores/sets/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | import numpy as np 3 | 4 | from numpy.typing import NDArray 5 | from mapie._machine_precision import EPSILON 6 | 7 | 8 | def get_true_label_position( 9 | y_pred_proba: NDArray, 10 | y: NDArray 11 | ) -> NDArray: 12 | """ 13 | Return the sorted position of the true label in the prediction 14 | 15 | Parameters 16 | ---------- 17 | y_pred_proba: NDArray of shape (n_samples, n_classes) 18 | Model prediction. 19 | 20 | y: NDArray of shape (n_samples) 21 | Labels. 22 | 23 | Returns 24 | ------- 25 | NDArray of shape (n_samples, 1) 26 | Position of the true label in the prediction. 27 | """ 28 | index = np.argsort(np.fliplr(np.argsort(y_pred_proba, axis=1))) 29 | position = np.take_along_axis(index, y.reshape(-1, 1), axis=1) 30 | 31 | return position 32 | 33 | 34 | def check_include_last_label( 35 | include_last_label: Optional[Union[bool, str]] 36 | ) -> Optional[Union[bool, str]]: 37 | """ 38 | Check if ``include_last_label`` is a boolean or a string. 39 | Else raise error. 40 | 41 | Parameters 42 | ---------- 43 | include_last_label: Optional[Union[bool, str]] 44 | Whether or not to include last label in 45 | prediction sets for the ``"aps"`` method. Choose among: 46 | 47 | - ``False``, does not include label whose cumulated score is just 48 | over the quantile. 49 | 50 | - ``True``, includes label whose cumulated score is just over the 51 | quantile, unless there is only one label in the prediction set. 52 | 53 | - ``"randomized"``, randomly includes label whose cumulated score 54 | is just over the quantile based on the comparison of a uniform 55 | number and the difference between the cumulated score of the last 56 | label and the quantile. 57 | 58 | Returns 59 | ------- 60 | Optional[Union[bool, str]] 61 | 62 | Raises 63 | ------ 64 | ValueError 65 | "Invalid include_last_label argument. " 66 | "Should be a boolean or 'randomized'." 67 | """ 68 | if ( 69 | (not isinstance(include_last_label, bool)) and 70 | (not include_last_label == "randomized") 71 | ): 72 | raise ValueError( 73 | "Invalid include_last_label argument. " 74 | "Should be a boolean or 'randomized'." 75 | ) 76 | else: 77 | return include_last_label 78 | 79 | 80 | def check_proba_normalized( 81 | y_pred_proba: NDArray, 82 | axis: int = 1 83 | ) -> NDArray: 84 | """ 85 | Check if for all the samples the sum of the probabilities is equal to one. 86 | 87 | Parameters 88 | ---------- 89 | y_pred_proba: NDArray of shape (n_samples, n_classes) or 90 | (n_samples, n_train_samples, n_classes) 91 | Softmax output of a model. 92 | 93 | Returns 94 | ------- 95 | ArrayLike of shape (n_samples, n_classes) 96 | Softmax output of a model if the scores all sum to one. 97 | 98 | Raises 99 | ------ 100 | ValueError 101 | If the sum of the scores is not equal to one. 102 | """ 103 | np.testing.assert_allclose( 104 | np.sum(y_pred_proba, axis=axis), 105 | 1, 106 | err_msg="The sum of the scores is not equal to one.", 107 | rtol=1e-5 108 | ) 109 | return y_pred_proba.astype(np.float64) 110 | 111 | 112 | def get_last_index_included( 113 | y_pred_proba_cumsum: NDArray, 114 | threshold: NDArray, 115 | include_last_label: Optional[Union[bool, str]] 116 | ) -> NDArray: 117 | """ 118 | Return the index of the last included sorted probability 119 | depending if we included the first label over the quantile 120 | or not. 121 | 122 | Parameters 123 | ---------- 124 | y_pred_proba_cumsum: NDArray of shape (n_samples, n_classes) 125 | Cumsumed probabilities in the original order. 126 | 127 | threshold: NDArray of shape (n_alpha,) or shape (n_samples_train,) 128 | Threshold to compare with y_proba_last_cumsum, can be either: 129 | 130 | - the quantiles associated with alpha values when 131 | ``cv`` == "prefit", ``cv`` == "split" 132 | or ``agg_scores`` is "mean" 133 | 134 | - the conformity score from training samples otherwise 135 | (i.e., when ``cv`` is a CV splitter and 136 | ``agg_scores`` is "crossval") 137 | 138 | include_last_label: Union[bool, str] 139 | Whether or not include the last label. If 'randomized', 140 | the last label is included. 141 | 142 | Returns 143 | ------- 144 | NDArray of shape (n_samples, n_alpha) 145 | Index of the last included sorted probability. 146 | """ 147 | if include_last_label or include_last_label == 'randomized': 148 | y_pred_index_last = ( 149 | np.ma.masked_less( 150 | y_pred_proba_cumsum 151 | - threshold[np.newaxis, :], 152 | -EPSILON 153 | ).argmin(axis=1) 154 | ) 155 | else: 156 | max_threshold = np.maximum( 157 | threshold[np.newaxis, :], 158 | np.min(y_pred_proba_cumsum, axis=1) 159 | ) 160 | y_pred_index_last = np.argmax( 161 | np.ma.masked_greater( 162 | y_pred_proba_cumsum - max_threshold[:, np.newaxis, :], 163 | EPSILON 164 | ), axis=1 165 | ) 166 | return y_pred_index_last[:, np.newaxis, :] 167 | -------------------------------------------------------------------------------- /mapie/conformity_scores/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, no_type_check 2 | 3 | from sklearn.utils.multiclass import ( 4 | check_classification_targets, 5 | type_of_target, 6 | ) 7 | 8 | from .regression import BaseRegressionScore 9 | from .classification import BaseClassificationScore 10 | from .bounds import ( 11 | AbsoluteConformityScore, 12 | GammaConformityScore, 13 | ResidualNormalisedScore, 14 | ) 15 | from .sets import ( 16 | LACConformityScore, 17 | TopKConformityScore, 18 | APSConformityScore, 19 | RAPSConformityScore, 20 | ) 21 | 22 | from numpy.typing import ArrayLike 23 | 24 | 25 | CONFORMITY_SCORES_STRING_MAP = { 26 | BaseRegressionScore: { 27 | "absolute": AbsoluteConformityScore, 28 | "gamma": GammaConformityScore, 29 | "residual_normalized": ResidualNormalisedScore, 30 | }, 31 | BaseClassificationScore: { 32 | "lac": LACConformityScore, 33 | "top_k": TopKConformityScore, 34 | "aps": APSConformityScore, 35 | "raps": RAPSConformityScore, 36 | }, 37 | } 38 | 39 | 40 | @no_type_check # Cumbersome to type 41 | def check_and_select_conformity_score(conformity_score, conformity_score_type): 42 | if isinstance(conformity_score, conformity_score_type): 43 | return conformity_score 44 | elif conformity_score in CONFORMITY_SCORES_STRING_MAP[conformity_score_type]: 45 | return CONFORMITY_SCORES_STRING_MAP[conformity_score_type][conformity_score]() 46 | else: 47 | raise ValueError("Invalid conformity_score parameter") 48 | 49 | 50 | def check_regression_conformity_score( 51 | conformity_score: Optional[BaseRegressionScore], 52 | sym: bool = True, 53 | ) -> BaseRegressionScore: 54 | """ 55 | Check parameter ``conformity_score`` for regression task. 56 | By default, return a AbsoluteConformityScore instance. 57 | 58 | Parameters 59 | ---------- 60 | conformity_score: BaseClassificationScore 61 | Conformity score function. 62 | 63 | By default, `None`. 64 | 65 | sym: bool 66 | Whether to use symmetric bounds. 67 | 68 | By default, `True`. 69 | 70 | Raises 71 | ------ 72 | ValueError 73 | If parameters are not valid. 74 | 75 | Examples 76 | -------- 77 | >>> from mapie.conformity_scores.utils import ( 78 | ... check_regression_conformity_score 79 | ... ) 80 | >>> try: 81 | ... check_regression_conformity_score(1) 82 | ... except Exception as exception: 83 | ... print(exception) 84 | ... 85 | Invalid conformity_score argument. 86 | Must be None or a BaseRegressionScore instance. 87 | """ 88 | if conformity_score is None: 89 | return AbsoluteConformityScore(sym=sym) 90 | elif isinstance(conformity_score, BaseRegressionScore): 91 | return conformity_score 92 | else: 93 | raise ValueError( 94 | "Invalid conformity_score argument.\n" 95 | "Must be None or a BaseRegressionScore instance." 96 | ) 97 | 98 | 99 | def check_target( 100 | conformity_score: BaseClassificationScore, 101 | y: ArrayLike 102 | ) -> None: 103 | """ 104 | Check that if the type of target is binary, 105 | (then the method have to be ``"lac"``), or multi-class. 106 | 107 | Parameters 108 | ---------- 109 | conformity_score: BaseClassificationScore 110 | Conformity score function. 111 | 112 | y: NDArray of shape (n_samples,) 113 | Training labels. 114 | 115 | Raises 116 | ------ 117 | ValueError 118 | If type of target is binary and method is not ``"lac"`` 119 | or ``"score"`` or if type of target is not multi-class. 120 | """ 121 | check_classification_targets(y) 122 | if ( 123 | type_of_target(y) == "binary" and 124 | not isinstance(conformity_score, LACConformityScore) 125 | ): 126 | raise ValueError( 127 | "Invalid conformity score for binary target. " 128 | "The only valid score is 'lac'." 129 | ) 130 | 131 | 132 | def check_classification_conformity_score( 133 | conformity_score: Optional[BaseClassificationScore] = None, 134 | ) -> BaseClassificationScore: 135 | """ 136 | Check parameter ``conformity_score`` for classification task. 137 | By default, return a LACConformityScore instance. 138 | 139 | Parameters 140 | ---------- 141 | conformity_score: BaseClassificationScore 142 | Conformity score function. 143 | 144 | By default, `None`. 145 | 146 | Raises 147 | ------ 148 | ValueError 149 | If conformity_score is not valid. 150 | """ 151 | if conformity_score is not None: 152 | if isinstance(conformity_score, BaseClassificationScore): 153 | return conformity_score 154 | else: 155 | raise ValueError( 156 | "Invalid conformity_score argument.\n" 157 | "Must be None or a BaseClassificationScore instance." 158 | ) 159 | else: 160 | return LACConformityScore() 161 | -------------------------------------------------------------------------------- /mapie/control_risk/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/mapie/control_risk/__init__.py -------------------------------------------------------------------------------- /mapie/control_risk/ltt.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from typing import Any, List, Optional, Tuple 3 | 4 | import numpy as np 5 | 6 | from numpy.typing import ArrayLike, NDArray 7 | 8 | from .p_values import compute_hoeffdding_bentkus_p_value 9 | 10 | 11 | def ltt_procedure( 12 | r_hat: NDArray, 13 | alpha_np: NDArray, 14 | delta: Optional[float], 15 | n_obs: int 16 | ) -> Tuple[List[List[Any]], NDArray]: 17 | """ 18 | Apply the Learn-Then-Test procedure for risk control. 19 | Note that we will do a multiple test for ``r_hat`` that are 20 | less than level ``alpha_np``. 21 | The procedure follows the instructions in [1]: 22 | - Calculate p-values for each lambdas descretized 23 | - Apply a family wise error rate algorithm, 24 | here Bonferonni correction 25 | - Return the index lambdas that give you the control 26 | at alpha level 27 | 28 | Parameters 29 | ---------- 30 | r_hat: NDArray of shape (n_lambdas, ). 31 | Empirical risk with respect 32 | to the lambdas. 33 | Here lambdas are thresholds that impact decision making, 34 | therefore empirical risk. 35 | 36 | alpha_np: NDArray of shape (n_alpha, ). 37 | Contains the different alphas control level. 38 | The empirical risk should be less than alpha with 39 | probability 1-delta. 40 | 41 | delta: float. 42 | Probability of not controlling empirical risk. 43 | Correspond to proportion of failure we don't 44 | want to exceed. 45 | 46 | Returns 47 | ------- 48 | valid_index: List[List[Any]]. 49 | Contain the valid index that satisfy fwer control 50 | for each alpha (length aren't the same for each alpha). 51 | 52 | p_values: NDArray of shape (n_lambda, n_alpha). 53 | Contains the values of p_value for different alpha. 54 | 55 | References 56 | ---------- 57 | [1] Angelopoulos, A. N., Bates, S., Candès, E. J., Jordan, 58 | M. I., & Lei, L. (2021). Learn then test: 59 | "Calibrating predictive algorithms to achieve risk control". 60 | """ 61 | if delta is None: 62 | raise ValueError( 63 | "Invalid delta: delta cannot be None while" 64 | + " controlling precision with LTT. " 65 | ) 66 | p_values = compute_hoeffdding_bentkus_p_value(r_hat, n_obs, alpha_np) 67 | N = len(p_values) 68 | valid_index = [] 69 | for i in range(len(alpha_np)): 70 | l_index = np.where(p_values[:, i] <= delta/N)[0].tolist() 71 | valid_index.append(l_index) 72 | return valid_index, p_values 73 | 74 | 75 | def find_lambda_control_star( 76 | r_hat: NDArray, 77 | valid_index: List[List[Any]], 78 | lambdas: NDArray 79 | ) -> Tuple[ArrayLike, ArrayLike]: 80 | """ 81 | Return the lambda that give the minimum precision along 82 | the lambdas that satisfy FWER control. 83 | Note: When a list in valid_index is empty, we can assume that 84 | lambda is equal to 1, as no lambda can verify the FWER control. 85 | And so to make our statement clear we also assume that the risk is 86 | high. 87 | 88 | Parameters 89 | ---------- 90 | r_hat: NDArray of shape (n_lambdas, n_alpha) 91 | Empirical risk with respect 92 | to the lambdas and to the alphas. 93 | Here, lambdas are thresholds that impact decision-making 94 | and therefore the empirical risk. 95 | Alphas are levels of empirical risk control such that 96 | the empirical risk is less than alpha. 97 | 98 | valid_index: List[List[Any]]. 99 | Contain the valid index that satisfy fwer control 100 | for each alpha (length aren't the same for each alpha). 101 | 102 | lambdas: NDArray of shape (n_lambda, ) 103 | Discretize parameters use for ltt procedure. 104 | 105 | Returns 106 | ------- 107 | l_lambda_star: ArrayLike of shape (n_alpha, ). 108 | The lambda that give the highest precision 109 | for a given alpha. 110 | 111 | r_star: ArrayLike of shape (n_alpha, ). 112 | The value of lowest risk for a given alpha. 113 | """ 114 | if [] in valid_index: 115 | warnings.warn( 116 | """ 117 | Warning: At least one sequence is empty! 118 | """ 119 | ) 120 | l_lambda_star = [] # type: List[Any] 121 | l_r_star = [] # type: List[Any] 122 | for i in range(len(valid_index)): 123 | if len(valid_index[i]) == 0: 124 | l_lambda_star.append(1) 125 | l_r_star.append(1) 126 | else: 127 | idx = np.argmin(valid_index[i]) 128 | l_lambda_star.append(lambdas[valid_index[i][idx]]) 129 | l_r_star.append(r_hat[valid_index[i][idx]]) 130 | 131 | return l_lambda_star, l_r_star 132 | -------------------------------------------------------------------------------- /mapie/control_risk/p_values.py: -------------------------------------------------------------------------------- 1 | from typing import Union, cast 2 | 3 | import numpy as np 4 | from scipy.stats import binom 5 | 6 | from numpy.typing import NDArray 7 | from mapie.utils import _check_alpha 8 | 9 | 10 | def compute_hoeffdding_bentkus_p_value( 11 | r_hat: NDArray, 12 | n_obs: int, 13 | alpha: Union[float, NDArray] 14 | ) -> NDArray: 15 | """ 16 | The method computes the p_values according to 17 | the Hoeffding_Bentkus inequality for each 18 | alpha. 19 | We return the minimum between the Hoeffding and 20 | Bentkus p-values (Note that it depends on 21 | scipy.stats). The p_value is introduced in 22 | learn then test paper [1]. 23 | 24 | Parameters 25 | ---------- 26 | r_hat: NDArray of shape (n_lambdas, ) 27 | Empirical risk with respect 28 | to the lambdas. 29 | Here lambdas are thresholds that impact decision 30 | making and therefore empirical risk. 31 | 32 | n_obs: int. 33 | Correspond to the number of observations in 34 | dataset. 35 | 36 | alpha: Union[float, Iterable[float]]. 37 | Contains the different alphas control level. 38 | The empirical risk must be less than alpha. 39 | If it is a iterable, it is a NDArray of shape 40 | (n_alpha, ). 41 | 42 | Returns 43 | ------- 44 | hb_p_values: NDArray of shape (n_lambda, n_alpha). 45 | 46 | References 47 | ---------- 48 | [1] Angelopoulos, A. N., Bates, S., Candès, E. J., Jordan, 49 | M. I., & Lei, L. (2021). Learn then test: 50 | "Calibrating predictive algorithms to achieve risk control". 51 | """ 52 | alpha_np = cast(NDArray, _check_alpha(alpha)) 53 | alpha_np = alpha_np[:, np.newaxis] 54 | r_hat_repeat = np.repeat( 55 | np.expand_dims(r_hat, axis=1), 56 | len(alpha_np), 57 | axis=1 58 | ) 59 | alpha_repeat = np.repeat( 60 | alpha_np.reshape(1, -1), 61 | len(r_hat), 62 | axis=0 63 | ) 64 | hoeffding_p_value = np.exp( 65 | -n_obs * _h1( 66 | np.where( 67 | r_hat_repeat > alpha_repeat, 68 | alpha_repeat, 69 | r_hat_repeat 70 | ), 71 | alpha_repeat 72 | ) 73 | ) 74 | bentkus_p_value = np.e * binom.cdf( 75 | np.ceil(n_obs * r_hat_repeat), n_obs, alpha_repeat 76 | ) 77 | hb_p_value = np.where( 78 | bentkus_p_value > hoeffding_p_value, 79 | hoeffding_p_value, 80 | bentkus_p_value 81 | ) 82 | return hb_p_value 83 | 84 | 85 | def _h1( 86 | r_hats: NDArray, 87 | alphas: NDArray 88 | ) -> NDArray: 89 | """ 90 | This function allow us to compute 91 | the tighter version of hoeffding inequality. 92 | This function is then used in the 93 | hoeffding_bentkus_p_value function for the 94 | computation of p-values. 95 | 96 | Parameters 97 | ---------- 98 | r_hats: NDArray of shape (n_lambdas, n_alpha). 99 | Empirical risk with respect 100 | to the lambdas. 101 | Here lambdas are thresholds that impact decision 102 | making and therefore empirical risk. 103 | The value table has an extended dimension of 104 | shape (n_lambda, n_alpha). 105 | 106 | alphas: NDArray of shape (n_lambdas, n_alpha). 107 | Contains the different alphas control level. 108 | In other words, empirical risk must be less 109 | than each alpha in alphas. 110 | The value table has an extended dimension of 111 | shape (n_lambda, n_alpha). 112 | 113 | Returns 114 | ------- 115 | NDArray of shape a(n_lambdas, n_alpha). 116 | """ 117 | elt1 = r_hats * np.log(r_hats/alphas) 118 | elt2 = (1-r_hats) * np.log((1-r_hats)/(1-alphas)) 119 | return elt1 + elt2 120 | -------------------------------------------------------------------------------- /mapie/control_risk/risks.py: -------------------------------------------------------------------------------- 1 | from typing import cast 2 | 3 | import numpy as np 4 | from sklearn.utils.validation import column_or_1d 5 | 6 | from numpy.typing import NDArray 7 | 8 | 9 | def compute_risk_recall( 10 | lambdas: NDArray, 11 | y_pred_proba: NDArray, 12 | y: NDArray 13 | ) -> NDArray: 14 | """ 15 | In `PrecisionRecallController` when `metric_control=recall`, 16 | compute the recall per observation for each different 17 | thresholds lambdas. 18 | 19 | Parameters 20 | ---------- 21 | y_pred_proba: NDArray of shape (n_samples, n_labels, 1) 22 | Predicted probabilities for each label and each observation. 23 | 24 | y: NDArray of shape (n_samples, n_labels) 25 | True labels. 26 | 27 | lambdas: NDArray of shape (n_lambdas, ) 28 | Threshold that permit to compute recall. 29 | 30 | Returns 31 | ------- 32 | NDArray of shape (n_samples, n_labels, n_lambdas) 33 | Risks for each observation and each value of lambda. 34 | """ 35 | if y_pred_proba.ndim != 3: 36 | raise ValueError( 37 | "y_pred_proba should be a 3d array, got an array of shape " 38 | "{} instead.".format(y_pred_proba.shape) 39 | ) 40 | if y.ndim != 2: 41 | raise ValueError( 42 | "y should be a 2d array, got an array of shape " 43 | "{} instead.".format(y_pred_proba.shape) 44 | ) 45 | if not np.array_equal(y_pred_proba.shape[:-1], y.shape): 46 | raise ValueError( 47 | "y and y_pred_proba could not be broadcast." 48 | ) 49 | lambdas = cast(NDArray, column_or_1d(lambdas)) 50 | 51 | n_lambdas = len(lambdas) 52 | y_pred_proba_repeat = np.repeat( 53 | y_pred_proba, 54 | n_lambdas, 55 | axis=2 56 | ) 57 | y_pred_th = (y_pred_proba_repeat > lambdas).astype(int) 58 | 59 | y_repeat = np.repeat(y[..., np.newaxis], n_lambdas, axis=2) 60 | risks = 1 - ( 61 | _true_positive(y_pred_th, y_repeat) / 62 | y.sum(axis=1)[:, np.newaxis] 63 | ) 64 | return risks 65 | 66 | 67 | def compute_risk_precision( 68 | lambdas: NDArray, 69 | y_pred_proba: NDArray, 70 | y: NDArray 71 | ) -> NDArray: 72 | """ 73 | In `PrecisionRecallController` when `metric_control=precision`, 74 | compute the precision per observation for each different 75 | thresholds lambdas. 76 | 77 | Parameters 78 | ---------- 79 | y_pred_proba: NDArray of shape (n_samples, n_labels, 1) 80 | Predicted probabilities for each label and each observation. 81 | 82 | y: NDArray of shape (n_samples, n_labels) 83 | True labels. 84 | 85 | lambdas: NDArray of shape (n_lambdas, ) 86 | Threshold that permit to compute precision score. 87 | 88 | Returns 89 | ------- 90 | NDArray of shape (n_samples, n_labels, n_lambdas) 91 | Risks for each observation and each value of lambda. 92 | """ 93 | if y_pred_proba.ndim != 3: 94 | raise ValueError( 95 | "y_pred_proba should be a 3d array, got an array of shape " 96 | "{} instead.".format(y_pred_proba.shape) 97 | ) 98 | if y.ndim != 2: 99 | raise ValueError( 100 | "y should be a 2d array, got an array of shape " 101 | "{} instead.".format(y_pred_proba.shape) 102 | ) 103 | if not np.array_equal(y_pred_proba.shape[:-1], y.shape): 104 | raise ValueError( 105 | "y and y_pred_proba could not be broadcast." 106 | ) 107 | lambdas = cast(NDArray, column_or_1d(lambdas)) 108 | 109 | n_lambdas = len(lambdas) 110 | y_pred_proba_repeat = np.repeat( 111 | y_pred_proba, 112 | n_lambdas, 113 | axis=2 114 | ) 115 | y_pred_th = (y_pred_proba_repeat > lambdas).astype(int) 116 | 117 | y_repeat = np.repeat(y[..., np.newaxis], n_lambdas, axis=2) 118 | with np.errstate(divide='ignore', invalid="ignore"): 119 | risks = 1 - _true_positive(y_pred_th, y_repeat)/y_pred_th.sum(axis=1) 120 | risks[np.isnan(risks)] = 1 # nan value indicate high risks. 121 | 122 | return risks 123 | 124 | 125 | def _true_positive( 126 | y_pred_th: NDArray, 127 | y_repeat: NDArray 128 | ) -> NDArray: 129 | """ 130 | Compute the number of true positive. 131 | 132 | Parameters 133 | ---------- 134 | y_pred_proba : NDArray of shape (n_samples, n_labels, 1) 135 | Predicted probabilities for each label and each observation. 136 | 137 | y: NDArray of shape (n_samples, n_labels) 138 | True labels. 139 | 140 | Returns 141 | ------- 142 | tp: float 143 | The number of true positive. 144 | """ 145 | tp = (y_pred_th * y_repeat).sum(axis=1) 146 | return tp 147 | -------------------------------------------------------------------------------- /mapie/estimator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/mapie/estimator/__init__.py -------------------------------------------------------------------------------- /mapie/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from . import classification 2 | from . import calibration 3 | from . import regression 4 | 5 | __all__ = [ 6 | "classification", 7 | "calibration", 8 | "regression", 9 | ] 10 | -------------------------------------------------------------------------------- /mapie/regression/__init__.py: -------------------------------------------------------------------------------- 1 | from .quantile_regression import ConformalizedQuantileRegressor 2 | from .regression import ( 3 | SplitConformalRegressor, 4 | CrossConformalRegressor, 5 | JackknifeAfterBootstrapRegressor, 6 | ) 7 | from .time_series_regression import TimeSeriesRegressor 8 | 9 | __all__ = [ 10 | "TimeSeriesRegressor", 11 | "SplitConformalRegressor", 12 | "CrossConformalRegressor", 13 | "JackknifeAfterBootstrapRegressor", 14 | "ConformalizedQuantileRegressor", 15 | ] 16 | -------------------------------------------------------------------------------- /mapie/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/MAPIE/d236caf6be82a786c4562752462830b9f88ce41e/mapie/tests/__init__.py -------------------------------------------------------------------------------- /mapie/tests/test_aggregations_functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from mapie.aggregation_functions import phi1D, phi2D 4 | 5 | 6 | def test_phi1D() -> None: 7 | """Test the result of phi1D.""" 8 | x = np.array([1, 2, 3, 4, 5]) 9 | B = np.array([[1, 1, 1, np.nan, np.nan], [np.nan, np.nan, np.nan, 1, 1]]) 10 | res = phi1D(x, B, fun=lambda x: np.nanmean(x, axis=1)) 11 | assert res[0] == 2.0 12 | 13 | 14 | def test_phi2D() -> None: 15 | """Test the result of phi2D.""" 16 | A = np.array([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]) 17 | B = np.array([[1, 1, 1, np.nan, np.nan], [np.nan, np.nan, np.nan, 1, 1]]) 18 | res = phi2D(A, B, fun=lambda x: np.nanmean(x, axis=1)) 19 | assert res[0, 0] == 2.0 20 | assert res[1, 0] == 7.0 21 | -------------------------------------------------------------------------------- /mapie/tests/test_conformity_scores_sets.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, cast 2 | 3 | import pytest 4 | import numpy as np 5 | from sklearn.datasets import make_classification 6 | from sklearn.linear_model import LogisticRegression 7 | 8 | from numpy.typing import NDArray 9 | from mapie.classification import _MapieClassifier 10 | from mapie.conformity_scores import BaseClassificationScore 11 | from mapie.conformity_scores.sets import ( 12 | APSConformityScore, LACConformityScore, NaiveConformityScore, 13 | RAPSConformityScore, TopKConformityScore 14 | ) 15 | from mapie.conformity_scores.utils import check_classification_conformity_score 16 | from mapie.utils import _check_alpha 17 | 18 | 19 | random_state = 42 20 | 21 | cs_list = [ 22 | None, LACConformityScore(), APSConformityScore(), RAPSConformityScore(), 23 | NaiveConformityScore(), TopKConformityScore() 24 | ] 25 | wrong_cs_list = [object(), "LAC", 1] 26 | valid_method_list = ['naive', 'aps', 'raps', 'lac', 'top_k'] 27 | all_method_list = valid_method_list + [None] 28 | wrong_method_list = ['naive_', 'aps_', 'raps_', 'lac_', 'top_k_'] 29 | 30 | REGULARIZATION_PARAMETERS = [ 31 | [.001, [1]], 32 | [[.01, .2], [1, 3]], 33 | [.1, [2, 4]] 34 | ] 35 | 36 | X_toy = np.arange(9).reshape(-1, 1) 37 | y_toy = np.array([0, 0, 1, 0, 1, 1, 2, 1, 2]) 38 | y_toy_string = np.array(["0", "0", "1", "0", "1", "1", "2", "1", "2"]) 39 | 40 | n_classes = 4 41 | X, y = make_classification( 42 | n_samples=500, 43 | n_features=10, 44 | n_informative=3, 45 | n_classes=n_classes, 46 | random_state=random_state, 47 | ) 48 | 49 | 50 | def test_error_mother_class_initialization() -> None: 51 | """ 52 | Test that the mother class BaseClassificationScore cannot be instantiated. 53 | """ 54 | with pytest.raises(TypeError): 55 | BaseClassificationScore() # type: ignore 56 | 57 | 58 | @pytest.mark.parametrize("conformity_score", cs_list) 59 | def test_check_classification_conformity_score( 60 | conformity_score: Optional[BaseClassificationScore] 61 | ) -> None: 62 | """ 63 | Test that the function check_classification_conformity_score returns 64 | an instance of BaseClassificationScore when using conformity_score. 65 | """ 66 | assert isinstance( 67 | check_classification_conformity_score(conformity_score), 68 | BaseClassificationScore 69 | ) 70 | 71 | 72 | @pytest.mark.parametrize("score", wrong_cs_list) 73 | def test_check_wrong_classification_score( 74 | score: Any 75 | ) -> None: 76 | """ 77 | Test that the function check_classification_conformity_score raises 78 | a ValueError when using a wrong score. 79 | """ 80 | with pytest.raises(ValueError, match="Invalid conformity_score argument*"): 81 | check_classification_conformity_score(conformity_score=score) 82 | 83 | 84 | @pytest.mark.parametrize("k_lambda", REGULARIZATION_PARAMETERS) 85 | def test_regularize_conf_scores_shape(k_lambda) -> None: 86 | """ 87 | Test that the conformity scores have the correct shape. 88 | """ 89 | lambda_, k = k_lambda[0], k_lambda[1] 90 | conf_scores = np.random.rand(100, 1) 91 | cutoff = np.cumsum(np.ones(conf_scores.shape)) - 1 92 | reg_conf_scores = RAPSConformityScore._regularize_conformity_score( 93 | k, lambda_, conf_scores, cutoff 94 | ) 95 | 96 | assert reg_conf_scores.shape == (100, 1, len(k)) 97 | 98 | 99 | def test_get_true_label_cumsum_proba_shape() -> None: 100 | """ 101 | Test that the true label cumsumed probabilities 102 | have the correct shape. 103 | """ 104 | clf = LogisticRegression() 105 | clf.fit(X, y) 106 | y_pred = clf.predict_proba(X) 107 | mapie_clf = _MapieClassifier( 108 | estimator=clf, random_state=random_state 109 | ) 110 | mapie_clf.fit(X, y) 111 | classes = mapie_clf.classes_ 112 | cumsum_proba, cutoff = APSConformityScore.get_true_label_cumsum_proba( 113 | y, y_pred, classes 114 | ) 115 | assert cumsum_proba.shape == (len(X), 1) 116 | assert cutoff.shape == (len(X), ) 117 | 118 | 119 | def test_get_true_label_cumsum_proba_result() -> None: 120 | """ 121 | Test that the true label cumsumed probabilities 122 | are the expected ones. 123 | """ 124 | clf = LogisticRegression() 125 | clf.fit(X_toy, y_toy) 126 | y_pred = clf.predict_proba(X_toy) 127 | mapie_clf = _MapieClassifier( 128 | estimator=clf, random_state=random_state 129 | ) 130 | mapie_clf.fit(X_toy, y_toy) 131 | classes = mapie_clf.classes_ 132 | cumsum_proba, cutoff = APSConformityScore.get_true_label_cumsum_proba( 133 | y_toy, y_pred, classes 134 | ) 135 | np.testing.assert_allclose( 136 | cumsum_proba, 137 | np.array( 138 | [ 139 | y_pred[0, 0], y_pred[1, 0], 140 | y_pred[2, 0] + y_pred[2, 1], 141 | y_pred[3, 0] + y_pred[3, 1], 142 | y_pred[4, 1], y_pred[5, 1], 143 | y_pred[6, 1] + y_pred[6, 2], 144 | y_pred[7, 1] + y_pred[7, 2], 145 | y_pred[8, 2] 146 | ] 147 | )[:, np.newaxis] 148 | ) 149 | np.testing.assert_allclose(cutoff, np.array([1, 1, 2, 2, 1, 1, 2, 2, 1])) 150 | 151 | 152 | @pytest.mark.parametrize("k_lambda", REGULARIZATION_PARAMETERS) 153 | @pytest.mark.parametrize("include_last_label", [True, False]) 154 | def test_get_last_included_proba_shape(k_lambda, include_last_label): 155 | """ 156 | Test that the outputs of _get_last_included_proba method 157 | have the correct shape. 158 | """ 159 | lambda_, k = k_lambda[0], k_lambda[1] 160 | if len(k) == 1: 161 | thresholds = .2 162 | else: 163 | thresholds = np.random.rand(len(k)) 164 | thresholds = cast(NDArray, _check_alpha(thresholds)) 165 | clf = LogisticRegression() 166 | clf.fit(X, y) 167 | y_pred_proba = clf.predict_proba(X) 168 | y_pred_proba = np.repeat( 169 | y_pred_proba[:, :, np.newaxis], len(thresholds), axis=2 170 | ) 171 | 172 | y_p_p_c, y_p_i_l, y_p_p_i_l = \ 173 | RAPSConformityScore._get_last_included_proba( 174 | RAPSConformityScore(), y_pred_proba, thresholds, 175 | include_last_label, lambda_=lambda_, k_star=k 176 | ) 177 | 178 | assert y_p_p_c.shape == (len(X), len(np.unique(y)), len(thresholds)) 179 | assert y_p_i_l.shape == (len(X), 1, len(thresholds)) 180 | assert y_p_p_i_l.shape == (len(X), 1, len(thresholds)) 181 | -------------------------------------------------------------------------------- /mapie/tests/test_conformity_scores_utils.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from mapie.conformity_scores.sets.utils import get_true_label_position 7 | from numpy.typing import NDArray 8 | 9 | Y_TRUE_PROBA_PLACE = [ 10 | [ 11 | np.array([2, 0]), 12 | np.array([ 13 | [.1, .3, .6], 14 | [.2, .7, .1] 15 | ]), 16 | np.array([[0], [1]]) 17 | ], 18 | [ 19 | np.array([1, 0]), 20 | np.array([ 21 | [.7, .12, .18], 22 | [.5, .24, .26] 23 | ]), 24 | np.array([[2], [0]]) 25 | ] 26 | ] 27 | 28 | 29 | def test_shape_get_true_label_position() -> None: 30 | """ 31 | Check the shape returned by the function 32 | """ 33 | y_pred_proba = np.random.rand(5, 3) 34 | y = np.random.randint(0, 3, size=(5, 1)) 35 | position = get_true_label_position(y_pred_proba, y) 36 | assert position.shape == y.shape 37 | 38 | 39 | @pytest.mark.parametrize("y_true_proba_place", Y_TRUE_PROBA_PLACE) 40 | def test_get_true_label_position( 41 | y_true_proba_place: List[NDArray] 42 | ) -> None: 43 | """ 44 | Check that the returned true label position the good. 45 | """ 46 | y_true = y_true_proba_place[0] 47 | y_pred_proba = y_true_proba_place[1] 48 | place = y_true_proba_place[2] 49 | 50 | found_place = get_true_label_position(y_pred_proba, y_true) 51 | 52 | assert (found_place == place).all() 53 | -------------------------------------------------------------------------------- /mapie/tests/test_control_risk.py: -------------------------------------------------------------------------------- 1 | """ 2 | Testing for control_risk module. 3 | Testing for now risks for multilabel classification 4 | """ 5 | from typing import List, Optional, Union 6 | 7 | import numpy as np 8 | import pytest 9 | 10 | from numpy.typing import NDArray 11 | from mapie.control_risk.ltt import find_lambda_control_star, ltt_procedure 12 | from mapie.control_risk.p_values import compute_hoeffdding_bentkus_p_value 13 | from mapie.control_risk.risks import (compute_risk_precision, 14 | compute_risk_recall) 15 | 16 | lambdas = np.array([0.5, 0.9]) 17 | 18 | y_toy = np.stack([ 19 | [1, 0, 1], 20 | [0, 1, 0], 21 | [1, 1, 0], 22 | [1, 1, 1], 23 | ]) 24 | 25 | y_preds_proba = np.stack([ 26 | [0.2, 0.6, 0.9], 27 | [0.8, 0.2, 0.6], 28 | [0.4, 0.8, 0.1], 29 | [0.6, 0.8, 0.7] 30 | ]) 31 | 32 | y_preds_proba = np.expand_dims(y_preds_proba, axis=2) 33 | 34 | test_recall = np.array([ 35 | [1/2, 1.], 36 | [1., 1.], 37 | [1/2, 1.], 38 | [0., 1.] 39 | ]) 40 | 41 | test_precision = np.array([ 42 | [1/2, 1.], 43 | [1., 1.], 44 | [0., 1.], 45 | [0., 1.] 46 | ]) 47 | 48 | r_hat = np.array([0.5, 0.8]) 49 | 50 | n = 1100 51 | 52 | alpha = np.array([0.6]) 53 | 54 | valid_index = [[0, 1]] 55 | 56 | wrong_alpha = 0 57 | 58 | wrong_alpha_shape = np.array([ 59 | [0.1, 0.2], 60 | [0.3, 0.4] 61 | ]) 62 | 63 | random_state = 42 64 | prng = np.random.RandomState(random_state) 65 | 66 | 67 | def test_compute_recall_equal() -> None: 68 | """Test that compute_recall give good result""" 69 | recall = compute_risk_recall(lambdas, y_preds_proba, y_toy) 70 | np.testing.assert_equal(recall, test_recall) 71 | 72 | 73 | def test_compute_precision() -> None: 74 | """Test that compute_precision give good result""" 75 | precision = compute_risk_precision(lambdas, y_preds_proba, y_toy) 76 | np.testing.assert_equal(precision, test_precision) 77 | 78 | 79 | def test_recall_with_zero_sum_is_equal_nan() -> None: 80 | """Test compute_recall with nan values""" 81 | y_toy = np.zeros((4, 3)) 82 | y_preds_proba = prng.rand(4, 3, 1) 83 | recall = compute_risk_recall(lambdas, y_preds_proba, y_toy) 84 | np.testing.assert_array_equal(recall, np.empty_like(recall)) 85 | 86 | 87 | def test_precision_with_zero_sum_is_equal_ones() -> None: 88 | """Test compute_precision with nan values""" 89 | y_toy = prng.rand(4, 3) 90 | y_preds_proba = np.zeros((4, 3, 1)) 91 | precision = compute_risk_precision(lambdas, y_preds_proba, y_toy) 92 | np.testing.assert_array_equal(precision, np.ones_like(precision)) 93 | 94 | 95 | def test_compute_recall_shape() -> None: 96 | """Test shape when using _compute_recall""" 97 | recall = compute_risk_recall(lambdas, y_preds_proba, y_toy) 98 | np.testing.assert_equal(recall.shape, test_recall.shape) 99 | 100 | 101 | def test_compute_precision_shape() -> None: 102 | """Test shape when using _compute_precision""" 103 | precision = compute_risk_precision(lambdas, y_preds_proba, y_toy) 104 | np.testing.assert_equal(precision.shape, test_precision.shape) 105 | 106 | 107 | def test_compute_recall_with_wrong_shape() -> None: 108 | """Test error when wrong shape in _compute_recall""" 109 | with pytest.raises(ValueError, match=r".*y_pred_proba should be a 3d*"): 110 | compute_risk_recall(lambdas, y_preds_proba.squeeze(), y_toy) 111 | with pytest.raises(ValueError, match=r".*y should be a 2d*"): 112 | compute_risk_recall(lambdas, y_preds_proba, np.expand_dims(y_toy, 2)) 113 | with pytest.raises(ValueError, match=r".*could not be broadcast*"): 114 | compute_risk_recall(lambdas, y_preds_proba, y_toy[:-1]) 115 | 116 | 117 | def test_compute_precision_with_wrong_shape() -> None: 118 | """Test shape when using _compute_precision""" 119 | with pytest.raises(ValueError, match=r".*y_pred_proba should be a 3d*"): 120 | compute_risk_precision(lambdas, y_preds_proba.squeeze(), y_toy) 121 | with pytest.raises(ValueError, match=r".*y should be a 2d*"): 122 | compute_risk_precision( 123 | lambdas, y_preds_proba, np.expand_dims(y_toy, 2) 124 | ) 125 | with pytest.raises(ValueError, match=r".*could not be broadcast*"): 126 | compute_risk_precision(lambdas, y_preds_proba, y_toy[:-1]) 127 | 128 | 129 | @pytest.mark.parametrize("alpha", [0.5, [0.5], [0.5, 0.9]]) 130 | def test_p_values_different_alpha(alpha: Union[float, NDArray]) -> None: 131 | """Test type for different alpha for p_values""" 132 | result = compute_hoeffdding_bentkus_p_value(r_hat, n, alpha) 133 | assert isinstance(result, np.ndarray) 134 | 135 | 136 | @pytest.mark.parametrize("delta", [0.1, 0.2]) 137 | def test_ltt_different_delta(delta: float) -> None: 138 | """Test _ltt_procedure for different delta""" 139 | assert ltt_procedure(r_hat, alpha, delta, n) 140 | 141 | 142 | def test_find_lambda_control_star() -> None: 143 | """Test _find_lambda_control_star""" 144 | assert find_lambda_control_star(r_hat, valid_index, lambdas) 145 | 146 | 147 | @pytest.mark.parametrize("delta", [0.1, 0.8]) 148 | @pytest.mark.parametrize("alpha", [[0.5], [0.6, 0.8]]) 149 | def test_ltt_type_output_alpha_delta( 150 | alpha: NDArray, 151 | delta: float 152 | ) -> None: 153 | """Test type output _ltt_procedure""" 154 | valid_index, p_values = ltt_procedure(r_hat, alpha, delta, n) 155 | assert isinstance(valid_index, list) 156 | assert isinstance(p_values, np.ndarray) 157 | 158 | 159 | @pytest.mark.parametrize("valid_index", [[[0, 1]]]) 160 | def test_find_lambda_control_star_output(valid_index: List[List[int]]) -> None: 161 | """Test _find_lambda_control_star with a list of list""" 162 | assert find_lambda_control_star(r_hat, valid_index, lambdas) 163 | 164 | 165 | def test_warning_valid_index_empty() -> None: 166 | """Test warning sent when empty list""" 167 | valid_index = [[]] # type: List[List[int]] 168 | with pytest.warns( 169 | UserWarning, match=r".*At least one sequence is empty*" 170 | ): 171 | find_lambda_control_star(r_hat, valid_index, lambdas) 172 | 173 | 174 | def test_invalid_alpha_hb() -> None: 175 | """Test error message when invalid alpha""" 176 | with pytest.raises(ValueError, match=r".*Invalid confidence_level"): 177 | compute_hoeffdding_bentkus_p_value(r_hat, n, wrong_alpha) 178 | 179 | 180 | def test_invalid_shape_alpha_hb() -> None: 181 | """Test error message when invalid alpha shape""" 182 | with pytest.raises(ValueError, match=r".*Invalid confidence_level"): 183 | compute_hoeffdding_bentkus_p_value(r_hat, n, wrong_alpha_shape) 184 | 185 | 186 | @pytest.mark.parametrize("delta", [None]) 187 | def test_delta_none_ltt(delta: Optional[float]) -> None: 188 | """Test error message when invalid delta""" 189 | with pytest.raises(ValueError, match=r".*Invalid delta"): 190 | ltt_procedure(r_hat, alpha, delta, n) 191 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.9 3 | ignore_missing_imports = True 4 | 5 | [mypy-sklearn.*] 6 | ignore_errors = True 7 | 8 | [mypy-doc.*] 9 | ignore_errors = True 10 | -------------------------------------------------------------------------------- /notebooks/Makefile: -------------------------------------------------------------------------------- 1 | convert2rst: 2 | jupyter nbconvert --to rst $(dir)/$(file).ipynb 3 | gsed -i -e'/error/d' $(dir)/$(file).rst 4 | gsed -i -e'/warning/d' $(dir)/$(file).rst 5 | gsed -i -e'/import os/d' $(dir)/$(file).rst 6 | gsed -i -e'/os.environ/d' $(dir)/$(file).rst 7 | gsed -i -e'/UserWarning/d' $(dir)/$(file).rst 8 | gsed -i -e'/WARNING:tensorflow/d' $(dir)/$(file).rst 9 | gsed -i -e's/.. code:: ipython3/.. code-block:: python/g' $(dir)/$(file).rst 10 | gsed -i -e's/``/`/g' $(dir)/$(file).rst 11 | gsed -i -e's/`TensorflowToMapie`/``TensorflowToMapie``/g' $(dir)/$(file).rst 12 | gsed -i -e'/=1.4", 23 | "scipy", 24 | "numpy>=1.23" 25 | ] 26 | classifiers = [ 27 | "Intended Audience :: Science/Research", 28 | "Intended Audience :: Developers", 29 | "Topic :: Software Development", 30 | "Topic :: Scientific/Engineering", 31 | "Operating System :: Microsoft :: Windows", 32 | "Operating System :: POSIX", 33 | "Operating System :: Unix", 34 | "Operating System :: MacOS", 35 | "Programming Language :: Python :: 3.9", 36 | "Programming Language :: Python :: 3.10", 37 | "Programming Language :: Python :: 3.11" 38 | ] 39 | 40 | [project.urls] 41 | Homepage = "https://github.com/scikit-learn-contrib/MAPIE" 42 | Documentation = "https://mapie.readthedocs.io/en/latest/" 43 | Repository = "https://github.com/scikit-learn-contrib/MAPIE" 44 | Issues = "https://github.com/scikit-learn-contrib/MAPIE/issues" 45 | Changelog = "https://github.com/scikit-learn-contrib/MAPIE/releases" 46 | DOWNLOAD = "https://pypi.org/project/MAPIE/#files" 47 | 48 | [tool.setuptools] 49 | include-package-data = true 50 | 51 | [tool.setuptools.packages.find] 52 | include = ["mapie", "mapie.*"] 53 | -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- 1 | bump2version==1.0.1 2 | build 3 | flake8 4 | mypy<1.15 5 | numpydoc 6 | numpy 7 | pandas 8 | pytest 9 | pytest-cov 10 | scikit-learn 11 | twine 12 | wheel -------------------------------------------------------------------------------- /requirements.doc.txt: -------------------------------------------------------------------------------- 1 | lightgbm==4.4.0 2 | matplotlib==3.5.1 3 | numpy==1.22.3 4 | numpydoc==1.1.0 5 | pandas==1.3.5 6 | scikit-learn==1.5.2 7 | sphinx==5.3.0 8 | sphinx-gallery==0.10.1 9 | sphinx_rtd_theme==1.0.0 10 | typing_extensions==4.0.1 -------------------------------------------------------------------------------- /tests_v1/test_functional/README.md: -------------------------------------------------------------------------------- 1 | # Scope 2 | 3 | Folder for testing the main functionalities of the API as seen from a user point-of-view. 4 | 5 | # Philosophy 6 | 7 | - New tests here should be added wisely. 8 | - Group tests in a class if more than one test is needed for a given functionality. 9 | - Be careful of test time. Testing varied scenarios is more important than trying to test all scenarios. 10 | - Write black-box tests if possible (no mocks): don't test implementation details. 11 | -------------------------------------------------------------------------------- /tests_v1/test_functional/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Dict, Any, Optional, Tuple, Union 2 | 3 | import numpy as np 4 | from sklearn.base import BaseEstimator, ClassifierMixin 5 | from typing_extensions import Self 6 | 7 | from numpy.typing import NDArray, ArrayLike 8 | import inspect 9 | from sklearn.model_selection import ShuffleSplit 10 | 11 | 12 | def train_test_split_shuffle( 13 | X: NDArray, 14 | y: NDArray, 15 | test_size: float = None, 16 | random_state: int = 42, 17 | sample_weight: Optional[NDArray] = None, 18 | ) -> Union[Tuple[Any, Any, Any, Any], Tuple[Any, Any, Any, Any, Any, Any]]: 19 | splitter = ShuffleSplit( 20 | n_splits=1, 21 | test_size=test_size, 22 | random_state=random_state 23 | ) 24 | train_idx, test_idx = next(splitter.split(X)) 25 | 26 | X_train, X_test = X[train_idx], X[test_idx] 27 | y_train, y_test = y[train_idx], y[test_idx] 28 | if sample_weight is not None: 29 | sample_weight_train = sample_weight[train_idx] 30 | sample_weight_test = sample_weight[test_idx] 31 | return X_train, X_test, y_train, y_test, sample_weight_train, sample_weight_test 32 | 33 | return X_train, X_test, y_train, y_test 34 | 35 | 36 | def filter_params( 37 | function: Callable, 38 | params: Optional[Dict[str, Any]] = None 39 | ) -> Dict[str, Any]: 40 | if params is None: 41 | return {} 42 | 43 | model_params = inspect.signature(function).parameters 44 | return {k: v for k, v in params.items() if k in model_params} 45 | 46 | 47 | class DummyClassifierWithFitAndPredictParams(BaseEstimator, ClassifierMixin): 48 | def __init__(self): 49 | self.classes_ = None 50 | self._dummy_fit_param = None 51 | 52 | def fit(self, X: ArrayLike, y: ArrayLike, dummy_fit_param: bool = False) -> Self: 53 | self.classes_ = np.unique(y) 54 | if len(self.classes_) < 2: 55 | raise ValueError("Dummy classifier needs at least 3 classes") 56 | self._dummy_fit_param = dummy_fit_param 57 | return self 58 | 59 | def predict_proba(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: 60 | probas = np.zeros((len(X), len(self.classes_))) 61 | if self._dummy_fit_param & dummy_predict_param: 62 | probas[:, 0] = 0.1 63 | probas[:, 1] = 0.9 64 | elif self._dummy_fit_param: 65 | probas[:, 1] = 0.1 66 | probas[:, 2] = 0.9 67 | elif dummy_predict_param: 68 | probas[:, 1] = 0.1 69 | probas[:, 0] = 0.9 70 | else: 71 | probas[:, 2] = 0.1 72 | probas[:, 0] = 0.9 73 | return probas 74 | 75 | def predict(self, X: ArrayLike, dummy_predict_param: bool = False) -> NDArray: 76 | y_preds_proba = self.predict_proba(X, dummy_predict_param) 77 | return np.amax(y_preds_proba, axis=0) 78 | -------------------------------------------------------------------------------- /tests_v1/test_unit/README.md: -------------------------------------------------------------------------------- 1 | # Scope 2 | 3 | Folder for testing small functions ("unit" = function). 4 | 5 | # Philosophy 6 | 7 | - Group tests in a class if more than one test is needed for a given function. 8 | - Focus on the function goal to define the test cases (the function name is a good hint). 9 | - Prefer black-box tests (no mocks) if possible, to avoid testing implementation details. 10 | -------------------------------------------------------------------------------- /tests_v1/test_unit/test_conformity_scores_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mapie.conformity_scores.utils import ( 4 | check_and_select_conformity_score, 5 | ) 6 | from mapie.conformity_scores.regression import BaseRegressionScore 7 | from mapie.conformity_scores.classification import BaseClassificationScore 8 | from mapie.conformity_scores.bounds import ( 9 | AbsoluteConformityScore, 10 | GammaConformityScore, 11 | ) 12 | from mapie.conformity_scores.sets import ( 13 | LACConformityScore, 14 | TopKConformityScore, 15 | ) 16 | 17 | 18 | class TestCheckAndSelectConformityScore: 19 | 20 | @pytest.mark.parametrize( 21 | "score, score_type, expected_class", [ 22 | (AbsoluteConformityScore(), BaseRegressionScore, AbsoluteConformityScore), 23 | ("gamma", BaseRegressionScore, GammaConformityScore), 24 | (LACConformityScore(), BaseClassificationScore, LACConformityScore), 25 | ("top_k", BaseClassificationScore, TopKConformityScore), 26 | ] 27 | ) 28 | def test_with_valid_inputs(self, score, score_type, expected_class): 29 | result = check_and_select_conformity_score(score, score_type) 30 | assert isinstance(result, expected_class) 31 | 32 | @pytest.mark.parametrize( 33 | "score_type", [BaseRegressionScore, BaseClassificationScore] 34 | ) 35 | def test_with_invalid_input(self, score_type): 36 | with pytest.raises(ValueError): 37 | check_and_select_conformity_score("I'm not a valid input :(", score_type) 38 | -------------------------------------------------------------------------------- /tests_v1/test_unit/test_regression.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mapie.subsample import Subsample 3 | from mapie.regression import JackknifeAfterBootstrapRegressor 4 | 5 | 6 | class TestCheckAndConvertResamplingToCv: 7 | def test_with_integer(self): 8 | regressor = JackknifeAfterBootstrapRegressor() 9 | cv = regressor._check_and_convert_resampling_to_cv(50) 10 | 11 | assert isinstance(cv, Subsample) 12 | assert cv.n_resamplings == 50 13 | 14 | def test_with_subsample(self): 15 | custom_subsample = Subsample(n_resamplings=25, random_state=42) 16 | regressor = JackknifeAfterBootstrapRegressor() 17 | cv = regressor._check_and_convert_resampling_to_cv(custom_subsample) 18 | 19 | assert cv is custom_subsample 20 | 21 | def test_with_invalid_input(self): 22 | regressor = JackknifeAfterBootstrapRegressor() 23 | 24 | with pytest.raises( 25 | ValueError, 26 | match="resampling must be an integer or a Subsample instance" 27 | ): 28 | regressor._check_and_convert_resampling_to_cv("invalid_input") 29 | --------------------------------------------------------------------------------