├── .coveragerc ├── .github └── workflows │ ├── docs.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── README.rst ├── benchmarks ├── test_bootstrap.py ├── test_jackknife.py ├── test_rcont.py └── test_usp.py ├── doc ├── Makefile ├── _static │ └── logo.svg ├── bench_rcont.json ├── changelog.rst ├── conf.py ├── example │ └── tag_and_probe.ipynb ├── examples.rst ├── index.rst ├── make.bat ├── make_raw_logo.py ├── plot_bench.py ├── reference.rst ├── tutorial │ ├── confidence_intervals.ipynb │ ├── jackknife_vs_bootstrap.ipynb │ ├── leave-one-out-cross-validation.ipynb │ ├── permutation_tests.ipynb │ ├── sklearn.ipynb │ ├── usp_continuous_data.ipynb │ └── variance_fit_parameters.ipynb └── tutorials.rst ├── pyproject.toml ├── src └── resample │ ├── __init__.py │ ├── _util.py │ ├── bootstrap.py │ ├── empirical.py │ ├── jackknife.py │ └── permutation.py └── tests ├── test_bootstrap.py ├── test_empirical.py ├── test_jackknife.py ├── test_permutation.py └── test_util.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = src/resample 3 | relative_files = True 4 | 5 | [report] 6 | exclude_lines = 7 | pragma: no cover 8 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | pull_request: 5 | push: 6 | tags: 7 | - '**' 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.head_ref }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | build: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 21 | # must come after checkout 22 | - uses: actions/setup-python@v5 23 | with: 24 | python-version: "3.11" 25 | - run: sudo apt-get install pandoc 26 | - run: pip install .[doc] 27 | - run: cd doc; make html 28 | - uses: actions/upload-pages-artifact@v3 29 | with: 30 | path: 'doc/_build/html' 31 | 32 | deploy: 33 | if: github.event_name == 'workflow_dispatch' || contains(github.event.ref, '/tags/') 34 | needs: build 35 | # Set permissions to allow deployment to GitHub Pages 36 | permissions: 37 | contents: read 38 | pages: write 39 | id-token: write 40 | 41 | environment: 42 | name: github-pages 43 | url: ${{ steps.deployment.outputs.page_url }} 44 | 45 | runs-on: ubuntu-latest 46 | 47 | steps: 48 | - uses: actions/configure-pages@v4 49 | - uses: actions/deploy-pages@v4 50 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '**' 7 | workflow_dispatch: 8 | 9 | concurrency: 10 | group: ${{ github.workflow }}-${{ github.head_ref }} 11 | cancel-in-progress: true 12 | 13 | env: 14 | PIP_ONLY_BINARY: ":all:" 15 | 16 | jobs: 17 | release: 18 | runs-on: ubuntu-latest 19 | 20 | environment: 21 | name: pypi 22 | url: https://pypi.org/p/resample 23 | permissions: 24 | id-token: write 25 | 26 | steps: 27 | - uses: actions/checkout@v4 28 | with: 29 | fetch-depth: 0 # needed by setuptools_scm 30 | - uses: actions/setup-python@v5 31 | with: 32 | python-version: '3.11' 33 | 34 | - run: python -m pip install --upgrade pip build 35 | - run: python -m build 36 | - run: python -m pip install --prefer-binary $(echo dist/*.whl)'[test]' 37 | - run: python -m pytest 38 | 39 | - uses: pypa/gh-action-pypi-publish@release/v1 40 | if: github.event_name == 'push' && contains(github.event.ref, '/tags/') 41 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | 6 | concurrency: 7 | group: ${{ github.workflow }}-${{ github.head_ref }} 8 | cancel-in-progress: true 9 | 10 | env: 11 | PIP_ONLY_BINARY: ":all:" 12 | 13 | jobs: 14 | test: 15 | runs-on: ${{ matrix.os }} 16 | 17 | strategy: 18 | matrix: 19 | os: [ubuntu-latest] 20 | # version number must be string, otherwise 3.10 becomes 3.1 21 | python-version: ["3.8", "3.10", "3.13"] 22 | include: 23 | - os: windows-latest 24 | python-version: "3.12" 25 | - os: macos-latest 26 | python-version: "3.9" 27 | - os: macos-13 28 | python-version: "3.11" 29 | fail-fast: false 30 | steps: 31 | - uses: actions/checkout@v4 32 | - uses: actions/setup-python@v5 33 | with: 34 | python-version: ${{ matrix.python-version }} 35 | allow-prereleases: true 36 | 37 | - uses: astral-sh/setup-uv@v3 38 | 39 | - run: uv pip install --system -e .[test] 40 | 41 | - if: matrix.os != 'ubuntu-latest' 42 | run: python -m pytest 43 | 44 | - if: matrix.os == 'ubuntu-latest' 45 | env: 46 | JUPYTER_PLATFORM_DIRS: 1 47 | run: coverage run -m pytest 48 | 49 | - if: matrix.os == 'ubuntu-latest' 50 | uses: coverallsapp/github-action@v2 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .DS_Store 3 | *.swp 4 | *checkpoints* 5 | build 6 | dist 7 | prof 8 | resample.egg-info 9 | install_log.txt 10 | *__pycache__ 11 | .pytest_cache 12 | .mypy_cache 13 | .benchmarks 14 | .coverage 15 | coverage.xml 16 | html* 17 | .doctrees 18 | _build 19 | .idea 20 | benchmarks/.asv/ 21 | junit 22 | py[0-9]* 23 | venv 24 | resample/_ext.cpython* 25 | src/resample/_ext.cpython-*.so 26 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | files: 'resample' 2 | 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v5.0.0 6 | hooks: 7 | - id: check-case-conflict 8 | - id: check-docstring-first 9 | - id: check-merge-conflict 10 | - id: check-symlinks 11 | - id: check-yaml 12 | - id: debug-statements 13 | - id: end-of-file-fixer 14 | - id: mixed-line-ending 15 | - id: sort-simple-yaml 16 | - id: file-contents-sorter 17 | - id: trailing-whitespace 18 | 19 | # Ruff linter, replacement for flake8, isort, pydocstyle 20 | - repo: https://github.com/astral-sh/ruff-pre-commit 21 | rev: 'v0.11.6' 22 | hooks: 23 | - id: ruff 24 | args: [--fix, --show-fixes, --exit-non-zero-on-fix] 25 | - id: ruff-format 26 | 27 | # Python type checking 28 | - repo: https://github.com/pre-commit/mirrors-mypy 29 | rev: 'v1.15.0' 30 | hooks: 31 | - id: mypy 32 | args: [--allow-redefinition, --ignore-missing-imports] 33 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 2 | 3 | version: 2 4 | 5 | sphinx: 6 | configuration: doc/conf.py 7 | 8 | python: 9 | version: 3.8 10 | install: 11 | - method: pip 12 | path: . 13 | extra_requirements: 14 | - doc 15 | system_packages: false 16 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # This CITATION.cff file was generated with cffinit. 2 | # Visit https://bit.ly/cffinit to generate yours today! 3 | 4 | cff-version: 1.2.0 5 | title: scikit-hep/resample 6 | message: >- 7 | If you use this software, please cite it using the 8 | metadata from this file. 9 | type: software 10 | authors: 11 | - given-names: Hans 12 | family-names: Dembinski 13 | email: hans.dembinski@gmail.com 14 | affiliation: TU Dortmund 15 | orcid: 'https://orcid.org/0000-0003-3337-3850' 16 | - given-names: Daniel 17 | family-names: Saxton 18 | - given-names: Henry 19 | family-names: Schreiner 20 | - given-names: Joshua 21 | family-names: Adelman 22 | - given-names: Eduardo 23 | family-names: Rodrigues 24 | identifiers: 25 | - type: doi 26 | value: 10.5281/zenodo.7750255 27 | repository-code: 'https://github.com/scikit-hep/resample' 28 | url: 'https://resample.readthedocs.io/en/stable/' 29 | abstract: 'Randomization-based inference in Python ' 30 | keywords: 31 | - Python 32 | - statistics 33 | - data analysis 34 | - Scikit-HEP 35 | license: BSD-3-Clause 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, Daniel D. Saxton 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, 9 | this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | * Neither the name of {{ project }} nor the names of its contributors 14 | may be used to endorse or promote products derived from this software 15 | without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. |resample| image:: doc/_static/logo.svg 2 | :alt: resample 3 | :target: http://resample.readthedocs.io 4 | 5 | |resample| 6 | ========== 7 | 8 | .. image:: https://img.shields.io/pypi/v/resample.svg 9 | :target: https://pypi.org/project/resample 10 | .. image:: https://img.shields.io/conda/vn/conda-forge/resample.svg 11 | :target: https://github.com/conda-forge/resample-feedstock 12 | .. image:: https://github.com/resample-project/resample/actions/workflows/test.yml/badge.svg 13 | :target: https://github.com/resample-project/resample/actions/workflows/tests.yml 14 | .. image:: https://coveralls.io/repos/github/resample-project/resample/badge.svg 15 | :target: https://coveralls.io/github/resample-project/resample 16 | .. image:: https://readthedocs.org/projects/resample/badge/?version=stable 17 | :target: https://resample.readthedocs.io/en/stable 18 | .. image:: https://img.shields.io/pypi/l/resample 19 | :target: https://pypi.org/project/resample 20 | .. image:: https://zenodo.org/badge/145776396.svg 21 | :target: https://zenodo.org/badge/latestdoi/145776396 22 | 23 | `Link to full documentation`_ 24 | 25 | .. _Link to full documentation: http://resample.readthedocs.io 26 | 27 | .. skip-marker-do-not-remove 28 | 29 | Resampling-based inference in Python based on data resampling and permutation. 30 | 31 | This package was created by Daniel Saxton and is now maintained by Hans Dembinski. 32 | 33 | Features 34 | -------- 35 | 36 | - Bootstrap resampling: ordinary or balanced with optional stratification 37 | - Extended bootstrap resampling: also varies sample size 38 | - Parametric resampling: Gaussian, Poisson, gamma, etc.) 39 | - Jackknife estimates of bias and variance of any estimator 40 | - Compute bootstrap confidence intervals (percentile or BCa) for any estimator 41 | - Permutation-based variants of traditional statistical tests (**USP test of independence** and others) 42 | - Tools for working with empirical distributions (CDF, quantile, etc.) 43 | - Depends only on `numpy`_ and `scipy`_ 44 | 45 | Example 46 | ------- 47 | 48 | We bootstrap the uncertainty of the arithmetic mean, an estimator for the expectation. In this case, we know the formula to compute this uncertainty and can compare it to the bootstrap result. More complex examples can be found `in the documentation `_. 49 | 50 | .. code-block:: python 51 | 52 | from resample.bootstrap import variance 53 | import numpy as np 54 | 55 | # data 56 | d = [1, 2, 6, 3, 5] 57 | 58 | # this call is all you need 59 | stdev_of_mean = variance(np.mean, d) ** 0.5 60 | 61 | print(f"bootstrap {stdev_of_mean:.2f}") 62 | print(f"exact {np.std(d) / len(d) ** 0.5:.2f}") 63 | # bootstrap 0.82 64 | # exact 0.83 65 | 66 | The amazing thing is that the bootstrap works as well for arbitrarily complex estimators. 67 | The bootstrap often provides good results even when the sample size is small. 68 | 69 | .. _numpy: http://www.numpy.org 70 | .. _scipy: https://www.scipy.org 71 | 72 | Installation 73 | ------------ 74 | You can install with pip. 75 | 76 | .. code-block:: shell 77 | 78 | pip install resample 79 | -------------------------------------------------------------------------------- /benchmarks/test_bootstrap.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | from resample.bootstrap import confidence_interval, resample 5 | 6 | 7 | def run_resample(n, method): 8 | x = np.arange(n) 9 | r = [] 10 | for b in resample(x, method=method): 11 | r.append(b) 12 | return r 13 | 14 | 15 | @pytest.mark.benchmark(group="bootstrap-100") 16 | @pytest.mark.parametrize("method", ("ordinary", "balanced", "normal")) 17 | def test_resample_100(benchmark, method): 18 | benchmark(run_resample, 100, method) 19 | 20 | 21 | @pytest.mark.benchmark(group="bootstrap-1000") 22 | @pytest.mark.parametrize("method", ("ordinary", "balanced", "normal")) 23 | def test_bootstrap_resample_1000(benchmark, method): 24 | benchmark(run_resample, 1000, method) 25 | 26 | 27 | @pytest.mark.benchmark(group="bootstrap-10000") 28 | @pytest.mark.parametrize("method", ("ordinary", "balanced", "normal")) 29 | def test_bootstrap_resample_10000(benchmark, method): 30 | benchmark(run_resample, 10000, method) 31 | 32 | 33 | def run_confidence_interval(n, ci_method): 34 | x = np.arange(n) 35 | confidence_interval(np.mean, x, ci_method=ci_method) 36 | 37 | 38 | @pytest.mark.benchmark(group="confidence-interval-100") 39 | @pytest.mark.parametrize("ci_method", ("percentile", "bca")) 40 | def test_bootstrap_confidence_interval_100(benchmark, ci_method): 41 | benchmark(run_confidence_interval, 100, ci_method) 42 | 43 | 44 | @pytest.mark.benchmark(group="confidence-interval-1000") 45 | @pytest.mark.parametrize("ci_method", ("percentile", "bca")) 46 | def test_bootstrap_confidence_interval_1000(benchmark, ci_method): 47 | benchmark(run_confidence_interval, 1000, ci_method) 48 | 49 | 50 | @pytest.mark.benchmark(group="confidence-interval-10000") 51 | @pytest.mark.parametrize("ci_method", ("percentile", "bca")) 52 | def test_bootstrap_confidence_interval_10000(benchmark, ci_method): 53 | benchmark(run_confidence_interval, 10000, ci_method) 54 | -------------------------------------------------------------------------------- /benchmarks/test_jackknife.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: D100 D103 2 | import numpy as np 3 | import pytest 4 | from numpy.testing import assert_equal 5 | from resample.jackknife import resample 6 | 7 | 8 | def run_resample(n, copy): 9 | x = np.arange(n) 10 | r = [] 11 | for b in resample(x, copy=copy): 12 | r.append(np.mean(b)) 13 | return r 14 | 15 | 16 | @pytest.mark.benchmark(group="jackknife-100") 17 | @pytest.mark.parametrize("copy", (True, False)) 18 | def test_jackknife_resample_100(benchmark, copy): 19 | result = benchmark(run_resample, 100, copy) 20 | assert_equal(result, run_resample(100, resample)) 21 | 22 | 23 | @pytest.mark.benchmark(group="jackknife-1000") 24 | @pytest.mark.parametrize("copy", (True, False)) 25 | def test_jackknife_resample_1000(benchmark, copy): 26 | result = benchmark(run_resample, 1000, copy) 27 | assert_equal(result, run_resample(1000, resample)) 28 | -------------------------------------------------------------------------------- /benchmarks/test_rcont.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | from scipy.stats import random_table 5 | 6 | 7 | @pytest.mark.parametrize("n", (10, 100, 1000, 10000, 100000)) 8 | @pytest.mark.parametrize("k", (2, 4, 10, 20, 40, 100)) 9 | @pytest.mark.parametrize("method", (None, "boyett", "patefield")) 10 | def test_rcont(k, n, method, benchmark): 11 | w = np.zeros((k, k)) 12 | rng = np.random.default_rng(1) 13 | for _ in range(n): 14 | i = rng.integers(k) 15 | j = rng.integers(k) 16 | w[i, j] += 1 17 | r = np.sum(w, axis=1) 18 | c = np.sum(w, axis=0) 19 | assert np.sum(r) == n 20 | assert np.sum(c) == n 21 | benchmark(lambda: random_table(r, c).rvs(100, method=method, random_state=rng)) 22 | -------------------------------------------------------------------------------- /benchmarks/test_usp.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | from resample.permutation import usp 5 | 6 | 7 | @pytest.mark.parametrize("n", (10, 100, 1000, 10000)) 8 | @pytest.mark.parametrize("k", (2, 10, 100)) 9 | @pytest.mark.parametrize("method", ("patefield", "shuffle")) 10 | def test_usp(k, n, method, benchmark): 11 | w = np.zeros((k, k)) 12 | rng = np.random.default_rng(1) 13 | for _ in range(n): 14 | i = rng.integers(k) 15 | j = rng.integers(k) 16 | w[i, j] += 1 17 | assert np.sum(w) == n 18 | benchmark(lambda: usp(w, method=method, size=100, random_state=1)) 19 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make html". 11 | html: 12 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /doc/_static/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 18 | 39 | 41 | 42 | 43 | 45 | 2021-05-12T12:33:45.232806 46 | image/svg+xml 47 | 48 | 49 | Matplotlib v3.3.3, https://matplotlib.org/ 50 | 51 | 52 | 53 | 54 | 55 | 57 | 60 | 62 | 65 | 68 | 69 | 71 | 74 | 77 | 80 | 83 | 84 | 87 | 90 | 92 | 96 | 100 | 104 | 108 | 112 | 116 | 120 | 124 | 125 | 126 | 127 | 128 | 133 | 137 | 138 | 143 | 152 | 161 | 170 | 179 | 188 | 197 | 206 | 215 | 216 | 221 | 224 | 225 | 229 | 236 | 243 | 244 | 245 | 249 | 250 | 254 | 256 | 259 | 260 | 268 | 276 | 284 | 292 | 300 | 308 | 309 | 310 | 311 | 312 | -------------------------------------------------------------------------------- /doc/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | For more recent versions, please look into `the release notes on Github `_. 5 | 6 | 1.5.1 (March 1, 2022) 7 | --------------------- 8 | 9 | - Documentation improvements 10 | 11 | 1.5.0 (March 1, 2022) 12 | --------------------- 13 | 14 | This is an API breaking release. The backward-incompatible changes are limited to the 15 | ``resample.permutation`` submodule. Other modules are not affected. 16 | 17 | Warning: It was discovered that the tests implemented in ``resample.permutation`` had 18 | various issues and gave wrong results, so any results obtained with these tests should 19 | be revised. Since the old implementations were broken anyway, the API of 20 | ``resample.permutation`` was altered to fix some design issues as well. 21 | 22 | Installing resample now requires compiling a C extension. This is needed for the 23 | computation of the new USP test. This makes the installation of this package less 24 | convenient, since now a C compiler is required on the target machine (or we have to 25 | start building binary wheels). The plan is to move the compiled code to SciPy, which 26 | would allows us to drop the C extension again in the future. 27 | 28 | New features 29 | ~~~~~~~~~~~~ 30 | - ``resample`` functions in ``resample.bootstrap`` and ``resample.jackknife``, and all 31 | convenience functions which depend on them, can now resample from several arrays of 32 | equal length, example: ``for ai, bi in resample(a, b): ...`` 33 | - USP test of independence for binned data was added to ``resample.permutation`` 34 | - ``resample.permutation.same_population`` was added as a generic permutation-based test 35 | to compute the p-value that two or more samples were drawn from the same population 36 | 37 | API changes in submodule ``resample.permutation`` 38 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 39 | - All functions now only accept keyword arguments for their configuration and return a 40 | tuple-like ``TestResult`` object instead of a dictionary 41 | - Keyword ``b`` in all tests was replaced by ``size`` 42 | - p-values computed by all tests are now upper limits to the true Type I error rate 43 | - ``corr_test`` was replaced by two separate functions ``pearsonr`` and ``spearmanr`` 44 | - ``kruskal_wallis`` was renamed to ``kruskal`` to follow SciPy naming 45 | - ``anova`` and ``kruskal`` now accept multiple samples directly instead of using a list 46 | of samples; example: ``anova(x, y)`` instead of ``anova([x, y])`` 47 | - ``wilcoxon`` and ``ks_test`` were removed, since the corresponding tests in Scipy 48 | (``wilcoxon`` and ``ks_2samp``) compute exact p-values for small samples and use 49 | asymptotic formulas only when the same size is large; we cannot do better than that 50 | 51 | 1.0.1 (August 23, 2020) 52 | ----------------------- 53 | 54 | - Minor fix to allow building from source. 55 | 56 | 1.0.0 (August 22, 2020) 57 | ----------------------- 58 | 59 | API Changes 60 | ~~~~~~~~~~~ 61 | 62 | - Bootstrap and jackknife generators ``resample.bootstrap.resample`` and ``resample.jackknife.resample`` are now exposed to compute replicates lazily. 63 | - Jackknife functions have been split into their own namespace ``resample.jackknife``. 64 | - Empirical distribution helper functions moved to a ``resample.empirical`` namespace. 65 | - Random number seeding is now done through using ``numpy`` generators rather than a global random state. As a result the minimum ``numpy`` version is now 1.17. 66 | - Parametric bootstrap now estimates both parameters of the t distribution. 67 | - Default confidence interval method changed from ``"percentile"`` to ``"bca"``. 68 | - Empirical quantile function no longer performs interpolation between quantiles. 69 | 70 | Enhancements 71 | ~~~~~~~~~~~~ 72 | 73 | - Added bootstrap estimate of bias. 74 | - Added ``bias_corrected`` function for jackknife and bootstrap, which computes the bias corrected estimates. 75 | - Performance of jackknife computation was increased. 76 | 77 | Bug fixes 78 | ~~~~~~~~~ 79 | 80 | - Removed incorrect implementation of Studentized bootstrap. 81 | 82 | Deprecations 83 | ~~~~~~~~~~~~ 84 | 85 | - Smoothing of bootstrap samples is no longer supported. 86 | - Supremum norm and MISE functionals removed. 87 | 88 | Other 89 | ~~~~~ 90 | 91 | - Benchmarks were added to track and compare performance of bootstrap and jackknife methods. 92 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | # -- Project information ----------------------------------------------------- 20 | import resample 21 | 22 | version = resample.__version__ # noqa 23 | 24 | project = "resample" 25 | copyright = "2018, Daniel Saxton" 26 | author = "Daniel Saxton and Hans Dembinski" 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # If your documentation needs a minimal Sphinx version, state it here. 31 | # 32 | # needs_sphinx = '1.0' 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | "sphinx.ext.autodoc", 39 | "sphinx.ext.napoleon", 40 | "nbsphinx", 41 | ] 42 | 43 | autoclass_content = "both" 44 | autosummary_generate = True 45 | autodoc_member_order = "groupwise" 46 | autodoc_type_aliases = {"ArrayLike": "ArrayLike"} 47 | 48 | # Add any paths that contain templates here, relative to this directory. 49 | templates_path = ["_templates"] 50 | 51 | # The suffix(es) of source filenames. 52 | # You can specify multiple suffix as a list of string: 53 | # 54 | # source_suffix = ['.rst', '.md'] 55 | source_suffix = ".rst" 56 | 57 | # The master toctree document. 58 | master_doc = "index" 59 | 60 | # The language for content autogenerated by Sphinx. Refer to documentation 61 | # for a list of supported languages. 62 | # 63 | # This is also used if you do content translation via gettext catalogs. 64 | # Usually you set "language" from the command line for these cases. 65 | language = "en" 66 | 67 | # List of patterns, relative to source directory, that match files and 68 | # directories to ignore when looking for source files. 69 | # This pattern also affects html_static_path and html_extra_path. 70 | exclude_patterns = [ 71 | "_build", 72 | "Thumbs.db", 73 | ".DS_Store", 74 | "__pycache__", 75 | ".ipynb_checkpoints", 76 | ] 77 | 78 | # The name of the Pygments (syntax highlighting) style to use. 79 | pygments_style = None 80 | 81 | 82 | # -- Options for HTML output ------------------------------------------------- 83 | 84 | import sphinx_rtd_theme 85 | 86 | html_theme = "sphinx_rtd_theme" 87 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | 93 | html_logo = "_static/logo.svg" 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ["_static"] 99 | 100 | # Custom sidebar templates, must be a dictionary that maps document names 101 | # to template names. 102 | # 103 | # The default sidebars (for documents that don't match any pattern) are 104 | # defined by theme itself. Builtin themes are using these templates by 105 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 106 | # 'searchbox.html']``. 107 | # 108 | # html_sidebars = {} 109 | 110 | # -- Options for HTMLHelp output --------------------------------------------- 111 | 112 | # Output file base name for HTML help builder. 113 | htmlhelp_basename = "resampledoc" 114 | -------------------------------------------------------------------------------- /doc/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | These are specific examples which show how to use resample in practice. In contrast to tutorials they are not designed to showcase particular functions in resample. 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | example/tag_and_probe 10 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. |resample| image:: _static/logo.svg 2 | 3 | |resample| 4 | ========== 5 | 6 | .. include:: ../README.rst 7 | :start-after: skip-marker-do-not-remove 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :hidden: 12 | 13 | reference 14 | tutorials 15 | examples 16 | changelog 17 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /doc/make_raw_logo.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generate prototypes of the resample logo. 3 | 4 | The final logo was further edited in Inkscape. The chosen version uses the Gentium Plus 5 | Regular font from https://fontlibrary.org/en/font/gentium-plus#Gentium%20Plus-Regular. 6 | """ 7 | 8 | import numpy as np 9 | from matplotlib import pyplot as plt 10 | 11 | for font, x0 in (("ubuntu", 0.2), ("gentium", 0.17)): 12 | 13 | plt.figure(figsize=(5, 1.4)) 14 | ax = plt.subplot() 15 | for k in ax.spines: 16 | ax.spines[k].set_visible(False) 17 | plt.tick_params( 18 | **{k: False for k in ax.spines}, **{f"label{k}": False for k in ax.spines} 19 | ) 20 | plt.gca().set_facecolor("none") 21 | 22 | size = 70 23 | w = 0.05 24 | h = 0.15 25 | y0 = 0.1 26 | 27 | # original 28 | plt.figtext(0, y0, "re", color="r", name=font, size=size, weight="bold") 29 | plt.figtext(x0, y0, "sample", color="0.2", name=font, size=size) 30 | 31 | # copies 32 | rng = np.random.default_rng(1) 33 | s = np.fromiter("resample", "U1") 34 | n = 2 35 | for i, col in enumerate(("0.8", "0.9")): 36 | x = (i + 1) * w 37 | y = y0 + (i + 1) * h 38 | s2 = rng.choice(s, size=len(s)) 39 | plt.figtext(x, y, "".join(s2), color=col, name=font, size=size, zorder=-(i + 1)) 40 | 41 | plt.savefig(f"{font}.svg") 42 | -------------------------------------------------------------------------------- /doc/plot_bench.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import json 3 | from pathlib import Path 4 | import numpy as np 5 | 6 | d = Path(__file__).parent if "__file__" in globals() else Path() 7 | 8 | with open(d / "bench_rcont.json") as f: 9 | data = json.load(f) 10 | 11 | vs = [[[], [], []], [[], [], []]] 12 | 13 | benchs = data["benchmarks"] 14 | for b in benchs: 15 | params = b["params"] 16 | m = params["method"] 17 | n = params["n"] 18 | k = params["k"] 19 | stats = b["stats"] 20 | val = stats["mean"] 21 | err = stats["stddev"] / stats["rounds"] ** 0.5 22 | vs[m][0].append(n) 23 | vs[m][1].append(k) 24 | vs[m][2].append(val) 25 | 26 | fig, ax = plt.subplots(1, 2, figsize=(10, 4), sharey=True) 27 | for i, label in enumerate(("shuffle", "patefield")): 28 | ax[0].scatter(vs[i][0], vs[i][2], s=np.add(vs[i][1], 1), label=label) 29 | ax[1].scatter(vs[i][1], vs[i][2], s=10 * np.log(vs[i][0]) - 10) 30 | ax[0].loglog() 31 | ax[1].loglog() 32 | ax[0].set_xlabel("N") 33 | ax[1].set_xlabel("K") 34 | ax[0].set_ylabel("t/sec") 35 | plt.figlegend(loc="upper center", ncol=2, frameon=False) 36 | -------------------------------------------------------------------------------- /doc/reference.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | bootstrap 5 | --------- 6 | 7 | .. automodule:: resample.bootstrap 8 | :members: 9 | 10 | jackknife 11 | --------- 12 | 13 | .. automodule:: resample.jackknife 14 | :members: 15 | 16 | permutation 17 | ----------- 18 | 19 | .. automodule:: resample.permutation 20 | :members: 21 | 22 | empirical 23 | --------- 24 | 25 | .. automodule:: resample.empirical 26 | :members: 27 | -------------------------------------------------------------------------------- /doc/tutorial/confidence_intervals.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Confidence intervals\n", 8 | "\n", 9 | "In this notebook, we look at the confidence interval methods in `resample`. We try them on the median of an exponential distribution." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": { 16 | "jupyter": { 17 | "outputs_hidden": false, 18 | "source_hidden": false 19 | }, 20 | "nteract": { 21 | "transient": { 22 | "deleting": false 23 | } 24 | } 25 | }, 26 | "outputs": [], 27 | "source": [ 28 | "import numpy as np\n", 29 | "from resample.bootstrap import confidence_interval as ci, bootstrap\n", 30 | "import matplotlib.pyplot as plt" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 2, 36 | "metadata": { 37 | "jupyter": { 38 | "outputs_hidden": false, 39 | "source_hidden": false 40 | }, 41 | "nteract": { 42 | "transient": { 43 | "deleting": false 44 | } 45 | } 46 | }, 47 | "outputs": [ 48 | { 49 | "data": { 50 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGdCAYAAAAIbpn/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtXUlEQVR4nO3de3SU1b3/8c/kNgmQSwPkpgECcrMCxlBCvMGRVAIur/QolAoqhSpJi6QoB39FEFvipYoVEXQVA64qFE9brGhRCAZUwi0VFdQIMceokATBEBPMdfbvD0/mMCRIgpNMdni/1nrWmnmePfvZX3Ym+fBcZhzGGCMAAAAL+fl6AAAAAGeLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsFaArwdwNlwulw4dOqTQ0FA5HA5fDwcAALSAMUbffPON4uLi5OfnnWMpVgaZQ4cOKT4+3tfDAAAAZ+Hzzz/X+eef75W+rAwyoaGhkr77hwgLC/PxaNBSJ2rrNeIPOZKkXf9vjLoEtfOPX22V9NjA7x7/tkAK6tq++z+HffXVV+rXr5/HusLCQvXo0cNHI7JLq947/JyjA6uoqFB8fLz777g3WBlkGk8nhYWFEWQsElBbLz9nF0nfzV37Bxl/yfm/pyLDwvgF345qamqarAsNDeX920Kteu/wcw4LePOyEC72BQAA1iLIAAAAaxFkAACAtay8RgYA0Dk0NDSorq7O18OAl/j7+ysgIKBdPxqFIAMA8InKykp98cUXMsb4eijwoi5duig2NlZBQUHtsj+CDACg3TU0NOiLL75Qly5d1LNnTz7ctBMwxqi2tlZHjhxRUVGR+vfv77UPvfs+BBkAQLurq6uTMUY9e/ZUSEiIr4cDLwkJCVFgYKA+++wz1dbWKjg4uM33ycW+AACf4UhM59MeR2E89teuewMAAPAiggwAAOeohQsX6uKLL3Y/v+2223TDDTf4bDxngyADAMA5wOFwaP369R7r5syZo5ycHN8MyEu42BcAAB9paGiQw+Fo9+tKGnXr1k3dunXzyb69hSMyAAC00OjRo5WRkaGMjAyFh4erR48emj9/vvuzcGpqajRnzhydd9556tq1q5KTk5Wbm+t+/apVqxQREaF//vOfuvDCC+V0OlVcXKyamhrNnTtX8fHxcjqduuCCC7Ry5Ur36/bt26dx48apW7duio6O1q233qqvvvrKY1y/+c1vdO+99yoyMlIxMTFauHChe3ufPn0kSTfeeKMcDof7+amnlk7lcrmUlZWlhIQEhYSEaNiwYfrv//7vH/zv6E0EGQCAzxljdKK23idLaz+Qb/Xq1QoICNCuXbv0pz/9SY8//rj+/Oc/S5IyMjKUl5entWvX6v3339d//ud/Ki0tTQcOHHC//sSJE3r44Yf15z//Wfv371dUVJSmTJmiNWvW6Mknn9RHH32kZ555xn2kpLy8XFdddZUSExO1Z88ebdy4UaWlpbr55pubjKtr167auXOnHnnkES1atEibNm2SJO3evVuSlJ2drcOHD7ufn0lWVpaef/55rVixQvv379fs2bP1i1/8Qlu3bm3Vv1lb4tQS0Ar5+fm+HkKrJSUl+XoIwBl9W9egC+9/3Sf7/nDRWHUJavmfw/j4eC1ZskQOh0MDBw7UBx98oCVLlmjs2LHKzs5WcXGx4uLiJH13DcrGjRuVnZ2txYsXS/ruM3SefvppDRs2TJL0ySefaN26ddq0aZNSU1MlSX379nXv76mnnlJiYqL79ZL03HPPKT4+Xp988okGDBggSRo6dKgWLFggSerfv7+eeuop5eTk6Kc//al69uwpSYqIiFBMTEyL6qypqdHixYu1efNmpaSkuMf19ttv65lnntGoUaNa/G/WlggyAAC0wsiRIz0+/yYlJUWPPfaYPvjgAzU0NLiDRaOamhp1797d/TwoKEhDhw51P9+7d6/8/f1PGwzee+89vfnmm81ey1JYWOgRZE4WGxursrKy1hf4vw4ePKgTJ07opz/9qcf62tpaJSYmnnW/3kaQAQD4XEigvz5cNNZn+/aGyspK+fv7Kz8/X/7+nn2eHEJCQkI8gtCZPtm4srJS1157rR5++OEm22JjY92PAwMDPbY5HA65XK5W1XDqfiXp1Vdf1Xnnneexzel0nnW/3kaQAQD4nMPhaNXpHV/auXOnx/MdO3aof//+SkxMVENDg8rKynTFFVe0uL8hQ4bI5XJp69at7lNLJ7vkkkv0t7/9TX369FFAwNn/GwUGBqqhoaHF7U++GLmjnEZqDhf7AgDQCsXFxcrMzFRBQYHWrFmjpUuXatasWRowYIAmT56sKVOm6O9//7uKioq0a9cuZWVl6dVXXz1tf3369NHUqVN1xx13aP369SoqKlJubq7WrVsnSUpPT9exY8c0adIk7d69W4WFhXr99dd1++23tyqY9OnTRzk5OSopKdHXX399xvahoaGaM2eOZs+erdWrV6uwsFD//ve/tXTpUq1evbrF+21rBBkAAFphypQp+vbbbzVixAilp6dr1qxZmjFjhqTv7gqaMmWKfvvb32rgwIG64YYbtHv3bvXq1et7+1y+fLl+9rOfaebMmRo0aJCmT5+uqqoqSVJcXJzeeecdNTQ06Oqrr9aQIUN09913KyIiolWfP/PYY49p06ZNio+Pb/E1Lg8++KDmz5+vrKwsDR48WGlpaXr11VeVkJDQ4v22NYdp7X1nHUBFRYXCw8N1/PhxhYWF+Xo4aKETtfXuuxJae5eAV9RWSYu/u5NA9x2Sgrq2ugvuWjo7R44cUVRUlMe6srIy950U+H6teu944ee8PVRXV6uoqEgJCQnt8g3J3jJ69GhdfPHFeuKJJ3w9lA7r++a2Lf5+c0QGAABYiyADAACsZccl4gAAdAAnf90AOgaOyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwBAC40ePVp33323r4eBkxBkAACAtQgyAADAWgQZAABaob6+XhkZGQoPD1ePHj00f/58NX7/ck1NjebOnav4+Hg5nU5dcMEFWrlypSSpoaFB06ZNU0JCgkJCQjRw4ED96U9/8mUpnQJfUQAA8D1jpLoTvtl3YBfJ4Whx89WrV2vatGnatWuX9uzZoxkzZqhXr16aPn26pkyZory8PD355JMaNmyYioqK9NVXX0mSXC6Xzj//fL300kvq3r27tm/frhkzZig2NlY333xzW1XX6RFkAAC+V3dCWhznm33fd0gK6tri5vHx8VqyZIkcDocGDhyoDz74QEuWLNGoUaO0bt06bdq0SampqZKkvn37ul8XGBioBx54wP08ISFBeXl5WrduHUHmB+DUEgAArTBy5Eg5TjqCk5KSogMHDujdd9+Vv7+/Ro0addrXLlu2TElJSerZs6e6deumZ599VsXFxe0x7E6LIzIAAN8L7PLdkRFf7dsLgoODv3f72rVrNWfOHD322GNKSUlRaGioHn30Ue3cudMr+z9XEWQAAL7ncLTq9I4vnRo8duzYof79+2vYsGFyuVzaunWr+9TSyd555x1deumlmjlzpntdYWFhm4+3s+PUEgAArVBcXKzMzEwVFBRozZo1Wrp0qWbNmqU+ffpo6tSpuuOOO7R+/XoVFRUpNzdX69atkyT1799fe/bs0euvv65PPvlE8+fP1+7du31cjf0IMgAAtMKUKVP07bffasSIEUpPT9esWbM0Y8YMSdLy5cv1s5/9TDNnztSgQYM0ffp0VVVVSZJ+9atf6aabbtItt9yi5ORkHT161OPoDM4Op5YAAGih3Nxc9+Ply5c32R4cHKzHH39cjz/+eJNtTqdT2dnZys7O9liflZXl9XGeSzgiAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAIDPNH5rNDqP9p5TggwAoN35+/tLkmpra308EnjbiRPffYt5YGBgu+yPz5EBALS7gIAAdenSRUeOHFFgYKD8/Ph/te2MMTpx4oTKysoUERHhDqttjSADAGh3DodDsbGxKioq0meffebr4cCLIiIiFBMT0277I8gAAHwiKChI/fv35/RSJxIYGNhuR2IaEWQAAD7j5+en4OBgXw8DFuOkJAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWalWQycrK0k9+8hOFhoYqKipKN9xwgwoKCjzaVFdXKz09Xd27d1e3bt00YcIElZaWerQpLi7WNddcoy5duigqKkr33HOP6uvrf3g1AADgnNKqILN161alp6drx44d2rRpk+rq6nT11VerqqrK3Wb27Nl65ZVX9NJLL2nr1q06dOiQbrrpJvf2hoYGXXPNNaqtrdX27du1evVqrVq1Svfff7/3qgIAAOeEgNY03rhxo8fzVatWKSoqSvn5+bryyit1/PhxrVy5Ui+++KKuuuoqSVJ2drYGDx6sHTt2aOTIkXrjjTf04YcfavPmzYqOjtbFF1+sBx98UHPnztXChQsVFBTkveoAAECn9oOukTl+/LgkKTIyUpKUn5+vuro6paamutsMGjRIvXr1Ul5eniQpLy9PQ4YMUXR0tLvN2LFjVVFRof379ze7n5qaGlVUVHgsAAAAZx1kXC6X7r77bl122WW66KKLJEklJSUKCgpSRESER9vo6GiVlJS425wcYhq3N25rTlZWlsLDw91LfHz82Q4bAAB0ImcdZNLT07Vv3z6tXbvWm+Np1rx583T8+HH38vnnn7f5PgEAQMfXqmtkGmVkZGjDhg3atm2bzj//fPf6mJgY1dbWqry83OOoTGlpqWJiYtxtdu3a5dFf411NjW1O5XQ65XQ6z2aoAACgE2vVERljjDIyMvSPf/xDW7ZsUUJCgsf2pKQkBQYGKicnx72uoKBAxcXFSklJkSSlpKTogw8+UFlZmbvNpk2bFBYWpgsvvPCH1AIAAM4xrToik56erhdffFEvv/yyQkND3de0hIeHKyQkROHh4Zo2bZoyMzMVGRmpsLAw/frXv1ZKSopGjhwpSbr66qt14YUX6tZbb9UjjzyikpIS/e53v1N6ejpHXQAAQKu0KsgsX75ckjR69GiP9dnZ2brtttskSUuWLJGfn58mTJigmpoajR07Vk8//bS7rb+/vzZs2KC77rpLKSkp6tq1q6ZOnapFixb9sEoAAMA5p1VBxhhzxjbBwcFatmyZli1bdto2vXv31muvvdaaXQMAADTBdy0BAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYK8PUAAOBU+fn5vh5CqyUlJfl6CMA5iSMyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWanWQ2bZtm6699lrFxcXJ4XBo/fr1Httvu+02ORwOjyUtLc2jzbFjxzR58mSFhYUpIiJC06ZNU2Vl5Q8qBAAAnHtaHWSqqqo0bNgwLVu27LRt0tLSdPjwYfeyZs0aj+2TJ0/W/v37tWnTJm3YsEHbtm3TjBkzWj96AABwTmv1B+KNGzdO48aN+942TqdTMTExzW776KOPtHHjRu3evVvDhw+XJC1dulTjx4/XH//4R8XFxbV2SAAA4BzVJtfI5ObmKioqSgMHDtRdd92lo0ePurfl5eUpIiLCHWIkKTU1VX5+ftq5c2ez/dXU1KiiosJjAQAA8HqQSUtL0/PPP6+cnBw9/PDD2rp1q8aNG6eGhgZJUklJiaKiojxeExAQoMjISJWUlDTbZ1ZWlsLDw91LfHy8t4cNAAAs5PXvWpo4caL78ZAhQzR06FD169dPubm5GjNmzFn1OW/ePGVmZrqfV1RUEGYAAEDb337dt29f9ejRQwcPHpQkxcTEqKyszKNNfX29jh07dtrrapxOp8LCwjwWAACANg8yX3zxhY4eParY2FhJUkpKisrLyz2+3XbLli1yuVxKTk5u6+EAAIBOpNWnliorK91HVySpqKhIe/fuVWRkpCIjI/XAAw9owoQJiomJUWFhoe69915dcMEFGjt2rCRp8ODBSktL0/Tp07VixQrV1dUpIyNDEydO5I4lAADQKq0+IrNnzx4lJiYqMTFRkpSZmanExETdf//98vf31/vvv6/rrrtOAwYM0LRp05SUlKS33npLTqfT3ccLL7ygQYMGacyYMRo/frwuv/xyPfvss96rCgAAnBNafURm9OjRMsacdvvrr79+xj4iIyP14osvtnbXAAAAHviuJQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaAb4eALwjPz/f10M4o+p6l/vxu+++q+CA9s3RfvXfKvGk/bsCQtp1/wAA7+OIDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWCvA1wMA0Lby8/N9PQR9/fXXTda99957+tGPfuSD0QDoTDgiAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsF+HoAANqZaVC3ox8osOao6pzdVdl9iOTw9/WoAOCstPqIzLZt23TttdcqLi5ODodD69ev99hujNH999+v2NhYhYSEKDU1VQcOHPBoc+zYMU2ePFlhYWGKiIjQtGnTVFlZ+YMKAXBmEYe3acjmn2tgXqb6/vsPGpiXqSGbf66Iw9t8PTQAOCutDjJVVVUaNmyYli1b1uz2Rx55RE8++aRWrFihnTt3qmvXrho7dqyqq6vdbSZPnqz9+/dr06ZN2rBhg7Zt26YZM2acfRUAziji8Db13bNQgdVHPNYHVh9R3z0LCTMArNTqU0vjxo3TuHHjmt1mjNETTzyh3/3ud7r++uslSc8//7yio6O1fv16TZw4UR999JE2btyo3bt3a/jw4ZKkpUuXavz48frjH/+ouLi4H1AOOhO/+m+93F91s4/PCaZB8fuekiQ5TtnkkGQkxe97ShU9LmmT00x+DdXqEth0nbfn2Kdqq5quC+ra/uMAzjFevUamqKhIJSUlSk1Nda8LDw9XcnKy8vLyNHHiROXl5SkiIsIdYiQpNTVVfn5+2rlzp2688cYm/dbU1Kimpsb9vKKiwpvDRgeV+K9r2qzvYZsmtFnfNnJICqr+Sokbr2uzfVTdF+a5Im9ym+3LJ/7VzLqFx9t9GMC5xqt3LZWUlEiSoqOjPdZHR0e7t5WUlCgqKspje0BAgCIjI91tTpWVlaXw8HD3Eh8f781hAwAAS1lx19K8efOUmZnpfl5RUUGYOQe8O+5Vr/bnV1/tPhLz3k//JldAsFf778i6HX1f/XfNO2O7AyOyVNl9qNf3/3V5ua699lqPda+88op+FBHh9X35SmJioq+H0Gr5+fm+HkKrJSUl+XoI6GC8GmRiYmIkSaWlpYqNjXWvLy0t1cUXX+xuU1ZW5vG6+vp6HTt2zP36UzmdTjmdTm8OFRZwBYS0Yd/Bbdp/R1MRNVy1wT0VWH2kyTUy0nfXyNQF91RF1PA2uUbG5V+tE3Wnrutkc8D1MIBPePXUUkJCgmJiYpSTk+NeV1FRoZ07dyolJUWSlJKSovLyco//CWzZskUul0vJycneHA6ARg5/fX5RuqTvQsvJGp9/flE6nycDwDqtPiJTWVmpgwcPup8XFRVp7969ioyMVK9evXT33Xfr97//vfr376+EhATNnz9fcXFxuuGGGyRJgwcPVlpamqZPn64VK1aorq5OGRkZmjhxIncsAW2oPPZKfTp8oeL3LVPQSbdg1wX31OcXpas89kofjg4Azk6rg8yePXv0H//xH+7njdeuTJ06VatWrdK9996rqqoqzZgxQ+Xl5br88su1ceNGBQf/3/UIL7zwgjIyMjRmzBj5+flpwoQJevLJJ71QDoDvUx57pcpjLuOTfQF0Gq0OMqNHj5Yxpx6c/j8Oh0OLFi3SokWLTtsmMjJSL774Ymt3DcAbHP6q7HGxr0cBAF7Bl0YCAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsFaArwcAAJ1Bfn5+m/VdXe9yP3733XcVHHD6/4P61X+rxJPaugJC2mxcQEfAERkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtrweZhQsXyuFweCyDBg1yb6+urlZ6erq6d++ubt26acKECSotLfX2MAAAwDmgTY7I/PjHP9bhw4fdy9tvv+3eNnv2bL3yyit66aWXtHXrVh06dEg33XRTWwwDAAB0cgFt0mlAgGJiYpqsP378uFauXKkXX3xRV111lSQpOztbgwcP1o4dOzRy5Mi2GA4AAOik2uSIzIEDBxQXF6e+fftq8uTJKi4uliTl5+errq5Oqamp7raDBg1Sr169lJeXd9r+ampqVFFR4bEAAAB4PcgkJydr1apV2rhxo5YvX66ioiJdccUV+uabb1RSUqKgoCBFRER4vCY6OlolJSWn7TMrK0vh4eHuJT4+3tvDBgAAFvL6qaVx48a5Hw8dOlTJycnq3bu31q1bp5CQkLPqc968ecrMzHQ/r6ioIMwAAIC2v/06IiJCAwYM0MGDBxUTE6Pa2lqVl5d7tCktLW32mppGTqdTYWFhHgsAAECbB5nKykoVFhYqNjZWSUlJCgwMVE5Ojnt7QUGBiouLlZKS0tZDAQAAnYzXTy3NmTNH1157rXr37q1Dhw5pwYIF8vf316RJkxQeHq5p06YpMzNTkZGRCgsL069//WulpKRwxxIAAGg1rweZL774QpMmTdLRo0fVs2dPXX755dqxY4d69uwpSVqyZIn8/Pw0YcIE1dTUaOzYsXr66ae9PQwAAHAO8HqQWbt27fduDw4O1rJly7Rs2TJv7xoAAJxj+K4lAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArBXg6wF0RPn5+b4eAgAAaAGOyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYK8PUAAABoqfz8fF8PodWSkpJ8PYROjSMyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFo+DTLLli1Tnz59FBwcrOTkZO3atcuXwwEAAJYJ8NWO//rXvyozM1MrVqxQcnKynnjiCY0dO1YFBQWKiory1bAAAPCq/Px8Xw/hrCQlJfl6CC3isyMyjz/+uKZPn67bb79dF154oVasWKEuXbroueee89WQAACAZXxyRKa2tlb5+fmaN2+ee52fn59SU1OVl5fXpH1NTY1qamrcz48fPy5JqqioaJPxVVZWtkm/57qaBpdcNSckSVVVlar3b98c7ddQrYoaI0mqrKqSy7+hXfd/Lquqqmp2XWBgoA9GY5/WvHf4OYe3tMXf2MY+jTHe69T4wJdffmkkme3bt3usv+eee8yIESOatF+wYIGRxMLCwsLCwtIJls8//9xrmcJn18i0xrx585SZmel+7nK5dOzYMXXv3l0Oh8Nr+6moqFB8fLw+//xzhYWFea3fjoY6Oxfq7Fyos/M5V2ptSZ3GGH3zzTeKi4vz2n59EmR69Oghf39/lZaWeqwvLS1VTExMk/ZOp1NOp9NjXURERJuNLywsrFP/sDWizs6FOjsX6ux8zpVaz1RneHi4V/fnk4t9g4KClJSUpJycHPc6l8ulnJwcpaSk+GJIAADAQj47tZSZmampU6dq+PDhGjFihJ544glVVVXp9ttv99WQAACAZXwWZG655RYdOXJE999/v0pKSnTxxRdr48aNio6O9tWQ5HQ6tWDBgiansTob6uxcqLNzoc7O51yp1Vd1Oozx5j1QAAAA7YfvWgIAANYiyAAAAGsRZAAAgLUIMgAAwFqdLsgsW7ZMffr0UXBwsJKTk7Vr167vbV9eXq709HTFxsbK6XRqwIABeu2111rVZ3V1tdLT09W9e3d169ZNEyZMaPJhf97m7TqzsrL0k5/8RKGhoYqKitINN9yggoICjz5Gjx4th8Phsdx5551tUl8jb9e5cOHCJjUMGjTIo4/OMJ99+vRpUqfD4VB6erq7TUefz+bG53A4dM0117jbGGN0//33KzY2ViEhIUpNTdWBAwc8+jl27JgmT56ssLAwRUREaNq0aW3+fWrerLOurk5z587VkCFD1LVrV8XFxWnKlCk6dOiQRz/NzflDDz1kTZ2SdNtttzXZnpaW5tGP7fMpqdntDodDjz76qLtNR59PSXriiSc0cOBAhYSEKD4+XrNnz1Z1dXWr+vTK71uvfdlBB7B27VoTFBRknnvuObN//34zffp0ExERYUpLS5ttX1NTY4YPH27Gjx9v3n77bVNUVGRyc3PN3r17W9XnnXfeaeLj401OTo7Zs2ePGTlypLn00kutqnPs2LEmOzvb7Nu3z+zdu9eMHz/e9OrVy1RWVrrbjBo1ykyfPt0cPnzYvRw/ftyqOhcsWGB+/OMfe9Rw5MgRj346w3yWlZV51Lhp0yYjybz55pvuNh19Po8ePeoxtn379hl/f3+TnZ3tbvPQQw+Z8PBws379evPee++Z6667ziQkJJhvv/3W3SYtLc0MGzbM7Nixw7z11lvmggsuMJMmTbKmzvLycpOammr++te/mo8//tjk5eWZESNGmKSkJI9+evfubRYtWuTR18nv345epzHGTJ061aSlpXm0O3bsmEc/ts+nMcZj++HDh81zzz1nHA6HKSwsdLfp6PP5wgsvGKfTaV544QVTVFRkXn/9dRMbG2tmz57dqj698fu2UwWZESNGmPT0dPfzhoYGExcXZ7Kyspptv3z5ctO3b19TW1t71n2Wl5ebwMBA89JLL7nbfPTRR0aSycvL+6ElndWYTtWSOk9VVlZmJJmtW7e6140aNcrMmjXrrMfdWm1R54IFC8ywYcNOu72zzuesWbNMv379jMvlcq/r6PN5qiVLlpjQ0FD3L3OXy2ViYmLMo48+6m5TXl5unE6nWbNmjTHGmA8//NBIMrt373a3+de//mUcDof58ssvvVFWE96uszm7du0yksxnn33mXte7d2+zZMmSsx53a7VFnVOnTjXXX3/9aV/TWefz+uuvN1dddZXHuo4+n+np6U3GnJmZaS677LIW9+mt37ed5tRSbW2t8vPzlZqa6l7n5+en1NRU5eXlNfuaf/7zn0pJSVF6erqio6N10UUXafHixWpoaGhxn/n5+aqrq/NoM2jQIPXq1eu0++1odTbn+PHjkqTIyEiP9S+88IJ69Oihiy66SPPmzdOJEye8UFVTbVnngQMHFBcXp759+2ry5MkqLi52b+uM81lbW6u//OUvuuOOO5p8yWpHns9TrVy5UhMnTlTXrl0lSUVFRSopKfHoMzw8XMnJye4+8/LyFBERoeHDh7vbpKamys/PTzt37vRGaR7aos7mHD9+XA6Ho8l3zj300EPq3r27EhMT9eijj6q+vv6s6jiTtqwzNzdXUVFRGjhwoO666y4dPXrUva0zzmdpaaleffVVTZs2rcm2jjyfl156qfLz892nij799FO99tprGj9+fIv79NbvWyu+/bolvvrqKzU0NDT5ZODo6Gh9/PHHzb7m008/1ZYtWzR58mS99tprOnjwoGbOnKm6ujotWLCgRX2WlJQoKCioyS+U6OholZSUeK/A/9UWdZ7K5XLp7rvv1mWXXaaLLrrIvf7nP/+5evfurbi4OL3//vuaO3euCgoK9Pe//927Rart6kxOTtaqVas0cOBAHT58WA888ICuuOIK7du3T6GhoZ1yPtevX6/y8nLddtttHus7+nyebNeuXdq3b59WrlzpXtc4H8312bitpKREUVFRHtsDAgIUGRnZYebzZM3Vearq6mrNnTtXkyZN8vhivt/85je65JJLFBkZqe3bt2vevHk6fPiwHn/88bMv6DTaqs60tDTddNNNSkhIUGFhoe677z6NGzdOeXl58vf375TzuXr1aoWGhuqmm27yWN/R5/PnP/+5vvrqK11++eUyxqi+vl533nmn7rvvvhb36a3ft50myJwNl8ulqKgoPfvss/L391dSUpK+/PJLPfroo83+QbBVa+tMT0/Xvn379Pbbb3usnzFjhvvxkCFDFBsbqzFjxqiwsFD9+vVr8zrOpCV1jhs3zt1+6NChSk5OVu/evbVu3bpm/0fUEbV2PleuXKlx48YpLi7OY31Hn8+TrVy5UkOGDNGIESN8PZQ2daY66+rqdPPNN8sYo+XLl3tsy8zMdD8eOnSogoKC9Ktf/UpZWVkd7qPxT1fnxIkT3Y+HDBmioUOHql+/fsrNzdWYMWPae5g/WEt+bp977jlNnjxZwcHBHus7+nzm5uZq8eLFevrpp5WcnKyDBw9q1qxZevDBBzV//vx2HUunObXUo0cP+fv7N7naubS0VDExMc2+JjY2VgMGDJC/v7973eDBg1VSUqLa2toW9RkTE6Pa2lqVl5e3eL8/RFvUebKMjAxt2LBBb775ps4///zvHUtycrIk6eDBg2dTyvdq6zobRUREaMCAAe4aOtt8fvbZZ9q8ebN++ctfnnEsHW0+G1VVVWnt2rVNgmbj6870/iwrK/PYXl9fr2PHjnWY+Wx0ujobNYaYzz77TJs2bfI4GtOc5ORk1dfX63/+539aVUNLtGWdJ+vbt6969Ojh8f7sLPMpSW+99ZYKCgpa/P7sSPM5f/583XrrrfrlL3+pIUOG6MYbb9TixYuVlZUll8vVrn8/O02QCQoKUlJSknJyctzrXC6XcnJylJKS0uxrLrvsMh08eFAul8u97pNPPlFsbKyCgoJa1GdSUpICAwM92hQUFKi4uPi0+/0h2qJO6bvbWDMyMvSPf/xDW7ZsUUJCwhnHsnfvXknf/WH1traq81SVlZUqLCx019BZ5rNRdna2oqKiPG79PJ2ONp+NXnrpJdXU1OgXv/iFx/qEhATFxMR49FlRUaGdO3e6+0xJSVF5ebny8/PdbbZs2SKXy+UObt7UFnVK/xdiDhw4oM2bN6t79+5nHMvevXvl5+fX5FSMN7RVnaf64osvdPToUffPZGeZz0YrV65UUlKShg0bdsaxdLT5PHHihPz8PCNE43+ujDHt+/ezxZcFW2Dt2rXG6XSaVatWmQ8//NDMmDHDREREmJKSEmOMMbfeeqv5r//6L3f74uJiExoaajIyMkxBQYHZsGGDiYqKMr///e9b3Kcx390+1qtXL7NlyxazZ88ek5KSYlJSUqyq86677jLh4eEmNzfX43a/EydOGGOMOXjwoFm0aJHZs2ePKSoqMi+//LLp27evufLKK62q87e//a3Jzc01RUVF5p133jGpqammR48epqyszN2mM8ynMd/dIdCrVy8zd+7cJvu0YT4bXX755eaWW25pts+HHnrIREREmJdfftm8//775vrrr2/29uvExESzc+dO8/bbb5v+/fu3+e263qyztrbWXHfddeb88883e/fu9Xh/1tTUGGOM2b59u1myZInZu3evKSwsNH/5y19Mz549zZQpU6yp85tvvjFz5swxeXl5pqioyGzevNlccsklpn///qa6utrdzvb5bHT8+HHTpUsXs3z58ibbbJjPBQsWmNDQULNmzRrz6aefmjfeeMP069fP3HzzzS3u0xjv/L7tVEHGGGOWLl1qevXqZYKCgsyIESPMjh073NtGjRplpk6d6tF++/btJjk52TidTtO3b1/zhz/8wdTX17e4T2OM+fbbb83MmTPNj370I9OlSxdz4403msOHD7dZjWca09nUKanZpfGzD4qLi82VV15pIiMjjdPpNBdccIG555572vRzR9qizltuucXExsaaoKAgc95555lbbrnFHDx40KOPzjCfxhjz+uuvG0mmoKCgyf5smc+PP/7YSDJvvPFGs/25XC4zf/58Ex0dbZxOpxkzZkyTeo8ePWomTZpkunXrZsLCwsztt99uvvnmG6/XdjJv1llUVHTa92fj5wLl5+eb5ORkEx4eboKDg83gwYPN4sWLPQJAW/BmnSdOnDBXX3216dmzpwkMDDS9e/c206dP9/ijZ4z989nomWeeMSEhIaa8vLzJNhvms66uzixcuND069fPBAcHm/j4eDNz5kzz9ddft7hPY7zz+9ZhjDEtP34DAADQcXSaa2QAAMC5hyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGv9f7aA/VTrO/LRAAAAAElFTkSuQmCC", 51 | "text/plain": [ 52 | "
" 53 | ] 54 | }, 55 | "metadata": {}, 56 | "output_type": "display_data" 57 | } 58 | ], 59 | "source": [ 60 | "rng = np.random.default_rng(1)\n", 61 | "\n", 62 | "# generate data\n", 63 | "data = rng.exponential(size=1000)\n", 64 | "\n", 65 | "# generate confidence intervals\n", 66 | "cis = {\n", 67 | " m: ci(np.median, data, cl=0.68, size=100, ci_method=m, random_state=rng)\n", 68 | " for m in (\"percentile\", \"bca\")\n", 69 | "}\n", 70 | "\n", 71 | "# compute mean and std. deviation of replicates\n", 72 | "rep = bootstrap(np.median, data, size=1000, random_state=rng)\n", 73 | "mr = np.mean(rep)\n", 74 | "sr = np.std(rep)\n", 75 | "\n", 76 | "# draw everything\n", 77 | "for i, (m, v) in enumerate(cis.items()):\n", 78 | " for j in (0, 1):\n", 79 | " plt.axvline(v[j], color=f\"C{i}\", label=m if j == 0 else None)\n", 80 | "\n", 81 | "plt.hist(rep, facecolor=\"0.8\")\n", 82 | "plt.axvline(np.log(2), lw=3, color=\"k\")\n", 83 | "plt.errorbar(mr, 100, 0, sr, fmt=\"o\") \n", 84 | "plt.legend();" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "The mean of the replicates and its standard deviation is shown with the dot and the horizontal error bar. The three interval methods are shown as thin vertical lines. The thick black line is the true value of the median for an exponential distribution." 92 | ] 93 | } 94 | ], 95 | "metadata": { 96 | "kernel_info": { 97 | "name": "python3" 98 | }, 99 | "kernelspec": { 100 | "display_name": "Python 3 (ipykernel)", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.11.9" 115 | }, 116 | "nteract": { 117 | "version": "0.23.3" 118 | } 119 | }, 120 | "nbformat": 4, 121 | "nbformat_minor": 4 122 | } 123 | -------------------------------------------------------------------------------- /doc/tutorial/jackknife_vs_bootstrap.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Bootstrap and Jackknife comparison\n", 8 | "\n", 9 | "In this notebook we compare the bootstrap to the jackknife. Bootstrap resampling is superior to jackknifing, but the jackknife is deterministic, which may be helpful, and it can exactly remove biases of order 1/N from an estimator. The bootstrap does not have a simple bias estimator.\n", 10 | "\n", 11 | "We consider as estimators the arithmetic mean and the naive variance $\\hat V = \\langle x^2 \\rangle - \\langle x \\rangle^2$ from a sample of inputs. We use `resample` to compute the variances of these two estimators and their bias. This can be done elegantly by defining a single function `fn` which returns both estimates.\n", 12 | "\n", 13 | "The exact bias is known for both estimators. It is zero for the mean, because it is a linear function of the sample. For $\\hat V$, the bias-corrected estimate is $\\frac N{N-1} \\hat V$, and thus the bias is $\\frac{- 1}{N - 1} \\hat V$.\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 10, 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "name": "stdout", 23 | "output_type": "stream", 24 | "text": [ 25 | "estimates [0.22 0.636]\n", 26 | "std.dev. (jackknife) [0.399 0.539]\n", 27 | "std.dev. (bootstrap) [0.345 0.36 ]\n", 28 | "bias (jackknife) [ 0. -0.159]\n", 29 | "bias (exact) [ 0. -0.159]\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "from resample import jackknife as j, bootstrap as b\n", 35 | "import numpy as np\n", 36 | "from scipy import stats\n", 37 | "\n", 38 | "rng = np.random.default_rng(1)\n", 39 | "data = rng.normal(size=5)\n", 40 | "\n", 41 | "\n", 42 | "def fn(d):\n", 43 | " return np.mean(d), np.var(d, ddof=0) # we return the biased variance\n", 44 | "\n", 45 | "\n", 46 | "print(\"estimates \", np.round(fn(data), 3))\n", 47 | "print(\"std.dev. (jackknife)\", np.round(j.variance(fn, data) ** 0.5, 3))\n", 48 | "print(\"std.dev. (bootstrap)\", np.round(b.variance(fn, data, random_state=1) ** 0.5, 3))\n", 49 | "print(\"bias (jackknife) \", np.round(j.bias(fn, data), 3))\n", 50 | "print(\"bias (exact) \", np.round((0, -1 / (len(data) - 1) * np.var(data, ddof=0)), 3))" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "The standard deviations for the estimates computed by bootstrap and jackknife differ by about 10 %. This difference shrinks for larger data sets.\n", 58 | "\n", 59 | "The Jackknife find the correct bias for both estimators." 60 | ] 61 | } 62 | ], 63 | "metadata": { 64 | "kernelspec": { 65 | "display_name": "Python 3 (ipykernel)", 66 | "language": "python", 67 | "name": "python3" 68 | }, 69 | "language_info": { 70 | "codemirror_mode": { 71 | "name": "ipython", 72 | "version": 3 73 | }, 74 | "file_extension": ".py", 75 | "mimetype": "text/x-python", 76 | "name": "python", 77 | "nbconvert_exporter": "python", 78 | "pygments_lexer": "ipython3", 79 | "version": "3.11.9" 80 | } 81 | }, 82 | "nbformat": 4, 83 | "nbformat_minor": 4 84 | } 85 | -------------------------------------------------------------------------------- /doc/tutorial/variance_fit_parameters.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "controversial-sally", 6 | "metadata": {}, 7 | "source": [ 8 | "# Variance of fit parameters\n", 9 | "\n", 10 | "We use the bootstrap and the jackknife to compute the uncertainties of a non-linear least-squares fit. The bootstrap is generally superior to the jackknife, which we will also see here. We use `scipy.optimize.curve_fit` to perform the fit, which also estimates the parameter uncertainties with asymptotic theory. For reference, we also doing a Monte-Carlo simulation of the experiment with a large number of tries, to have a reference for the parameter uncertainties.\n", 11 | "\n", 12 | "In this case, the asymptotic theory estimate is very accurate, while the bootstrap and the jackknife estimates are similar and off. The accuracy of the non-parametric methods improves with the sample size." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "major-companion", 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np\n", 23 | "import matplotlib.pyplot as plt\n", 24 | "from scipy.optimize import curve_fit\n", 25 | "from resample import bootstrap, jackknife\n", 26 | "\n", 27 | "rng = np.random.default_rng(1)\n", 28 | "\n", 29 | "# generate some random data, each y value scatters randomly\n", 30 | "x = np.linspace(0, 1, 100)\n", 31 | "y = 1 + 10 * x ** 2\n", 32 | "ye = 0.5 + x\n", 33 | "y += rng.normal(0, ye)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 2, 39 | "id": "intermediate-currency", 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "def model(x, a, b, c):\n", 44 | " return a + b * x + c * x ** 2\n", 45 | "\n", 46 | "def fit(x, y, ye):\n", 47 | " return curve_fit(model, x, y, sigma=ye, absolute_sigma=True)\n", 48 | "\n", 49 | "# fit original data and compute covariance estimate from asymptotic theory\n", 50 | "par, cov = fit(x, y, ye)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 3, 56 | "id": "successful-inquiry", 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "data": { 61 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABWAElEQVR4nO3deXwTdf4/8NckzdFCWyjQNmC5KgK1KIKACChIkct6rXIoCqwLirAu8P25gIqlqIDHuvVAUFYBRcQTlmurBQUWL5BSVyiCQkGUttwtUNpc8/ujJDRpjkkyk6uv5+PRXWYymflkqMw7n+P9FkRRFEFEREQUJKpQN4CIiIgaFgYfREREFFQMPoiIiCioGHwQERFRUDH4ICIioqBi8EFERERBxeCDiIiIgorBBxEREQVVTKgb4MxqteLYsWOIj4+HIAihbg4RERFJIIoizp07h5YtW0Kl8ty3EXbBx7Fjx5CWlhbqZhAREZEfjh49iiuuuMLjMWEXfMTHxwOobXxCQkKIW0NERERSVFZWIi0tzf4c9yTsgg/bUEtCQgKDDyIioggjZcoEJ5wSERFRUDH4ICIioqBi8EFERERBFXZzPqQQRRFmsxkWiyXUTYk4arUaMTExXMZMREQhE3HBh9FoRGlpKaqqqkLdlIgVFxcHg8EArVYb6qYQEVEDFFHBh9VqRUlJCdRqNVq2bAmtVstv8D4QRRFGoxEnTpxASUkJOnTo4DURDBERkdwiKvgwGo2wWq1IS0tDXFxcqJsTkWJjY6HRaHDkyBEYjUbo9fpQN4mIiBqYiPzay2/rgeH9IyKiUOJTiIiIiIKKwQcREREFFYOPIBFFERMnTkRSUhIEQUCTJk0wderUUDeLiIgo6CJqwmkky8/Px7Jly7Blyxa0b98eKpUKsbGx9tfbtm2LqVOnMiAhIqKox+AjSA4ePAiDwYAbb7wx1E0hIiIKqcgPPkQRMIUo4ZgmDpCQZ2TcuHFYvnw5gNpqf23atEHbtm3RtWtX5OXloX///jhy5AimTZuGadOmAagdpiEiIpJTldGMjKc/BwAUzx2MOG1owoDIDz5MVcC8lqG59hPHAG0jr4e98sorSE9Px1tvvYWdO3dCrVbj3nvvtb/+2Wef4dprr8XEiRMxYcIEJVtMRERhLlwCBCVF3ycKQ4mJiYiPj4darUZqamq915OSkqBWqxEfH+/ydSIiomgS+cGHJq62ByJU1yYiIiKfRH7wIQiShj6IiIgoPDDPR5jQarWwWCyhbgYREZHiGHyEibZt22Lbtm34448/cPLkyVA3h4iISDEMPsLE3LlzcfjwYaSnp6NFixahbg4REZFiIn/OR4Rwzl66ZcsWh9dvuOEG/Pjjj8FtFBERUQiw54OIiIiCisEHERERBRWDDyIiIgoqBh9EREQUVAw+iIiIKKgYfBAREVFQMfggIiKioGqwwUeV0Yy2Mzeg7cwNqDKaQ90cIiKiBqPBBh/hoH///g6Jx4iIiBqCBht8WKyi/c87Sk47bIejLVu2QBAEnD17NtRNISIiCkiDDD7y95Qi6+Wt9u1xS3ei7/NfIn9PaQhbRURE1DD4HHxs27YN2dnZaNmyJQRBwJo1a+yvmUwmzJgxA126dEGjRo3QsmVLPPjggzh27JicbQ5I/p5STFpRiPLKGof9ZRXVmLSiULEA5MKFC3jwwQfRuHFjGAwG/OMf/3B4/b333sP111+P+Ph4pKam4r777sPx48cBAIcPH8aAAQMAAE2bNoUgCBg3blzt58nPR9++fdGkSRM0a9YMt912Gw4ePKjIZyAiIpKDz8HHhQsXcO2112LhwoX1XquqqkJhYSFmz56NwsJCfPbZZ9i/fz9uv/12WRobKItVRO66YrgaYLHty11XrMgQzOOPP46tW7fi3//+N7744gts2bIFhYWF9tdNJhOeeeYZ/Pjjj1izZg0OHz5sDzDS0tLw6aefAgD279+P0tJSvPLKKwBq/z6mT5+OH374AZs3b4ZKpcJdd90Fq9Uq+2cgIqLwFUkLKXyuajt06FAMHTrU5WuJiYkoKChw2Pf666+jZ8+e+O2339C6dWv/WimTHSWnUVpR7fZ1EUBpRTV2lJxG7/Rmsl33/PnzePvtt7FixQoMHDgQALB8+XJcccUV9mP+/Oc/2//cvn17vPrqq+jRowfOnz+Pxo0bIykpCQCQnJyMJk2a2I/905/+5HCtd955By1atEBxcTEyMzNl+wxERERyUXzOR0VFBQRBcHhg1lVTU4PKykqHH6UcP+c+8PDnOKkOHjwIo9GIXr162fclJSWhY8eO9u1du3YhOzsbrVu3Rnx8PG6++WYAwG+//ebx3L/88gtGjx6N9u3bIyEhAW3btpX0PiIiolBRNPiorq7GjBkzMHr0aCQkJLg8Zv78+UhMTLT/pKWlKdae5Hi9rMfJ5cKFCxg8eDASEhLw/vvvY+fOnVi9ejUAwGg0enxvdnY2Tp8+jSVLluD777/H999/L+l9REREoaJY8GEymTBixAiIoohFixa5PW7WrFmoqKiw/xw9elSpJqFnuyQYEvUQ3LwuADAk6tGzXZKs101PT4dGo7EHBgBw5swZHDhwAADw888/49SpU1iwYAH69euHTp062Seb2mi1WgCAxWKx7zt16hT279+Pp556CgMHDkTnzp1x5swZWdtOREQkN0WCD1vgceTIERQUFLjt9QAAnU6HhIQEhx+lqFUCcrIzAKBeAGLbzsnOgFrlLjzxT+PGjfHQQw/h8ccfx5dffok9e/Zg3LhxUKlqb3/r1q2h1Wrx2muv4dChQ1i7di2eeeYZh3O0adMGgiBg/fr1OHHiBM6fP4+mTZuiWbNmeOutt/Drr7/iyy+/xPTp02VtOxERSRNJEz5DTfbgwxZ4/PLLL9i0aROaNZNv4qYchmQasGhMNyQn6Bz2pybqsWhMNwzJNChy3RdffBH9+vVDdnY2srKy0LdvX3Tv3h0A0KJFCyxbtgwff/wxMjIysGDBArz00ksO72/VqhVyc3Mxc+ZMpKSkYMqUKVCpVFi1ahV27dqFzMxMTJs2DS+++KIi7SciIpKLz6tdzp8/j19//dW+XVJSgqKiIiQlJcFgMOCee+5BYWEh1q9fD4vFgrKyMgC1EyxtQwehNiTTgD5XNkeXOV8AAJaN74F+HVrI3uNRV+PGjfHee+/hvffes+97/PHH7X8ePXo0Ro8e7fAeUXRc8jt79mzMnj3bYV9WVhaKi4s9vo+IiCic+Bx8/PDDD/aEVwDs3fxjx47FnDlzsHbtWgBA165dHd731VdfoX///v63VGZ1A42e7ZIUDTyIiIjoMp+Dj/79+3v8Zh0p37rjtDE4vGB4qJtBRETkwLn2mNI986HQIGu7EBERhSOla4+FS1FVBh9ERERhwJfaY/6srAmnoqoMPoiIiEJM6dpjoSqq6k5EBh+RMq8kXPH+ERGFF19qj/kqlEVV3Ymo4EOj0QCorZ5L/rPdP9v9JCKi0FKy9piSgY2/fF7tEkpqtRpNmjSxpx6Pi4uDIETXDGAliaKIqqoqHD9+HE2aNIFarQ51k4iICMrWHgtVUVVPIir4AIDU1FQAqFf7hKRr0qSJ/T4SEVHo2WqPlVVUuxweEVCbiduf2mPhWFQ14oIPQRBgMBiQnJwMk8kU6uZEHI1Gwx4PIqIwY6s9NmlFIQTAIQAJtPaYkoGNvyIu+LBRq9V8iBIRUdSw1R7LWbvXYVVKaqIeOdkZftceUzKw8VdETTglIiKKZkMyDdg0/Wb79rLxPbB9xi0BFz0NVVFVdyK254OIiCgaKVV7LBRFVd1hzwcREZEC/MlCqrRwKarK4IOIiIiCisEHERERBRWDDyIiIh+F45BKJGHwQUREREHF4IOIiIiCisEHERFRA6KCNdRNYPBBRETUYBgv4APtsxit3hzSZjDJGBERkRdVRjMynv4cAFA8d7Cs56m7HadV8LFsroHu07HopfoZnYTfgKoZgDZFuet5wJ4PIiKiKGCxXq7asqPktMM2LGbg04egLvkKVaIO441/B+KahaCVtRh8EBERRbj8PaXIenmrfXvc0p3o+/yXyN9TClitwNq/AvvWQVRrMcE0HYXiVSFsLYddiIiIIlpBcTmmripyqFYLAGUV1Zi0ohCLMvZiyKGVgKCG8c5/4ev3Q//oZ88HERFFpYaSCGzexn31Ag8Al/aJyC1OgUUUgDvfgKXj8OA2zg0GH0RERDLwOOdCQeWVNW5fEyGgFM2xo+crwLWjgtIeKULf90JERBQkzqtN5Fpdkr+nFDlr99q3xy3diZQEnSznlsPxVlmhboIDBh9EREQBcDfn4riHHolgS47Xh7oJDjjsQkREFADPcy5qKTkEk5Kgg+DmNQGAIVGPnu2SFLu+Pxh8EBERBcDTnAubXUfOKHb9J4Z1BgAITiGQLSDJyc6AWuUuPAkNBh9EREQKO3FOuSGYQRkpWDQwBinCaYf9qYl6LBrTDUMyDYpd21+c80FERKSwFvHKTT5VHf4vhnz/IAZpa/CK+W68ZrkLS8f3Qr8OLcKux8OGPR9EREQB8DTnwqZ7m6ayXtM2h6QtSrH7g1xYTDUQrxqMNyx3QIQKPdslhW3gATD4ICIiCsjlOReO6m7LGQjUTaV+GAaMvvg4+lrexIZOC2COkAENBh9EREQBGJSRgkVjuiHZKa9HSoL8y1vz95Ri0orCepNcy8yNMfXjYtmvpxQGH0RERAEakmnApuk327eXje+Bguk3yXoNi1VE7rpir8t6I0Fk9M8QERGFubpDK0rk1dhRchqlFdVuX4+kAMTnno9t27YhOzsbLVu2hCAIWLNmjcProiji6aefhsFgQGxsLLKysvDLL7/I1V4iIqIG6XjZ76Fugmx8Dj4uXLiAa6+9FgsXLnT5+gsvvIBXX30Vixcvxvfff49GjRph8ODBqK52H60RERGRB2ePInn70z69JZyr+vo87DJ06FAMHTrU5WuiKCIvLw9PPfUU7rjjDgDAu+++i5SUFKxZswajRoVPRT0iIiJXlCo+J1WcNgaHFwy/vKPiD2B5NnpWHYZBNRJl1kSXQywCImfoRdYJpyUlJSgrK0NW1uXqeYmJiejVqxe+/fZbOS9FREQU/SpLgeXZwJkSqJPaIOfOawF4XtYbCWQNPsrKygAAKSkpDvtTUlLsrzmrqalBZWWlww8REVGkq1tMbkfJad+Ly50rrw08Th8EmrQGxq7HkJ5Xu1zWm5qoR96orjK0OjhCvtpl/vz5yM3NDXUziIiIZFNQXI55G/fZt8ct3QlDoh4zh3aSdoLzJ2oDj1O/AIlpwNj1QJM0ALXLevtc2Rxd5nwBoHZZb78OLVBjtng9bb0hnRCRtecjNTUVAFBeXu6wv7y83P6as1mzZqGiosL+c/ToUTmbREREJDtvvRpTVxXVTwRWUY2pq4q8n/zCSeDd24GT+4GEVsDYtUDTNg6HOC/rDedU6q7IGny0a9cOqamp2Lx5s31fZWUlvv/+e/Tu3dvle3Q6HRISEhx+iIiIwlVBcbk9vTlQ26vR9/kvUVB8+Yu334nAqk4D794BHC8G4g3A2HVAUvuA2xxufB52OX/+PH799Vf7dklJCYqKipCUlITWrVtj6tSpePbZZ9GhQwe0a9cOs2fPRsuWLXHnnXfK2W4iIopSoV5t4s3UVUX1AgmpvRoeA5CLZ4AP7gLK9wCNU2oDj2bpAbQ0fPn8N/rDDz9gwIAB9u3p06cDAMaOHYtly5bh73//Oy5cuICJEyfi7Nmz6Nu3L/Lz86HXy5/jnoiIKNjc9WoEMvCRgAvQffAnoOwnoFGL2sCjeYcAzhjefA4++vfvD1F0H7sJgoC5c+di7ty5ATWMiIgokvibYyMeVXhXuwDqsoNAXLPawKNFR1nbFm5YWI6IiMgLX5fJSukF2VFyGpaqs3hPOw9dVQchxiYBD64Fkjv718gIEl4DaURERGEmf08pctbu9fl93jKOjlu6E6nqSsxRNcNp8Thi71uN2NRMv9sZSdjzQURE5EZBcTkmrSist2zWlbq9HXmjutZLBOZKmaUxHjFNxcCal/B9VUvfE5FFKAYfREQUlQLOMApg3sZ9kuZyOA+zDMpIwabpN9u3m8Zp3LxTBUDAGSTYl+zm7yn1uZ2RhsEHERFFnfw9pV5zcUghpccDcJ3evG7irzNVJknnKauoxqQVhVEfgDD4ICKiqJK/p9TlUElZRTX+VicXh7+9Ic4euak9ts+4BYMyUrwf7IWtNbnriqN6CIbBBxERRQ2LVUTuumJJGUblGua4Ib2ZrOnNRQClFdXYUXJatnOGGwYfREQUNXaUnEZpRbXk470Nc6Qk6Lwum+3epqnH15vjDFJVFRBgldwuADh+TvrniDQMPoiIKGr4+sD2NszxxLDanBvOAUjdbU+9Hsk4gw+1z2KO+h0Agk9ZUJPjozczOIMPIiKKGv48sD0NcwzKSMGiMd3qLZtNSfB+HeHcMazSPoN0VSlubVqKRXemSVp+KwAwJOrRs12S1I8QcRh8EBFR1OjZLgmGRL1fdVbc9ZoMyTQ4LJtdNr4HCqbf5PFcQuUf0K24He1VZThqbYGaMesw5IZrHc4DuO9RycnOkHUeSbhh8EFERGGvymhG25kb0HbmBlQZzW6PU6sE5GRnAPC90JunXpO6gUDPdkkeA4M0oRy6926D6kwJfrO2wCjjUxCbtK53nldcJCJLTdRj0ZhuGJJp8LH1kYXBBxERRZUhmQaXQyXuyDnM0V44ho+0z0BV8RusTdtjpPFp/IEWLo91TkS2bHwPbJ9xS9QHHgCDDyIiikLOQyVTBqRDgLLDHMLxYnyonQuDcBrW5h1R/cA6lKKZx/f40qPiTZw2BocXDMfhBcMRpw3v0m0MPoiIyGdSh0FCqe6D/NEBV7rsDZFtmOOPQujfvx0thErstbZB9f1rgcapgZ0zioV3aERERCSTIZkG9LmyObrM+QJA7TBHvw4tAu7xUP2+A/hwJISaSqDV9bh6zCdAbNOwDcrCAXs+iIiowXA1zBFIL05v1V7oPrgHqKkE2vQBHlwDxHpOOkbs+SAiIvJLf1URFmv+CcFkAtoPAEatBLRxoW5WRGDPBxERkY/U+9fjLc0/oBdMMHcYAoxexcDDBww+iIiI6qibZt1V5Vv13k+h/ezP0AoWrLf0gvHuZYAmelOhK4HBBxER0SUFxeXIenmrfdtW+baguBwAcK96C7T/fhiCaMGnln74m2kKoNaEqLWeeQuiQonBBxER0SVTVxWhvLLGYV9ZRTWmrirCzaoivKh5CwJEmK4bh/9nehgWqEPUUs/y95S6DKLcVe8NNgYfREREl7jqGxAv/e8B6xWwiAJMPSfBNOQliGH6CM3fU4pJKwpdBlGTVhSGRQASnneOiIgarHAcLhAhoBTNMcM0AaaBzwBCeBZ9s1hF5K4r9hBEAbnrikN+T7nUloiIwkb+nlLkrN1r3x63dCcMiXrMHNophK267BNrf8wNg8DDlkrd2Y6S0yitcF2dF6gNQEorqrGj5DR6p3tO/a4kBh9ERFGsymhGxtOfAwCK5w4O65oftuEC5+/ktjkX5N3xc+4DD3+OUwqHXYiIKOSkDBcEg7s+jdD3dUiTHC9tya/U45TC4IOIiByEomiclOGCYBGcrhYpgQdQmzLekKj3GEQZEvXo2S4pmM2qh8EHERGFXKiHAQCgBc5iYbOPkYrTDvtTE/XIG9U1NI3ykVolICc7A0D9oMm2nZOdEXAxvUAx+CAiopAL9TDAFcJxfKzNxbALq/HfpOeQhtqkYsvG98D2GbdgUEZKSNvniyGZBiwa0w3JCTqH/amJeiwa0w1DMg0hatllDD6IiCjkpAwXKEU48TM+0eairaoc1iZtYBq7HkeRYm9XqHsJ/DEk04BN02+2b9uCqHAIPAAGH0REFAakDBco4o9d0K/IRqpwBgesrVDzwAaITdspecWgqRs0hVsQxeCDiIjCgqfhAiXmXKhKtgDLb4dw8TSKrOkYYXwaYrx8PQO2XByHFwwPeIlzKCYBKyl8F3wTEVHU8ZZ3ZEimAX2ubI4uc74AUDtc0K9DC9SYLbK24zbVt9B9uBiwmmBp0w/37x+HC4iV/H53Sb5IGvZ8EBFRWFF6uGCs+nO8qnkdgtUEZNyJmpEf+hR4UOAYfBARUcMgitBseRa5muVQCSJM3R8C7nkHiNF5fy/JisEHERFFP6sZWDsFmm/+CQB40TQCplufB1TqEDesYZI9+LBYLJg9ezbatWuH2NhYpKen45lnnoEohr4qIRER+SfUEx4DqXSrRw20n44Fdq+AKKgwwzQBCy13hm1l2oZA9gmnzz//PBYtWoTly5fj6quvxg8//IDx48cjMTERjz32mNyXIyKiKOeu0m1OdobXvBWJOI9/aV9CzC8HgBg9jHcuwYcrfO/t4ARTeckefHzzzTe44447MHx47V9S27Zt8cEHH2DHjh1yX4qIiELEuSeiX4cWfk8MdV4BU1dBcTmmripyWel20opCjxk7hco/8JF2LjqqfoeoT4Qw+kNYDD0AfO5XO+XgHMREw7JZf8g+7HLjjTdi8+bNOHDgAADgxx9/xPbt2zF06FCXx9fU1KCystLhh4iIwldBcTmyXt5q3x63dCf6Pv8l8veUyn6teRv3eax0m7uu2PUQzIn9iH13GDqqfgfiDRDG5wNtesvePvKP7D0fM2fORGVlJTp16gS1Wg2LxYLnnnsO999/v8vj58+fj9zcXLmbQURECvG3J8If5ZU1bl8TAZRWVGNHyWn0Tm92+YWjO4GV9wIXzwDNOgAPfAY0aS1bm+TUUIdzZO/5+Oijj/D+++9j5cqVKCwsxPLly/HSSy9h+fLlLo+fNWsWKioq7D9Hjx6Vu0lERCQjv3oiFORQEffnDcDy7NrAo9X1wJ8/D9vAoyGTvefj8ccfx8yZMzFq1CgAQJcuXXDkyBHMnz8fY8eOrXe8TqeDTsc11kREkc5tT4TC7BVxdywB/vN3QLQCHW4F7l0GaBs5HNtQexrCjew9H1VVVVCpHE+rVqthtVrlvhQREYUhh56IAKUk6DxWujUk6tGzTROg4Glg4/+rDTy6jQVGfVAv8HAnkGW85B/Zez6ys7Px3HPPoXXr1rj66quxe/duvPzyy/jzn/8s96WIiCgM2XsiZPDEsM6YuqoIAhyHe2wBSc6wq6BeMwHY82ntjltmA/3+T3IOD1fLeFMS2BuvNNmDj9deew2zZ8/Go48+iuPHj6Nly5Z4+OGH8fTTT8t9KSIiCgHnQKDu/tREPXq2S5LtWoMyUrBoTDfkrN3rMPk0NVGPnMFtMKTwYeDI14AqBrhjIXDtKMnnzt9TikkrCut9luMeJrmSPGQPPuLj45GXl4e8vDy5T01ERGHCbU9EdobsheBcVrptcRHqlfcCJ/cDugRg5HuouqIvMmZuAOC6Ym5dFquI3HXFHifP2o4j+bG2CxFRFFNiPkPeqK5IdhqaSE3UI29UVzyyolCRFOx1A5pesUehfmdQbeAR3xL4cz7Qvr9P59tRchqlFd7npuw6csbXppIEsvd8EBGR/JyzgHr6Vm8TSFpyTwZlpOCWTsmOPREdWqDGbPH7nFLdrPoR+vcmAKYLQEomcN9HQGIrn88jdVLsiXMcglECez6IiKKQbT6Dc5IuWzKwQLOR1u2J6NkuSfahFldGqL/C25oXIZguAO1uBsZv9CvwAKRPim0Rz8mnSmDwQUQUZaTMZ/CUDCzslp6KIjRb5+MFzRLECFaYu4wE7v8E0Cf6fcqe7ZJgSNS7XcZr071NU7+vQe4x+CAiijLe5jPUTQbmLH9Pqcu6LQXF5Uo01TtzNfDpX6D5+iUAwCvmu2C8bSEQow3otGqVgJzsDACoF4AITseR/Bh8EBFFGanzGZyP8zRUM3VVkVzNkywJldC9fyew5xOIqhg8bpqIf5rvlZzDw5shmQYsGtOt3uTZlAT58pSQaww+iIiijNT5DHWPk7r0NFjShT+wRjsb6j92AvpE1Iz6BB9b+st+nSGZBmyafrN9e9n4HiiYfpPs1yFHDD6IiKKMt/kM9rTkdZKBSRmqCRZVyVas1uagteoErE3bAX/ZDGvbfopdLxSTZxs6Bh9ERFGiymhG25kbkP7ERswc2gmA+/kMzsnA5KzHEgj17neh+3AEEoQq7LReheqxnwPNO4S6WSQzBh9ERFHIlpbcVTKwRWO61cvzIWc9Fn8IsGJmzEro/jMNgtWM1ZY+uN/4JBAXvOq4NrbKt4cXDJeUT4V8x7tKRBSlXKYl79DC5bCCbaimrKLabd0WOYZenJfx9uvQAjBVYZHmFQxR7wQAGPvNwLSCa1C/34aiBYMPIqIoJnU+g23p6aQVhW7rtgTKZcbVeA1mx36EYeqdqBFjgDteh67baBweKNNFKSxx2IWIiAC4X3pqq9sSCLfLeM8ZMfn4nfjYfBPuMz4JS+a9AV2HLgvn4aPwag0REYWUu6GaQOq2eF7GK0CAiJnmCbBA7fc1KLKw54OIiBy4GqoJJOW692W8AgMPL8Iu5X2A2PNBREQeuZqrkZIgveBauCzjjVRKVScOJfZ8EBGRWwXF5S7nahyvlF5qXollvLacJm1nbkCV0Sz7+cOF0tWJQ4XBBxFRA+LrQ3vexn1eU657GwLoqTkEg+oMBFi9Xi8ahhTkEmh14nDG4IOIiNxy/sbtyq4jZ9y/+L+PoF4+HDnqZQAEr8t2bVV0I/UbvZwCqU4c7hh8EBGFmUgbUjhxzkWAIlqBTXOAzyYAlhoM6dwci0ZeXW8ZryuRPqQgF3+rE0cCBh9ERBSQFvGOAUUjXIT2kweA7f+s3dF3OjBqJYZc186hgmzTOI3L80X6kIJc/KlOHCkYfBARkVspCTqvQyXd2zS1/zlNKMen2jmI+SUfUOuAu5cAWTmAqvZxU3cZ75kqk9tzRvKQglz8qU4cKRh8EBGRW08M6wzAfXVc4HJAoTr0FdZpn0In1VFYG6cA4/8DXDMioOtH4pCCXGwp7wHp1YkjBYMPIiJyy1113JSEOl39oghsz4PuwxFoIlxAkTUdNeM3AVd0D/j6kTikICdPKe9dVSeOFAw+iIgiQCgzXA7JNDjM1Vg2vgcKpt8EAIhFNbRr/gJsyoEgWvGhuT9GGmdDjG/p9byehnQieUhBbq7u//YZt0Rs4AEw+CAiCnv5e0qR9fJW+3YolqO6SrmeJpTjM20OYvatAVQxuHjri5hhnoAaaCUFSN6GdHwdUoi2FOR1Sa1OHCkYfBARhTF3GUZDvRw17ret+G9iLjqrjgKNkpHf7zP039IettBBSoDkbkjH1ZCCtwqt4RCgkXQMPoiIgkjuDKPBX44qIubbV4H37wGqzwKtrkf+zf/GpM/P+xUgyTGkEK4BGrnH4IOIKIx5yjAq13JUqQFRLKrxuuY1aL/KrU0idt0DsIzdgNzNpQEFSIEOKYRfgEbeMPggIopwwViOKpw5jM+0ObhN/R1ElQYY/jJw+2vYcfRCyFOAByNAI3kx+CAiinCKL0f9pQD6pQPRWXUUJ8RE1Nz/b6DHQ4AgSA58Ri/5LqSp4htyvpBwxOCDiCiMhXI5qgpWaLYtAN6/F0L1Wey2Xonbap6DNa2X/ZhIycMRKe1sKBh8EBGFMV+Wo8q51LQpKrFM8zw0218EIMLU7c8YaZyNcjgGOlJSgCuN+UIiD4MPIqIwJnU5qrulpgXF5T5fU/XHD1ivexI3qX+CGBML3PUWTENehBH1C8FJSQGutEDzhXhbxkvyY/BBRBTmvC1Hzd9T6nap6dRVRVg8ppvEB6uImF1vQ/febWglnMIhayqqx30BXDvSa/vcBUh5o7pK/pz+8iVfCIUHhnhEFNWqjGZkPP05AKB47mBZv9kqeW5n7pajWqwictcVu11qKqB2qemgjFSP3/5jUY15mreh/fxrAMB/LD3wuOlh7EjOkNS+IZkG9LmyObrM+QJAbYDUr0ML1Jgtkt4fKHfXj/RMoNFKkZ6PP/74A2PGjEGzZs0QGxuLLl264IcfflDiUkREDdqOktMBL3UVTv2CNdqncZf6a4iCGsaBczHJNBXnEedTW0KdAjzU1yfpZA/Tz5w5gz59+mDAgAH4z3/+gxYtWuCXX35B06ZN5b4UEVGDJ3UJqdvjiv8N/ZpH0VF1HsfFJkgYswL6K/vhcD8ZG0nkRPbg4/nnn0daWhqWLl1q39euXTu5L0NERJC+hNT5uBiYodk0G9jxBgQA31s7YYrxMWxt3VuBVhI5kn3YZe3atbj++utx7733Ijk5Gddddx2WLFni9viamhpUVlY6/BARkTRSlro6LzVthRP4SDsXmh1vAABMN/wV9xmfxAk0Uby9RIACwcehQ4ewaNEidOjQAZ9//jkmTZqExx57DMuXL3d5/Pz585GYmGj/SUtLk7tJRERRS8pS17pLTdUHNmKD7gl0U/0KUZcAjFwB0y1zYIE6aG0mkj34sFqt6NatG+bNm4frrrsOEydOxIQJE7B48WKXx8+aNQsVFRX2n6NHj8rdJCKiqOZpqat9qanZCOTPQszHD2KftTVeNd2JbbdugKXjbSFqNTVkss/5MBgMyMhwXJrVuXNnfPrppy6P1+l00Ol0Ll8jIvIkmEtdleT8OZwzlfbr0MLrOTwuNT1zGPh4PPKPqpFrehWlaFb7po+PwvDFCcwc2kn2z0Tkiew9H3369MH+/fsd9h04cABt2rSR+1JERFGnoLjc70ylLpeaFq8FFt+E/KNqTDJNRalTenRbIrJII2cqeQo+2b8mTJs2DTfeeCPmzZuHESNGYMeOHXjrrbfw1ltvyX0pIqKgCVYvy9RVRfUShvkVIJhrgE1PADvehEUUkGv9C0QX01JticgiSf6eUuSs3WvfHrd0JwyJevbgRBDZ/+vp0aMHVq9ejVmzZmHu3Llo164d8vLycP/998t9KSKiqOMpU6lUrYVy6N4dCpT9CADYkfEkSnfH+3TNcGVLJS9LgEYho0joftttt+G22ziJiYhILlIDhOGq7zBfswTqsotAbBJw12Icr8oEdhcp2TzJbEXc/CEllTxFhsicnUVERI5qzkO74XEs1K4EAFiu6AX1vUuBxFZIPngqxI2Th5RU8hQZWNWWiCjSHdsNvHkTYv63EhZRwCvmu1AzZi2Q2AqAtERkkUBqKnkKf+z5ICIKIwJcf4N3ud9qBb59Hdg8F7CaYI1vidEnH8IOsTMmqC7/825LRDZpRWG980gNPAIZLpGL1FTyFP7Y80FEFERSloi6y1Tq4FwZsOJuoGA2YDUBnbNR/Zdt2CF2dnldT4nI8kZ19e1DhEi09OAQgw8ioqDJ31PqNYdH3qiuXgME1a9fAIv6AIe+AmJigexXgBHvAbGeq4cPyTRg0/Sb7dvLxvfA9hm3YFBGSoCfLDikpJKnyMDgg4jCUpXRjLYzN6DtzA2oMppD3ZyA2ZaIllfWOOx3XiI6KCPFbYCggxE5Mcuh/2g0UHUSSOkCPLwV6D4OEKQ9fl0lIoukhF3R0INDnPNBRKQ4X5eIugoQhBM/Y432aXRW/Vb7wg2PAgNzAE1g8yDcJezKyc6orQkTBL7OJ3GXSr7GbFGqiSEXDnNu5MSeDyIihQW0RFS0At8thn7pQHRW/YaTYgKqR6wChswPOPAoKC532xszaUUh8veUBnR+JblMJU8Rg8EHEUU1V0MKwR7S8XeJaCpOQffBPUD+DAjmamy1XIOhNQtgvXKQLO2at3Gf294YAMhdVxzWQzAUuRh8EJGiQjl3Q8oEz2DwZ4nobapv8bluBtSHtwIxsTAOfgFjTTNwAk1ka5dzj0ddIoDSimrsKDkd0DUiaT4JBQ/nfBBRVAq0BohzITlXnB+s9hL2TmxLRMsqqr3n8Lh4FtovZuB17Se11zBcB/WflkDbvAMO9/babNkFktgrHOaTUHhizwcRRR1vEzzl4K5XxdU8CalLRG9U7YH+X/0Qs/cTmEUV8sx3o+bB/wDNO8jUat/5m9grkueTkPIYfBBR1FG6Bog/D1aPS0TvycDsmPewUjsPqnPHYG3aHvcY5yDPfA+g1gTYWvdSEnQeE3YZEvXo2S7Jr3NzPgl5wuCDiKKO0jVA/H2wukzy9WAzZH83Eg/F/AcAYLpuHKof2oIi8Ur5G+7kiWG12VDd9cbkZGf4tIrEthz0gwk3BGU+CUUuBh9EFHWUrgESyIPV9jCPgRk3/vEO1G9nQXVyP06IiRhvfBymof8AtI2UaHY9gzJS3PbGLBrTze95GVKDPxaKa7g44ZSIIpa7CZ8+TfBUiKcHa0fhN7ykWQzttsMAAPNVwzH4f9k4jQSFW1Wfu4RdgeTNkBr8yR0kRlsirmjGng8iikieJnyGQw0Qlw9WixkxX7+Mddon0UV1GKK+CXD3v2D80/KQBB42cifsklIALpD5JBT5GHwQUcTxVCfFNuFTyRogfk3UPP4z8HYWtFufg1awoMDSDRcnfg1cc6/kuiyRQkrw5+t8EoouDD6IKKJIWUZrm/CpVBVXnyZqWszA9n8Cb/YDju2GqE/ENOMkTDD9H9A41f6ZbHxJxOWcwM027HB4wXDEaUM7qu4p+AtkPglFBwYfRCQrpTOaSllGW3fCp1xDCnUDgsRYDRbeV//BmpKghwjgkRWFtZ/9xH5Y3h4EbJoDWIywXHkrqid8jdXWfrCFKq6Gj+puRzJ3wR8DD2LwQURBF0iAEoqVFAXF5fUChGc2FGPmkE72fcvG90DB9JsAACpYEfPda8DiflAfK0SlGIf/Z3oYNfeuhBhvcDivq+Gj4x5W00QaFoAjV7jahYgiSihWUkxdVeQyTfv0j360b9vmeFwlHMXzmiXQfvkrAMDSfiBuLb4TZWiGuU5zO7zlCwHARFwUldjzQUQRJRQrKSSlaTfXQLNtAdZrn8B1ql8h6uKB219HzcgPUYZmLs/rKV+Iza4jZ3xuL1G4Y/BBRBElnFZS2AKQDjgK/TsDoNn+4qWVLN1RPfEboNsDAa9kOXEueoZgiGwYfBBRWPK0AiTcVlJMjvk3VCf3Q4xrgcnGxzDBNB1ifEtZzt0iXuf9IKIIw+CDiMKOlIqx4bSSIkU4A3OXkbg48RtssN4AqanMPOULsenepmnA7SMKNww+iAKg9LLShkhKAjGbYK2kcD+/xIpknMEb5tthzH4DiPNtnom3fCEAuDqEohKDDyIKG74kEAu2+gGCFYCAs2iE/4rX+nVOd4XdUhKULYxHFGpcaktEinJV/M0dXxKI9U53vYJECXm3GTA//xeUmRvb96U2jsHM267B31YVBXRuJQq7ecLiaxQOGHwQkWIKissxb+M++/a4pTthSNRj5tBOLo8PVgIxd9VwnWlgxkT1ety+ZS1uU9fga+FqrDQPxKgHJqJfp1aoMVsCaocNE3FRQ8Pgg4gU4y4511Q3vQXBSCCWv6cUOWv32rdtAVFOdobDZFXVka+xUTsLHVR/AGYA7W5Czv47UCIa8PKVqQwQiALAOR9El3DyqPwkJeeqQ+kEYu7SmTtMZr1wClg9CZoVd+CkmIAV5luw7YYlqBr5KUpE1iQhkgN7Pogo6NwFILYEYpNWFEJwOs62XVpRjRqzxa+qrZ7SmQsAcj/diUG6v6HgQjpyTa+i1JaZdAuQUrjN5+uRcjh3JbKx54OIwoq7BGJyrADxlM5cBFB6UY3Xz92MSaaplwOPS6Kp2BtRqDH4IKKw4yqBmK1irNKWqu6G6GLgh8XeiOTDYRciUozz0Im3/XU5rwAJlrMmtddjdh05g/4dk/06P4cLiNjzQUQK85S9M9g8pTMXADSJ1Ug6D4u9EQVG8eBjwYIFEAQBU6dOVfpSRKQgqauB6g5JTB6Q7rL4W96orko1060EXMDslj8AEC9lJ73M1hNz9qJJ0rlY7I0oMIoOu+zcuRNvvvkmrrnmGiUvQ0QKqDKakfH05wCA4rmDJb3HOYfG618dRHK81r5ty94pV3IuKeJiVDg84jSwaQ5w+CRiND0wBw+jzBRnPyYlQY+yyupLf9bheGWNx2EhFnsjCoxiPR/nz5/H/fffjyVLlqBpU/6HShTt3BWEO3HOaP+zLXunc4ZRxSZw/lEIvJ0FrJ0CVJ0Eml+FIeOeRMGTd9oPcZ7M2hCLvdnmoRxeMNyvJcxEvlIs+Jg8eTKGDx+OrKwsj8fV1NSgsrLS4YcoUgTtIRrmpBSEsx2Xv6cUWS9vte8bt3Qn+j7/pUO12oCdPw6s/Suw5Bbgj12AtjFw67PAI18D6QM8pjMPpNgbfx+IpFEkxF21ahUKCwuxc+dOr8fOnz8fubm5SjSDSFFS03Q3BN4Kwtm8ufUgFn510GXK9UkrCrFoTLeA7l1V1Xm89tz/4dGYfyNeuFi785qRQFYukCD9vK6KvXVv09S+bVN35Yq734e/D+5o3+epjgxRQyJ7z8fRo0fxt7/9De+//z70eu/fFGbNmoWKigr7z9GjR+VuEpHs3A0xOKTpbkCkFnp777vfPPaO5K4r9rO3QIR631ro3+yNGZpViBcuwmLoCozPB+5+y6fAw8aXYm/ufh9KK6ox7aMf7dty9/Kwp4UilezBx65du3D8+HF069YNMTExiImJwdatW/Hqq68iJiYGFovjRDOdToeEhASHH6JwJmWIwf+HaGSSWuitwsNqElvq9B0lp326dqZwCB9qn4Fu9XioKn5DmdgU042PoGZcAdCmt0/n8oen3wdX5ApQgzJ8RaQQ2YddBg4ciJ9++slh3/jx49GpUyfMmDEDarX3BD5E4czbEEPdh2jv9GZuj4smtoJwZRXVkh/C7kjtRUFlKbSbcrFWuwoqQYQYEwvzDX/FgE2dcBF6PCsEJ42R1CEnG3sdmXXFGJThX3VcW0+LUsNXcmJSNXJF9v864+PjkZmZ6fDTqFEjNGvWDJmZmXJfjkg2UruwpT4cJT9Ew0CgFX1tBeGAwJOKJXpJ9KWDETHbXwJe646Y/30AlSACXUZA+OsPMN00AxcReA0YX/jz9+xvLw/AnjeKDsxwSgTfurClDjFIPS5aSCkI5ynDqI3bHBpWC+5WbcOXuv+Ddtt8wHQBuKIn8JfNwJ+WAIlXBPgJ/BPI37M/gYsvPW9E4SoowceWLVuQl5cXjEsR+czXyaO2IQZPaboNifqg1iMJFeceE28F4fzLoSFCdXAT9O8MwMvaxWglnII14QrgT28DD30BXHG9vB/KR95+HzzxJ3CJxp43anjY80ENmj9d2FKGGHKyMxrscko5c2h0EQ5hpeY56D8cCdXxvagU4zDfNBrVD38HdLkHEEJ/jz39PrgTSIDKnjeKBgw+qEHztwvb3RBDaqK+3mS/QOdTBCrU13fmrXcEAHC6BNo1E7BO9xRuVBdDVGth6jUZ/Wry8KYlG9DEBrnVnrn7fXAl0ACVPW8UDZhHlxq0QLqwXSWiYgIpaZx7R2ySUAnNF7OAwqWIsZpgFQWstvbBsMmvQ0xMQ8XWz0PRXADeV224+n2oqrEgd/1ehyG9VBeJ6HxZEWLraZm0otBeEM+GPW8UKRh8UIMWaBe2qyEGfwqyNXjG85isXoNHYtZB80NtZlJLuwHI/nkQisW2GJqYVm81UjgGes6/D3HaGPS7Sv4A1dbTkrPWe2BDFI4YfFCD5i0/hYDaf9DDpQvbObCJ9CJgOhgRs2MxNN/k4XHNCQCANeUaqG6di5q0fii+9FkLissxb+M++/siKZW9L5lSfcGeN4pknPNBDVo4Tx4Nt7kasrKYMFq9GV/ppkO76UkIVSdw2JqCx4yTUf3nzUD6APtQxOIx3TB1VRFT2bugVGBDpDQGH9Tg+TJ5lAJktQA/fgj9W70xX/M2WgqnYY1viZqh/0SW8UWstfYB6mQmZUItoujE4IMIrldgbJ9xCwOPOgIrYiZisGoH9P/qB6yeCNWZEpwQE5BregDVk3bCct2DMLsYBWZCLaLoFNkDxkQyYhe2o7rBxRtf/YqPd/1u37bNuZg5tJPnk4giVIc2Y632KVyjKgFOAtAnwnjDX3FTfntchB6Px7if9MuEWkTRicEHEdWTv6cUOWv32rdf/+pgvWPKKqoxdVWRmzOIGKAqgm75S1AfK8Q1KgDaxsANk4DeU2BWN8bFfO/LZplQiyg6MfggIgcFxeWYuqrIa3VaW3VWx50i1Af+g7Xa2bU9HccAxMQCPR4C+k4DGjWvPU7iBNpgrUZytYyXiJTD4IOIHMzbuM9r4GFz+Tgr1Ps3AF+/BF3Z/3CNCqgSddDcMAGafn8DGif71RalEmrVTerl3MsjeUiJiPzGCadE5MB5SasUM9QfQvfpg0DZ/yBqGmGRORt9a16BaWCu34GHjZKrkTwVFXQ/pEREgWLwQSER1TksGqCuql8hahsD/f4PFyfvxvPm0TiNBNnOL3U1ki8rcqQs4yUiZXDYhYj8zpMhwIoUnMF31k64dvI6xCW2kDyfw1feViO5Gz5xlwVVyjJeAPhgwg0Rn0mWKNyw54OogcvfU4qsl7f6/D7h0pTTCsThFcu9QGxT+RsnkafhE3dZULmMlyh0GHxQ2IjWoZjAknMpy91D2zXHdqcm6JE36jpcRODl7QO5R/5mQeUyXqLQYV8iUQC8lUIPRUE0qVV1PT20azkupk0VKpCEM9gvtsbb429Avw4tHIJEf5eoBnqPfMmC2ju9mX1/pBUVJIom7Pkgklndb9h/C4OCaK56FaqMZqQ/sdHjQ9sWeNyr+grvDjShYPafUCy2gwVq9GyXhILiMofhmnFLd6Lv819i24ETOLxgOA4vGC5prkSgReP8HT4J56KCRNGOwQeRjKTMnwhmQbSC4nKXAUJBcbnkc3xs7Y/rb74N6pjLgURBcbnPcyzcCbRoXCDDJywqSBQaDD6IZOLL/Al/C6L5OjfCXa/C1FVFUEPqvJr63/zdJSKTGjTEaWPwwYQbPF5V6j2yDZ+4658QABg8DJ+wqCBR8DH4oAbJ1eRW2/wNqcMFdXmfP+GapyED50Bj4/9KXfZieOplcB8giGiGSqTiFARYXb7X02CDpwBLatAg12oTOYZPWFSQKLgYfBDJwNukR3fcDRk4D9+MW7oTj66UZ5gDAEQIOI4kDFN9D0Bw+9AOhLegQc7VJhw+IYosDD4o6gVjCa+vuSBsD/fRS76r1yZfh28A/+ePLLUOQd6o61w+tPNGdfX5fHV5CxoCHS5xxuETosjB4INIBr7kgvDUq+DP8I2rYY5EnJf4XhUGZaS4fGgPykhx+76UBF3AQYMSq01CPXwSyNAdUUPC4INCIpwTb/nD27f4ujz1Kvg7fAMAxyurgd++h3bNRHynfRQGH+Zz+PrQfmJYZ5fn8TVoaAjDJQxIiOpj8EFB52o+g6/LP8ONp2/xdXnrVQgklXfylr8D79yKmOJPEasy4/6YTVBqPsegjBTZggYOlxA1PAw+KKgivYS5p/kjbr/FJ1wekvHWq+BPKm8BIgw4iZ5nNwAxepivvR/Da57DS+aRis3nAOQNGkI9XBKp2KtCkYrBBwVNQyhh7uqBXDD9Jsnv92X4BoB9WCWnyX+gvjUXmL4PxuGvYq/YDgD8ms/hCwYNROQPBh8UNFJLmNuOjdR5IIE8kD0P34ioV9xNcxGLBmox5O8rgT5/A+LqT/JkgEBE4YbBBwWNL/MZpCTQilb24Zt4rcN+A07hjZhX8M+Y1zFQ2IVlo67EF0/ehUc2m9D2if9EVSVgIopuHCSkoPF1PoMtgVa0rHqQxGoFDm/DkAMrkSWux05NaxxHEySrzqPn1R1gunYmOi+tgggVXstID3VriYj8wuCDgsZbCXNntoLuueuKMSgjNbqHC04dBIpWAj+uAip/B1D7H2eycBZfWq7DtOk5UDdNRY3RDBGfh7atREQBYvBBQWObzzBpRSEESJtkWjeBVu/0ZvVerzKakfF07cO4eO7giJrxH48qDFd/B927ecDvOy6/oEsEuvwJ1VePwMA3TwEQMK1Rc8nndc6h0r1NU/kaHUK2lR1EFPki519qklWoHtq2+Qw5a/dKSh9uE0j+i7BiMQEHvoS2aBV26tZCL5iA3wEIKiB9IND1PqDjMECjh9VoBnzs5cjfU4qctXvt2+OW7kSK01JbIqJQY/BBQTck04A+VzZHlzlfSH6PP/kvPHHuHejXoYVywzqiFT2FfbhD/Q1iX50CXDyNGAAxArDfegXaZf0F2utGA/GpAV2moLgcU1cV1etROu4iyPOnd4Q9D0QkF9mDj/nz5+Ozzz7Dzz//jNjYWNx44414/vnn0bFjR7kvRRGs7oM+JUGH45U1LodhBNQmxZJaXAyo36vj/KCtqrEgd71j74AhUY+c7AwZJ7aKyBRKoNn8NNTFq/GR7ljt7osAGiXD1PkO3P11G/wktkPxDUOglaHnad7GfV5zqFisouTeEQYbRKQU2YOPrVu3YvLkyejRowfMZjOeeOIJ3HrrrSguLkajRo3kvhxFgSeGdcbUVUX15oH4W1ysroLicszbuM++PW7pTpfHybay5uQv0BR9hM3a95CuKgW+r91dKcYh39IDtz/wGPRX9ofJAvy0Xd6Jo1KGsd7cehALvzooqXeEiEgpsgcf+fn5DtvLli1DcnIydu3ahZtukp7pkSKXr/NJbHVCnOeBpMrQG+FqGMIVv1fWiCJw4meg+N9A8Vrg+F5oAKSrgGpRg5jOw2C5+k/o8b4VNdDitnb9AXUMLCaT/RSKD/vU8d53v0nqHSEiUpLicz4qKioAAElJrrvNa2pqUFNz+YFTWVmpdJMCEurVFaG+vlKc54EsG99Dlgeyv6XpXa2sqXvk1cJhaLY8C+xfD5z65fJLghqW9gPwf/uuQoG1O3befScAoKbOxFFXwx7yD/u4VnHR5PWYXUfOoH/HZEXbQUQNm6JPLqvViqlTp6JPnz7IzMx0ecz8+fORm5urZDMoQoRLGnCXK2usVuCPH6DZswb/1X6ENNUJ4JtLr6m1QPsBQMbtQMdhqIlJwJqnXQ+puJsUKsewj6e5M744cY5DMESkLEWDj8mTJ2PPnj3Yvn2722NmzZqF6dOn27crKyuRlpamZLNCIlp7LKRy/vzhzL6yxnQRKNkGHMgH9ucD545BAyBNBVwUtdB0GoyYzDuBDrcC+oTLJ3BKc26buGmxiuj7/Jduhz0CTajmae6MLwFJi/iGtzSXk2uJgkuxJ+CUKVOwfv16bNu2DVdccYXb43Q6HXS64P9j19CDAVca+j0RAKQmaNDzzHrgg3zg0BbAVHX5AG08aq4cjDG7O6NIvBJLut3o09CQlMJ60oZ9XHM3dyYlQY+yyupLf/a+sqhfhxY+X5uIyBeyP11EUcRf//pXrF69Glu2bEG7du3kvgSFkXAPWKR+6xcuHZVz8QWo119eEXNMTMJmSzfce99fsMWUiZwN+1Eu1j7Y3c3VcPctWmqitEASqrmaO9O9TVP7tpIri8IBezCIIoPsVW0nT56MFStWYOXKlYiPj0dZWRnKyspw8eJFuS8V0aqMZrSduQFtZ26wVyN1tU8pzrkvonmFg+tHqVNpepzCIk0ehqh3Aq26AwOewsWHtuDGmtcw2/xnfG68BpM++F+95ay2uRpSqu9KTZSWGKuRdJw7nubO2HpHkp3yeqQm6htWAT8iCinZv6YuWrQIANC/f3+H/UuXLsW4cePkvlxYCPdv/87crbaYObRTCFuljLxRXTFv4z6HoMGAU5gd8x6aCudqK8bGVKNnh1ZQd3oQ6PA+EJ8CABCNZgC1ycE8JfCSOldDamE9uWuxOPcGyLmySK6eBvZYEDUsigy7UPjK31OKSSsKXa62mLqqyGFfJARVrtKkQxSRJpSjn2oPhhe/j+H4L3ZqWtUGGjiLnqqfIbToiHfK2mGb9RosnvUY1HGNPV7HUwIvqXM1PBXWq7vtKQhwfkj720MWLiuLiKhhCr+nSZhz+bBTiL8Pf3fvs1hF5K4r9vgNPljk+KbrsgdHexGz9R/hv7pLS10P1P5fJ5UVJ62J6Hb7JKivykJVbAqetS2HjZGnboyUuRruCuvVnRRqI+UeBfP3kYhILg0m+JCy1NPbwz4chisCKYgmZbVFpMjfsQeTPjt8qc2XP3+ZUYfJxgfxWsxZJKvOomv/u2FtPwDdFx+HFSoUXzsY0MbUWw4rB6lzOrxNCpUqHH4fiYj80WCCD2e+fmP0ZbhCKYFmxgxWWXrZv41brcDJ/cBv3wFHv4flyPfILZsGEUlw7q8RL82hnmJ+DICAZa16oHtqU1jh24PdmdzF75yHPXwVDr+PRET+apDBh6tiY56+McoxXOFPCXPnNgeaGVPusvSuyPFtXI8aqI5sB0p3Ar99D/y+A6iusL++w9IZpfCWB0OwX99VxVZfhdMS1XAaPiMi8keDDD7cPcTdfWMMdLhCaglzTwJdbVFlNGP0ku8AuM994WsmTGeeAiS338atVuDUr8CxQsT9UYjD7XcCZf8D3ncaFtHE1S6BTeuF46YewBaL5HbJUbFVyeJ3voqm4TMiapgaZPDh6zfGQIYr3HWP132ASRmakGO1RV3eUnD70zvjLUACRLTCSaj3/RsoLwKO7QaOFQHGc/XfFN8SaN0LSLv0k9oFUNfmv0g+eArY8p3kdslVsVWp4ne+CtbwGRGRUhpk8OGOu8eSL8MVdSeBeuoer8s2NPH3wR0dzuPrw1/qQ8lV7ovEOA0gAmcvVT111zvjaeKutwAJEHCX+r8o/GQveqp+hlq4dGdiYgHDtUCrbkDLbrVBR2IaILh+qEvNl+FKoBVbw2GJajCGz4iIlMTgQwJfHnZ1J4Emxmo9do/XVVpRjWkf/ehwHl+HZqQ+lAZlpOCWTsn2b/BTBqRj4VcH6302qcMVCTiPFqjAQbTyeuzrlrvxuuVuGLTVyOluwpCemUCLToBa+q+ip3wZ3kRDxVZvv4+BDp8RESlN9vTq4crX7va6KcdtDztA2mQ+2yTQTcVlvjbTwXGHPBA6t9cWABgCWG3x8a7f3Q6X2FgsVuBcOVSH/4sx6gLMiVkG3cq7oHklE29q/omh6u8lXxsAyox6TPomHvknm/kUeNjY8mU4pwn3Jhoqtnr6feRkUyKKBA0i+MjfU4qsl7f69J5xS3ei7/Nf2mt2+PKwsz20Vxf94WtTXZ4HAO7pVlsZ2N3DJpDVFp6GS2z2vDQU+MdV0K+8E89qlmJczBcoOHgRN516AqNNs/G65W4XrXbPdlTuumK/52EMyTRg0/Sb7dtN4zReH75ypy4PFXe/j6mJeuSN6hqaRhERSRT1wYdtwqeUB6wz56Jhzg87T0QApy+YJD0QpVi45SAS4zT1io7ZHjaPrCi0F6TzVDROBSuEc6VQ/b4Dd6n+iyEqaT0WX1e3hUVUwdq0HTZbrsN04yOYZJrqYsmr9E9bd6Ksv+oGXHNuv9plCwQ3x3sT7sX3nH8fl43vge0zbsGgjJQQtoqIyLuonvMhdcKnO66Wsfrau5B9bUu89+0RWcbhK6pMDuewrbaoMdcuO1XBioLCA5hfcMR+zLilO2HQVGF2s6+wXfc5UnAGmtdqj/+nFvjW0hn51l5er/265W582mg0Zt7cGX+TOYmVXKs33C2H9Sd1eaRkDw2HCbBERL6K6p4Pb/kQpAj02/ktnZL9mpvgri0CRDRCFWaqV6LvT09AveIu6P91E3bqJuH1mFcwdc0hlF1wzIFRZtJjctkw7LG2g0awQBTUsCZcgW8tGSixpiJVZ4IgITQqq6xRJHumnKs3XPUGFEy/yadzuOstY/ZQIiJ5RHXPh5z5EI6fPQ9YEgCLGWpYYIWA5IRYtym3ARExsKCH5hAaaY24KbsaD63ci1NiAg4g7VJPiOBwvJQhCxECLiAO16oOImZPbZZWFYAkCHjG/GC9Wie171FBgIi/mqYgyVSJr+aMAFQxGH1pyewr93Z1mb2z/rV9n9Aod1pyKZx7A3wpYheq7KGB1OxxxvL0RBTuojr4kPMbdfKakcC6fYgDcPDSafMv9sAkTIUA0V5TBAAEWAEIeF3zGhq9uxMAEAfgg0udH/mWHsg1PehivoT0x9uHlgHonjUC2sRUVOuaYcjy3z2mHBchwAQNytEMUDn+tbsbrnB9Ht+EU1pyKUKRPdRduv9gZ04lIgqWqB52seVDcP9oE5GM00gVTl8KGOoTYIUBJ9FT9XO914aod2KRJg+pOOOwP1U4ixdiFuM61S+wJqYBLTrBYrgO31oysNlyHbIyW2FLz+8wVpWP21Tf4L3rD+GNnqeQGiv90bbG2hfm3o8BXe+DNX0gDiNV8ntdTZ70ZTKtjaelvza2wMbVqgwp9WiCLRTZQ6euKnI5xFN3srPcbL0jhxcMr1e9mYhIaVH9r46nZFS2YY/jSMIrI10PO9Q+RFXIGdEX6s6HAdRm+Ow9fzMEiPjmiUEYoo9FHxPQ5ZkvAQh1yqMnAWagePJgxGljUGM024c5iu+qzQy6fEft9gu31x7Tb5jJa1l1OSauuktg5k8PhLsejbrCJS25FKHIHhpIzR4iokgU1T0fgPt8CCkJlx8yXr+dd0sHYpvYfyrQGGcRD+gTAW0c1BotbI/dQFYc1H2fAM9LRt2RemVv2UulnGfygHTJeSYiZVWGt94yuVodp43BBxNu8HiMHEuRiYjCUdQHH4C0FRDuciY4DwtI6a6WIz9E3qiufieQkvKAtH2zNiTqoYtR+3We1786CFG8/NmCkWeiymhG25kb7DlN5BbM7KFSh3hYSI6Iok1UD7vUJWUFhBzfzt1NHvQ1P4Rz/RXnnB511Q1uJg9Ix8e7fpeUVM1TNVxXxedcOXHOaP9zOPdo+MLWW+Y8ATdV5kmgUod4WEiOiKJNg+j5CCZ3kwf9yQ8hJRhyTh3v3Bshhatv1oMyUuqlLndFrnL14UZqT1ggpAzx+Fqzh4goEjD48MLXbn5vBdoCVXfYZ9uBEy6TYdXtjZDC3TfrusHOmSqT1/PsOnLG6zGRROl5KlKGeMJtKTIRkRwYfASJLQD5YMINsixt9JYMy0buariehKJcfaQvGfVUIC4clyITEcmBwUeQyTV5UGrq+Hu7K1cN11k0lKsPhWAM8RARhRMGHzKQsmzSxjbEEeg3dqlBTJtmjWT5Zu2pB8XGVq4+0nsjQiFSliITEcmhwTwZpNS7CKQmhm3yYFlFtd91TJyv72mOidQVEC3idejfMTngJF+e0qTbtsPlgcnaJkRE4Y09HzIJ9uRB76nja9l6I6R8s/bUY+EuEVvdZG0UfOxlIqJIxOBDRsGcPCg1GZacvRFylKsnIiLiVyUvfC11LmcdE2/DB+6SYaUk6FFWqUxWTOceFKk4FEJERDbs+fDAOYHXuKU70ff5L71WGg3m5MFQ90aw25+IiHzFp4Ub+XtKMWlFYb3Jo7ZS5+GUg0FK6ngiIqJwwZ4PF6Qk8MpdVxxV6cRdCcdeDTmK9hERUWgx+HDBWwIvljoPDX+HwYiIKLww+HCBpc7DQ926Ov8u+sNlHRvbMBgDECKiyMHgwwWWOg8/8zbua/DDYERE0SI8BvLDjBzZSsNNpE9Cde7xqKvuMFjv9GayXTPS7xkRUbhiz4cLLHUemTgMRkQUGRQLPhYuXIi2bdtCr9ejV69e2LFjh1KXUkQg2UrDcZVIQ8BhMCKiyKDIk/HDDz/E9OnTsXjxYvTq1Qt5eXkYPHgw9u/fj+TkZCUuqQg5s5VGi1ANRaQk6HC8siZqhsGIiBoyRXo+Xn75ZUyYMAHjx49HRkYGFi9ejLi4OLzzzjtKXE5RkVDqvCH0tDwxrDOA6B0Gawh/h0RENrIHH0ajEbt27UJWVtbli6hUyMrKwrffflvv+JqaGlRWVjr8EDlzV1VXiaJ9RESkLNmDj5MnT8JisSAlJcVhf0pKCsrKyuodP3/+fCQmJtp/0tLS5G4SRQlXdWy2z7iFgQcRUYQJ+WqXWbNmoaKiwv5z9OjRUDeJwlgkDIMREZFnsg8uN2/eHGq1GuXl5Q77y8vLkZqaWu94nU4HnU5Xbz8RERFFJ9l7PrRaLbp3747Nmzfb91mtVmzevBm9e/eW+3JEREQUYRSZVj99+nSMHTsW119/PXr27Im8vDxcuHAB48ePV+JyREREFEEUCT5GjhyJEydO4Omnn0ZZWRm6du2K/Pz8epNQIwFTbIdO3VotO0pOo1+HFiFsDRERyUWxhAJTpkzBlClTlDo9Rbn8PaXIWbvXvj1u6U4YEvWYObRTCFtFRERyYDYjCjv5e0oxaUVhvWymZRXVmLqqKBRNIiIiGYV8qS1RXRariNx1xS7TqLvaR0REkYfBB4WVHSWnUVrhvjotAxAiosjH4IPCyvFz7gMPIiKKDgw+KKwkx+tD3QQiIlIYgw8KKz3bJcGQqK9XvdaGydSJiCIfgw8KK2qVgJzsDAD1Aw3b9uIx3Vh6nogogjH4oLAzJNOARWO6ITnBseZPaqIei8Z0YxVbIqIIx6+OFJaGZBrQ58rm6DLnCwDAsvE90K9DC1axJSKKAuz5oLBVN9Do2S6JgQcRUZRg8EFERERBxeCDiIiIgorBBxEREQUVgw8iIiIKKgYfREREFFQMPoiIiCioGHwQERFRUDH4ICIioqBi8EFERERBxeCDiIiIgoq1XShsxWljcHjB8FA3g4iIZMaeDyIiIgoqBh9EREQUVAw+iIiIKKgYfBAREVFQMfggIiKioGLwQUREREHF4IOIiIiCisEHERERBRWDDyIiIgoqBh9EREQUVAw+iIiIKKgYfBAREVFQMfggIiKioGLwQUREREHF4IOIiIiCKibUDXAmiiIAoLKyMsQtISIiIqlsz23bc9yTsAs+zp07BwBIS0sLcUuIiIjIV+fOnUNiYqLHYwRRSogSRFarFceOHUN8fDwEQZD13JWVlUhLS8PRo0eRkJAg67npMt7n4OB9Dg7e5+DhvQ4Ope6zKIo4d+4cWrZsCZXK86yOsOv5UKlUuOKKKxS9RkJCAn+xg4D3OTh4n4OD9zl4eK+DQ4n77K3Hw4YTTomIiCioGHwQERFRUDWo4EOn0yEnJwc6nS7UTYlqvM/BwfscHLzPwcN7HRzhcJ/DbsIpERERRbcG1fNBREREocfgg4iIiIKKwQcREREFFYMPIiIiCqqoCz4WLlyItm3bQq/Xo1evXtixY4fH4z/++GN06tQJer0eXbp0wcaNG4PU0sjmy31esmQJ+vXrh6ZNm6Jp06bIysry+vdCtXz9fbZZtWoVBEHAnXfeqWwDo4Sv9/ns2bOYPHkyDAYDdDodrrrqKv7bIYGv9zkvLw8dO3ZEbGws0tLSMG3aNFRXVweptZFp27ZtyM7ORsuWLSEIAtasWeP1PVu2bEG3bt2g0+lw5ZVXYtmyZYq3E2IUWbVqlajVasV33nlH3Lt3rzhhwgSxSZMmYnl5ucvjv/76a1GtVosvvPCCWFxcLD711FOiRqMRf/rppyC3PLL4ep/vu+8+ceHCheLu3bvFffv2iePGjRMTExPF33//Pcgtjyy+3mebkpISsVWrVmK/fv3EO+64IziNjWC+3ueamhrx+uuvF4cNGyZu375dLCkpEbds2SIWFRUFueWRxdf7/P7774s6nU58//33xZKSEvHzzz8XDQaDOG3atCC3PLJs3LhRfPLJJ8XPPvtMBCCuXr3a4/GHDh0S4+LixOnTp4vFxcXia6+9JqrVajE/P1/RdkZV8NGzZ09x8uTJ9m2LxSK2bNlSnD9/vsvjR4wYIQ4fPtxhX69evcSHH35Y0XZGOl/vszOz2SzGx8eLy5cvV6qJUcGf+2w2m8Ubb7xR/Ne//iWOHTuWwYcEvt7nRYsWie3btxeNRmOwmhgVfL3PkydPFm+55RaHfdOnTxf79OmjaDujiZTg4+9//7t49dVXO+wbOXKkOHjwYAVbJopRM+xiNBqxa9cuZGVl2fepVCpkZWXh22+/dfmeb7/91uF4ABg8eLDb48m/++ysqqoKJpMJSUlJSjUz4vl7n+fOnYvk5GQ89NBDwWhmxPPnPq9duxa9e/fG5MmTkZKSgszMTMybNw8WiyVYzY44/tznG2+8Ebt27bIPzRw6dAgbN27EsGHDgtLmhiJUz8GwKyznr5MnT8JisSAlJcVhf0pKCn7++WeX7ykrK3N5fFlZmWLtjHT+3GdnM2bMQMuWLev9wtNl/tzn7du34+2330ZRUVEQWhgd/LnPhw4dwpdffon7778fGzduxK+//opHH30UJpMJOTk5wWh2xPHnPt933304efIk+vbtC1EUYTab8cgjj+CJJ54IRpMbDHfPwcrKSly8eBGxsbGKXDdqej4oMixYsACrVq3C6tWrodfrQ92cqHHu3Dk88MADWLJkCZo3bx7q5kQ1q9WK5ORkvPXWW+jevTtGjhyJJ598EosXLw5106LKli1bMG/ePLzxxhsoLCzEZ599hg0bNuCZZ54JddNIBlHT89G8eXOo1WqUl5c77C8vL0dqaqrL96Smpvp0PPl3n21eeuklLFiwAJs2bcI111yjZDMjnq/3+eDBgzh8+DCys7Pt+6xWKwAgJiYG+/fvR3p6urKNjkD+/D4bDAZoNBqo1Wr7vs6dO6OsrAxGoxFarVbRNkcif+7z7Nmz8cADD+Avf/kLAKBLly64cOECJk6ciCeffBIqFb87y8HdczAhIUGxXg8gino+tFotunfvjs2bN9v3Wa1WbN68Gb1793b5nt69ezscDwAFBQVujyf/7jMAvPDCC3jmmWeQn5+P66+/PhhNjWi+3udOnTrhp59+QlFRkf3n9ttvx4ABA1BUVIS0tLRgNj9i+PP73KdPH/z666/24A4ADhw4AIPBwMDDDX/uc1VVVb0AwxbwiSxJJpuQPQcVnc4aZKtWrRJ1Op24bNkysbi4WJw4caLYpEkTsaysTBRFUXzggQfEmTNn2o//+uuvxZiYGPGll14S9+3bJ+bk5HCprQS+3ucFCxaIWq1W/OSTT8TS0lL7z7lz50L1ESKCr/fZGVe7SOPrff7tt9/E+Ph4ccqUKeL+/fvF9evXi8nJyeKzzz4bqo8QEXy9zzk5OWJ8fLz4wQcfiIcOHRK/+OILMT09XRwxYkSoPkJEOHfunLh7925x9+7dIgDx5ZdfFnfv3i0eOXJEFEVRnDlzpvjAAw/Yj7cttX388cfFffv2iQsXLuRSW3+89tprYuvWrUWtViv27NlT/O677+yv3XzzzeLYsWMdjv/oo4/Eq666StRqteLVV18tbtiwIcgtjky+3Oc2bdqIAOr95OTkBL/hEcbX3+e6GHxI5+t9/uabb8RevXqJOp1ObN++vfjcc8+JZrM5yK2OPL7cZ5PJJM6ZM0dMT08X9Xq9mJaWJj766KPimTNngt/wCPLVV1+5/PfWdm/Hjh0r3nzzzfXe07VrV1Gr1Yrt27cXly5dqng7BVFk/xUREREFT9TM+SAiIqLIwOCDiIiIgorBBxEREQUVgw8iIiIKKgYfREREFFQMPoiIiCioGHwQERFRUDH4ICIioqBi8EFERERBxeCDiIiIgorBBxEREQUVgw8iIiIKqv8PC+A1dk48l3QAAAAASUVORK5CYII=", 62 | "text/plain": [ 63 | "
" 64 | ] 65 | }, 66 | "metadata": {}, 67 | "output_type": "display_data" 68 | } 69 | ], 70 | "source": [ 71 | "plt.errorbar(x, y, ye, fmt=\"o\", label=\"data\")\n", 72 | "xm = np.linspace(np.min(x), np.max(x), 1000)\n", 73 | "plt.plot(xm, model(xm, *par), label=\"fit\")\n", 74 | "plt.legend();" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 4, 80 | "id": "passive-cowboy", 81 | "metadata": { 82 | "scrolled": true 83 | }, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "a = 1.10 +/- 0.18 jackknife=0.13 bootstrap=0.13 MC=0.18\n", 90 | "b = -0.72 +/- 1.09 jackknife=0.93 bootstrap=0.88 MC=1.04\n", 91 | "c = 10.52 +/- 1.22 jackknife=1.11 bootstrap=1.05 MC=1.19\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "# now only return fit parameters\n", 97 | "def fit2(x, y, ye):\n", 98 | " return fit(x, y, ye)[0]\n", 99 | "\n", 100 | "# jackknife and bootstrap\n", 101 | "jvar = jackknife.variance(fit2, x, y, ye)\n", 102 | "bvar = bootstrap.variance(fit2, x, y, ye, size=1000, random_state=1)\n", 103 | "\n", 104 | "# Monte-Carlo simulation for reference\n", 105 | "mvar = []\n", 106 | "for itry in range(1000):\n", 107 | " y2 = 1 + 10 * x ** 2 + rng.normal(0, ye)\n", 108 | " mvar.append(fit2(x, y2, ye))\n", 109 | "mvar = np.var(mvar, axis=0)\n", 110 | "\n", 111 | "for n, p, e, ej, eb, em in zip(\"abc\", par,\n", 112 | " np.diag(cov) ** 0.5,\n", 113 | " jvar ** 0.5,\n", 114 | " bvar ** 0.5,\n", 115 | " mvar ** 0.5):\n", 116 | " print(f\"{n} = {p:5.2f} +/- {e:1.2f} \"\n", 117 | " f\"jackknife={ej:1.2f} \"\n", 118 | " f\"bootstrap={eb:1.2f} \"\n", 119 | " f\"MC={em:1.2f}\")" 120 | ] 121 | } 122 | ], 123 | "metadata": { 124 | "kernelspec": { 125 | "display_name": "Python 3 (ipykernel)", 126 | "language": "python", 127 | "name": "python3" 128 | }, 129 | "language_info": { 130 | "codemirror_mode": { 131 | "name": "ipython", 132 | "version": 3 133 | }, 134 | "file_extension": ".py", 135 | "mimetype": "text/x-python", 136 | "name": "python", 137 | "nbconvert_exporter": "python", 138 | "pygments_lexer": "ipython3", 139 | "version": "3.11.9" 140 | } 141 | }, 142 | "nbformat": 4, 143 | "nbformat_minor": 5 144 | } 145 | -------------------------------------------------------------------------------- /doc/tutorials.rst: -------------------------------------------------------------------------------- 1 | Tutorials 2 | ========= 3 | 4 | The following tutorials show how to use resample. 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | tutorial/jackknife_vs_bootstrap 10 | tutorial/permutation_tests 11 | tutorial/sklearn 12 | tutorial/usp_continuous_data 13 | tutorial/variance_fit_parameters 14 | tutorial/confidence_intervals 15 | tutorial/leave-one-out-cross-validation -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools >= 60", "setuptools_scm[toml] >= 8.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "resample" 7 | requires-python = ">=3.8" 8 | dependencies = ["numpy >= 1.21", "scipy >= 1.10"] 9 | authors = [ 10 | { name = "Daniel Saxton", email = "dsaxton@pm.me" }, 11 | { name = "Hans Dembinski", email = "hans.dembinski@gmail.com" }, 12 | ] 13 | readme = "README.rst" 14 | description = "Resampling-based inference in Python" 15 | license = { text = "BSD-3-Clause" } 16 | classifiers = [ 17 | # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers 18 | 'Development Status :: 5 - Production/Stable', 19 | 'Intended Audience :: Science/Research', 20 | "License :: OSI Approved :: BSD License", 21 | "Operating System :: OS Independent", 22 | "Programming Language :: Python :: 3", 23 | 'Programming Language :: Python :: 3', 24 | 'Programming Language :: Python :: 3 :: Only', 25 | 'Programming Language :: Python :: 3.8', 26 | 'Programming Language :: Python :: 3.9', 27 | 'Programming Language :: Python :: 3.10', 28 | 'Programming Language :: Python :: 3.11', 29 | 'Programming Language :: Python :: 3.12', 30 | 'Programming Language :: Python :: 3.13', 31 | 'Programming Language :: Python :: Implementation :: CPython', 32 | 'Programming Language :: Python :: Implementation :: PyPy', 33 | ] 34 | dynamic = ["version"] 35 | 36 | [project.urls] 37 | repository = "http://github.com/resample-project/resample" 38 | documentation = "https://resample.readthedocs.io/en/stable/" 39 | 40 | [project.optional-dependencies] 41 | test = ["pytest", "pytest-cov", "coverage[toml]"] 42 | doc = ["ipython", "nbsphinx", "sphinx_rtd_theme"] 43 | 44 | [tool.setuptools.packages.find] 45 | where = ["src"] 46 | 47 | [tool.setuptools_scm] 48 | 49 | [tool.ruff.lint] 50 | extend-select = ["D", "I"] 51 | ignore = ["D212", "D211", "D203"] 52 | 53 | [tool.ruff.lint.per-file-ignores] 54 | "test_*.py" = ["D"] 55 | 56 | [tool.mypy] 57 | strict = true 58 | no_implicit_optional = false 59 | allow_redefinition = true 60 | ignore_missing_imports = true 61 | files = "src/resample/*.py" 62 | 63 | [tool.pytest.ini_options] 64 | addopts = "--doctest-modules --strict-config --strict-markers -q -ra --ff" 65 | testpaths = ["src/resample", "tests"] 66 | log_cli_level = "INFO" 67 | xfail_strict = true 68 | filterwarnings = ["error::DeprecationWarning", "error::FutureWarning"] 69 | -------------------------------------------------------------------------------- /src/resample/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Resampling tools for Python. 3 | 4 | This library offers randomisation-based inference in Python based on data resampling 5 | and permutation. The following functionality is implemented. 6 | 7 | - Bootstrap samples (ordinary or balanced with optional stratification) from N-D arrays 8 | - Apply parametric bootstrap (Gaussian, Poisson, gamma, etc.) on samples 9 | - Compute bootstrap confidence intervals (percentile or BCa) for any estimator 10 | - Jackknife estimates of bias and variance of any estimator 11 | - Permutation-based variants of traditional statistical tests (t-test, K-S test, etc.) 12 | - Tools for working with empirical distributions (CDF, quantile, etc.) 13 | """ 14 | 15 | from importlib.metadata import version 16 | 17 | from resample import bootstrap, empirical, jackknife, permutation 18 | 19 | __version__ = version("resample") 20 | 21 | __all__ = [ 22 | "jackknife", 23 | "bootstrap", 24 | "permutation", 25 | "empirical", 26 | "__version__", 27 | ] 28 | -------------------------------------------------------------------------------- /src/resample/_util.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple, Union 2 | 3 | import numpy as np 4 | from numpy.typing import ArrayLike 5 | 6 | __all__ = ["normalize_rng", "wilson_score_interval"] 7 | 8 | 9 | def normalize_rng( 10 | random_state: Optional[Union[int, np.random.Generator]], 11 | ) -> np.random.Generator: 12 | """Return normalized RNG object.""" 13 | if random_state is None: 14 | return np.random.default_rng() 15 | if isinstance(random_state, np.random.Generator): 16 | return random_state 17 | return np.random.default_rng(random_state) 18 | 19 | 20 | def wilson_score_interval( 21 | n1: "ArrayLike", n: "ArrayLike", z: float 22 | ) -> Tuple[np.ndarray, Tuple[np.ndarray, np.ndarray]]: 23 | """Return binomial fraction and Wilson score interval.""" 24 | p = n1 / n 25 | norm = 1 / (1 + z**2 / n) 26 | a = p + 0.5 * z**2 / n 27 | b = z * np.sqrt(p * (1 - p) / n + 0.25 * (z / n) ** 2) 28 | return p, ((a - b) * norm, (a + b) * norm) 29 | -------------------------------------------------------------------------------- /src/resample/bootstrap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Bootstrap resampling tools. 3 | 4 | Compute estimator bias, variance, confidence intervals with bootstrap resampling. 5 | 6 | Several forms of bootstrapping on N-dimensional data are supported: ordinary, balanced, 7 | extended, parametric, and stratified sampling, see :func:`resample` for details. 8 | Parametric bootstrapping fits a user-specified distribution to the data and samples 9 | from the parametric distribution. The distributions are taken from scipy.stats. 10 | 11 | Confidence intervals can be computed with the ordinary percentile method and with the 12 | more efficient BCa method, see :func:`confidence_interval` for details. 13 | """ 14 | 15 | __all__ = [ 16 | "resample", 17 | "bootstrap", 18 | "variance", 19 | "covariance", 20 | "confidence_interval", 21 | ] 22 | 23 | from typing import ( 24 | Any, 25 | Callable, 26 | Collection, 27 | Dict, 28 | Generator, 29 | List, 30 | Optional, 31 | Tuple, 32 | Union, 33 | ) 34 | 35 | import numpy as np 36 | from numpy.typing import ArrayLike 37 | from scipy import stats 38 | 39 | from . import _util 40 | from .empirical import quantile_function_gen 41 | from .jackknife import jackknife 42 | 43 | 44 | def resample( 45 | sample: "ArrayLike", 46 | *args: "ArrayLike", 47 | size: int = 100, 48 | method: str = "balanced", 49 | strata: Optional["ArrayLike"] = None, 50 | random_state: Optional[Union[np.random.Generator, int]] = None, 51 | ) -> Generator[np.ndarray, None, None]: 52 | """ 53 | Return generator of bootstrap samples. 54 | 55 | Parameters 56 | ---------- 57 | sample : array-like 58 | Original sample. 59 | *args : array-like 60 | Optional additional arrays of the same length to resample. 61 | size : int, optional 62 | Number of bootstrap samples to generate. Default is 100. 63 | method : str or None, optional 64 | How to generate bootstrap samples. Supported are 'ordinary', 'balanced', 65 | 'extended', or a distribution name for a parametric bootstrap. 66 | Default is 'balanced'. Supported distribution names: 'normal' (also: 67 | 'gaussian', 'norm'), 'student' (also: 't'), 'laplace', 'logistic', 68 | 'F' (also: 'f'), 'beta', 'gamma', 'log-normal' (also: 'lognorm', 69 | 'log-gaussian'), 'inverse-gaussian' (also: 'invgauss'), 'pareto', 'poisson'. 70 | strata : array-like, optional 71 | Stratification labels. Must have the same shape as `sample`. Default is None. 72 | random_state : numpy.random.Generator or int, optional 73 | Random number generator instance. If an integer is passed, seed the numpy 74 | default generator with it. Default is to use `numpy.random.default_rng()`. 75 | 76 | Yields 77 | ------ 78 | ndarray 79 | Bootstrap sample. 80 | 81 | Examples 82 | -------- 83 | Compute uncertainty of arithmetic mean. 84 | 85 | >>> from resample.bootstrap import resample 86 | >>> import numpy as np 87 | >>> x = np.arange(10) 88 | >>> fx = np.mean(x) 89 | >>> fb = [] 90 | >>> for b in resample(x, size=10000, random_state=1): 91 | ... fb.append(np.mean(b)) 92 | >>> print(f"f(x) = {fx:.1f} +/- {np.std(fb):.1f}") 93 | f(x) = 4.5 +/- 0.9 94 | 95 | Compute uncertainty of function applied to multivariate data. 96 | 97 | >>> from resample.bootstrap import resample 98 | >>> import numpy as np 99 | >>> x = np.arange(10) 100 | >>> y = np.arange(10, 20) 101 | >>> fx = np.mean((x, y)) 102 | >>> fb = [] 103 | >>> for bx, by in resample(x, y, size=10000, random_state=1): 104 | ... fb.append(np.mean((bx, by))) 105 | >>> print(f"f(x, y) = {fx:.1f} +/- {np.std(fb):.1f}") 106 | f(x, y) = 9.5 +/- 0.9 107 | 108 | Notes 109 | ----- 110 | Balanced vs. ordinary bootstrap: 111 | 112 | The balanced bootstrap produces more accurate results for the same number of 113 | bootstrap samples than the ordinary bootstrap, but needs to allocate memory for 114 | B integers, where B is the number of bootstrap samples. Since values of B larger 115 | than 10000 are rarely needed, this is usually not an issue. 116 | 117 | Non-parametric vs. parametric bootstrap: 118 | 119 | If you know that the data follow a particular parametric distribution, it is 120 | better to sample from this parametric distribution, but in most cases it is 121 | sufficient and more convenient to do a non-parametric bootstrap (using "balanced", 122 | "ordinary", "extended"). The parametric bootstrap is essential for estimators 123 | sensitive to the tails of a distribution (for example, a quantile close to 0 or 1). 124 | In this case, only a parametric bootstrap will give reasonable answers, since the 125 | non-parametric bootstrap cannot include rare events in the tail if the original 126 | sample did not have them. 127 | 128 | Extended bootstrap: 129 | 130 | In particle physics and perhaps also in other fields, estimators are used which are 131 | that are a function of both the size and shape of a sample (for example, fit of a 132 | peak over smooth background to the mass distribution of decay candidates). In this 133 | case, the normal bootstrap (parametric or non-parametric) is not correct, since the 134 | sample size is kept constant. For such cases, one needs the "extended" bootstrap. 135 | The name alludes to the so-called extended maximum-likelihood (EML) method in 136 | particle physics. Estimates obtained with the EML need to be bootstrapped with the 137 | "extended" bootstrap. 138 | 139 | Stratification: 140 | 141 | If the sample consists of several distinct classes, stratification 142 | ensures that the relative proportions of each class are maintained in each 143 | replicated sample. This is a stricter constraint than that offered by the 144 | balanced bootstrap, which only guarantees that classes have the original 145 | proportions over all replicates, but not within each one replicate. 146 | 147 | """ 148 | sample_np = np.atleast_1d(sample) 149 | n_sample = len(sample_np) 150 | args_np: List[np.ndarray] = [] 151 | 152 | if args: 153 | if not isinstance(args[0], Collection): 154 | import warnings 155 | 156 | warnings.warn( 157 | "Calling resample with positional instead of keyword parameters is " 158 | "deprecated", 159 | FutureWarning, 160 | ) 161 | kwargs: Dict[str, Any] = { 162 | "size": size, 163 | "method": method, 164 | "strata": strata, 165 | "random_state": random_state, 166 | } 167 | if len(args) > len(kwargs): 168 | raise ValueError("too many arguments") 169 | for key, val in zip(kwargs, args): 170 | kwargs[key] = val 171 | size = kwargs["size"] 172 | method = kwargs["method"] 173 | strata = kwargs["strata"] 174 | random_state = kwargs["random_state"] 175 | del args 176 | else: 177 | args_np.append(sample_np) 178 | for arg in args: 179 | arg = np.atleast_1d(arg) 180 | n_arg = len(arg) 181 | if n_arg != n_sample: 182 | raise ValueError( 183 | f"extra argument has wrong length {n_arg} != {n_sample}" 184 | ) 185 | args_np.append(arg) 186 | 187 | rng = _util.normalize_rng(random_state) 188 | 189 | if strata is not None: 190 | strata_np = np.atleast_1d(strata) 191 | if args_np: 192 | raise ValueError("Stratified resampling only works with one sample array") 193 | if len(strata_np) != n_sample: 194 | raise ValueError("a and strata must have the same shape") 195 | return _resample_stratified(sample_np, size, method, strata_np, rng) 196 | 197 | if method == "balanced": 198 | if args_np: 199 | return _resample_balanced_n(args_np, size, rng) 200 | else: 201 | return _resample_balanced_1(sample_np, size, rng) 202 | if method == "ordinary": 203 | if args_np: 204 | return _resample_ordinary_n(args_np, size, rng) 205 | else: 206 | return _resample_ordinary_1(sample_np, size, rng) 207 | if method == "extended": 208 | if args_np: 209 | return _resample_extended_n(args_np, size, rng) 210 | else: 211 | return _resample_extended_1(sample_np, size, rng) 212 | 213 | if args_np: 214 | raise ValueError("Parametric resampling only works with one sample array") 215 | 216 | dist = { 217 | # put aliases here 218 | "gaussian": stats.norm, 219 | "normal": stats.norm, 220 | "log-normal": stats.lognorm, 221 | "log-gaussian": stats.lognorm, 222 | "inverse-gaussian": stats.invgauss, 223 | "student": stats.t, 224 | }.get(method) 225 | 226 | # fallback to scipy.stats name 227 | if dist is None: 228 | try: 229 | dist = getattr(stats, method.lower()) 230 | except AttributeError: 231 | raise ValueError(f"Invalid family: '{method}'") 232 | 233 | if sample_np.ndim > 1: 234 | if dist != stats.norm: 235 | raise ValueError(f"family '{method}' only supports 1D samples") 236 | dist = stats.multivariate_normal 237 | if sample_np.ndim > 2: 238 | raise ValueError("multivariate normal only works with 2D samples") 239 | 240 | return _resample_parametric(sample_np, size, dist, rng) 241 | 242 | 243 | def bootstrap( 244 | fn: Callable[..., np.ndarray], 245 | sample: "ArrayLike", 246 | *args: "ArrayLike", 247 | **kwargs: Any, 248 | ) -> np.ndarray: 249 | """ 250 | Calculate function values from bootstrap samples. 251 | 252 | This is equivalent to ``numpy.array([fn(b) for b in resample(sample)])`` and 253 | implemented for convenience. 254 | 255 | Parameters 256 | ---------- 257 | fn : Callable 258 | Bootstrap samples are passed to this function. 259 | sample : array-like 260 | Original sample. 261 | *args : array-like 262 | Optional additional arrays of the same length to resample. 263 | **kwargs 264 | Keywords are forwarded to :func:`resample`. 265 | 266 | Returns 267 | ------- 268 | np.array 269 | Results of `fn` applied to each bootstrap sample. 270 | 271 | Examples 272 | -------- 273 | >>> from resample.bootstrap import bootstrap 274 | >>> import numpy as np 275 | >>> x = np.arange(10) 276 | >>> fx = np.mean(x) 277 | >>> fb = bootstrap(np.mean, x, size=10000, random_state=1) 278 | >>> print(f"f(x) = {fx:.1f} +/- {np.std(fb):.1f}") 279 | f(x) = 4.5 +/- 0.9 280 | 281 | """ 282 | gen = resample(sample, *args, **kwargs) 283 | if args: 284 | return np.array([fn(*b) for b in gen]) 285 | return np.array([fn(x) for x in gen]) 286 | 287 | 288 | def variance( 289 | fn: Callable[..., np.ndarray], 290 | sample: "ArrayLike", 291 | *args: "ArrayLike", 292 | **kwargs: Any, 293 | ) -> np.ndarray: 294 | """ 295 | Calculate bootstrap estimate of variance. 296 | 297 | If the function returns a vector, the variance is computed elementwise. 298 | 299 | Parameters 300 | ---------- 301 | fn : callable 302 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 303 | and k is the length of the output array. 304 | sample : array-like 305 | Original sample. 306 | *args : array-like 307 | Optional additional arrays of the same length to resample. 308 | **kwargs 309 | Keyword arguments forwarded to :func:`resample`. 310 | 311 | Returns 312 | ------- 313 | ndarray 314 | Bootstrap estimate of variance. 315 | 316 | Examples 317 | -------- 318 | Compute variance of arithmetic mean. 319 | 320 | >>> from resample.bootstrap import variance 321 | >>> import numpy as np 322 | >>> x = np.arange(10) 323 | >>> v = variance(np.mean, x, size=10000, random_state=1) 324 | >>> f"{v:.1f}" 325 | '0.8' 326 | 327 | """ 328 | thetas = bootstrap(fn, sample, *args, **kwargs) 329 | return np.var(thetas, ddof=1, axis=0) 330 | 331 | 332 | def covariance( 333 | fn: Callable[..., np.ndarray], 334 | sample: "ArrayLike", 335 | *args: "ArrayLike", 336 | **kwargs: Any, 337 | ) -> np.ndarray: 338 | """ 339 | Calculate bootstrap estimate of covariance. 340 | 341 | Parameters 342 | ---------- 343 | fn : callable 344 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 345 | and k is the length of the output array. 346 | sample : array-like 347 | Original sample. 348 | *args : array-like 349 | Optional additional arrays of the same length to resample. 350 | **kwargs 351 | Keyword arguments forwarded to :func:`resample`. 352 | 353 | Returns 354 | ------- 355 | ndarray 356 | Bootstrap estimate of covariance. In general, this is a matrix, but if the 357 | function maps to a scalar, it is scalar as well. 358 | 359 | Examples 360 | -------- 361 | Compute covariance of sample mean and sample variance. 362 | 363 | >>> from resample.bootstrap import variance 364 | >>> import numpy as np 365 | >>> x = np.arange(10) 366 | >>> def fn(x): 367 | ... return np.mean(x), np.var(x) 368 | >>> np.round(covariance(fn, x, size=10000, random_state=1), 1) 369 | array([[0.8, 0. ], 370 | [0. , 5.5]]) 371 | 372 | """ 373 | thetas = bootstrap(fn, sample, *args, **kwargs) 374 | return np.cov(thetas, rowvar=False, ddof=1) 375 | 376 | 377 | def confidence_interval( 378 | fn: Callable[..., np.ndarray], 379 | sample: "ArrayLike", 380 | *args: "ArrayLike", 381 | cl: float = 0.95, 382 | ci_method: str = "bca", 383 | **kwargs: Any, 384 | ) -> Tuple[float, float]: 385 | """ 386 | Calculate bootstrap confidence intervals. 387 | 388 | Parameters 389 | ---------- 390 | fn : callable 391 | Function to be bootstrapped. 392 | sample : array-like 393 | Original sample. 394 | *args : array-like 395 | Optional additional arrays of the same length to resample. 396 | cl : float, default : 0.95 397 | Confidence level. Asymptotically, this is the probability that the interval 398 | contains the true value. 399 | ci_method : str, {'bca', 'percentile'}, optional 400 | Confidence interval method. Default is 'bca'. See notes for details. 401 | **kwargs 402 | Keyword arguments forwarded to :func:`resample`. 403 | 404 | Returns 405 | ------- 406 | (float, float) 407 | Upper and lower confidence limits. 408 | 409 | Examples 410 | -------- 411 | Compute confidence interval for arithmetic mean. 412 | 413 | >>> from resample.bootstrap import confidence_interval 414 | >>> import numpy as np 415 | >>> x = np.arange(10) 416 | >>> a, b = confidence_interval(np.mean, x, size=10000, random_state=1) 417 | >>> f"{a:.1f} to {b:.1f}" 418 | '2.6 to 6.2' 419 | 420 | Notes 421 | ----- 422 | Both the 'percentile' and 'bca' methods produce intervals that are invariant to 423 | monotonic transformations of the data values, a desirable and consistent property. 424 | 425 | The 'percentile' method is straightforward and useful as a fallback. The 'bca' 426 | method is 2nd order accurate (to O(1/n) where n is the sample size) and generally 427 | preferred. It computes a jackknife estimate in addition to the bootstrap, which 428 | increases the number of function evaluations in a direct comparison to 429 | 'percentile'. However the increase in accuracy should compensate for this, with the 430 | result that less bootstrap replicas are needed overall to achieve the same accuracy. 431 | 432 | """ 433 | if args and not isinstance(args[0], Collection): 434 | import warnings 435 | 436 | warnings.warn( 437 | "Calling confidence_interval with positional instead of keyword " 438 | "arguments is deprecated", 439 | FutureWarning, 440 | ) 441 | 442 | if len(args) == 1: 443 | (cl,) = args 444 | elif len(args) == 2: 445 | cl, ci_method = args 446 | else: 447 | raise ValueError("too many arguments") 448 | args = () 449 | 450 | if not 0 < cl < 1: 451 | raise ValueError("cl must be between zero and one") 452 | 453 | thetas = bootstrap(fn, sample, *args, **kwargs) 454 | alpha = 1 - cl 455 | 456 | if ci_method == "percentile": 457 | return _confidence_interval_percentile(thetas, alpha / 2) 458 | 459 | if ci_method == "bca": 460 | theta = fn(sample, *args) 461 | j_thetas = jackknife(fn, sample, *args) 462 | return _confidence_interval_bca(theta, thetas, j_thetas, alpha / 2) 463 | 464 | raise ValueError( 465 | f"ci_method must be 'percentile' or 'bca', but '{ci_method}' was supplied" 466 | ) 467 | 468 | 469 | def _resample_stratified( 470 | sample: np.ndarray, 471 | size: int, 472 | method: str, 473 | strata: np.ndarray, 474 | rng: np.random.Generator, 475 | ) -> Generator[np.ndarray, None, None]: 476 | # call resample on sub-samples and merge the replicates 477 | sub_samples = [sample[strata == x] for x in np.unique(strata)] 478 | for sub_replicates in zip( 479 | *[resample(s, size=size, method=method, random_state=rng) for s in sub_samples] 480 | ): 481 | yield np.concatenate(sub_replicates, axis=0) 482 | 483 | 484 | def _resample_ordinary_1( 485 | sample: np.ndarray, size: int, rng: np.random.Generator 486 | ) -> Generator[np.ndarray, None, None]: 487 | # i.i.d. sampling from empirical cumulative distribution of sample 488 | n = len(sample) 489 | for _ in range(size): 490 | yield rng.choice(sample, size=n, replace=True) 491 | 492 | 493 | def _resample_ordinary_n( 494 | samples: List[np.ndarray], size: int, rng: np.random.Generator 495 | ) -> Generator[np.ndarray, None, None]: 496 | n = len(samples[0]) 497 | indices = np.arange(n) 498 | for _ in range(size): 499 | m = rng.choice(indices, size=n, replace=True) 500 | yield tuple(s[m] for s in samples) 501 | 502 | 503 | def _resample_balanced_1( 504 | sample: np.ndarray, size: int, rng: np.random.Generator 505 | ) -> Generator[np.ndarray, None, None]: 506 | # effectively computes a random permutation of `size` concatenated 507 | # copies of `sample` and returns `size` equal chunks of that 508 | n = len(sample) 509 | indices = rng.permutation(n * size) 510 | for i in range(size): 511 | m = indices[i * n : (i + 1) * n] % n 512 | yield sample[m] 513 | 514 | 515 | def _resample_balanced_n( 516 | samples: List[np.ndarray], size: int, rng: np.random.Generator 517 | ) -> Generator[np.ndarray, None, None]: 518 | n = len(samples[0]) 519 | indices = rng.permutation(n * size) 520 | for i in range(size): 521 | m = indices[i * n : (i + 1) * n] % n 522 | yield tuple(s[m] for s in samples) 523 | 524 | 525 | def _resample_extended_1( 526 | sample: np.ndarray, size: int, rng: np.random.Generator 527 | ) -> Generator[np.ndarray, None, None]: 528 | # randomly generates the sample size from a Poisson distribution 529 | n = len(sample) 530 | for i in range(size): 531 | k = rng.poisson(1, size=n) 532 | yield np.repeat(sample, k, axis=0) 533 | 534 | 535 | def _resample_extended_n( 536 | samples: List[np.ndarray], size: int, rng: np.random.Generator 537 | ) -> Generator[np.ndarray, None, None]: 538 | n = len(samples[0]) 539 | for i in range(size): 540 | k = rng.poisson(1, size=n) 541 | yield tuple(np.repeat(s, k, axis=0) for s in samples) 542 | 543 | 544 | def _fit_parametric_family( 545 | dist: stats.rv_continuous, sample: np.ndarray 546 | ) -> Tuple[float, ...]: 547 | if dist == stats.multivariate_normal: 548 | # has no fit method... 549 | return np.mean(sample, axis=0), np.cov(sample.T, ddof=1) 550 | 551 | if dist in {stats.f, stats.beta}: 552 | fit_kwargs = {"floc": 0, "fscale": 1} 553 | elif dist in {stats.gamma, stats.lognorm, stats.invgauss, stats.pareto}: 554 | fit_kwargs = {"floc": 0} 555 | else: 556 | fit_kwargs = {} 557 | 558 | return dist.fit(sample, **fit_kwargs) # type: ignore 559 | 560 | 561 | def _resample_parametric( 562 | sample: np.ndarray, size: int, dist: stats.rv_continuous, rng: np.random.Generator 563 | ) -> Generator[np.ndarray, None, None]: 564 | n = len(sample) 565 | 566 | # fit parameters by maximum likelihood and sample from that 567 | if dist == stats.poisson: 568 | # - poisson has no fit method and there is no scale parameter 569 | # - random number generation for poisson distribution in scipy seems to be buggy 570 | mu = np.mean(sample) 571 | for _ in range(size): 572 | yield rng.poisson(mu, size=n) 573 | else: 574 | args = _fit_parametric_family(dist, sample) 575 | dist = dist(*args) 576 | for _ in range(size): 577 | yield dist.rvs(size=n, random_state=rng) 578 | 579 | 580 | def _confidence_interval_percentile( 581 | thetas: np.ndarray, alpha_half: float 582 | ) -> Tuple[float, float]: 583 | quant = quantile_function_gen(thetas) 584 | return quant(alpha_half), quant(1 - alpha_half) 585 | 586 | 587 | def _confidence_interval_bca( 588 | theta: float, thetas: np.ndarray, j_thetas: np.ndarray, alpha_half: float 589 | ) -> Tuple[float, float]: 590 | norm = stats.norm 591 | 592 | # bias correction; implementation notes: 593 | # - if prop_less is zero, z_naught would become -inf; 594 | # we set z_naught to zero then (no bias) 595 | prop_less = np.mean(thetas < theta) # proportion of replicates less than obs 596 | z_naught = norm.ppf(prop_less) if prop_less > 0 else 0.0 597 | 598 | # acceleration; implementation notes: 599 | # - np.mean returns float even if j_thetas are int, 600 | # must convert type explicity to make -= operator work 601 | # - it is possible that all j_thetas are zero, it then follows 602 | # that den and num are zero; we set acc to zero then (no acceleration) 603 | j_mean = np.mean(j_thetas) 604 | j_thetas = j_thetas.astype(j_mean.dtype, copy=False) 605 | j_thetas -= j_mean 606 | num = np.sum((-j_thetas) ** 3) 607 | den = np.sum(j_thetas**2) 608 | acc = num / (6 * den**1.5) if den > 0 else 0.0 609 | 610 | z_low = z_naught + norm.ppf(alpha_half) 611 | z_high = z_naught + norm.ppf(1 - alpha_half) 612 | 613 | p_low = norm.cdf(z_naught + z_low / (1 - acc * z_low)) 614 | p_high = norm.cdf(z_naught + z_high / (1 - acc * z_high)) 615 | 616 | quant = quantile_function_gen(thetas) 617 | return quant(p_low), quant(p_high) 618 | 619 | 620 | def __getattr__(key: str) -> Any: 621 | for match in ("bias", "bias_corrected"): 622 | if key == match: 623 | msg = ( 624 | f"resample.bootstrap.{match} has been removed. The implementation was " 625 | "discovered to be faulty, and a generic fix is not in sight. " 626 | "Please use resample.jackknife.bias instead." 627 | ) 628 | raise NotImplementedError(msg) 629 | raise AttributeError 630 | -------------------------------------------------------------------------------- /src/resample/empirical.py: -------------------------------------------------------------------------------- 1 | """ 2 | Empirical functions. 3 | 4 | Empirical functions based on a data sample instead of a parameteric density function, 5 | like the empirical CDF. Implemented here are mostly tools used internally. 6 | """ 7 | 8 | __all__ = ["cdf_gen", "quantile_function_gen", "influence"] 9 | 10 | from typing import Callable, Union 11 | 12 | import numpy as np 13 | from numpy.typing import ArrayLike 14 | 15 | from .jackknife import jackknife 16 | 17 | 18 | def cdf_gen(sample: "ArrayLike") -> Callable[[np.ndarray], np.ndarray]: 19 | """ 20 | Return the empirical distribution function for the given sample. 21 | 22 | Parameters 23 | ---------- 24 | sample : array-like 25 | Sample. 26 | 27 | Returns 28 | ------- 29 | callable 30 | Empirical distribution function. 31 | 32 | """ 33 | sample_np = np.sort(sample) 34 | n = len(sample_np) 35 | return lambda x: np.searchsorted(sample_np, x, side="right", sorter=None) / n 36 | 37 | 38 | def quantile_function_gen( 39 | sample: "ArrayLike", 40 | ) -> Callable[[Union[float, "ArrayLike"]], Union[float, np.ndarray]]: 41 | """ 42 | Return the empirical quantile function for the given sample. 43 | 44 | Parameters 45 | ---------- 46 | sample : array-like 47 | Sample. 48 | 49 | Returns 50 | ------- 51 | callable 52 | Empirical quantile function. 53 | 54 | """ 55 | 56 | class QuantileFn: 57 | def __init__(self, sample: "ArrayLike"): 58 | self._sorted = np.sort(sample, axis=0) 59 | 60 | def __call__(self, p: Union[float, "ArrayLike"]) -> Union[float, np.ndarray]: 61 | ndim = np.ndim(p) # must come before atleast_1d 62 | p = np.atleast_1d(p) 63 | result = np.empty(len(p)) 64 | valid = (p >= 0) & (p <= 1) 65 | n = len(self._sorted) 66 | idx = np.maximum(np.ceil(p[valid] * n).astype(int) - 1, 0) 67 | result[valid] = self._sorted[idx] 68 | result[~valid] = np.nan 69 | if ndim == 0: 70 | return result[0] 71 | return result 72 | 73 | return QuantileFn(sample) 74 | 75 | 76 | def influence( 77 | fn: Callable[["ArrayLike"], np.ndarray], sample: "ArrayLike" 78 | ) -> np.ndarray: 79 | """ 80 | Calculate the empirical influence function for a given sample and estimator. 81 | 82 | Parameters 83 | ---------- 84 | fn : callable 85 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 86 | and k is the length of the output array. 87 | sample : array-like 88 | Sample. Must be one-dimensional. 89 | 90 | Returns 91 | ------- 92 | ndarray 93 | Empirical influence values. 94 | 95 | """ 96 | sample = np.atleast_1d(sample) 97 | n = len(sample) 98 | return (n - 1) * (fn(sample) - jackknife(fn, sample)) 99 | -------------------------------------------------------------------------------- /src/resample/jackknife.py: -------------------------------------------------------------------------------- 1 | """ 2 | Jackknife resampling tools. 3 | 4 | Compute estimator bias and variance with jackknife resampling. The implementation 5 | supports resampling of N-dimensional data. The interface of this module mimics that of 6 | the bootstrap module, so that you can easily switch between bootstrapping and 7 | jackknifing bias and variance of an estimator. 8 | 9 | The jackknife is an approximation to the bootstrap, so in general bootstrapping is 10 | preferred, especially when the sample is small. The computational cost of the jackknife 11 | increases quadratically with the sample size, but only linearly for the bootstrap. An 12 | advantage of the jackknife can be the deterministic outcome, since no random sampling 13 | is involved, but this can be overcome by fixing the seed for the bootstrap. 14 | 15 | The jackknife should be used to estimate the bias, since the bootstrap cannot (easily) 16 | estimate bias. The bootstrap should be preferred when computing the variance. 17 | """ 18 | 19 | __all__ = [ 20 | "resample", 21 | "jackknife", 22 | "bias", 23 | "bias_corrected", 24 | "variance", 25 | "cross_validation", 26 | ] 27 | 28 | from typing import Any, Callable, Collection, Generator, List 29 | 30 | import numpy as np 31 | from numpy.typing import ArrayLike 32 | 33 | 34 | def resample( 35 | sample: "ArrayLike", *args: "ArrayLike", copy: bool = True 36 | ) -> Generator[Any, None, None]: 37 | """ 38 | Generate jackknifed samples. 39 | 40 | Parameters 41 | ---------- 42 | sample : array-like 43 | Sample. If the sequence is multi-dimensional, the first dimension must 44 | walk over i.i.d. observations. 45 | *args: array-like 46 | Optional additional arrays of the same length to resample. 47 | copy : bool, optional 48 | If `True`, return the replicated sample as a copy, otherwise return a view into 49 | the internal array buffer of the generator. Setting this to `False` avoids 50 | `len(sample)` copies, which is more efficient, but see notes for caveats. 51 | 52 | Yields 53 | ------ 54 | ndarray 55 | Array with same shape and type as input, but with the size of the first 56 | dimension reduced by one. Replicates are missing one value of the original in 57 | ascending order, e.g. for a sample (1, 2, 3), one gets (2, 3), (1, 3), (1, 2). 58 | 59 | See Also 60 | -------- 61 | resample.bootstrap.resample : Generate bootstrap samples. 62 | resample.jackknife.jackknife : Generate jackknife estimates. 63 | 64 | Notes 65 | ----- 66 | On performance: 67 | 68 | The generator interally keeps a single array to the replicates, which is updated 69 | on each iteration of the generator. The safe default is to return copies of this 70 | internal state. To increase performance, it also possible to return a view into 71 | the generator state, by setting the `copy=False`. However, this will only produce 72 | correct results if the generator is called strictly sequentially in a single- 73 | threaded program and the loop body consumes the view and does not try to store it. 74 | The following program shows what happens otherwise: 75 | 76 | >>> from resample.jackknife import resample 77 | >>> r1 = [] 78 | >>> for x in resample((1, 2, 3)): # works as expected 79 | ... r1.append(x) 80 | >>> print(r1) 81 | [array([2, 3]), array([1, 3]), array([1, 2])] 82 | >>> 83 | >>> r2 = [] 84 | >>> for x in resample((1, 2, 3), copy=False): 85 | ... r2.append(x) # x is now a view into the same array in memory 86 | >>> print(r2) 87 | [array([1, 2]), array([1, 2]), array([1, 2])] 88 | 89 | """ 90 | sample_np = np.atleast_1d(sample) 91 | n_sample = len(sample_np) 92 | 93 | args_np = [] 94 | if args: 95 | if not isinstance(args[0], Collection): 96 | import warnings 97 | 98 | warnings.warn( 99 | "Calling resample with positional instead of keyword arguments is " 100 | "deprecated", 101 | FutureWarning, 102 | ) 103 | if len(args) == 1: 104 | (copy,) = args 105 | else: 106 | raise ValueError("too many arguments") 107 | else: 108 | args_np.append(sample_np) 109 | for arg in args: 110 | arg_np = np.atleast_1d(arg) 111 | n_arg = len(arg_np) 112 | if n_arg != n_sample: 113 | raise ValueError( 114 | f"extra argument has wrong length {n_arg} != {n_sample}" 115 | ) 116 | args_np.append(arg_np) 117 | 118 | if args_np: 119 | return _resample_n(args_np, copy) 120 | return _resample_1(sample_np, copy) 121 | 122 | 123 | def _resample_1(sample: np.ndarray, copy: bool) -> Generator[np.ndarray, None, None]: 124 | # yield x0 125 | x = sample[1:].copy() 126 | yield x.copy() if copy else x 127 | 128 | # update of x needs to change only value at index i 129 | # for a = [0, 1, 2, 3] 130 | # x0 = [1, 2, 3] (yielded above) 131 | # x1 = [0, 2, 3] # override first index 132 | # x2 = [0, 1, 3] # override second index 133 | # x3 = [0, 1, 2] # ... 134 | for i in range(len(sample) - 1): 135 | x[i] = sample[i] 136 | yield x.copy() if copy else x 137 | 138 | 139 | def _resample_n(samples: List[np.ndarray], copy: bool) -> Generator[Any, None, None]: 140 | x = [a[1:].copy() for a in samples] 141 | yield (xi.copy() for xi in x) 142 | for i in range(len(samples[0]) - 1): 143 | for xi, ai in zip(x, samples): 144 | xi[i] = ai[i] 145 | yield (xi.copy() for xi in x) 146 | 147 | 148 | def jackknife( 149 | fn: Callable[..., np.ndarray], 150 | sample: "ArrayLike", 151 | *args: "ArrayLike", 152 | ) -> np.ndarray: 153 | """ 154 | Calculate jackknife estimates for a given sample and estimator. 155 | 156 | The jackknife is a linear approximation to the bootstrap. In contrast to the 157 | bootstrap it is deterministic and does not use random numbers. The caveat is the 158 | computational cost of the jackknife, which is O(n²) for n observations, compared 159 | to O(n x k) for k bootstrap replicates. For large samples, the bootstrap is more 160 | efficient. 161 | 162 | Parameters 163 | ---------- 164 | fn : callable 165 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 166 | and k is the length of the output array. 167 | sample : array-like 168 | Original sample. 169 | *args: array-like 170 | Optional additional arrays of the same length to resample. 171 | 172 | Returns 173 | ------- 174 | ndarray 175 | Jackknife samples. 176 | 177 | Examples 178 | -------- 179 | >>> from resample.jackknife import jackknife 180 | >>> import numpy as np 181 | >>> x = np.arange(10) 182 | >>> fx = np.mean(x) 183 | >>> fb = jackknife(np.mean, x) 184 | >>> print(f"f(x) = {fx:.1f} +/- {np.std(fb):.1f}") 185 | f(x) = 4.5 +/- 0.3 186 | 187 | """ 188 | gen = resample(sample, *args, copy=False) 189 | if args: 190 | return np.array([fn(*b) for b in gen]) 191 | return np.asarray([fn(b) for b in gen]) 192 | 193 | 194 | def bias( 195 | fn: Callable[..., np.ndarray], sample: "ArrayLike", *args: "ArrayLike" 196 | ) -> np.ndarray: 197 | """ 198 | Calculate jackknife estimate of bias. 199 | 200 | The bias estimate is accurate to O(1/n), where n is the number of samples. 201 | If the bias is exactly O(1/n), then the estimate is exact. 202 | 203 | Wikipedia: 204 | https://en.wikipedia.org/wiki/Jackknife_resampling 205 | 206 | Parameters 207 | ---------- 208 | fn : callable 209 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 210 | and k is the length of the output array. 211 | sample : array-like 212 | Original sample. 213 | *args: array-like 214 | Optional additional arrays of the same length to resample. 215 | 216 | Returns 217 | ------- 218 | ndarray 219 | Jackknife estimate of bias (= expectation of estimator - true value). 220 | 221 | Examples 222 | -------- 223 | Compute bias of numpy.var with and without bias-correction. 224 | 225 | >>> from resample.jackknife import bias 226 | >>> import numpy as np 227 | >>> x = np.arange(10) 228 | >>> b1 = bias(np.var, x) 229 | >>> b2 = bias(lambda x: np.var(x, ddof=1), x) 230 | >>> f"bias of naive sample variance {b1:.1f}, bias of corrected variance {b2:.1f}" 231 | 'bias of naive sample variance -0.9, bias of corrected variance 0.0' 232 | 233 | """ 234 | sample = np.atleast_1d(sample) 235 | n = len(sample) 236 | theta = fn(sample) 237 | mean_theta = np.mean(jackknife(fn, sample, *args), axis=0) 238 | return (n - 1) * (mean_theta - theta) 239 | 240 | 241 | def bias_corrected( 242 | fn: Callable[..., np.ndarray], sample: "ArrayLike", *args: "ArrayLike" 243 | ) -> np.ndarray: 244 | """ 245 | Calculate bias-corrected estimate of the function with the jackknife. 246 | 247 | Removes a bias of O(1/n), where n is the sample size, leaving bias of 248 | order O(1/n²). If the original function has a bias of exactly O(1/n), 249 | the corrected result is now unbiased. 250 | 251 | Wikipedia: 252 | https://en.wikipedia.org/wiki/Jackknife_resampling 253 | 254 | Parameters 255 | ---------- 256 | fn : callable 257 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 258 | and k is the length of the output array. 259 | sample : array-like 260 | Original sample. 261 | *args: array-like 262 | Optional additional arrays of the same length to resample. 263 | 264 | Returns 265 | ------- 266 | ndarray 267 | Estimate with O(1/n) bias removed. 268 | 269 | Examples 270 | -------- 271 | Compute bias-corrected estimate of numpy.var. 272 | 273 | >>> from resample.jackknife import bias_corrected 274 | >>> import numpy as np 275 | >>> x = np.arange(10) 276 | >>> v1 = np.var(x) 277 | >>> v2 = bias_corrected(np.var, x) 278 | >>> f"naive variance {v1:.1f}, bias-corrected variance {v2:.1f}" 279 | 'naive variance 8.2, bias-corrected variance 9.2' 280 | 281 | """ 282 | sample = np.atleast_1d(sample) 283 | n = len(sample) 284 | theta = fn(sample) 285 | mean_theta = np.mean(jackknife(fn, sample, *args), axis=0) 286 | return n * theta - (n - 1) * mean_theta 287 | 288 | 289 | def variance( 290 | fn: Callable[..., np.ndarray], sample: "ArrayLike", *args: "ArrayLike" 291 | ) -> np.ndarray: 292 | """ 293 | Calculate jackknife estimate of variance. 294 | 295 | Wikipedia: 296 | https://en.wikipedia.org/wiki/Jackknife_resampling 297 | 298 | Parameters 299 | ---------- 300 | fn : callable 301 | Estimator. Can be any mapping ℝⁿ → ℝᵏ, where n is the sample size 302 | and k is the length of the output array. 303 | sample : array-like 304 | Original sample. 305 | *args: array-like 306 | Optional additional arrays of the same length to resample. 307 | 308 | Returns 309 | ------- 310 | ndarray 311 | Jackknife estimate of variance. 312 | 313 | Examples 314 | -------- 315 | Compute variance of arithmetic mean. 316 | 317 | >>> from resample.jackknife import variance 318 | >>> import numpy as np 319 | >>> x = np.arange(10) 320 | >>> v = variance(np.mean, x) 321 | >>> f"{v:.1f}" 322 | '0.9' 323 | 324 | """ 325 | # formula is (n - 1) / n * sum((fj - mean(fj)) ** 2) 326 | # = np.var(fj, ddof=0) * (n - 1) 327 | sample = np.atleast_1d(sample) 328 | thetas = jackknife(fn, sample, *args) 329 | n = len(sample) 330 | return (n - 1) * np.var(thetas, ddof=0, axis=0) 331 | 332 | 333 | def cross_validation( 334 | predict: Callable[..., float], x: "ArrayLike", y: "ArrayLike", *args: "ArrayLike" 335 | ) -> float: 336 | """ 337 | Calculate mean-squared error of model with leave-one-out-cross-validation. 338 | 339 | Wikipedia: 340 | https://en.wikipedia.org/wiki/Cross-validation_(statistics) 341 | 342 | Parameters 343 | ---------- 344 | predict : callable 345 | Function with the signature (x_in, y_in, x_out, *args). It takes x_in, y_in, 346 | which are arrays with the same length. x_out should be one element of the x 347 | array. *args are further optional arguments for the function. The function 348 | should return the prediction y(x_out). 349 | x : array-like 350 | Explanatory variable. Must be an array of shape (N, ...), where N is the number 351 | of samples. 352 | y : array-like 353 | Observations. Must be an array of shape (N, ...). 354 | *args: 355 | Optional arguments which are passed unmodified to predict. 356 | 357 | Returns 358 | ------- 359 | float 360 | Variance of the difference (y[i] - predict(..., x[i], *args)). 361 | 362 | """ 363 | deltas = [] 364 | for i, (x_in, y_in) in enumerate(resample(x, y, copy=False)): 365 | yip = predict(x_in, y_in, x[i], *args) 366 | deltas.append((y[i] - yip)) 367 | return np.var(deltas) # type:ignore 368 | -------------------------------------------------------------------------------- /src/resample/permutation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Permutation-based tests. 3 | 4 | A collection of statistical tests that use permutated samples. Permutations are used to 5 | compute the distribution of a test statistic under some null hypothesis to obtain 6 | p-values without relying on approximate asymptotic formulas. 7 | 8 | The permutation method is generic, it can be used with any test statistic, therefore we 9 | also provide a generic test function that accepts a user-defined function to compute the 10 | test statistic and then automatically computes the p-value for that statistic. The other 11 | tests internally also call this generic test function. 12 | 13 | All tests return a TestResult object, which mimics the interface of the result 14 | objects returned by tests in scipy.stats, but has a third field to return the 15 | estimated distribution of the test statistic under the null hypothesis. 16 | 17 | Further reading: 18 | 19 | - https://en.wikipedia.org/wiki/P-value 20 | - https://en.wikipedia.org/wiki/Test_statistic 21 | - https://en.wikipedia.org/wiki/Paired_difference_test 22 | """ 23 | 24 | __all__ = [ 25 | "TestResult", 26 | "usp", 27 | "same_population", 28 | "anova", 29 | "kruskal", 30 | "pearsonr", 31 | "spearmanr", 32 | "ttest", 33 | ] 34 | 35 | import sys 36 | import warnings 37 | from dataclasses import dataclass 38 | from typing import Any, Callable, Optional, Tuple, Union 39 | 40 | import numpy as np 41 | from numpy.typing import ArrayLike, NDArray 42 | from scipy import stats as _stats 43 | 44 | from . import _util 45 | 46 | _dataclass_kwargs = {"frozen": True, "repr": False} 47 | if sys.version_info >= (3, 10): 48 | _dataclass_kwargs["slots"] = True # pragma: no cover 49 | 50 | 51 | @dataclass(**_dataclass_kwargs) 52 | class TestResult: 53 | """ 54 | Holder of the result of the permutation test. 55 | 56 | This class acts like a tuple, which means its can be unpacked and the fields can be 57 | accessed by name or by index. 58 | 59 | Attributes 60 | ---------- 61 | statistic: float 62 | Value of the test statistic computed on the original data 63 | pvalue: float 64 | Estimated chance probability (aka Type I error) for rejecting the null 65 | hypothesis. See https://en.wikipedia.org/wiki/P-value for details. 66 | samples: array 67 | Values of the test statistic from the permutated samples. 68 | 69 | """ 70 | 71 | statistic: float 72 | pvalue: float 73 | samples: NDArray 74 | 75 | def __repr__(self) -> str: 76 | """Return (potentially shortened) representation.""" 77 | s = None 78 | if len(self.samples) < 7: 79 | s = str(self.samples) 80 | else: 81 | s = "[{}, {}, {}, ..., {}, {}, {}]".format( 82 | *self.samples[:3], *self.samples[-3:] 83 | ) 84 | return ( 85 | f"" 86 | ) 87 | 88 | def __len__(self) -> int: 89 | """Return length of tuple.""" 90 | return 3 91 | 92 | def __getitem__(self, idx: int) -> Union[float, NDArray]: 93 | """Return fields by index.""" 94 | if idx == 0: 95 | return self.statistic 96 | elif idx == 1: 97 | return self.pvalue 98 | elif idx == 2: 99 | return self.samples 100 | raise IndexError 101 | 102 | 103 | def usp( 104 | w: "ArrayLike", 105 | *, 106 | size: int = 9999, 107 | method: str = "auto", 108 | random_state: Optional[Union[np.random.Generator, int]] = None, 109 | ) -> TestResult: 110 | """ 111 | Test independence of two discrete data sets with the U-statistic. 112 | 113 | The USP test is described in this paper: https://doi.org/10.1098/rspa.2021.0549. 114 | According to the paper, it outperforms the Pearson's χ² and the G-test in both in 115 | stability and power. 116 | 117 | It requires that the input is a contigency table (a 2D histogram of value pairs). 118 | Whether the original values were discrete or continuous does not matter for the 119 | test. In case of continuous values, using a large number of bins is safe, since the 120 | test is not negatively affected by bins with zero entries. 121 | 122 | Parameters 123 | ---------- 124 | w : array-like 125 | Two-dimensional array which represents the counts in a histogram. The counts 126 | can be of floating point type, but must have integral values. 127 | size : int, optional 128 | Number of permutations. Default 9999. 129 | method : str, optional 130 | Method used to generate random tables under the null hypothesis. 131 | 'auto': Use heuristic to select fastest algorithm for given table. 132 | 'boyett': Boyett's algorithm, which requires extra space to store N + 1 133 | integers for N entries in total and has O(N) time complexity. It performs 134 | poorly when N is large, but does not depend on the number of K table cells. 135 | 'patefield': Patefield's algorithm, which does not require extra space and 136 | has O(K log(N)) time complexity. It performs well even if N is huge. For 137 | small N and large K, the shuffling algorithm is faster. 138 | Default is 'auto'. 139 | random_state : numpy.random.Generator or int, optional 140 | Random number generator instance. If an integer is passed, seed the numpy 141 | default generator with it. Default is to use ``numpy.random.default_rng()``. 142 | 143 | Returns 144 | ------- 145 | TestResult 146 | 147 | """ 148 | if size <= 0: 149 | raise ValueError("size must be positive") 150 | 151 | if method == "shuffle": 152 | warnings.warn( 153 | "method 'shuffle' is deprecated, please use 'boyett'", FutureWarning 154 | ) 155 | method = "boyett" 156 | 157 | rng = _util.normalize_rng(random_state) 158 | 159 | w = np.array(w, dtype=float) 160 | if w.ndim != 2: 161 | raise ValueError("w must be two-dimensional") 162 | 163 | r = np.sum(w, axis=1) 164 | c = np.sum(w, axis=0) 165 | ntot = np.sum(r) 166 | 167 | m = np.outer(r, c) / ntot 168 | 169 | f1 = 1.0 / (ntot * (ntot - 3)) 170 | f2 = 4.0 / (ntot * (ntot - 2) * (ntot - 3)) 171 | 172 | t = _usp(f1, f2, w, m) 173 | 174 | ts = np.empty(size) 175 | for b, w in enumerate( 176 | _stats.random_table(r, c).rvs( 177 | size, method=None if method == "auto" else method, random_state=rng 178 | ) 179 | ): 180 | # m stays the same, since r and c remain unchanged 181 | ts[b] = _usp(f1, f2, w, m) 182 | 183 | # Thomas B. Berrett, Ioannis Kontoyiannis, Richard J. Samworth 184 | # Ann. Statist. 49(5): 2457-2490 (October 2021). DOI: 10.1214/20-AOS2041 185 | # Eq. 5 says we need to add 1 to n_pass and n_total 186 | pvalue = (np.sum(t <= ts) + 1) / (size + 1) 187 | 188 | return TestResult(t, pvalue, ts) 189 | 190 | 191 | def _usp(f1: float, f2: float, w: NDArray, m: NDArray) -> NDArray: 192 | # Eq. 2.1 from https://doi.org/10.1098/rspa.2021.0549 193 | return f1 * np.sum((w - m) ** 2) - f2 * np.sum(w * m) 194 | 195 | 196 | def same_population( 197 | fn: Callable[..., float], 198 | x: "ArrayLike", 199 | y: "ArrayLike", 200 | *args: "ArrayLike", 201 | transform: Optional[Callable[[NDArray], NDArray]] = None, 202 | size: int = 9999, 203 | random_state: Optional[Union[np.random.Generator, int]] = None, 204 | ) -> TestResult: 205 | """ 206 | Compute p-value for hypothesis that samples originate from same population. 207 | 208 | The computation is based on a user-defined test statistic. The distribution of the 209 | test statistic under the null hypothesis is generated by generating random 210 | permutations of the inputs, to simulate that they are actually drawn from the same 211 | population. The test statistic is recomputed on these permutations and the p-value 212 | is computed as the fraction of these resampled test statistics which are larger 213 | than the original value. 214 | 215 | Some test statistics need to be transformed to fulfill the condition above, for 216 | example if they are signed. A transform can be passed to this function for those 217 | cases. 218 | 219 | Parameters 220 | ---------- 221 | fn : Callable 222 | Function with signature f(x, ...), where the number of arguments corresponds to 223 | the number of data samples passed to the test. 224 | x : array-like 225 | First sample. 226 | y : array-like 227 | Second sample. 228 | *args: array-like 229 | Further samples, if the test allows to compare more than two. 230 | transform : Callable, optional 231 | Function with signature f(x) for the test statistic to turn it into a measure of 232 | deviation. Must be vectorised. 233 | size : int, optional 234 | Number of permutations. Default 9999. 235 | random_state : numpy.random.Generator or int, optional 236 | Random number generator instance. If an integer is passed, seed the numpy 237 | default generator with it. Default is to use `numpy.random.default_rng()`. 238 | 239 | Returns 240 | ------- 241 | TestResult 242 | 243 | """ 244 | if size <= 0: 245 | raise ValueError("max_size must be positive") 246 | 247 | rng = _util.normalize_rng(random_state) 248 | 249 | r = [] 250 | for arg in (x, y) + args: 251 | a = np.array(arg) 252 | if a.ndim != 1: 253 | raise ValueError("input samples must be 1D arrays") 254 | if len(a) < 2: 255 | raise ValueError("input arrays must have at least two items") 256 | if a.dtype.kind == "f" and np.any(np.isnan(a)): 257 | raise ValueError("input contains NaN") 258 | r.append(a) 259 | args = r 260 | del r 261 | 262 | # compute test statistic for original input 263 | t = fn(*args) 264 | 265 | # compute test statistic for permutated inputs 266 | slices = [] 267 | start = 0 268 | for a in args: 269 | stop = start + len(a) 270 | slices.append(slice(start, stop)) 271 | start = stop 272 | 273 | joined_sample = np.concatenate(args) 274 | 275 | # For algorithm below, see comment in usp function. 276 | ts = np.empty(size) 277 | for b in range(size): 278 | rng.shuffle(joined_sample) 279 | ts[b] = fn(*(joined_sample[sl] for sl in slices)) 280 | if transform is None: 281 | u = t 282 | us = ts 283 | else: 284 | u = transform(t) 285 | us = transform(ts) 286 | # see usp for why we need to add 1 to both numerator and denominator 287 | pvalue = (np.sum(u <= us) + 1) / (size + 1) 288 | 289 | return TestResult(t, pvalue, ts) 290 | 291 | 292 | def anova( 293 | x: "ArrayLike", y: "ArrayLike", *args: "ArrayLike", **kwargs: Any 294 | ) -> TestResult: 295 | """ 296 | Test whether the means of two or more samples are compatible. 297 | 298 | This test uses one-way analysis of variance (one-way ANOVA) which tests whether the 299 | samples have the same mean. This test is typically used when one has three groups 300 | or more. For two groups, Welch's ttest is preferred, because ANOVA assumes equal 301 | variances for the samples. 302 | 303 | Parameters 304 | ---------- 305 | x : array-like 306 | First sample. 307 | y : array-like 308 | Second sample. 309 | *args : array-like 310 | Further samples. 311 | **kwargs : 312 | Keyword arguments are forward to :meth:`same_population`. 313 | 314 | Returns 315 | ------- 316 | TestResult 317 | 318 | Notes 319 | ----- 320 | https://en.wikipedia.org/wiki/One-way_analysis_of_variance 321 | https://en.wikipedia.org/wiki/F-test 322 | 323 | """ 324 | kwargs["transform"] = None 325 | return same_population(_ANOVA(), x, y, *args, **kwargs) 326 | 327 | 328 | def kruskal( 329 | x: "ArrayLike", y: "ArrayLike", *args: "ArrayLike", **kwargs: Any 330 | ) -> TestResult: 331 | """ 332 | Test whether two or more samples have the same mean rank. 333 | 334 | This performs a permutation-based Kruskal-Wallis test. In a sense, it extends the 335 | Mann-Whitney U test, which also uses ranks, to more than two groups. It does so by 336 | comparing the means of the rank distributions. 337 | 338 | Parameters 339 | ---------- 340 | x : array-like 341 | First sample. 342 | y : array-like 343 | Second sample. 344 | *args : array-like 345 | Further samples. 346 | **kwargs : 347 | Keyword arguments are forward to :meth:`same_population`. 348 | 349 | Returns 350 | ------- 351 | TestResult 352 | 353 | Notes 354 | ----- 355 | https://en.wikipedia.org/wiki/Kruskal%E2%80%93Wallis_one-way_analysis_of_variance 356 | 357 | """ 358 | kwargs["transform"] = None 359 | return same_population(_kruskal, x, y, *args, **kwargs) 360 | 361 | 362 | def pearsonr(x: "ArrayLike", y: "ArrayLike", **kwargs: Any) -> TestResult: 363 | """ 364 | Test whether two samples are drawn from same population using correlation. 365 | 366 | The test statistic is the Pearson correlation coefficient. The test is very 367 | sensitive to linear relationship of x and y. If the relationship is very non-linear 368 | but monotonic, :func:`spearmanr` may be more sensitive. 369 | 370 | https://en.wikipedia.org/wiki/Pearson_correlation_coefficient 371 | 372 | Parameters 373 | ---------- 374 | x : array-like 375 | First sample. 376 | y : array-like 377 | Second sample. 378 | **kwargs : 379 | Keyword arguments are forward to :meth:`same_population`. 380 | 381 | Returns 382 | ------- 383 | TestResult 384 | 385 | """ 386 | if len(x) != len(y): 387 | raise ValueError("x and y must have have the same length") 388 | kwargs["transform"] = np.abs 389 | return same_population(_pearson, x, y, **kwargs) 390 | 391 | 392 | def spearmanr(x: "ArrayLike", y: "ArrayLike", **kwargs: Any) -> TestResult: 393 | """ 394 | Test whether two samples are drawn from same population using rank correlation. 395 | 396 | The test statistic is Spearman's rank correlation coefficient. The test is very 397 | sensitive to monotonic relationships between x and y, even if it is very non-linear. 398 | 399 | Parameters 400 | ---------- 401 | x : array-like 402 | First sample. 403 | y : array-like 404 | Second sample. 405 | **kwargs : 406 | Keyword arguments are forward to :meth:`same_population`. 407 | 408 | Returns 409 | ------- 410 | TestResult 411 | 412 | """ 413 | if len(x) != len(y): 414 | raise ValueError("x and y must have have the same length") 415 | kwargs["transform"] = np.abs 416 | return same_population(_spearman, x, y, **kwargs) 417 | 418 | 419 | def ttest(x: "ArrayLike", y: "ArrayLike", **kwargs: Any) -> TestResult: 420 | """ 421 | Test whether the means of two samples are compatible with Welch's t-test. 422 | 423 | See https://en.wikipedia.org/wiki/Welch%27s_t-test for details on this test. The 424 | p-value computed is for the null hypothesis that the two population means are equal. 425 | The test is two-sided, which means that swapping x and y gives the same p-value. 426 | Welch's t-test does not require the sample sizes to be equal and it does not require 427 | the samples to have the same variance. 428 | 429 | Parameters 430 | ---------- 431 | x : array-like 432 | First sample. 433 | y : array-like 434 | Second sample. 435 | **kwargs : 436 | Keyword arguments are forward to :meth:`same_population`. 437 | 438 | Returns 439 | ------- 440 | TestResult 441 | 442 | """ 443 | kwargs["transform"] = np.abs 444 | return same_population(_ttest, x, y, **kwargs) 445 | 446 | 447 | def _ttest(x: NDArray, y: NDArray) -> float: 448 | n1 = len(x) 449 | n2 = len(y) 450 | m1 = np.mean(x) 451 | m2 = np.mean(y) 452 | v1 = np.var(x, ddof=1) 453 | v2 = np.var(y, ddof=1) 454 | r: float = (m1 - m2) / np.sqrt(v1 / n1 + v2 / n2) 455 | return r 456 | 457 | 458 | def _pearson(x: NDArray, y: NDArray) -> float: 459 | m1 = np.mean(x) 460 | m2 = np.mean(y) 461 | s1 = np.mean((x - m1) ** 2) 462 | s2 = np.mean((y - m2) ** 2) 463 | r: float = np.mean((x - m1) * (y - m2)) / np.sqrt(s1 * s2) 464 | return r 465 | 466 | 467 | def _spearman(x: NDArray, y: NDArray) -> float: 468 | x = _stats.rankdata(x) 469 | y = _stats.rankdata(y) 470 | return _pearson(x, y) 471 | 472 | 473 | def _kruskal(*args: NDArray) -> float: 474 | # see https://en.wikipedia.org/wiki/ 475 | # Kruskal%E2%80%93Wallis_one-way_analysis_of_variance 476 | # method 3 and 4 477 | joined = np.concatenate(args) 478 | r = _stats.rankdata(joined) 479 | n = len(r) 480 | start = 0 481 | r_args = [] 482 | for i, a in enumerate(args): 483 | r_args.append(r[start : start + len(a)]) 484 | start += len(a) 485 | 486 | # method 3 (assuming no ties) 487 | h: float = 12.0 / (n * (n + 1)) * sum( 488 | len(r) * np.mean(r) ** 2 for r in r_args 489 | ) - 3 * (n + 1) 490 | 491 | # apply tie correction 492 | h /= _stats.tiecorrect(r) 493 | return h 494 | 495 | 496 | class _ANOVA: 497 | # see https://en.wikipedia.org/wiki/F-test 498 | km1: int = -2 499 | nmk: int = 0 500 | a_bar: float = 0.0 501 | 502 | def __call__(self, *args: NDArray) -> float: 503 | if self.km1 == -2: 504 | self._init(args) 505 | 506 | between_group_variability: float = ( 507 | sum(len(a) * (np.mean(a) - self.a_bar) ** 2 for a in args) / self.km1 508 | ) 509 | within_group_variability: float = sum(len(a) * np.var(a) for a in args) / ( 510 | self.nmk 511 | ) 512 | return between_group_variability / within_group_variability 513 | 514 | def _init(self, args: Tuple[NDArray, ...]) -> None: 515 | n = sum(len(a) for a in args) 516 | k = len(args) 517 | self.km1 = k - 1 518 | self.nmk = n - k 519 | self.a_bar = np.mean(np.concatenate(args)) 520 | -------------------------------------------------------------------------------- /tests/test_bootstrap.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: D100 D103 2 | import numpy as np 3 | import pytest 4 | from numpy.testing import assert_equal, assert_allclose 5 | from scipy import stats 6 | 7 | from resample.bootstrap import ( 8 | _fit_parametric_family, 9 | bootstrap, 10 | confidence_interval, 11 | resample, 12 | variance, 13 | covariance, 14 | ) 15 | 16 | PARAMETRIC_CONTINUOUS = { 17 | # use scipy.stats names here 18 | "norm", 19 | "t", 20 | "laplace", 21 | "logistic", 22 | "f", 23 | "beta", 24 | "gamma", 25 | "lognorm", 26 | "invgauss", 27 | "pareto", 28 | } 29 | PARAMETRIC_DISCRETE = {"poisson"} 30 | PARAMETRIC = PARAMETRIC_CONTINUOUS | PARAMETRIC_DISCRETE 31 | NON_PARAMETRIC = {"ordinary", "balanced"} 32 | ALL_METHODS = NON_PARAMETRIC | PARAMETRIC 33 | 34 | 35 | def chisquare( 36 | obs, exp=None 37 | ): # we do not use scipy.stats.chisquare, because it is broken 38 | n = len(obs) 39 | if exp is None: 40 | exp = 1.0 / n 41 | t = np.sum(obs**2 / exp) - n 42 | return stats.chi2(n - 1).cdf(t) 43 | 44 | 45 | @pytest.fixture 46 | def rng(): 47 | return np.random.Generator(np.random.PCG64(1)) 48 | 49 | 50 | @pytest.mark.parametrize("method", ALL_METHODS) 51 | def test_resample_shape_1d(method): 52 | if method == "beta": 53 | x = (0.1, 0.2, 0.3) 54 | else: 55 | x = (1.0, 2.0, 3.0) 56 | n_rep = 5 57 | count = 0 58 | with np.errstate(invalid="ignore"): 59 | for bx in resample(x, size=n_rep, method=method): 60 | assert len(bx) == len(x) 61 | count += 1 62 | assert count == n_rep 63 | 64 | 65 | @pytest.mark.parametrize("method", NON_PARAMETRIC | {"norm"}) 66 | def test_resample_shape_2d(method): 67 | x = [(1.0, 2.0), (4.0, 3.0), (6.0, 5.0)] 68 | n_rep = 5 69 | count = 0 70 | for bx in resample(x, size=n_rep, method=method): 71 | assert bx.shape == np.shape(x) 72 | count += 1 73 | assert count == n_rep 74 | 75 | 76 | @pytest.mark.parametrize("method", NON_PARAMETRIC) 77 | def test_resample_shape_4d(method): 78 | x = np.ones((2, 3, 4, 5)) 79 | n_rep = 5 80 | count = 0 81 | for bx in resample(x, size=n_rep, method=method): 82 | assert bx.shape == np.shape(x) 83 | count += 1 84 | assert count == n_rep 85 | 86 | 87 | @pytest.mark.parametrize("method", NON_PARAMETRIC | PARAMETRIC_CONTINUOUS) 88 | def test_resample_1d_statistical_test(method, rng): 89 | # distribution parameters for parametric families 90 | args = { 91 | "t": (2,), 92 | "f": (25, 20), 93 | "beta": (2, 1), 94 | "gamma": (1.5,), 95 | "lognorm": (1.0,), 96 | "invgauss": (1,), 97 | "pareto": (1,), 98 | }.get(method, ()) 99 | 100 | if method in NON_PARAMETRIC: 101 | dist = stats.norm 102 | else: 103 | dist = getattr(stats, method) 104 | 105 | x = dist.rvs(*args, size=1000, random_state=rng) 106 | 107 | # make equidistant bins in quantile space for this particular data set 108 | with np.errstate(invalid="ignore"): 109 | par = _fit_parametric_family(dist, x) 110 | prob = np.linspace(0, 1, 11) 111 | xe = dist(*par).ppf(prob) 112 | 113 | # - in case of parametric bootstrap, wref is exactly uniform 114 | # - in case of ordinary and balanced, it needs to be computed from original sample 115 | if method in NON_PARAMETRIC: 116 | wref = np.histogram(x, bins=xe)[0] 117 | else: 118 | wref = len(x) / (len(xe) - 1) 119 | 120 | # compute P values for replicates compared to original 121 | prob = [] 122 | wsum = 0 123 | with np.errstate(invalid="ignore"): 124 | for bx in resample(x, size=100, method=method, random_state=rng): 125 | w = np.histogram(bx, bins=xe)[0] 126 | wsum += w 127 | pvalue = chisquare(w, wref) 128 | prob.append(pvalue) 129 | 130 | if method == "balanced": 131 | # balanced bootstrap exactly reproduces frequencies in original sample 132 | assert_equal(wref * 100, wsum) 133 | 134 | # check whether P value distribution is flat 135 | # - test has chance probability of 1 % to fail randomly 136 | # - if it fails due to programming error, value is typically < 1e-20 137 | wp = np.histogram(prob, range=(0, 1))[0] 138 | pvalue = chisquare(wp) 139 | assert pvalue > 0.01 140 | 141 | 142 | def test_resample_1d_statistical_test_poisson(rng): 143 | # poisson is behaving super weird in scipy 144 | x = rng.poisson(1.5, size=1000) 145 | mu = np.mean(x) 146 | 147 | xe = (0, 1, 2, 3, 10) 148 | # somehow location 1 is needed here... 149 | wref = np.diff(stats.poisson(mu, 1).cdf(xe)) * len(x) 150 | 151 | # compute P values for replicates compared to original 152 | prob = [] 153 | for bx in resample(x, size=100, method="poisson", random_state=rng): 154 | w = np.histogram(bx, bins=xe)[0] 155 | 156 | pvalue = chisquare(w, wref) 157 | prob.append(pvalue) 158 | 159 | # check whether P value distribution is flat 160 | # - test has chance probability of 1 % to fail randomly 161 | # - if it fails due to programming error, value is typically < 1e-20 162 | wp = np.histogram(prob, range=(0, 1))[0] 163 | pvalue = chisquare(wp) 164 | assert pvalue > 0.01 165 | 166 | 167 | def test_resample_invalid_family_raises(): 168 | msg = "Invalid family" 169 | with pytest.raises(ValueError, match=msg): 170 | next(resample((1, 2, 3), method="foobar")) 171 | 172 | 173 | @pytest.mark.parametrize("method", PARAMETRIC - {"norm"}) 174 | def test_resample_2d_parametric_raises(method): 175 | with pytest.raises(ValueError): 176 | next(resample(np.ones((2, 2)), method=method)) 177 | 178 | 179 | def test_resample_3d_parametric_normal_raises(): 180 | with pytest.raises(ValueError): 181 | next(resample(np.ones((2, 2, 2)), method="normal")) 182 | 183 | 184 | def test_resample_equal_along_axis(): 185 | data = np.reshape(np.tile([0, 1, 2], 3), (3, 3)) 186 | for b in resample(data, size=2): 187 | assert_equal(data, b) 188 | 189 | 190 | @pytest.mark.parametrize("method", NON_PARAMETRIC) 191 | def test_resample_full_strata(method): 192 | data = np.arange(3) 193 | for b in resample(data, size=2, strata=data, method=method): 194 | assert_equal(data, b) 195 | 196 | 197 | def test_resample_invalid_strata_raises(): 198 | msg = "must have the same shape" 199 | with pytest.raises(ValueError, match=msg): 200 | next(resample((1, 2, 3), strata=np.arange(4))) 201 | 202 | 203 | def test_bootstrap_2d_balanced(rng): 204 | data = ((1, 2, 3), (2, 3, 4), (3, 4, 5)) 205 | 206 | def mean(x): 207 | return np.mean(x, axis=0) 208 | 209 | r = bootstrap(mean, data, method="balanced") 210 | 211 | # arithmetic mean is linear, therefore mean over all replicates in 212 | # balanced bootstrap is equal to mean of original sample 213 | assert_allclose(mean(data), mean(r)) 214 | 215 | 216 | @pytest.mark.parametrize("action", [bootstrap, variance, confidence_interval]) 217 | def test_bootstrap_several_args(action): 218 | x = [1, 2, 3] 219 | y = [4, 5, 6] 220 | xy = np.transpose([x, y]) 221 | 222 | if action is confidence_interval: 223 | 224 | def f1(x, y): 225 | return np.sum(x + y) 226 | 227 | def f2(xy): 228 | return np.sum(xy) 229 | 230 | else: 231 | 232 | def f1(x, y): 233 | return np.sum(x), np.sum(y) 234 | 235 | def f2(xy): 236 | return np.sum(xy, axis=0) 237 | 238 | r1 = action(f1, x, y, size=10, random_state=1) 239 | r2 = action(f2, xy, size=10, random_state=1) 240 | 241 | assert_equal(r1, r2) 242 | 243 | 244 | @pytest.mark.parametrize("ci_method", ["percentile", "bca"]) 245 | def test_confidence_interval(ci_method, rng): 246 | data = rng.normal(size=1000) 247 | par = stats.norm.fit(data) 248 | dist = stats.norm( 249 | par[0], par[1] / len(data) ** 0.5 250 | ) # accuracy of mean is sqrt(n) better 251 | cl = 0.9 252 | ci_ref = dist.ppf(0.05), dist.ppf(0.95) 253 | ci = confidence_interval( 254 | np.mean, data, cl=cl, size=1000, ci_method=ci_method, random_state=rng 255 | ) 256 | assert_allclose(ci_ref, ci, atol=6e-3) 257 | 258 | 259 | def test_confidence_interval_invalid_p_raises(): 260 | msg = "must be between zero and one" 261 | with pytest.raises(ValueError, match=msg): 262 | confidence_interval(np.mean, (1, 2, 3), cl=2) 263 | 264 | 265 | def test_confidence_interval_invalid_ci_method_raises(): 266 | msg = "method must be 'percentile' or 'bca'" 267 | with pytest.raises(ValueError, match=msg): 268 | confidence_interval(np.mean, (1, 2, 3), ci_method="foobar") 269 | 270 | 271 | def test_bca_confidence_interval_estimator_returns_int(rng): 272 | def fn(data): 273 | return int(np.mean(data)) 274 | 275 | data = (1, 2, 3) 276 | ci = confidence_interval(fn, data, ci_method="bca", size=5, random_state=rng) 277 | assert_allclose((1.0, 2.0), ci) 278 | 279 | 280 | @pytest.mark.parametrize("ci_method", ["percentile", "bca"]) 281 | def test_bca_confidence_interval_bounded_estimator(ci_method, rng): 282 | def fn(data): 283 | return max(np.mean(data), 0) 284 | 285 | data = (-3, -2, -1) 286 | ci = confidence_interval(fn, data, ci_method=ci_method, size=5, random_state=rng) 287 | assert_allclose((0.0, 0.0), ci) 288 | 289 | 290 | @pytest.mark.parametrize("method", NON_PARAMETRIC) 291 | def test_variance(method, rng): 292 | data = np.arange(100) 293 | v = np.var(data) / len(data) 294 | 295 | r = variance(np.mean, data, size=1000, method=method, random_state=rng) 296 | assert r == pytest.approx(v, rel=0.05) 297 | 298 | 299 | @pytest.mark.parametrize("method", NON_PARAMETRIC) 300 | def test_covariance(method, rng): 301 | cov = np.array([[1.0, 0.1], [0.1, 2.0]]) 302 | data = rng.multivariate_normal([0.1, 0.2], cov, size=1000) 303 | 304 | r = covariance( 305 | lambda x: np.mean(x, axis=0), data, size=1000, method=method, random_state=rng 306 | ) 307 | assert_allclose(r, cov / len(data), atol=1e-3) 308 | 309 | 310 | def test_resample_deprecation(rng): 311 | data = [1, 2, 3] 312 | 313 | with pytest.warns(FutureWarning): 314 | r = list(resample(data, 10)) 315 | assert np.shape(r) == (10, 3) 316 | 317 | with pytest.warns(FutureWarning): 318 | resample(data, 10, "balanced") 319 | 320 | with pytest.warns(FutureWarning): 321 | with pytest.raises(ValueError): 322 | resample(data, 10, "foo") 323 | 324 | with pytest.warns(FutureWarning): 325 | resample(data, 10, "balanced", [1, 1, 2]) 326 | 327 | with pytest.warns(FutureWarning): 328 | with pytest.raises(ValueError): 329 | resample(data, 10, "balanced", [1, 1]) 330 | 331 | with pytest.warns(FutureWarning): 332 | resample(data, 10, "balanced", [1, 1, 2], rng) 333 | 334 | with pytest.warns(FutureWarning): 335 | resample(data, 10, "balanced", [1, 1, 2], 1) 336 | 337 | with pytest.warns(FutureWarning): 338 | with pytest.raises(TypeError): 339 | resample(data, 10, "balanced", [1, 1, 2], 1.3) 340 | 341 | with pytest.warns(FutureWarning): 342 | with pytest.raises(ValueError): # too many arguments 343 | resample(data, 10, "balanced", [1, 1, 2], 1, 2) 344 | 345 | 346 | def test_confidence_interval_deprecation(rng): 347 | d = [1, 2, 3] 348 | with pytest.warns(FutureWarning): 349 | r = confidence_interval(np.mean, d, 0.6, random_state=1) 350 | assert_equal(r, confidence_interval(np.mean, d, cl=0.6, random_state=1)) 351 | 352 | with pytest.warns(FutureWarning): 353 | r = confidence_interval(np.mean, d, 0.6, "percentile", random_state=1) 354 | assert_equal( 355 | r, 356 | confidence_interval(np.mean, d, cl=0.6, ci_method="percentile", random_state=1), 357 | ) 358 | 359 | with pytest.warns(FutureWarning): 360 | with pytest.raises(ValueError): 361 | confidence_interval(np.mean, d, 0.6, "percentile", 1) 362 | 363 | 364 | def test_random_state(): 365 | d = [1, 2, 3] 366 | a = list(resample(d, size=5, random_state=np.random.default_rng(1))) 367 | b = list(resample(d, size=5, random_state=1)) 368 | c = list(resample(d, size=5, random_state=[2, 3])) 369 | assert_equal(a, b) 370 | assert not np.all([np.all(ai == ci) for (ai, ci) in zip(a, c)]) 371 | 372 | with pytest.raises(TypeError): 373 | resample(d, size=5, random_state=1.5) 374 | 375 | 376 | @pytest.mark.parametrize("method", NON_PARAMETRIC) 377 | def test_resample_several_args(method): 378 | a = [1, 2, 3] 379 | b = [(1, 2), (2, 3), (3, 4)] 380 | c = ["12", "3", "4"] 381 | r1 = [[], [], []] 382 | for ai, bi, ci in resample(a, b, c, size=5, method=method, random_state=1): 383 | r1[0].append(ai) 384 | r1[1].append(bi) 385 | r1[2].append(ci) 386 | 387 | r2 = [[], [], []] 388 | abc = np.empty(3, dtype=[("a", "i"), ("b", "i", 2), ("c", "U4")]) 389 | abc[:]["a"] = a 390 | abc[:]["b"] = b 391 | abc[:]["c"] = c 392 | for abci in resample(abc, size=5, method=method, random_state=1): 393 | r2[0].append(abci["a"]) 394 | r2[1].append(abci["b"]) 395 | r2[2].append(abci["c"]) 396 | 397 | for i in range(3): 398 | assert_equal(r1[i], r2[i]) 399 | 400 | 401 | def test_resample_several_args_incompatible_keywords(): 402 | a = [1, 2, 3] 403 | b = [(1, 2), (2, 3), (3, 4)] 404 | with pytest.raises(ValueError): 405 | resample(a, b, size=5, method="norm") 406 | 407 | resample(a, size=5, strata=[1, 1, 2]) 408 | 409 | with pytest.raises(ValueError): 410 | resample(a, b, size=5, strata=[1, 1, 2]) 411 | 412 | resample(a, b, a, b, size=5) 413 | 414 | with pytest.raises(ValueError): 415 | resample(a, [1, 2]) 416 | 417 | with pytest.raises(ValueError): 418 | resample(a, [1, 2, 3, 4]) 419 | 420 | with pytest.raises(ValueError): 421 | resample(a, b, 5) 422 | 423 | 424 | def test_resample_extended_1(): 425 | a = [1, 2, 3] 426 | bs = list(resample(a, size=100, method="extended", random_state=1)) 427 | 428 | # check that lengths of bootstrap samples are poisson distributed 429 | w, xe = np.histogram([len(b) for b in bs], bins=10, range=(0, 10)) 430 | wm = stats.poisson(len(a)).pmf(xe[:-1]) * np.sum(w) 431 | t = np.sum((w - wm) ** 2 / wm) 432 | pvalue = 1 - stats.chi2(len(w)).cdf(t) 433 | assert pvalue > 0.1 434 | 435 | 436 | def test_resample_extended_2(): 437 | n = 10 438 | a = np.arange(n) 439 | ts = [] 440 | for b in resample(a, size=1000, method="extended", random_state=1): 441 | ts.append(np.mean(b)) 442 | 443 | t = np.var(ts) 444 | expected_not_extended = np.var(a) / n 445 | 446 | k = np.arange(100) 447 | pk = stats.poisson(n).pmf(k) 448 | expected = expected_not_extended * np.sum(pk[1:] * n / k[1:]) / (1 - pk[0]) 449 | 450 | assert expected / expected_not_extended > 1.1 451 | assert t > expected_not_extended 452 | assert_allclose(t, expected, atol=0.02) 453 | 454 | 455 | def test_resample_extended_3(): 456 | n = 10 457 | a = np.arange(n) 458 | b = 5 + a 459 | ns = [] 460 | for ai, bi in resample(a, b, size=1000, method="extended", random_state=1): 461 | assert len(ai) == len(bi) 462 | assert_equal(bi - ai, 5) 463 | ns.append(len(ai)) 464 | assert_allclose(np.var(ns), 10, rtol=0.05) 465 | 466 | 467 | def test_resample_extended_4(): 468 | x = np.ones(10) 469 | a = np.transpose((x, 3 * x)) 470 | 471 | ts = [] 472 | for b in resample(a, size=1000, method="extended", random_state=1): 473 | ts.append(np.sum(b, axis=0)) 474 | 475 | t = np.var(ts, axis=0) 476 | 477 | mu = np.sum(x, axis=0) 478 | assert_allclose(t, (mu, 3**2 * mu), rtol=0.05) 479 | 480 | 481 | def test_resample_extended_5(): 482 | x = np.ones(10) 483 | a = np.transpose((x, 3 * x)) 484 | 485 | ts1 = [] 486 | ts2 = [] 487 | for b1, b2 in resample(a, 3 * a, size=1000, method="extended", random_state=1): 488 | ts1.append(np.sum(b1, axis=0)) 489 | ts2.append(np.sum(b2, axis=0)) 490 | 491 | t1 = np.var(ts1, axis=0) 492 | t2 = np.var(ts2, axis=0) 493 | 494 | mu1 = np.sum(x, axis=0) 495 | mu2 = 3**2 * np.sum(x, axis=0) 496 | assert_allclose(t1, (mu1, 3**2 * mu1), rtol=0.05) 497 | assert_allclose(t2, (mu2, 3**2 * mu2), rtol=0.05) 498 | 499 | 500 | def test_bias_error(): 501 | with pytest.raises(NotImplementedError): 502 | from resample.bootstrap import bias # noqa 503 | 504 | with pytest.raises(NotImplementedError): 505 | import resample.bootstrap as b 506 | 507 | b.bias_corrected # noqa 508 | -------------------------------------------------------------------------------- /tests/test_empirical.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: D100 D103 2 | import numpy as np 3 | import pytest 4 | from numpy.testing import assert_equal 5 | 6 | from resample.empirical import cdf_gen, influence, quantile_function_gen 7 | 8 | 9 | # high-quality platform-independent reproducible sequence of pseudo-random numbers 10 | @pytest.fixture 11 | def rng(): 12 | return np.random.Generator(np.random.PCG64(1)) 13 | 14 | 15 | def test_cdf_increasing(rng): 16 | x = rng.normal(size=100) 17 | cdf = cdf_gen(x) 18 | result = [cdf(s) for s in np.linspace(x.min(), x.max(), 100)] 19 | assert np.all(np.diff(result) >= 0) 20 | 21 | 22 | def test_cdf_at_infinity(): 23 | cdf = cdf_gen(np.arange(10)) 24 | assert cdf(-np.inf) == 0.0 25 | assert cdf(np.inf) == 1.0 26 | 27 | 28 | def test_cdf_simple_cases(): 29 | cdf = cdf_gen([0, 1, 2, 3]) 30 | assert cdf(0) == 0.25 31 | assert cdf(1) == 0.5 32 | assert cdf(2) == 0.75 33 | assert cdf(3) == 1.0 34 | 35 | 36 | def test_cdf_on_array(): 37 | x = np.arange(4) 38 | cdf = cdf_gen(x) 39 | assert_equal(cdf(x), (x + 1) / len(x)) 40 | assert_equal(cdf(x + 1e-10), (x + 1) / len(x)) 41 | assert_equal(cdf(x - 1e-10), x / len(x)) 42 | 43 | 44 | def test_quantile_simple_cases(): 45 | q = quantile_function_gen([0, 1, 2, 3]) 46 | assert q(0.25) == 0 47 | assert q(0.5) == 1 48 | assert q(0.75) == 2 49 | assert q(1.0) == 3 50 | 51 | 52 | def test_quantile_on_array(): 53 | x = np.arange(4) 54 | q = quantile_function_gen(x) 55 | prob = (x + 1) / len(x) 56 | assert_equal(q(prob), x) 57 | 58 | 59 | def test_quantile_is_inverse_of_cdf(rng): 60 | x = rng.normal(size=30) 61 | y = cdf_gen(x)(x) 62 | assert_equal(quantile_function_gen(x)(y), x) 63 | 64 | 65 | @pytest.mark.parametrize("arg", [-1, 1.5]) 66 | def test_quantile_out_of_bounds_is_nan(arg): 67 | q = quantile_function_gen(np.array([0, 1, 2, 3])) 68 | assert np.isnan(q(arg)) 69 | 70 | 71 | def test_influence_shape(): 72 | n = 100 73 | data = np.random.random(n) 74 | emp = influence(np.mean, data) 75 | assert len(emp) == n 76 | -------------------------------------------------------------------------------- /tests/test_jackknife.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | from numpy.testing import assert_almost_equal, assert_equal 4 | from scipy.optimize import curve_fit 5 | from resample.jackknife import ( 6 | bias, 7 | bias_corrected, 8 | jackknife, 9 | resample, 10 | variance, 11 | cross_validation, 12 | ) 13 | 14 | 15 | def test_resample_1d(): 16 | data = (1, 2, 3) 17 | 18 | r = [] 19 | for x in resample(data): 20 | r.append(x.copy()) 21 | assert_equal(r, [[2, 3], [1, 3], [1, 2]]) 22 | 23 | 24 | def test_resample_2d(): 25 | data = ((1, 2), (3, 4), (5, 6)) 26 | 27 | r = [] 28 | for x in resample(data): 29 | r.append(x.copy()) 30 | assert_equal(r, [[(3, 4), (5, 6)], [(1, 2), (5, 6)], [(1, 2), (3, 4)]]) 31 | 32 | 33 | def test_jackknife(): 34 | data = (1, 2, 3) 35 | r = jackknife(lambda x: x.copy(), data) 36 | assert_equal(r, [[2, 3], [1, 3], [1, 2]]) 37 | 38 | 39 | def test_bias_on_unbiased(): 40 | data = (0, 1, 2, 3) 41 | # bias is exactly zero for linear functions 42 | r = bias(np.mean, data) 43 | assert r == 0 44 | 45 | 46 | def test_bias_on_biased_order_n_minus_one(): 47 | # this "mean" has a bias of exactly O(n^{-1}) 48 | def bad_mean(x): 49 | return (np.sum(x) + 2) / len(x) 50 | 51 | data = (0, 1, 2) 52 | r = bias(bad_mean, data) 53 | mean_jk = np.mean([bad_mean([1, 2]), bad_mean([0, 2]), bad_mean([0, 1])]) 54 | # (5/2 + 4/2 + 3/2) / 3 = 12 / 6 = 2 55 | assert mean_jk == 2.0 56 | # f = 5/3 57 | # (n-1) * (mean_jk - f) 58 | # (3 - 1) * (6/3 - 5/3) = 2/3 59 | # note: 2/3 is exactly the bias of bad_mean for n = 3 60 | assert r == pytest.approx(2.0 / 3.0) 61 | 62 | 63 | def test_bias_on_array_map(): 64 | # compute mean and (biased) variance simultanously 65 | def fn(x): 66 | return np.mean(x), np.var(x, ddof=0) 67 | 68 | data = (0, 1, 2) 69 | r = bias(fn, data) 70 | assert_almost_equal(r, (0.0, -1.0 / 3.0)) 71 | 72 | 73 | def test_bias_corrected(): 74 | # this "mean" has a bias of exactly O(n^{-1}) 75 | def bad_mean(x): 76 | return (np.sum(x) + 2) / len(x) 77 | 78 | # bias correction is exact up to O(n^{-1}) 79 | data = (0, 1, 2) 80 | r = bias_corrected(bad_mean, data) 81 | assert r == 1.0 # which is the correct unbiased mean 82 | 83 | 84 | def test_variance(): 85 | data = (0, 1, 2) 86 | r = variance(np.mean, data) 87 | # formula is (n - 1) / n * sum((jf - mean(jf)) ** 2) 88 | # fj = [3/2, 1, 1/2] 89 | # mfj = 1 90 | # ((3/2 - 1)^2 + (1 - 1)^2 + (1/2 - 1)^2) * 2 / 3 91 | # (1/4 + 1/4) / 3 * 2 = 1/3 92 | assert r == pytest.approx(1.0 / 3.0) 93 | 94 | 95 | def test_resample_several_args(): 96 | a = [1, 2, 3] 97 | b = [(1, 2), (2, 3), (3, 4)] 98 | c = ["12", "3", "4"] 99 | for ai, bi, ci in resample(a, b, c): 100 | assert np.shape(ai) == (2,) 101 | assert np.shape(bi) == (2, 2) 102 | assert np.shape(ci) == (2,) 103 | assert set(ai) <= set(a) 104 | assert set(ci) <= set(c) 105 | bi = list(tuple(x) for x in bi) 106 | assert set(bi) <= set(b) 107 | 108 | 109 | def test_resample_several_args_incompatible_keywords(): 110 | a = [1, 2, 3] 111 | with pytest.raises(ValueError): 112 | resample(a, [1, 2]) 113 | 114 | with pytest.raises(ValueError): 115 | resample(a, [1, 2, 3, 4]) 116 | 117 | 118 | def test_resample_deprecation(): 119 | data = [1, 2, 3] 120 | 121 | with pytest.warns(FutureWarning): 122 | r = list(resample(data, False)) 123 | 124 | assert_equal(r, list(resample(data, copy=False))) 125 | 126 | with pytest.warns(FutureWarning): 127 | with pytest.raises(ValueError): # too many arguments 128 | resample(data, True, 1) 129 | 130 | 131 | @pytest.mark.filterwarnings("ignore:Covariance") 132 | def test_cross_validation(): 133 | x = [1, 2, 3] 134 | y = [3, 4, 5] 135 | 136 | def predict(xi, yi, xo, npar): 137 | def model(x, *par): 138 | return np.polyval(par, x) 139 | 140 | popt = curve_fit(model, xi, yi, p0=np.zeros(npar))[0] 141 | return model(xo, *popt) 142 | 143 | v = cross_validation(predict, x, y, 2) 144 | assert v == pytest.approx(0) 145 | 146 | v2 = cross_validation(predict, x, y, 1) 147 | assert v2 == pytest.approx(1.5) 148 | -------------------------------------------------------------------------------- /tests/test_permutation.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: D100 D101 D103 D105 D107 2 | import numpy as np 3 | from numpy.testing import assert_allclose 4 | from resample import permutation as perm 5 | from scipy import stats 6 | import pytest 7 | 8 | 9 | @pytest.fixture() 10 | def rng(): 11 | return np.random.Generator(np.random.PCG64(1)) 12 | 13 | 14 | def test_TestResult(): 15 | p = perm.TestResult(1, 2, [3, 4]) 16 | assert p.statistic == 1 17 | assert p.pvalue == 2 18 | assert p.samples == [3, 4] 19 | assert repr(p) == "" 20 | assert len(p) == 3 21 | first, *rest = p 22 | assert first == 1 23 | assert rest == [2, [3, 4]] 24 | 25 | p2 = perm.TestResult(1, 2, np.arange(10)) 26 | assert repr(p2) == ( 27 | "" 28 | ) 29 | 30 | 31 | class Scipy: 32 | def __init__(self, **kwargs): 33 | self.d = kwargs 34 | 35 | def __getitem__(self, key): 36 | if key in self.d: 37 | return self.d[key] 38 | return getattr(stats, key) 39 | 40 | 41 | scipy = Scipy( 42 | anova=stats.f_oneway, 43 | ttest=lambda x, y: stats.ttest_ind(x, y, equal_var=False), 44 | ) 45 | 46 | 47 | @pytest.mark.parametrize( 48 | "test_name", 49 | ( 50 | "anova", 51 | "kruskal", 52 | "pearsonr", 53 | "spearmanr", 54 | "ttest", 55 | ), 56 | ) 57 | @pytest.mark.parametrize("size", (10, 100)) 58 | def test_two_sample_same_size(test_name, size, rng): 59 | x = rng.normal(size=size) 60 | y = rng.normal(1, size=size) 61 | 62 | test = getattr(perm, test_name) 63 | scipy_test = scipy[test_name] 64 | 65 | for a, b in ((x, y), (y, x)): 66 | expected = scipy_test(a, b) 67 | got = test(a, b, size=999, random_state=1) 68 | assert_allclose(expected[0], got[0]) 69 | assert_allclose(expected[1], got[1], atol={10: 0.2, 100: 0.02}[size]) 70 | 71 | 72 | @pytest.mark.parametrize( 73 | "test_name", 74 | ( 75 | "anova", 76 | "kruskal", 77 | "pearsonr", 78 | "spearmanr", 79 | "ttest", 80 | ), 81 | ) 82 | @pytest.mark.parametrize("size", (10, 100)) 83 | def test_two_sample_different_size(test_name, size, rng): 84 | x = rng.normal(size=size) 85 | y = rng.normal(1, size=2 * size) 86 | 87 | test = getattr(perm, test_name) 88 | scipy_test = scipy[test_name] 89 | 90 | if test_name in ("pearsonr", "spearmanr"): 91 | with pytest.raises(ValueError): 92 | test(x, y) 93 | return 94 | 95 | for a, b in ((x, y), (y, x)): 96 | expected = scipy_test(a, b) 97 | got = test(a, b, size=999, random_state=1) 98 | assert_allclose(expected[0], got[0]) 99 | assert_allclose(expected[1], got[1], atol=5e-2) 100 | 101 | 102 | @pytest.mark.parametrize( 103 | "test_name", 104 | ( 105 | "anova", 106 | "kruskal", 107 | ), 108 | ) 109 | @pytest.mark.parametrize("size", (10, 100)) 110 | def test_three_sample_same_size(test_name, size, rng): 111 | x = rng.normal(size=size) 112 | y = rng.normal(1, size=size) 113 | z = rng.normal(0.5, size=size) 114 | 115 | test = getattr(perm, test_name) 116 | scipy_test = scipy[test_name] 117 | 118 | for a, b, c in ((x, y, z), (z, y, x)): 119 | expected = scipy_test(a, b, c) 120 | got = test(a, b, c, size=999, random_state=1) 121 | assert_allclose(expected[0], got[0]) 122 | assert_allclose(expected[1], got[1], atol=5e-2) 123 | 124 | 125 | @pytest.mark.parametrize( 126 | "test_name", 127 | ( 128 | "anova", 129 | "kruskal", 130 | ), 131 | ) 132 | @pytest.mark.parametrize("size", (10, 100)) 133 | def test_three_sample_different_size(test_name, size, rng): 134 | x = rng.normal(size=size) 135 | y = rng.normal(1, size=2 * size) 136 | z = rng.normal(0.5, size=size * 2) 137 | 138 | test = getattr(perm, test_name) 139 | scipy_test = scipy[test_name] 140 | 141 | for a, b, c in ((x, y, z), (z, y, x)): 142 | expected = scipy_test(a, b, c) 143 | got = test(a, b, c, size=500, random_state=1) 144 | assert_allclose(expected[0], got[0]) 145 | assert_allclose(expected[1], got[1], atol=5e-2) 146 | 147 | 148 | def test_bad_input(): 149 | with pytest.raises(ValueError): 150 | perm.ttest([1, 2, 3], [1.0, np.nan, 2.0]) 151 | 152 | 153 | @pytest.mark.parametrize("method", ("auto", "patefield", "boyett")) 154 | def test_usp_1(method, rng): 155 | x = rng.normal(0, 2, size=100) 156 | y = rng.normal(1, 3, size=100) 157 | 158 | w = np.histogram2d(x, y, bins=(5, 10))[0] 159 | r = perm.usp(w, method=method, size=100, random_state=1) 160 | assert r.pvalue > 0.05 161 | 162 | 163 | @pytest.mark.parametrize("method", ("auto", "patefield", "boyett")) 164 | def test_usp_2(method, rng): 165 | x = rng.normal(0, 2, size=100).astype(int) 166 | 167 | w = np.histogram2d(x, x, range=((-5, 5), (-5, 5)))[0] 168 | 169 | r = perm.usp(w, method=method, size=99, random_state=1) 170 | assert r.pvalue == 0.01 171 | 172 | 173 | @pytest.mark.parametrize("method", ("auto", "patefield", "boyett")) 174 | def test_usp_3(method, rng): 175 | cov = np.empty((2, 2)) 176 | cov[0, 0] = 2**2 177 | cov[1, 1] = 3**2 178 | rho = 0.5 179 | cov[0, 1] = rho * np.sqrt(cov[0, 0] * cov[1, 1]) 180 | cov[1, 0] = cov[0, 1] 181 | 182 | xy = rng.multivariate_normal([0, 1], cov, size=500).astype(int) 183 | 184 | w = np.histogram2d(*xy.T)[0] 185 | 186 | r = perm.usp(w, method=method, random_state=1) 187 | assert r.pvalue < 0.0012 188 | 189 | 190 | @pytest.mark.parametrize("method", ("auto", "patefield", "boyett")) 191 | def test_usp_4(method): 192 | # table1 from https://doi.org/10.1098/rspa.2021.0549 193 | w = [[18, 36, 21, 9, 6], [12, 36, 45, 36, 21], [6, 9, 9, 3, 3], [3, 9, 9, 6, 3]] 194 | r1 = perm.usp(w, method=method, size=9999, random_state=1) 195 | r2 = perm.usp(np.transpose(w), method=method, size=1, random_state=1) 196 | assert_allclose(r1.statistic, r2.statistic) 197 | expected = 0.004106 # checked against USP R package 198 | assert_allclose(r1.statistic, expected, atol=1e-6) 199 | # according to paper, pvalue is 0.001, but USP R package gives correct value 200 | expected = 0.0024 # computed from USP R package with b=99999 201 | assert_allclose(r1.pvalue, expected, atol=0.001) 202 | 203 | 204 | @pytest.mark.parametrize("method", ("auto", "patefield", "boyett")) 205 | def test_usp_5(method, rng): 206 | w = np.empty((100, 100)) 207 | for i in range(100): 208 | for j in range(100): 209 | w[i, j] = (i + j) % 2 210 | r = perm.usp(w, method=method, size=99, random_state=1) 211 | assert r.pvalue > 0.1 212 | 213 | 214 | def test_usp_bias(rng): 215 | # We compute the p-value as an upper limit to the type I error rate. 216 | # Therefore, the p-value is not unbiased. For size=1, we expect 217 | # an average p-value = (1 + 0.5) / (1 + 1) = 0.75 218 | got = [ 219 | perm.usp(rng.poisson(1000, size=(2, 2)), size=1, random_state=i).pvalue 220 | for i in range(1000) 221 | ] 222 | assert_allclose(np.mean(got), 0.75, atol=0.05) 223 | 224 | 225 | def test_usp_bad_input(): 226 | with pytest.raises(ValueError): 227 | perm.usp([[1, 2], [3, 4]], size=0) 228 | 229 | with pytest.raises(ValueError): 230 | perm.usp([[1, 2], [3, 4]], size=-1) 231 | 232 | with pytest.raises(ValueError): 233 | perm.usp([1, 2]) 234 | 235 | with pytest.raises(ValueError): 236 | perm.usp([[1, 2], [3, 4]], method="foo") 237 | 238 | 239 | def test_usp_deprecrated(): 240 | w = [[1, 2, 3], [4, 5, 6]] 241 | r1 = perm.usp(w, method="boyett", size=100, random_state=1) 242 | with pytest.warns(FutureWarning): 243 | r2 = perm.usp(w, method="shuffle", size=100, random_state=1) 244 | assert r1.statistic == r2.statistic 245 | 246 | 247 | def test_ttest_bad_input(): 248 | with pytest.raises(ValueError): 249 | perm.ttest([1, 2], [3, 4], size=0) 250 | 251 | with pytest.raises(ValueError): 252 | perm.ttest([1, 2], [3, 4], size=-1) 253 | 254 | with pytest.raises(ValueError): 255 | perm.ttest(1, 2) 256 | 257 | with pytest.raises(ValueError): 258 | perm.ttest([1], [2]) 259 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: D100 D103 2 | from resample import _util as u 3 | import numpy as np 4 | from numpy.testing import assert_allclose 5 | 6 | 7 | def test_wilson_score_interval(): 8 | n = 100 9 | for n1 in (10, 50, 90): 10 | p, lh = u.wilson_score_interval(n1, n, 1) 11 | s = np.sqrt(p * (1 - p) / n) 12 | assert_allclose(p, n1 / n) 13 | assert_allclose(lh, (p - s, p + s), atol=0.01) 14 | 15 | n = 10 16 | n1 = 0 17 | p, lh = u.wilson_score_interval(n1, n, 1) 18 | assert_allclose(p, 0.0) 19 | assert_allclose(lh, (0, 0.1), atol=0.01) 20 | 21 | n1 = 10 22 | p, lh = u.wilson_score_interval(n1, n, 1) 23 | assert_allclose(p, 1.0) 24 | assert_allclose(lh, (0.9, 1.0), atol=0.01) 25 | --------------------------------------------------------------------------------