├── mimic ├── tests │ ├── __init__.py │ ├── test_mimic_calibration.py │ └── data_set.csv ├── _version.py ├── __init__.py └── mimic_calibration.py ├── MANIFEST.in ├── requirements.txt ├── data ├── mimic_calib_prob.png ├── evaluation_calib_1.png ├── evaluation_calib_2.png └── merging_bins_history.png ├── examples ├── README.txt └── plot_calibration_eval.py ├── requirements_doc.txt ├── setup.cfg ├── enviornment.yml ├── .readthedocs.yml ├── doc ├── api.rst ├── _templates │ ├── function.rst │ ├── numpydoc_docstring.py │ └── class.rst ├── _static │ ├── css │ │ └── project-template.css │ └── js │ │ └── copybutton.js ├── quick_start.rst ├── index.rst ├── user_guide.rst ├── make.bat ├── Makefile └── conf.py ├── .coveragerc ├── appveyor.yml ├── .pep8speaks.yml ├── .gitignore ├── .circleci └── config.yml ├── .travis.yml ├── LICENSE ├── setup.py └── README.rst /mimic/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | -------------------------------------------------------------------------------- /mimic/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.0.3" 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scikit-learn 3 | coverage 4 | pytest 5 | matplotlib 6 | pandas 7 | -------------------------------------------------------------------------------- /data/mimic_calib_prob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/mimic/HEAD/data/mimic_calib_prob.png -------------------------------------------------------------------------------- /data/evaluation_calib_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/mimic/HEAD/data/evaluation_calib_1.png -------------------------------------------------------------------------------- /data/evaluation_calib_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/mimic/HEAD/data/evaluation_calib_2.png -------------------------------------------------------------------------------- /data/merging_bins_history.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scikit-learn-contrib/mimic/HEAD/data/merging_bins_history.png -------------------------------------------------------------------------------- /examples/README.txt: -------------------------------------------------------------------------------- 1 | .. _general_examples: 2 | 3 | General examples 4 | ================ 5 | 6 | Introductory examples. 7 | -------------------------------------------------------------------------------- /requirements_doc.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scikit-learn 3 | coverage 4 | pytest 5 | matplotlib 6 | sphinx_gallery 7 | sphinx 8 | numpydoc 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | 4 | [aliases] 5 | test = pytest 6 | 7 | [tool:pytest] 8 | addopts = --doctest-modules 9 | -------------------------------------------------------------------------------- /mimic/__init__.py: -------------------------------------------------------------------------------- 1 | from .mimic_calibration import _MimicCalibration 2 | from ._version import __version__ 3 | __all__ = ["_MimicCalibration", "__version__"] 4 | -------------------------------------------------------------------------------- /enviornment.yml: -------------------------------------------------------------------------------- 1 | name: mimic-env 2 | dependencies: 3 | - numpy 4 | - scipy 5 | - scikit-learn 6 | - coverage 7 | - matplotlib 8 | - pytest 9 | - pandas 10 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | formats: 2 | - none 3 | requirements_file: requirements_doc.txt 4 | python: 5 | pip_install: true 6 | extra_requirements: 7 | - tests 8 | - docs 9 | -------------------------------------------------------------------------------- /doc/api.rst: -------------------------------------------------------------------------------- 1 | #################### 2 | sklearn-mimic API 3 | #################### 4 | 5 | .. currentmodule:: sklearn-mimic 6 | 7 | Estimator 8 | ========= 9 | 10 | .. autosummary:: 11 | :toctree: generated/ 12 | :template: class.rst 13 | 14 | mimic._MimicCalibration 15 | -------------------------------------------------------------------------------- /doc/_templates/function.rst: -------------------------------------------------------------------------------- 1 | :mod:`{{module}}`.{{objname}} 2 | {{ underline }}==================== 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autofunction:: {{ objname }} 7 | 8 | .. include:: {{module}}.{{objname}}.examples 9 | 10 | .. raw:: html 11 | 12 |
13 | -------------------------------------------------------------------------------- /doc/_templates/numpydoc_docstring.py: -------------------------------------------------------------------------------- 1 | {{index}} 2 | {{summary}} 3 | {{extended_summary}} 4 | {{parameters}} 5 | {{returns}} 6 | {{yields}} 7 | {{other_parameters}} 8 | {{attributes}} 9 | {{raises}} 10 | {{warns}} 11 | {{warnings}} 12 | {{see_also}} 13 | {{notes}} 14 | {{references}} 15 | {{examples}} 16 | {{methods}} 17 | -------------------------------------------------------------------------------- /doc/_static/css/project-template.css: -------------------------------------------------------------------------------- 1 | @import url("theme.css"); 2 | 3 | .highlight a { 4 | text-decoration: underline; 5 | } 6 | 7 | .deprecated p { 8 | padding: 10px 7px 10px 10px; 9 | color: #b94a48; 10 | background-color: #F3E5E5; 11 | border: 1px solid #eed3d7; 12 | } 13 | 14 | .deprecated p span.versionmodified { 15 | font-weight: bold; 16 | } 17 | -------------------------------------------------------------------------------- /doc/_templates/class.rst: -------------------------------------------------------------------------------- 1 | :mod:`{{module}}`.{{objname}} 2 | {{ underline }}============== 3 | 4 | .. currentmodule:: {{ module }} 5 | 6 | .. autoclass:: {{ objname }} 7 | 8 | {% block methods %} 9 | .. automethod:: __init__ 10 | {% endblock %} 11 | 12 | .. include:: {{module}}.{{objname}}.examples 13 | 14 | .. raw:: html 15 | 16 |
17 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # Configuration for coverage.py 2 | 3 | [run] 4 | branch = True 5 | source = mimic 6 | include = */mimic/* 7 | omit = 8 | */setup.py 9 | 10 | [report] 11 | exclude_lines = 12 | pragma: no cover 13 | def __repr__ 14 | if self.debug: 15 | if settings.DEBUG 16 | raise AssertionError 17 | raise NotImplementedError 18 | if 0: 19 | if __name__ == .__main__.: 20 | if self.verbose: 21 | show_missing = True -------------------------------------------------------------------------------- /doc/quick_start.rst: -------------------------------------------------------------------------------- 1 | ##################################### 2 | Quick Start with the sklearn-mimic 3 | ##################################### 4 | 5 | 6 | 1. Quick start 7 | ------------------------------------- 8 | 9 | To use mimic calibration, please git clone the following repository:: 10 | 11 | $ git clone https://github.com/pinjutien/mimic.git 12 | 13 | This calibration method is for binary classification model. 14 | It requires the probabity prediction of binary model (`X`) and the binary target (`y`). 15 | Both `X` and `y` are 1-d array. 16 | >>> from mimic import _MimicCalibration 17 | >>> mimicObject = _MimicCalibration(threshold_pos=5, record_history=False) 18 | >>> mimicObject.fit(X, y) 19 | >>> y_pred = mimicObject.predict(X) 20 | 21 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | build: false 2 | 3 | environment: 4 | matrix: 5 | - PYTHON: "C:\\Miniconda3-x64" 6 | PYTHON_VERSION: "3.5.x" 7 | PYTHON_ARCH: "32" 8 | NUMPY_VERSION: "1.13.1" 9 | SCIPY_VERSION: "0.19.1" 10 | SKLEARN_VERSION: "0.19.1" 11 | 12 | - PYTHON: "C:\\Miniconda3-x64" 13 | PYTHON_VERSION: "3.6.x" 14 | PYTHON_ARCH: "64" 15 | NUMPY_VERSION: "*" 16 | SCIPY_VERSION: "*" 17 | SKLEARN_VERSION: "*" 18 | 19 | install: 20 | # Prepend miniconda installed Python to the PATH of this build 21 | # Add Library/bin directory to fix issue 22 | # https://github.com/conda/conda/issues/1753 23 | - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\Library\\bin;%PATH%" 24 | # install the dependencies 25 | - "conda install --yes pip numpy==%NUMPY_VERSION% scipy==%SCIPY_VERSION% scikit-learn==%SKLEARN_VERSION% nose pytest pytest-cov pandas" 26 | - pip install codecov coverage codacy-coverage 27 | - pip install . 28 | 29 | test_script: 30 | - pytest -v --cov=mimic --pyargs mimic 31 | 32 | after_test: 33 | - codecov 34 | -------------------------------------------------------------------------------- /.pep8speaks.yml: -------------------------------------------------------------------------------- 1 | scanner: 2 | diff_only: True 3 | linter: pycodestyle 4 | 5 | pycodestyle: # Same as scanner.linter value. Other option is flake8 6 | max-line-length: 100 # Default is 79 in PEP 8 7 | 8 | no_blank_comment: True # If True, no comment is made on PR without any errors. 9 | descending_issues_order: False # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file 10 | 11 | message: # Customize the comment made by the bot 12 | opened: # Messages when a new PR is submitted 13 | header: "Hello @{name}! Thanks for opening this PR. " 14 | # The keyword {name} is converted into the author's username 15 | footer: "Do see the [Hitchhiker's guide to code style](https://goo.gl/hqbW4r)" 16 | # The messages can be written as they would over GitHub 17 | updated: # Messages when new commits are added to the PR 18 | header: "Hello @{name}! Thanks for updating this PR. " 19 | footer: "" # Why to comment the link to the style guide everytime? :) 20 | no_errors: "There are currently no PEP 8 issues detected in this Pull Request. Cheers! :beers: " 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # scikit-learn specific 10 | doc/_build/ 11 | doc/auto_examples/ 12 | doc/modules/generated/ 13 | doc/datasets/generated/ 14 | 15 | # Distribution / packaging 16 | 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *,cover 53 | .hypothesis/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | 62 | # Sphinx documentation 63 | doc/_build/ 64 | doc/generated/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | 70 | # other 71 | *~ 72 | *.log 73 | *.pyc 74 | .#* 75 | .DS_Store 76 | .ipynb_checkpoints/ -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | python3: 5 | docker: 6 | - image: circleci/python:3.6.1 7 | steps: 8 | - checkout 9 | - run: 10 | command: | 11 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh 12 | chmod +x miniconda.sh && ./miniconda.sh -b -p ~/miniconda 13 | export PATH="~/miniconda/bin:$PATH" 14 | conda update --yes --quiet conda 15 | conda create -n testenv --yes --quiet python=3.6 16 | source activate testenv 17 | conda install --yes pip numpy pandas scipy scikit-learn matplotlib sphinx sphinx_rtd_theme numpydoc pillow 18 | pip install sphinx-gallery 19 | pip install . 20 | cd doc 21 | make html 22 | - store_artifacts: 23 | path: doc/_build/html 24 | destination: doc 25 | # - store_artifacts: 26 | # path: ~/log.txt 27 | - persist_to_workspace: 28 | root: doc/_build/html 29 | paths: . 30 | - attach_workspace: 31 | at: doc/_build/html 32 | - run: ls -ltrh doc/_build/html 33 | filters: 34 | branches: 35 | ignore: gh-pages 36 | 37 | workflows: 38 | version: 2 39 | build-doc-and-deploy: 40 | jobs: 41 | - python3 42 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: false 3 | 4 | language: python 5 | 6 | cache: 7 | directories: 8 | - $HOME/.cache/pip 9 | 10 | matrix: 11 | include: 12 | - env: PYTHON_VERSION="3.5" NUMPY_VERSION="1.13.1" SCIPY_VERSION="0.19.1" 13 | SKLEARN_VERSION="0.19.1" 14 | 15 | install: 16 | # install miniconda 17 | - deactivate 18 | - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh 19 | - MINICONDA_PATH=/home/travis/miniconda 20 | - chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH 21 | - export PATH=$MINICONDA_PATH/bin:$PATH 22 | - conda update --yes conda 23 | # create the testing environment 24 | - conda create -n testenv --yes python=$PYTHON_VERSION pip 25 | - source activate testenv 26 | - | 27 | if [ $SKLEARN_VERSION = "nightly" ]; then 28 | conda install --yes numpy==$NUMPY_VERSION scipy==$SCIPY_VERSION cython nose pytest pytest-cov pandas 29 | # install nightly wheels 30 | pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn 31 | else 32 | conda install --yes numpy==$NUMPY_VERSION scipy==$SCIPY_VERSION scikit-learn==$SKLEARN_VERSION cython nose pytest pytest-cov pandas 33 | fi 34 | - pip install codecov coverage codacy-coverage 35 | - pip install . 36 | 37 | script: 38 | - pytest -v --cov=mimic --pyargs mimic 39 | 40 | after_success: 41 | - coverage xml && python-codacy-coverage -r coverage.xml 42 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. project-template documentation master file, created by 2 | sphinx-quickstart on Mon Jan 18 14:44:12 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to mimic calibration's documentation! 7 | ============================================ 8 | 9 | This project is mimic calibration, one of calibration method for binary classification. 10 | This method was presented at NYC ML Meetup talk given by Sam Steingold. 11 | 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :hidden: 16 | :caption: Getting Started 17 | 18 | quick_start 19 | 20 | .. toctree:: 21 | :maxdepth: 2 22 | :hidden: 23 | :caption: Documentation 24 | 25 | user_guide 26 | api 27 | 28 | .. toctree:: 29 | :maxdepth: 2 30 | :hidden: 31 | :caption: Tutorial - Examples 32 | 33 | auto_examples/index 34 | 35 | `Getting started `_ 36 | ------------------------------------- 37 | 38 | A quick start on how to use mimic calibration. 39 | 40 | `User Guide `_ 41 | ------------------------------- 42 | 43 | It provide detail information of mimic calibration. And describe the implementation. 44 | 45 | `API Documentation `_ 46 | ------------------------------- 47 | 48 | API documentation of mimic calibration. 49 | 50 | `Examples `_ 51 | -------------------------------------- 52 | 53 | A set of examples. It complements the `User Guide `_. 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019, Pin-Ju Tien 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of mimic nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | """A template for scikit-learn compatible packages.""" 3 | 4 | import codecs 5 | import os 6 | from setuptools import find_packages, setup 7 | 8 | # get __version__ from _version.py 9 | ver_file = os.path.join('mimic', '_version.py') 10 | with open(ver_file) as f: 11 | exec(f.read()) 12 | 13 | DISTNAME = 'mimiccalib' 14 | DESCRIPTION = 'mimic is a calibration method in binary classification.' 15 | with codecs.open('README.rst', encoding='utf-8-sig') as f: 16 | LONG_DESCRIPTION = f.read() 17 | MAINTAINER = 'Pin-Ju Tien' 18 | MAINTAINER_EMAIL = 'pinju.tien@gmail.com' 19 | URL = 'https://github.com/scikit-learn-contrib/mimic' 20 | DOWNLOAD_URL = 'https://github.com/scikit-learn-contrib/mimic' 21 | LICENSE = 'new BSD' 22 | VERSION = __version__ 23 | INSTALL_REQUIRES = ['numpy', 'scikit-learn', 'matplotlib', 'pandas'] 24 | CLASSIFIERS = ['Intended Audience :: Science/Research', 25 | 'Intended Audience :: Developers', 26 | 'License :: OSI Approved', 27 | 'Programming Language :: Python', 28 | 'Topic :: Software Development', 29 | 'Topic :: Scientific/Engineering', 30 | 'Operating System :: Microsoft :: Windows', 31 | 'Operating System :: POSIX', 32 | 'Operating System :: Unix', 33 | 'Operating System :: MacOS', 34 | 'Programming Language :: Python :: 3.5'] 35 | 36 | setup(name=DISTNAME, 37 | packages=find_packages(exclude=['tests*']), 38 | maintainer=MAINTAINER, 39 | maintainer_email=MAINTAINER_EMAIL, 40 | description=DESCRIPTION, 41 | license=LICENSE, 42 | url=URL, 43 | version=VERSION, 44 | download_url=DOWNLOAD_URL, 45 | long_description=LONG_DESCRIPTION, 46 | # zip_safe=False, # the package can run out of an .egg file 47 | classifiers=CLASSIFIERS, 48 | install_requires=INSTALL_REQUIRES) 49 | -------------------------------------------------------------------------------- /doc/user_guide.rst: -------------------------------------------------------------------------------- 1 | .. title:: User guide : contents 2 | 3 | .. _user_guide: 4 | 5 | ================================================== 6 | User guide: mimic calibration 7 | ================================================== 8 | 9 | Introduction 10 | ------------ 11 | mimic calibration is a calibration method for binary classification model. 12 | This method was presented at NYC ML Meetup talk given by Sam Steingold, see [*]_. 13 | 14 | Here is how it is implemented. 15 | 16 | 1. Sort the probabitliy in the ascending. Merge neighbor data points into 17 | one bin until the number of positive equal to threshold positive at each bin. 18 | In this initial binning, only the last bin may have number of positive less than threshold positive. 19 | 2. Calculate the number of positive rate at each bin. Merge neighbor two bins 20 | if nPos rate in the left bin is greater than right bin. 21 | 3. Keep step 2. until the nPos rate in the increasing order. 22 | 4. In this step, we have information at each bin, such as nPos rate, the avg/min/max probability. 23 | we record those informations in two places. One is `boundary_table`. The other is `calibrated_model`. `boundary_table`: it records probability at each bin. The default is recording the avg prob of bin. `calibrated_model`: it records all the information of bin, such nPos rate, the avg/min/max probability. 24 | 5. The final step is linear interpolation. 25 | 26 | 27 | Estimator 28 | --------- 29 | `_MimicCalibration` is the main class of mimic calibration. 30 | 31 | >>> from mimic import _MimicCalibration 32 | 33 | Once imported, you can specify two parameters, `threshold_pos` and `record_history`. 34 | `threshold_pos` is the number of positive in the initial binning. 35 | `record_history`: a boolean flag to record merging-bin history. 36 | 37 | >>> from mimic import _MimicCalibration 38 | >>> mimicObject = _MimicCalibration(threshold_pos=5, record_history=False) 39 | >>> mimicObject.fit(X, y) 40 | >>> y_pred = mimicObject.predict(X) 41 | 42 | It also provides ``fit`` and ``predict`` methods. 43 | 44 | - ``fit``: it requires two inputs `X` and `y`. 45 | `X`: 1-d array, the probability from the binary model. 46 | `y`: 1-d array, binary target, its element is 0 or 1. 47 | 48 | - ``predict``: it requires one inputs `pre_calib_prob`. 49 | `pre_calib_prob`: 1-d array, the probability prediction from the binary model. 50 | It returns 1-d array, the mimic-calibrated probability. 51 | 52 | Reference 53 | ---------- 54 | .. [*] https://www.youtube.com/watch?v=Cg--SC76I1I 55 | -------------------------------------------------------------------------------- /doc/_static/js/copybutton.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | /* Add a [>>>] button on the top-right corner of code samples to hide 3 | * the >>> and ... prompts and the output and thus make the code 4 | * copyable. */ 5 | var div = $('.highlight-python .highlight,' + 6 | '.highlight-python3 .highlight,' + 7 | '.highlight-pycon .highlight,' + 8 | '.highlight-default .highlight') 9 | var pre = div.find('pre'); 10 | 11 | // get the styles from the current theme 12 | pre.parent().parent().css('position', 'relative'); 13 | var hide_text = 'Hide the prompts and output'; 14 | var show_text = 'Show the prompts and output'; 15 | var border_width = pre.css('border-top-width'); 16 | var border_style = pre.css('border-top-style'); 17 | var border_color = pre.css('border-top-color'); 18 | var button_styles = { 19 | 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', 20 | 'border-color': border_color, 'border-style': border_style, 21 | 'border-width': border_width, 'color': border_color, 'text-size': '75%', 22 | 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', 23 | 'border-radius': '0 3px 0 0' 24 | } 25 | 26 | // create and add the button to all the code blocks that contain >>> 27 | div.each(function(index) { 28 | var jthis = $(this); 29 | if (jthis.find('.gp').length > 0) { 30 | var button = $('>>>'); 31 | button.css(button_styles) 32 | button.attr('title', hide_text); 33 | button.data('hidden', 'false'); 34 | jthis.prepend(button); 35 | } 36 | // tracebacks (.gt) contain bare text elements that need to be 37 | // wrapped in a span to work with .nextUntil() (see later) 38 | jthis.find('pre:has(.gt)').contents().filter(function() { 39 | return ((this.nodeType == 3) && (this.data.trim().length > 0)); 40 | }).wrap(''); 41 | }); 42 | 43 | // define the behavior of the button when it's clicked 44 | $('.copybutton').click(function(e){ 45 | e.preventDefault(); 46 | var button = $(this); 47 | if (button.data('hidden') === 'false') { 48 | // hide the code output 49 | button.parent().find('.go, .gp, .gt').hide(); 50 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); 51 | button.css('text-decoration', 'line-through'); 52 | button.attr('title', show_text); 53 | button.data('hidden', 'true'); 54 | } else { 55 | // show the code output 56 | button.parent().find('.go, .gp, .gt').show(); 57 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); 58 | button.css('text-decoration', 'none'); 59 | button.attr('title', hide_text); 60 | button.data('hidden', 'false'); 61 | } 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /mimic/tests/test_mimic_calibration.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | import numpy as np 5 | from sklearn.datasets import make_classification 6 | from sklearn.naive_bayes import MultinomialNB 7 | from sklearn.utils.testing import assert_raises 8 | from sklearn.exceptions import NotFittedError 9 | # import sys 10 | # sys.path.append('../') 11 | import unittest 12 | from mimic import _MimicCalibration 13 | import pytest 14 | 15 | 16 | def test_mimic_example(): 17 | n_samples = 1000 18 | X, y = make_classification(n_samples=3 * n_samples, n_features=6, 19 | random_state=42) 20 | X -= X.min() # MultinomialNB only allows positive X 21 | # split train and test 22 | X_train, y_train = X[:n_samples], y[:n_samples] 23 | X, y = X[n_samples:2 * n_samples], y[n_samples:2 * n_samples] 24 | clf = MultinomialNB().fit(X_train, y_train) 25 | y_prob = clf.predict_proba(X) 26 | y_pre_calib_prob = np.array([y[1] for y in y_prob]) 27 | mimic_obj = _MimicCalibration(threshold_pos=5, record_history=False) 28 | 29 | # X: probability prediction from MultinomialNB 30 | # y: binary target, [0, 1] 31 | X = y_pre_calib_prob 32 | # use mimi calibration to calibrate X 33 | mimic_obj.fit(X, y) 34 | # y_calib_prob: the mimic-calibrated probaility 35 | y_calib_prob = mimic_obj.predict(X) 36 | assert(y_calib_prob.shape[0] == X.shape[0]), \ 37 | "The length of calibrated prob must be the same as \ 38 | pre-calibrated prob." 39 | 40 | 41 | def test_mimic_history_plots(): 42 | import numpy as np 43 | n_samples = 1000 44 | X, y = make_classification(n_samples=3 * n_samples, n_features=6,random_state=42) 45 | X -= X.min() 46 | 47 | # Train data: train binary model. 48 | X_train, y_train = X[:n_samples], y[:n_samples] 49 | # calibrate data. 50 | X_calib, y_calib = X[n_samples:2 * n_samples], y[n_samples:2 * n_samples] 51 | # test data. 52 | X_test, y_test = X[2 * n_samples:], y[2 * n_samples:] 53 | clf = MultinomialNB().fit(X_train, y_train) 54 | 55 | # y_calib_score: training in the calibration model. 56 | y_calib_score = clf.predict_proba(X_calib) 57 | y_calib_score = np.array([score[1] for score in y_calib_score]) 58 | 59 | # y_test_score: evaluation in the calibration model. 60 | y_test_score = clf.predict_proba(X_test) 61 | y_test_score = np.array([score[1] for score in y_test_score]) 62 | 63 | mimicObject = _MimicCalibration(threshold_pos=5, record_history=True) 64 | # assert_raises(NotFittedError, mimicObject.predict, y_test_score) 65 | mimicObject.fit(y_calib_score, y_calib) 66 | y_mimic_score = mimicObject.predict(y_test_score) 67 | history = mimicObject.history_record_table 68 | mimicObject.output_history_result([0, 5, 19]) 69 | 70 | def test_mimic_output(): 71 | import pandas as pd 72 | tolerance = 1e-6 73 | df = pd.read_csv("mimic/tests/data_set.csv") 74 | X = df["probability"].values 75 | y = df["y"].values 76 | y_mimic = df["mimic_probability"].values 77 | mimicObject = _MimicCalibration(threshold_pos=5, record_history=True) 78 | mimicObject.fit(X, y) 79 | pred = mimicObject.predict(X) 80 | error = abs(y_mimic - pred)/y_mimic 81 | pass_flag = (error < tolerance).all() 82 | assert(pass_flag), "The numerical error is greater than {x}".format(x = tolerance) 83 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. -*- mode: rst -*- 2 | 3 | |Travis|_ |AppVeyor|_ |CircleCI|_ |ReadTheDocs|_ |Codecov|_ 4 | 5 | .. |Travis| image:: https://travis-ci.org/scikit-learn-contrib/mimic.svg?branch=master 6 | .. _Travis: https://travis-ci.org/scikit-learn-contrib/mimic 7 | 8 | .. |AppVeyor| image:: https://ci.appveyor.com/api/projects/status/l27hftp120wehpai?svg=true 9 | .. _AppVeyor: https://ci.appveyor.com/project/pinjutien/mimic 10 | 11 | .. |CircleCI| image:: https://circleci.com/gh/scikit-learn-contrib/mimic.svg?style=shield&circle-token=:circle-token 12 | .. _CircleCI: https://circleci.com/gh/scikit-learn-contrib/mimic/tree/master 13 | 14 | .. |ReadTheDocs| image:: https://readthedocs.org/projects/sklearn-mimic/badge/?version=latest 15 | .. _ReadTheDocs: https://sklearn-mimic.readthedocs.io/en/latest/?badge=latest 16 | 17 | .. |Codecov| image:: https://codecov.io/gh/scikit-learn-contrib/mimic/branches/master/graph/badge.svg 18 | .. _Codecov: https://codecov.io/gh/scikit-learn-contrib/mimic/ 19 | 20 | mimic calibration 21 | ================================================== 22 | 23 | Introduction 24 | ------------ 25 | mimic calibration is a calibration method for binary classification model. 26 | This method was presented at NYC ML Meetup talk given by Sam Steingold, see [*]_. 27 | 28 | 29 | Implementation 30 | --------------- 31 | It requires two inputs, the probability prediction from binary classification model and the binary target (0 and 1). 32 | Here is how it is implemented 33 | 34 | 1. Sort the probabitliy in the ascending. Merge neighbor data points into 35 | one bin until the number of positive equal to threshold positive at each bin. 36 | In this initial binning, only the last bin may have number of positive less than threshold positive. 37 | 2. Calculate the number of positive rate at each bin. Merge neighbor two bins 38 | if nPos rate in the left bin is greater than right bin. 39 | 3. Keep step 2. until the nPos rate in the increasing order. 40 | 4. In this step, we have information at each bin, such as nPos rate, the avg/min/max probability. 41 | we record those informations in two places. One is ``boundary_table``. The other is ``calibrated_model``. 42 | ``boundary_table``: it records probability at each bin. The default is recording the avg prob of bin. 43 | ``calibrated_model``: it records all the information of bin, such nPos rate, the avg/min/max probability. 44 | 5. The final step is linear interpolation. 45 | 46 | Install: 47 | --------------- 48 | pip install -i https://test.pypi.org/simple/ mimiccalib 49 | 50 | Parameters: 51 | --------------- 52 | >>> _MimicCalibration(threshold_pos, record_history) 53 | 54 | * threshold_pos: the number of positive in the initial binning. default = 5. 55 | * record_history: boolean parameter, decide if record all the mergeing of bin history. default = False. 56 | 57 | Usage 58 | --------------- 59 | 60 | >>> from mimic import _MimicCalibration 61 | >>> mimicObject = _MimicCalibration(threshold_pos=5, record_history=True) 62 | >>> # y_calib_score: probability prediction from binary classification model 63 | >>> # y_calib: the binary target, 0 or 1. 64 | >>> mimicObject.fit(y_calib_score, y_calib) 65 | >>> y_mimic_calib_score = mimicObject.predict(y_calib_score) 66 | 67 | Results: calibration evaluation. 68 | ------------------------------------------------------------ 69 | - calibration curve and brier score. 70 | In our testing examples, mimic and isotonic have very similar brier score. 71 | But, as number of bins increase in calibration curve, mimic calibration has more smooth behavior. 72 | It is because the calibrated probability of mimic has more continuous prediction space compared to 73 | isotonic calibration which is step function. 74 | In the following plot, brier scores are 0.1028 (mimic) and 0.1027 (isotonic). 75 | 76 | 77 | >>> calibration_curve(y_test, y_output_score, n_bins=10) 78 | 79 | .. image:: https://github.com/pinjutien/mimic/blob/master/data/evaluation_calib_1.png 80 | 81 | >>> calibration_curve(y_test, y_output_score, n_bins=20) 82 | 83 | .. image:: https://github.com/pinjutien/mimic/blob/master/data/evaluation_calib_2.png 84 | 85 | 86 | The above behavior is similar in the followings cases. 87 | 1. base model = GaussianNB, LinearSVC 88 | 2. positive rate in the data = 0.5, 0.2 89 | 90 | Comparison :mimic, isotonic and platt calibration. 91 | ------------------------------------------------------------ 92 | .. image:: https://github.com/pinjutien/mimic/blob/master/data/mimic_calib_prob.png 93 | :width: 40pt 94 | 95 | History of merging bins. 96 | ------------------------------------------------------------ 97 | .. image:: https://github.com/pinjutien/mimic/blob/master/data/merging_bins_history.png 98 | :width: 40pt 99 | 100 | Run test 101 | ------------------------------------------------------------ 102 | >>> pytest -v --cov=mimic --pyargs mimic 103 | 104 | 105 | Reference 106 | ---------- 107 | .. [*] https://www.youtube.com/watch?v=Cg--SC76I1I 108 | -------------------------------------------------------------------------------- /examples/plot_calibration_eval.py: -------------------------------------------------------------------------------- 1 | """ 2 | =========================== 3 | Plotting Calibration curve 4 | =========================== 5 | Compare mimic and isotonic calibration. 6 | """ 7 | 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import pandas as pd 11 | from sklearn.datasets import make_classification 12 | from sklearn.naive_bayes import MultinomialNB, GaussianNB 13 | from sklearn.calibration import calibration_curve 14 | from sklearn.isotonic import IsotonicRegression 15 | from sklearn.linear_model import LogisticRegression 16 | from sklearn.metrics import (brier_score_loss, 17 | precision_score, 18 | recall_score, 19 | f1_score) 20 | from sklearn.svm import LinearSVC 21 | from copy import copy 22 | import sys 23 | # sys.path.append('../') 24 | from mimic.mimic_calibration import _MimicCalibration 25 | # plt.rcParams["figure.figsize"] = (20,10) 26 | 27 | 28 | def calibration_comparison(base_estimator, 29 | n_samples, 30 | weights=None, 31 | n_bins=10, 32 | detail=False): 33 | 34 | X, y = make_classification(n_samples=3*n_samples, 35 | n_features=6, 36 | random_state=42, 37 | weights=weights) 38 | base_estimator_dict = { 39 | "MultinomialNB": MultinomialNB(), 40 | "GaussianNB": GaussianNB(), 41 | "SVC": LinearSVC() 42 | } 43 | 44 | if (base_estimator == "MultinomialNB"): 45 | X -= X.min() 46 | # Train data: train binary model. 47 | X_train, y_train = X[:n_samples], y[:n_samples] 48 | print("Positive Rate: {x}".format(x=y_train.mean())) 49 | # calibrate data. 50 | X_calib, y_calib = X[n_samples:2 * n_samples], y[n_samples:2 * n_samples] 51 | # test data. 52 | X_test, y_test = X[2 * n_samples:], y[2 * n_samples:] 53 | # train the base estimator 54 | clf = base_estimator_dict[base_estimator].fit(X_train, y_train) 55 | 56 | if (base_estimator == "SVC"): 57 | # y_calib_score: training in the calibration model. 58 | y_calib_score = clf.decision_function(X_calib) 59 | y_calib_score = (y_calib_score - y_calib_score.min()) /\ 60 | (y_calib_score.max() - y_calib_score.min()) 61 | # y_test_score: evaluation in the calibration model. 62 | y_test_score = clf.decision_function(X_test) 63 | y_test_score = (y_test_score - y_test_score.min()) /\ 64 | (y_test_score.max() - y_test_score.min()) 65 | else: 66 | # y_calib_score: training in the calibration model. 67 | y_calib_score = clf.predict_proba(X_calib) 68 | y_calib_score = np.array([score[1] for score in y_calib_score]) 69 | 70 | # y_test_score: evaluation in the calibration model. 71 | y_test_score = clf.predict_proba(X_test) 72 | y_test_score = np.array([score[1] for score in y_test_score]) 73 | 74 | calibrate_model_dict = { 75 | "mimic": _MimicCalibration(threshold_pos=5, record_history=False), 76 | "isotonic": IsotonicRegression(y_min=0.0, 77 | y_max=1.0, 78 | out_of_bounds='clip'), 79 | # "platt": LogisticRegression() 80 | } 81 | 82 | result = {} 83 | result[base_estimator] = {} 84 | for cal_name, cal_object in calibrate_model_dict.items(): 85 | # import pdb; pdb.set_trace() 86 | print(cal_name) 87 | cal_object.fit(copy(y_calib_score), copy(y_calib)) 88 | if cal_name in ["mimic", "isotonic"]: 89 | y_output_score = cal_object.predict(copy(y_test_score)) 90 | else: 91 | raise "Please specify probability prediction function." 92 | 93 | frac_pos, predicted_value = calibration_curve( 94 | y_test, 95 | y_output_score, 96 | n_bins=n_bins) 97 | b_score = brier_score_loss(y_test, y_output_score, pos_label=1) 98 | # precsion = precision_score(y_test, y_output_score) 99 | # recall = recall_score(y_test, y_output_score) 100 | # f1 = f1_score(y_test, y_output_score) 101 | 102 | result[base_estimator][cal_name] = { 103 | "calibration_curve": [frac_pos, predicted_value], 104 | # "eval_score" : [b_score, precsion, recall, f1] 105 | "eval_score": [b_score] 106 | } 107 | 108 | if (detail): 109 | result[base_estimator][cal_name]["detail"] = { 110 | "y_test": y_test, 111 | "y_test_calibrate_score": y_output_score 112 | } 113 | 114 | return result 115 | 116 | 117 | def show_comparison_plots(base_estimator, 118 | n_samples, 119 | weights=None, 120 | n_bins=10, 121 | detail=False): 122 | res = calibration_comparison(base_estimator, 123 | n_samples, 124 | weights, 125 | n_bins, 126 | detail) 127 | all_calibration_methods = res[base_estimator] 128 | all_calibration_methods_names = list(all_calibration_methods.keys()) 129 | color_map = { 130 | "isotonic": 'orangered', 131 | "mimic": 'limegreen'} 132 | 133 | eval_df = [] 134 | fig = plt.figure() 135 | for i, calib_name in enumerate(all_calibration_methods_names): 136 | calib_model = all_calibration_methods[calib_name] 137 | frac_pos, predicted_value = calib_model["calibration_curve"] 138 | b_score = all_calibration_methods[calib_name]["eval_score"][0] 139 | if (i == 0): 140 | plt.plot(frac_pos, 141 | frac_pos, 142 | color='grey', 143 | label="perfect-calibration", 144 | alpha=0.3, 145 | linewidth=5) 146 | 147 | plt.plot(predicted_value, 148 | frac_pos, 149 | color=color_map[calib_name], 150 | label="%s: %1.4f" % (calib_name, b_score), 151 | alpha=0.7, linewidth=5) 152 | 153 | plt.legend(fontsize=20) 154 | plt.xlabel("calibrated probability", fontsize=18) 155 | plt.ylabel("fraction_of_positives", fontsize=18) 156 | plt.show() 157 | 158 | 159 | show_comparison_plots("GaussianNB", 10000, None, 10) 160 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\project-template.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\project-template.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | -rm -rf $(BUILDDIR)/* 51 | -rm -rf auto_examples/ 52 | -rm -rf generated/* 53 | -rm -rf modules/generated/* 54 | 55 | html: 56 | # These two lines make the build a bit more lengthy, and the 57 | # the embedding of images more robust 58 | rm -rf $(BUILDDIR)/html/_images 59 | #rm -rf _build/doctrees/ 60 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 63 | 64 | dirhtml: 65 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 66 | @echo 67 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 68 | 69 | singlehtml: 70 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 71 | @echo 72 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 73 | 74 | pickle: 75 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 76 | @echo 77 | @echo "Build finished; now you can process the pickle files." 78 | 79 | json: 80 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 81 | @echo 82 | @echo "Build finished; now you can process the JSON files." 83 | 84 | htmlhelp: 85 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 86 | @echo 87 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 88 | ".hhp project file in $(BUILDDIR)/htmlhelp." 89 | 90 | qthelp: 91 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 92 | @echo 93 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 94 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 95 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/project-template.qhcp" 96 | @echo "To view the help file:" 97 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/project-template.qhc" 98 | 99 | devhelp: 100 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 101 | @echo 102 | @echo "Build finished." 103 | @echo "To view the help file:" 104 | @echo "# mkdir -p $$HOME/.local/share/devhelp/project-template" 105 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/project-template" 106 | @echo "# devhelp" 107 | 108 | epub: 109 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 110 | @echo 111 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 112 | 113 | latex: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo 116 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 117 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 118 | "(use \`make latexpdf' here to do that automatically)." 119 | 120 | latexpdf: 121 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 122 | @echo "Running LaTeX files through pdflatex..." 123 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 124 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 125 | 126 | latexpdfja: 127 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 128 | @echo "Running LaTeX files through platex and dvipdfmx..." 129 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 130 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 131 | 132 | text: 133 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 134 | @echo 135 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 136 | 137 | man: 138 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 139 | @echo 140 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 141 | 142 | texinfo: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo 145 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 146 | @echo "Run \`make' in that directory to run these through makeinfo" \ 147 | "(use \`make info' here to do that automatically)." 148 | 149 | info: 150 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 151 | @echo "Running Texinfo files through makeinfo..." 152 | make -C $(BUILDDIR)/texinfo info 153 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 154 | 155 | gettext: 156 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 157 | @echo 158 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 159 | 160 | changes: 161 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 162 | @echo 163 | @echo "The overview file is in $(BUILDDIR)/changes." 164 | 165 | linkcheck: 166 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 167 | @echo 168 | @echo "Link check complete; look for any errors in the above output " \ 169 | "or in $(BUILDDIR)/linkcheck/output.txt." 170 | 171 | doctest: 172 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 173 | @echo "Testing of doctests in the sources finished, look at the " \ 174 | "results in $(BUILDDIR)/doctest/output.txt." 175 | 176 | xml: 177 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 178 | @echo 179 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 180 | 181 | pseudoxml: 182 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 183 | @echo 184 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 185 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # project-template documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jan 18 14:44:12 2016. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import sphinx 18 | from distutils.version import LooseVersion 19 | import sphinx_gallery 20 | import sphinx_rtd_theme 21 | 22 | # If extensions (or modules to document with autodoc) are in another directory, 23 | # add these directories to sys.path here. If the directory is relative to the 24 | # documentation root, use os.path.abspath to make it absolute, like shown here. 25 | # sys.path.insert(0, os.path.abspath('.')) 26 | 27 | # -- General configuration ------------------------------------------------ 28 | 29 | # If your documentation needs a minimal Sphinx version, state it here. 30 | # needs_sphinx = '1.0' 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = [ 36 | 'sphinx.ext.autodoc', 37 | 'sphinx.ext.autosummary', 38 | 'sphinx.ext.doctest', 39 | 'sphinx.ext.intersphinx', 40 | 'sphinx.ext.viewcode', 41 | 'numpydoc', 42 | 'sphinx_gallery.gen_gallery', 43 | ] 44 | 45 | # this is needed for some reason... 46 | # see https://github.com/numpy/numpydoc/issues/69 47 | numpydoc_show_class_members = False 48 | 49 | # pngmath / imgmath compatibility layer for different sphinx versions 50 | if LooseVersion(sphinx.__version__) < LooseVersion('1.4'): 51 | extensions.append('sphinx.ext.pngmath') 52 | else: 53 | extensions.append('sphinx.ext.imgmath') 54 | 55 | autodoc_default_flags = ['members', 'inherited-members'] 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = ['_templates'] 59 | 60 | # generate autosummary even if no references 61 | autosummary_generate = True 62 | 63 | # The suffix of source filenames. 64 | source_suffix = '.rst' 65 | 66 | # The encoding of source files. 67 | # source_encoding = 'utf-8-sig' 68 | 69 | # Generate the plots for the gallery 70 | plot_gallery = True 71 | 72 | # The master toctree document. 73 | master_doc = 'index' 74 | 75 | # General information about the project. 76 | project = u'sklearn-mimic' 77 | copyright = u'2019, Pin-Ju Tien' 78 | 79 | # The version info for the project you're documenting, acts as replacement for 80 | # |version| and |release|, also used in various other places throughout the 81 | # built documents. 82 | # 83 | # The short X.Y version. 84 | # ToDo: remove 85 | sys.path.append('../') 86 | from mimic import __version__ 87 | 88 | version = __version__ 89 | # The full version, including alpha/beta/rc tags. 90 | release = __version__ 91 | 92 | # The language for content autogenerated by Sphinx. Refer to documentation 93 | # for a list of supported languages. 94 | # language = None 95 | 96 | # There are two options for replacing |today|: either, you set today to some 97 | # non-false value, then it is used: 98 | # today = '' 99 | # Else, today_fmt is used as the format for a strftime call. 100 | # today_fmt = '%B %d, %Y' 101 | 102 | # List of patterns, relative to source directory, that match files and 103 | # directories to ignore when looking for source files. 104 | exclude_patterns = ['_build', '_templates'] 105 | 106 | # The reST default role (used for this markup: `text`) to use for all 107 | # documents. 108 | # default_role = None 109 | 110 | # If true, '()' will be appended to :func: etc. cross-reference text. 111 | # add_function_parentheses = True 112 | 113 | # If true, the current module name will be prepended to all description 114 | # unit titles (such as .. function::). 115 | # add_module_names = True 116 | 117 | # If true, sectionauthor and moduleauthor directives will be shown in the 118 | # output. They are ignored by default. 119 | # show_authors = False 120 | 121 | # The name of the Pygments (syntax highlighting) style to use. 122 | pygments_style = 'sphinx' 123 | 124 | # Custom style 125 | html_style = 'css/project-template.css' 126 | 127 | # A list of ignored prefixes for module index sorting. 128 | # modindex_common_prefix = [] 129 | 130 | # If true, keep warnings as "system message" paragraphs in the built documents. 131 | # keep_warnings = False 132 | 133 | 134 | # -- Options for HTML output ---------------------------------------------- 135 | 136 | # The theme to use for HTML and HTML Help pages. See the documentation for 137 | # a list of builtin themes. 138 | html_theme = 'sphinx_rtd_theme' 139 | 140 | # Theme options are theme-specific and customize the look and feel of a theme 141 | # further. For a list of options available for each theme, see the 142 | # documentation. 143 | # html_theme_options = {} 144 | 145 | # Add any paths that contain custom themes here, relative to this directory. 146 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 147 | 148 | # The name for this set of Sphinx documents. If None, it defaults to 149 | # " v documentation". 150 | # html_title = None 151 | 152 | # A shorter title for the navigation bar. Default is the same as html_title. 153 | # html_short_title = None 154 | 155 | # The name of an image file (relative to this directory) to place at the top 156 | # of the sidebar. 157 | # html_logo = None 158 | 159 | # The name of an image file (within the static path) to use as favicon of the 160 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 161 | # pixels large. 162 | # html_favicon = None 163 | 164 | # Add any paths that contain custom static files (such as style sheets) here, 165 | # relative to this directory. They are copied after the builtin static files, 166 | # so a file named "default.css" will overwrite the builtin "default.css". 167 | html_static_path = ['_static'] 168 | 169 | # Add any extra paths that contain custom files (such as robots.txt or 170 | # .htaccess) here, relative to this directory. These files are copied 171 | # directly to the root of the documentation. 172 | # html_extra_path = [] 173 | 174 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 175 | # using the given strftime format. 176 | # html_last_updated_fmt = '%b %d, %Y' 177 | 178 | # If true, SmartyPants will be used to convert quotes and dashes to 179 | # typographically correct entities. 180 | # html_use_smartypants = True 181 | 182 | # Custom sidebar templates, maps document names to template names. 183 | # html_sidebars = {} 184 | 185 | # Additional templates that should be rendered to pages, maps page names to 186 | # template names. 187 | # html_additional_pages = {} 188 | 189 | # If false, no module index is generated. 190 | # html_domain_indices = True 191 | 192 | # If false, no index is generated. 193 | # html_use_index = True 194 | 195 | # If true, the index is split into individual pages for each letter. 196 | # html_split_index = False 197 | 198 | # If true, links to the reST sources are added to the pages. 199 | # html_show_sourcelink = True 200 | 201 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 202 | # html_show_sphinx = True 203 | 204 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 205 | # html_show_copyright = True 206 | 207 | # If true, an OpenSearch description file will be output, and all pages will 208 | # contain a tag referring to it. The value of this option must be the 209 | # base URL from which the finished HTML is served. 210 | # html_use_opensearch = '' 211 | 212 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 213 | # html_file_suffix = None 214 | 215 | # Output file base name for HTML help builder. 216 | htmlhelp_basename = 'project-templatedoc' 217 | 218 | 219 | # -- Options for LaTeX output --------------------------------------------- 220 | 221 | latex_elements = { 222 | # The paper size ('letterpaper' or 'a4paper'). 223 | # 'papersize': 'letterpaper', 224 | 225 | # The font size ('10pt', '11pt' or '12pt'). 226 | # 'pointsize': '10pt', 227 | 228 | # Additional stuff for the LaTeX preamble. 229 | # 'preamble': '', 230 | } 231 | 232 | # Grouping the document tree into LaTeX files. List of tuples 233 | # (source start file, target name, title, 234 | # author, documentclass [howto, manual, or own class]). 235 | latex_documents = [ 236 | ('index', 'project-template.tex', u'project-template Documentation', 237 | u'Vighnesh Birodkar', 'manual'), 238 | ] 239 | 240 | # The name of an image file (relative to this directory) to place at the top of 241 | # the title page. 242 | # latex_logo = None 243 | 244 | # For "manual" documents, if this is true, then toplevel headings are parts, 245 | # not chapters. 246 | # latex_use_parts = False 247 | 248 | # If true, show page references after internal links. 249 | # latex_show_pagerefs = False 250 | 251 | # If true, show URL addresses after external links. 252 | # latex_show_urls = False 253 | 254 | # Documents to append as an appendix to all manuals. 255 | # latex_appendices = [] 256 | 257 | # If false, no module index is generated. 258 | # latex_domain_indices = True 259 | 260 | 261 | # -- Options for manual page output --------------------------------------- 262 | 263 | # One entry per manual page. List of tuples 264 | # (source start file, name, description, authors, manual section). 265 | # man_pages = [ 266 | # ('index', 'project-template', u'project-template Documentation', 267 | # [u'Vighnesh Birodkar'], 1) 268 | # ] 269 | 270 | # If true, show URL addresses after external links. 271 | # man_show_urls = False 272 | 273 | 274 | # -- Options for Texinfo output ------------------------------------------- 275 | 276 | # Grouping the document tree into Texinfo files. List of tuples 277 | # (source start file, target name, title, author, 278 | # dir menu entry, description, category) 279 | texinfo_documents = [ 280 | ('index', 'project-template', u'project-template Documentation', 281 | u'Vighnesh Birodkar', 'project-template', 282 | 'One line description of project.', 283 | 'Miscellaneous'), 284 | ] 285 | 286 | # Documents to append as an appendix to all manuals. 287 | # texinfo_appendices = [] 288 | 289 | # If false, no module index is generated. 290 | # texinfo_domain_indices = True 291 | 292 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 293 | # texinfo_show_urls = 'footnote' 294 | 295 | # If true, do not generate a @detailmenu in the "Top" node's menu. 296 | # texinfo_no_detailmenu = False 297 | 298 | 299 | # Example configuration for intersphinx: refer to the Python standard library. 300 | # intersphinx configuration 301 | intersphinx_mapping = { 302 | 'python': ('https://docs.python.org/{.major}'.format( 303 | sys.version_info), None), 304 | 'numpy': ('https://docs.scipy.org/doc/numpy/', None), 305 | 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), 306 | 'matplotlib': ('https://matplotlib.org/', None), 307 | 'sklearn': ('http://scikit-learn.org/stable', None) 308 | } 309 | 310 | # sphinx-gallery configuration 311 | sphinx_gallery_conf = { 312 | 'doc_module': 'skltemplate', 313 | 'backreferences_dir': os.path.join('generated'), 314 | 'reference_url': { 315 | 'skltemplate': None} 316 | } 317 | 318 | 319 | def setup(app): 320 | # a copy button to copy snippet of code from the documentation 321 | app.add_javascript('js/copybutton.js') 322 | -------------------------------------------------------------------------------- /mimic/mimic_calibration.py: -------------------------------------------------------------------------------- 1 | """Mimic Calibration of predicted probabilities.""" 2 | # Author: Pin-Ju Tien 3 | # ref: NYC ML Meetup talk given by Sam Steingold. 4 | # https://www.youtube.com/watch?v=Cg--SC76I1I 5 | # Acknowledgements: Special thanks to Ritesh Bansal for 6 | # the encouragment and support throughout the project. 7 | 8 | import numpy as np 9 | from sklearn.base import BaseEstimator, RegressorMixin 10 | from sklearn.utils import indexable, column_or_1d 11 | from sklearn.utils.validation import check_is_fitted 12 | import sys 13 | 14 | 15 | class _MimicCalibration(BaseEstimator, RegressorMixin): 16 | """ mimic calibration: 17 | A method to calibrate probability of binary classification model. 18 | """ 19 | def __init__(self, threshold_pos=5, record_history=False): 20 | """ 21 | Parameters 22 | ---------- 23 | threshold_pos: int 24 | the number of positive at each bin at initial binning step. 25 | record_history: bool 26 | to record the merging bin process. 27 | """ 28 | self.threshold_pos = threshold_pos 29 | self.boundary_choice = 2 30 | self.record_history = record_history 31 | self.history_record_table = [] 32 | 33 | def get_bin_boundary(self, current_binning, boundary_choice): 34 | """ 35 | Parameters 36 | ---------- 37 | current_binning: array-like, shape (num_bins, 7) 38 | [[bl_index, score_min, score_max, score_mean, 39 | nPos_temp, total_temp, PosRate_temp]] 40 | 41 | boundary_choice: int 42 | 0: choose socre_min, ie left boundary of bin 43 | 1: choose socre_max, ie right boundary of bin 44 | 2: choose socre_mean, ie mean score of bin 45 | 46 | Returns 47 | ---------- 48 | boundary_table: array-like, shape (num_bins, 1) 49 | 50 | """ 51 | num_rows = len(current_binning) 52 | boundary_table_temp = [] 53 | k = None 54 | if (boundary_choice == 0): 55 | k = 1 56 | elif (boundary_choice == 1): 57 | k = 2 58 | elif (boundary_choice == 2): 59 | k = 3 60 | else: 61 | raise Exception("Un-identified boundary choice: {x}" 62 | .format(x=boundary_choice)) 63 | for i in range(num_rows): 64 | boundary_table_temp += [current_binning[i][k]] 65 | return boundary_table_temp 66 | 67 | def construct_initial_bin(self, 68 | sorted_score, 69 | sorted_target, 70 | threshold_pos): 71 | """make each bin having the number of positives equal to 72 | threshold_pos.the default = 5. 73 | 74 | Parameters 75 | ---------- 76 | sorted_score: the sorted probability from the model, 77 | ie pre-calibrated score. 78 | sorted target: the target in the order of increasing score. 79 | the number of target = 2. 80 | threshold_pos: number of positive in each bin, default=5 81 | 82 | Returns 83 | ---------- 84 | bin_info: 2-D array, shape (number of bins, 6). 85 | [[bl_index, score_min, score_max, score_mean, 86 | nPos_temp, total_temp, nPosRate_temp]] 87 | total_number_pos: integer 88 | number of positive. 89 | """ 90 | bin_right_index_array = [] 91 | last_index = len(sorted_target)-1 92 | count = 0 93 | # make each bin having number of positive = threshold positive. 94 | # bin_right_index_array: right-boundary index of each bin. 95 | for i in range(len(sorted_target)): 96 | y = sorted_target[i] 97 | if y > 0: 98 | count += 1 99 | if (count == threshold_pos): 100 | bin_right_index_array += [i] 101 | count = 0 102 | if (len(sorted_target)-1 not in bin_right_index_array): 103 | bin_right_index_array += [last_index] 104 | 105 | # bl_index: left boundary index of each bin. 106 | bl_index = 0 107 | bin_info = [] 108 | total_number_pos = 0 109 | for br_index in bin_right_index_array: 110 | # score stats 111 | score_temp = sorted_score[bl_index: br_index + 1] 112 | score_min = min(score_temp) 113 | score_max = max(score_temp) 114 | score_mean = np.mean(score_temp) 115 | # target 116 | target_row = sorted_target[bl_index: br_index + 1] 117 | nPos_temp = np.sum(target_row) 118 | if (br_index != last_index): 119 | assert (nPos_temp == threshold_pos),\ 120 | "The sum of positive must be equal to threshold pos \ 121 | except the last index." 122 | total_number_per_bin = len(target_row) 123 | nPosRate_temp = 1.0*nPos_temp/total_number_per_bin 124 | bin_info += [[bl_index, score_min, score_max, score_mean, 125 | nPos_temp, total_number_per_bin, nPosRate_temp]] 126 | total_number_pos += nPos_temp 127 | bl_index = br_index + 1 128 | return bin_info, total_number_pos 129 | 130 | def merge_bins(self, binning_input, increasing_flag): 131 | """ 132 | Parameters 133 | ---------- 134 | binning_input: array-like, shape (number of bins, 7) 135 | [[bl_index, score_min, score_max, 136 | score_mean, nPos_temp, total_temp, PosRate_temp]] 137 | increasing_flag: bool 138 | 139 | Returns 140 | ---------- 141 | result: array-like, shape (number of bins, 7) 142 | It merge bins to make sure the positive at each bin increasing. 143 | increasing_flag: bool 144 | """ 145 | # binning_input 146 | # [[bl_index, score_min, score_max, 147 | # score_mean, nPos_temp, total_temp, PosRate_temp]] 148 | nbins = len(binning_input) 149 | result = [] 150 | for i in range(1, nbins): 151 | # current_bin: latest new bin in the result 152 | if (i == 1): 153 | result += [binning_input[0]] 154 | current_bin = result[-1] 155 | current_bin_PosRate = current_bin[-1] 156 | next_bin = binning_input[i] 157 | next_bin_PosRate = next_bin[-1] 158 | if(current_bin_PosRate > next_bin_PosRate): 159 | increasing_flag = False 160 | # merge two bins: 161 | # [[bl_index, score_min, score_max, score_mean, 162 | # nPos_temp, total_temp, PosRate_temp]] 163 | new_bin_index_temp = min(current_bin[0], next_bin[0]) 164 | new_score_min_temp = min(current_bin[1], next_bin[1]) 165 | new_score_max_temp = max(current_bin[2], next_bin[2]) 166 | new_score_mean_temp = (current_bin[3] + next_bin[3])/2.0 167 | new_pos_temp = current_bin[4] + next_bin[4] 168 | new_total_temp = current_bin[5] + next_bin[5] 169 | new_PosRate_temp = 1.0*new_pos_temp/new_total_temp 170 | # update the latest bin info in the latest result 171 | result[-1] = [new_bin_index_temp, new_score_min_temp, 172 | new_score_max_temp, new_score_mean_temp, 173 | new_pos_temp, new_total_temp, new_PosRate_temp] 174 | else: 175 | result += [next_bin] 176 | return result, increasing_flag 177 | 178 | def run_merge_function(self, current_binning, record_history=False): 179 | """ It keep merging bins together until 180 | the positive rate at each bin increasing. 181 | 182 | Parameters 183 | ---------- 184 | current_binning: array-like, shape (number of bins, 7) 185 | [[bl_index, score_min, score_max, 186 | score_mean, nPos_temp, total_temp, PosRate_temp]] 187 | record_history: bool 188 | 189 | Returns 190 | ---------- 191 | result: array-like, shape (number of bins, 7) 192 | it return the final binning result. 193 | """ 194 | 195 | # current_binning 196 | # [[bl_index, score_min, score_max, score_mean, 197 | # nPos_temp, total_temp, PosRate_temp]] 198 | self.history_record_table = [] 199 | if (record_history): 200 | self.history_record_table += [current_binning] 201 | 202 | keep_merge = True 203 | while(keep_merge): 204 | new_bin_temp, increasing_flag = self.merge_bins(current_binning, 205 | True) 206 | if (record_history): 207 | self.history_record_table += [new_bin_temp] 208 | 209 | # update the current_binning 210 | current_binning = new_bin_temp 211 | # if it increasing monotonically, we stop merge 212 | keep_merge = not increasing_flag 213 | # if (record_history): 214 | # return self.history_record_table 215 | return [new_bin_temp] 216 | 217 | def _mimic_calibration(self, 218 | y_score, 219 | y_target, 220 | number_positive_within_bin=5): 221 | """Perform mimic calibration. 222 | 223 | Parameters 224 | ---------- 225 | y_score: array-like, shape (number of row, 1) 226 | the probability prediction from binary model. 227 | y_target: array-like, shape (number of row, 1) 228 | the element of this array is 0 or 1. 229 | number_positive_within_bin: int 230 | number of positive in the initial binning. 231 | 232 | Returns 233 | ------- 234 | boundary_table: array-like, shape (number of bin, 1) 235 | a seris of boundary of each bin. 236 | calibrated_model: array-like, shape (number of bins, 7). 237 | [bl_index, score_min, score_max, score_mean, 238 | nPos, total_num, PosRate] 239 | """ 240 | assert ((y_score.min() >= 0) & (y_score.max() <= 1.0)), \ 241 | "y_score is a probability which is between 0 and 1." 242 | # assert (len(np.unique(y_score)) > 2), \ 243 | # "y_score should be at least 3 different probability." 244 | assert np.array_equal(np.unique(y_target), np.array([0, 1])), \ 245 | "y_traget must be 0 and 1." 246 | if (len(np.unique(y_score)) <= 2): 247 | print("[WARNING]: the unique number of probabilities is\ 248 | less or equal than 2. {x}".format(x=np.unique(y_score))) 249 | y_score = column_or_1d(y_score) 250 | y_target = column_or_1d(y_target) 251 | # sort y_score 252 | sorted_index = y_score.argsort() 253 | y_score = y_score[sorted_index] 254 | y_target = y_target[sorted_index] 255 | threshold_pos = number_positive_within_bin 256 | # initial binning 257 | initial_binning, total_number_pos = self.construct_initial_bin( 258 | y_score, 259 | y_target, 260 | threshold_pos) 261 | # start to merge bin 262 | final_binning = self.run_merge_function(initial_binning, 263 | self.record_history) 264 | calibrated_model = final_binning[-1] 265 | boundary_table = self.get_bin_boundary(calibrated_model, 266 | self.boundary_choice) 267 | return boundary_table, calibrated_model 268 | 269 | def fit(self, X, y, sample_weight=None): 270 | """ perform mimic calibration. 271 | 272 | Parameters 273 | ---------- 274 | X: array-like, shape (number of row, 1) 275 | the probability from the binary model. 276 | y: array-like, shape (number of row, 1) 277 | binary target, its element is 0 or 1. 278 | 279 | Returns 280 | ------- 281 | self : object 282 | Returns an instance of self. 283 | """ 284 | X = column_or_1d(X) 285 | y = column_or_1d(y) 286 | X, y = indexable(X, y) 287 | self.boundary_table, self.calibrated_model = self._mimic_calibration( 288 | X, 289 | y, 290 | self.threshold_pos) 291 | return self 292 | 293 | def predict(self, pre_calib_prob): 294 | """ prediction function of mimic calibration. 295 | It returns 1-d array, calibrated probability using mimic calibration. 296 | 297 | Parameters 298 | ---------- 299 | pre_calib_prob: array-like 300 | the probability prediction from the binary model. 301 | 302 | Returns 303 | ------- 304 | calib_prob : array-like 305 | the mimic-calibrated probability. 306 | """ 307 | pre_calib_prob = column_or_1d(pre_calib_prob) 308 | # check_is_fitted(self, "calibrated_model") 309 | 310 | boundary_table = [cali[3] for cali in self.calibrated_model] 311 | x_start = np.array([0] + boundary_table) 312 | x_end = np.array(boundary_table + [1]) 313 | 314 | calibration_table = [cali[6] for cali in self.calibrated_model] 315 | y_start = np.array([calibration_table[0]] + calibration_table) 316 | y_end = np.array(calibration_table + [calibration_table[-1]]) 317 | 318 | bin_idx = np.digitize(pre_calib_prob, boundary_table, right=True) 319 | x_start = x_start[bin_idx] 320 | x_end = x_end[bin_idx] 321 | y_start = y_start[bin_idx] 322 | y_end = y_end[bin_idx] 323 | 324 | calib_prob = (pre_calib_prob - x_start) / (x_end - x_start) *\ 325 | (y_end - y_start) + y_start 326 | 327 | return calib_prob 328 | 329 | def get_one_history(self, one_history): 330 | score_array = [] 331 | nP_array = [] 332 | for row in one_history: 333 | # the mean of score at each bin 334 | score = row[3] 335 | # the nPos rate at each bin 336 | nP = row[6] 337 | score_array += [score] 338 | nP_array += [nP] 339 | return score_array, nP_array 340 | 341 | def output_history_result(self, show_history_array=[]): 342 | """ Output merging history. 343 | Parameters 344 | ---------- 345 | show_history_array: array-like 346 | given history index. 347 | 348 | Returns 349 | ------- 350 | score-posRate-array : array-like 351 | [[score_array, nPosRate_array, i]] 352 | """ 353 | # import matplotlib.pyplot as plt 354 | # fig = plt.figure() 355 | data = None 356 | if (self.record_history): 357 | data = self.history_record_table 358 | else: 359 | data = self.calibrated_model 360 | 361 | number_of_history = len(data) 362 | print("plot history size: {x}".format(x=number_of_history)) 363 | if (len(show_history_array) == 0): 364 | show_history_array = range(number_of_history) 365 | 366 | assert(max(show_history_array) <= number_of_history-1), \ 367 | "The max of history index is {x}. \ 368 | Please choose indexs between 0 and {x}"\ 369 | .format(x=number_of_history-1) 370 | result = [] 371 | for i in show_history_array: 372 | one_history = data[i] 373 | score_array, nPosRate_array = self.get_one_history(one_history) 374 | result += [[score_array, nPosRate_array, i]] 375 | # plt.plot(score_array, nPosRate_array, label=str(i)) 376 | # plt.xlabel("pre calibrated prob", fontsize=18) 377 | # plt.ylabel("mimic calibrated prob", fontsize=18) 378 | # plt.legend() 379 | # fig.savefig('merging_bins_history.png') 380 | # plt.show() 381 | return result 382 | -------------------------------------------------------------------------------- /mimic/tests/data_set.csv: -------------------------------------------------------------------------------- 1 | ,Unnamed: 0,probability,y,mimic_probability 2 | 0,0,0.7251972934223173,1,0.9968865063778944 3 | 1,1,0.5620401636212927,0,0.6793570986061384 4 | 2,2,0.3779046318289263,0,0.07039398632931597 5 | 3,3,0.6149031614468732,1,0.8100861924288778 6 | 4,4,0.7086606458985453,1,0.992677474959846 7 | 5,5,0.6220232881421031,1,0.8294323100330148 8 | 6,6,0.5139928969351397,1,0.44411943566705453 9 | 7,7,0.6617391212597532,1,0.9373443193453062 10 | 8,8,0.3179547086426049,0,0.02674493016228863 11 | 9,9,0.4888897433766341,0,0.2593756473124928 12 | 10,10,0.3945698753569144,0,0.08252781590119963 13 | 11,11,0.6816413352071689,0,0.9714156190694545 14 | 12,12,0.5442378126754551,0,0.625 15 | 13,13,0.2341279157617694,0,0.02100840336134454 16 | 14,14,0.6183237899823001,1,0.8193803924844428 17 | 15,15,0.5521985329960718,1,0.6442426548081326 18 | 16,16,0.4644367233738772,0,0.17810304966527835 19 | 17,17,0.4394292883635593,0,0.13134320736555677 20 | 18,18,0.27422898016883474,0,0.02100840336134454 21 | 19,19,0.4893134208140554,0,0.26247617510387083 22 | 20,20,0.37784765572295503,0,0.0703525024855087 23 | 21,21,0.3812700091747631,0,0.07284429046623918 24 | 22,22,0.2627589017524129,0,0.02100840336134454 25 | 23,23,0.7269641068489044,1,0.9973362089679536 26 | 24,24,0.3897639986970223,0,0.0790286958171342 27 | 25,25,0.5165318215717769,1,0.4640254639862957 28 | 26,26,0.17100070260225134,0,0.02100840336134454 29 | 27,27,0.5770188807925386,1,0.7307114278125305 30 | 28,28,0.1980319972838074,0,0.02100840336134454 31 | 29,29,0.3429805705285316,0,0.044966058614648724 32 | 30,30,0.7753334711055164,1,1.0 33 | 31,31,0.5394167512085947,0,0.625 34 | 32,32,0.576432710861256,1,0.7303935966765311 35 | 33,33,0.3185278023934824,0,0.027162195105953007 36 | 34,34,0.3426122415220175,0,0.044697881231946984 37 | 35,35,0.29985139746538503,0,0.02100840336134454 38 | 36,36,0.5077815933529944,1,0.3976287160592402 39 | 37,37,0.7280851765998355,1,0.9976215520398616 40 | 38,38,0.592239056393208,0,0.7389640614655153 41 | 39,39,0.7576362742196717,1,1.0 42 | 40,40,0.44846818580491066,0,0.14848263774412016 43 | 41,41,0.4155155762551837,0,0.09445850290696119 44 | 42,42,0.3568486990880121,0,0.05506333130911043 45 | 43,43,0.4491477505577773,0,0.1497431782175401 46 | 44,44,0.5759129741313249,0,0.7288545897766092 47 | 45,45,0.4607267125199346,0,0.1712212642011345 48 | 46,46,0.2005721637514038,0,0.02100840336134454 49 | 47,47,0.6716190320354944,1,0.9591648211928369 50 | 48,48,0.7166135267970518,1,0.9947017017880282 51 | 49,49,0.3321406040218655,0,0.037073566318576515 52 | 50,50,0.39447635269917264,0,0.08245972280743583 53 | 51,51,0.4759186530325152,0,0.19940114817115365 54 | 52,52,0.21169631819824794,0,0.02100840336134454 55 | 53,53,0.3386013590677845,0,0.04177759002523352 56 | 54,54,0.7459334821441148,1,1.0 57 | 55,55,0.5589559579552197,0,0.6683528076865752 58 | 56,56,0.6140606999286059,1,0.8077971377101917 59 | 57,57,0.6013807599303476,1,0.7649602316341159 60 | 58,58,0.6305833660789305,1,0.8526909234755803 61 | 59,59,0.7129078375455669,1,0.993758501997224 62 | 60,60,0.5925614532578036,1,0.7391388704351478 63 | 61,61,0.3714692209355605,0,0.06570841549598228 64 | 62,62,0.7053922405929871,1,0.9918455759550305 65 | 63,63,0.33638353331703114,0,0.040162808961003556 66 | 64,64,0.4119397472217784,0,0.092453312059847 67 | 65,65,0.6112491463280706,1,0.800157856987458 68 | 66,66,0.6682573445441828,0,0.9550550033862352 69 | 67,67,0.63802792130712,1,0.8729185468199143 70 | 68,68,0.37123849620300703,0,0.06554042667672975 71 | 69,69,0.25503092145004225,0,0.02100840336134454 72 | 70,70,0.6079572263306421,1,0.7912133712866166 73 | 71,71,0.3458012443283543,0,0.04701976848738156 74 | 72,72,0.7488739998107206,1,1.0 75 | 73,73,0.7218229465438468,1,0.9960276423302415 76 | 74,74,0.30666856854146185,0,0.02100840336134454 77 | 75,75,0.4039656631926936,0,0.08798174472263198 78 | 76,76,0.3890738024410396,0,0.07852616948277966 79 | 77,77,0.6832094356524309,1,0.9733323922179093 80 | 78,78,0.6121383420429427,1,0.8025738933319426 81 | 79,79,0.3894702744694983,0,0.07881483757306332 82 | 80,80,0.3281719237525104,0,0.034184002182728226 83 | 81,81,0.27690737751996536,0,0.02100840336134454 84 | 82,82,0.5815060538898797,0,0.7331444480799352 85 | 83,83,0.6018905106565495,1,0.7670652421913836 86 | 84,84,0.6342206280254419,1,0.8625737388223077 87 | 85,85,0.2276492637943673,0,0.02100840336134454 88 | 86,86,0.7111612420983352,1,0.9933139454387407 89 | 87,87,0.5651494229654704,1,0.6904507797909167 90 | 88,88,0.4647580349176538,0,0.17869905792468097 91 | 89,89,0.6270922234641572,1,0.8432051294034792 92 | 90,90,0.5061721866849742,1,0.3858508641816475 93 | 91,91,0.7134436125351964,1,0.9938948714615373 94 | 92,92,0.359914968959312,0,0.057295857703175475 95 | 93,93,0.20967503859302136,0,0.02100840336134454 96 | 94,94,0.6155853625324132,1,0.8119398030450161 97 | 95,95,0.6779084534488701,1,0.9668527177946779 98 | 96,96,0.7394801318203144,1,1.0 99 | 97,97,0.3793898692699768,1,0.07147537574635157 100 | 98,98,0.5473162433566244,0,0.6268228905761142 101 | 99,99,0.6118856441801281,1,0.8018872872107904 102 | 100,100,0.4197481900392464,0,0.09683199401626331 103 | 101,101,0.5619570749642977,1,0.6790606424509127 104 | 102,102,0.4578058024703398,0,0.16580320018118389 105 | 103,103,0.6364415524924874,1,0.8686082193545365 106 | 104,104,0.6427688680287441,1,0.8858001873801777 107 | 105,105,0.44507919368040944,0,0.14207811321593578 108 | 106,106,0.5050819155377291,1,0.37787211503525414 109 | 107,107,0.4041683614474447,1,0.0880954103037758 110 | 108,108,0.14820982516397474,0,0.02100840336134454 111 | 109,109,0.6953822443827141,1,0.9882118682017228 112 | 110,110,0.6933149886180234,1,0.9856849507757888 113 | 111,111,0.4162614736704686,0,0.09487677421021505 114 | 112,112,0.36375463237539096,0,0.06009148570740201 115 | 113,113,0.5653950847846879,1,0.6913272888240969 116 | 114,114,0.3831872282631426,1,0.07424020224102304 117 | 115,115,0.6957658153349139,1,0.988680727517416 118 | 116,116,0.4112002556684389,0,0.0920386329237859 119 | 117,117,0.4871300317459816,0,0.2464978437436797 120 | 118,118,0.6344325784094537,1,0.8631496298465616 121 | 119,119,0.5862175315574863,1,0.7356990900351348 122 | 120,120,0.6502279860881056,1,0.906067379436778 123 | 121,121,0.5836971721354822,1,0.7343325090225784 124 | 122,122,0.5413842385971028,0,0.625 125 | 123,123,0.5121912192233273,1,0.4299936727216111 126 | 124,124,0.5746480929661616,1,0.7243415573049921 127 | 125,125,0.4697680861076006,0,0.18799231845974188 128 | 126,126,0.2442185360604228,0,0.02100840336134454 129 | 127,127,0.34686106801917405,0,0.047791417579640824 130 | 128,128,0.33777559383627576,0,0.0411763570118253 131 | 129,129,0.6101532631328548,0,0.7971802294700648 132 | 130,130,0.3194131574940978,0,0.027806815023053166 133 | 131,131,0.5492688326157086,1,0.6337896309883171 134 | 132,132,0.682351044623675,1,0.9722831348952901 135 | 133,133,0.6858152607727256,1,0.9765176318084225 136 | 134,134,0.6095666070408438,0,0.7955862244424812 137 | 135,135,0.7856198360614599,1,1.0 138 | 136,136,0.5031552117611628,1,0.3637722407503352 139 | 137,137,0.3439113162971131,0,0.045643727111466845 140 | 138,138,0.5908226769305948,1,0.7381960768426543 141 | 139,139,0.6478531018412295,1,0.8996145743248887 142 | 140,140,0.3736048704924924,0,0.06726336473543146 143 | 141,141,0.6372541877877106,1,0.8708162331531103 144 | 142,142,0.6560077623033693,1,0.9217716267307197 145 | 143,143,0.4309261537276654,0,0.115187122505659 146 | 144,144,0.585181131140371,0,0.7351371364030178 147 | 145,145,0.5576711229048149,1,0.6637685807545536 148 | 146,146,0.5969887582579047,1,0.746823504381339 149 | 147,147,0.3778987492866032,0,0.07038970329763522 150 | 148,148,0.3942165297678162,0,0.08227054782402488 151 | 149,149,0.4120535357023157,0,0.09251712037326154 152 | 150,150,0.5630143723703747,0,0.6828330264955165 153 | 151,151,0.5203915568882015,0,0.4942870949952756 154 | 152,152,0.5523352145209192,0,0.6447303276242794 155 | 153,153,0.5716223795036248,0,0.7135459636300129 156 | 154,154,0.4832804484467019,0,0.21832608157175845 157 | 155,155,0.6237696159577842,1,0.8341772625800219 158 | 156,156,0.6248568338525651,1,0.8371313455836071 159 | 157,157,0.6515305245080641,1,0.9096065104852896 160 | 158,158,0.7477520151412793,1,1.0 161 | 159,159,0.2936652659984077,0,0.02100840336134454 162 | 160,160,0.27090481645931,0,0.02100840336134454 163 | 161,161,0.4255159995415455,0,0.10490774744235974 164 | 162,162,0.666329471157324,1,0.9498167728231408 165 | 163,163,0.4202170406989773,0,0.09709490788680163 166 | 164,164,0.6852589559643478,1,0.975837630650157 167 | 165,165,0.4103100653016941,0,0.09153944754358166 168 | 166,166,0.6904609894053687,1,0.9821963547020763 169 | 167,167,0.3004697185099614,0,0.02100840336134454 170 | 168,168,0.4772944459515131,0,0.2019531384710123 171 | 169,169,0.625060148651271,1,0.8376837728331197 172 | 170,170,0.4597946634330078,1,0.16949238466120947 173 | 171,171,0.6467433085322772,1,0.8965991515957317 174 | 172,172,0.5322941507539822,1,0.58760745872158 175 | 173,173,0.4325185087354155,0,0.11821262118748585 176 | 174,174,0.3289615914235054,0,0.03475895285309723 177 | 175,175,0.5477597186413451,1,0.628405188119493 178 | 176,176,0.3335924231111297,0,0.03813062410286594 179 | 177,177,0.4435486976661273,0,0.13917014756052923 180 | 178,178,0.6739251520156295,0,0.9619837151408578 181 | 179,179,0.6567379807712388,1,0.9237557055204384 182 | 180,180,0.3995177827328044,0,0.08548754013289679 183 | 181,181,0.2807000244790399,0,0.02100840336134454 184 | 182,182,0.33320566229923304,0,0.03784902667113589 185 | 183,183,0.7803173402198856,1,1.0 186 | 184,184,0.33233877762371344,0,0.037217854921766186 187 | 185,185,0.3973708056220393,0,0.08428359586955204 188 | 186,186,0.3669034042886753,0,0.06238408117367052 189 | 187,187,0.26919364116071104,0,0.02100840336134454 190 | 188,188,0.7660696183787343,1,1.0 191 | 189,189,0.6602674813777264,1,0.9333457222069009 192 | 190,190,0.3633661918927009,0,0.05980866532089727 193 | 191,191,0.4655145772209338,1,0.18010238584761404 194 | 192,192,0.7130449561210546,1,0.9937934024443307 195 | 193,193,0.6692869105486471,1,0.9563141442107405 196 | 194,194,0.3939773556057415,0,0.08209640704319929 197 | 195,195,0.7450895929151182,1,1.0 198 | 196,196,0.5314227516177195,0,0.5807753946132164 199 | 197,197,0.5954126174108312,0,0.7406848193030717 200 | 198,198,0.602270550980761,0,0.7686346149328667 201 | 199,199,0.5761433396830138,1,0.7296765224954663 202 | 200,200,0.648561888336074,1,0.9015404202364049 203 | 201,201,0.5288656930063144,0,0.560727189795287 204 | 202,202,0.624866036771139,1,0.8371563508612735 205 | 203,203,0.7262892800289573,1,0.9971644469900305 206 | 204,204,0.4511457548768429,0,0.1534493229221116 207 | 205,205,0.6240571719981107,1,0.834958581959867 208 | 206,206,0.4101525388218625,1,0.09145111260010823 209 | 207,207,0.4493821791484889,1,0.15017802526555463 210 | 208,208,0.3731445577082488,0,0.06692821470510689 211 | 209,209,0.6404414907027726,1,0.8794764634656002 212 | 210,210,0.20772911993524773,0,0.02100840336134454 213 | 211,211,0.8020202830717023,1,1.0 214 | 212,212,0.7497802707573122,1,1.0 215 | 213,213,0.6878147919520134,1,0.9789617658468621 216 | 214,214,0.709990283170445,1,0.9930159041997503 217 | 215,215,0.39376006444299066,0,0.08193819909791082 218 | 216,216,0.2970448767583074,0,0.02100840336134454 219 | 217,217,0.5866630559280145,0,0.73594066079686 220 | 218,218,0.7663480961967429,1,1.0 221 | 219,219,0.6454665729214667,1,0.893130129437365 222 | 220,220,0.5938082909989371,1,0.7398149267023196 223 | 221,221,0.599035757907055,1,0.7552765690866899 224 | 222,222,0.4361466546011116,0,0.12510615339651832 225 | 223,223,0.5762162089402374,1,0.7299365163455529 226 | 224,224,0.6179632769581257,1,0.8184008414649121 227 | 225,225,0.4947213055397354,0,0.30205179420361394 228 | 226,226,0.2882931432798581,0,0.02100840336134454 229 | 227,227,0.3665652499684406,0,0.06213787373698751 230 | 228,228,0.4614269336159666,0,0.17252012060616068 231 | 229,229,0.2571004127709504,0,0.02100840336134454 232 | 230,230,0.7795166490989558,1,1.0 233 | 231,231,0.5907596403242321,1,0.738161897340874 234 | 232,232,0.4449393721780591,0,0.1418124502394054 235 | 233,233,0.6369696178897606,1,0.8700430274296748 236 | 234,234,0.6550154080166223,1,0.9190752979211159 237 | 235,235,0.7462488238563938,1,1.0 238 | 236,236,0.4541952601977974,0,0.15910592130329207 239 | 237,237,0.6824897647101565,1,0.9724526998856722 240 | 238,238,0.4948974759149879,0,0.30334103242175553 241 | 239,239,0.3430113388832122,0,0.04498846080584834 242 | 240,240,0.3057904292605406,0,0.02100840336134454 243 | 241,241,0.4139845982116462,0,0.093599987823786 244 | 242,242,0.6781369350807708,1,0.9671320031288958 245 | 243,243,0.2378000453952189,0,0.02100840336134454 246 | 244,244,0.5037269286624868,0,0.36795614101215846 247 | 245,245,0.35711185057332,0,0.055254929785841476 248 | 246,246,0.606723022299996,1,0.7870210509910432 249 | 247,247,0.4893379407191313,1,0.2626556150277726 250 | 248,248,0.6168086476793647,1,0.8152635947876787 251 | 249,249,0.6283187615740131,1,0.8465377597814087 252 | 250,250,0.7101825098243693,1,0.9930648311678428 253 | 251,251,0.5868489368294963,1,0.7360414485290955 254 | 252,252,0.3606656965643908,0,0.05784245642521851 255 | 253,253,0.7425858348200007,1,1.0 256 | 254,254,0.4522627304098138,1,0.15552122683000308 257 | 255,255,0.5785684399471736,1,0.7315516246908286 258 | 256,256,0.30949576755300195,0,0.02100840336134454 259 | 257,257,0.597780889757505,1,0.7500946035356783 260 | 258,258,0.5770380057140063,1,0.7307217976646699 261 | 259,259,0.3638066255771451,0,0.06012934153878676 262 | 260,260,0.3342949470196391,0,0.03864212610172425 263 | 261,261,0.6080560841568535,1,0.7914819781827913 264 | 262,262,0.7305083798548164,1,0.9982383238866261 265 | 263,263,0.578723946168982,0,0.7316359427639584 266 | 264,264,0.3549838399639242,0,0.05370554240256053 267 | 265,265,0.4502832110388072,0,0.15184937028572704 268 | 266,266,0.4489197556899549,0,0.14932026523182415 269 | 267,267,0.5495238864199249,0,0.6346996501522826 270 | 268,268,0.316830627817779,0,0.0259264959683596 271 | 269,269,0.6388510393485541,1,0.8751550433196662 272 | 270,270,0.2769283642380902,0,0.02100840336134454 273 | 271,271,0.7391449300086792,1,1.0 274 | 272,272,0.8089328066754246,1,1.0 275 | 273,273,0.7015562765449187,1,0.990869217646999 276 | 274,274,0.7124914759686252,1,0.9936525265293369 277 | 275,275,0.26459399635011904,0,0.02100840336134454 278 | 276,276,0.5715720724577777,1,0.7133664706169837 279 | 277,277,0.5986268033760014,1,0.7535877954154402 280 | 278,278,0.34626564427180073,0,0.047357894345399054 281 | 279,279,0.4589722577484706,0,0.1679668852197556 282 | 280,280,0.40346100717061256,0,0.08769875254740085 283 | 281,281,0.256202772044131,0,0.02100840336134454 284 | 282,282,0.3438378284864492,0,0.04559022122847699 285 | 283,283,0.34605291418471873,0,0.04720300728256496 286 | 284,284,0.6943354317103256,1,0.9869322930138201 287 | 285,285,0.707333494397249,1,0.9923396784168989 288 | 286,286,0.5596679240760367,1,0.6708930670488242 289 | 287,287,0.4520763383076925,0,0.15517548378247362 290 | 288,288,0.6940317737288849,1,0.9865611156017492 291 | 289,289,0.7589265556065217,1,1.0 292 | 290,290,0.5783175081077658,0,0.7314155652525649 293 | 291,291,0.5115547361748272,1,0.4252410766586988 294 | 292,292,0.2435663153714205,0,0.02100840336134454 295 | 293,293,0.41864652287270093,0,0.09621422037600079 296 | 294,294,0.4072626249458231,0,0.08983055725502676 297 | 295,295,0.5652180697452649,1,0.6906957080536 298 | 296,296,0.8103849880166766,1,1.0 299 | 297,297,0.30169077609009376,0,0.02100840336134454 300 | 298,298,0.3166176745162968,0,0.025771446384918753 301 | 299,299,0.6879655043902462,1,0.9791459897309989 302 | 300,300,0.407826497760304,0,0.09014675598711683 303 | 301,301,0.5051404562485824,0,0.37830052373700723 304 | 302,302,0.403029662037256,0,0.0874568703697929 305 | 303,303,0.5133817507700237,1,0.4393278429216674 306 | 304,304,0.4798779402196298,0,0.2067453221063056 307 | 305,305,0.7340638454008016,1,0.9991432876144511 308 | 306,306,0.4352980187950013,0,0.12349373248520049 309 | 307,307,0.3934394867363779,0,0.08170478905202389 310 | 308,308,0.6848106165941041,1,0.9752896014286395 311 | 309,309,0.6271632547342242,0,0.8433981286804737 312 | 310,310,0.7606086547165518,1,1.0 313 | 311,311,0.7618581678165132,1,1.0 314 | 312,312,0.658196019927189,1,0.9277173480852184 315 | 313,313,0.7084722048567313,1,0.992629511534109 316 | 314,314,0.605857510399953,0,0.7834469282024283 317 | 315,315,0.38395322025735534,0,0.07479791484183773 318 | 316,316,0.6980830969252759,1,0.9899851979496216 319 | 317,317,0.34661212846141826,0,0.04761016669288145 320 | 318,318,0.30725163842896025,0,0.02100840336134454 321 | 319,319,0.6550823774273854,1,0.9192572607080098 322 | 320,320,0.6234483554496312,1,0.833304364689531 323 | 321,321,0.33511093498064426,0,0.0392362403632361 324 | 322,322,0.2400689562354271,0,0.02100840336134454 325 | 323,323,0.7052700557270017,1,0.9918144765475477 326 | 324,324,0.4363440908000162,1,0.12548128517091744 327 | 325,325,0.6706704475748174,1,0.9580053156083822 328 | 326,326,0.22082727194700766,0,0.02100840336134454 329 | 327,327,0.2728015943077025,0,0.02100840336134454 330 | 328,328,0.7168151167774075,1,0.9947530119801128 331 | 329,329,0.4212377097282509,0,0.09766726080868708 332 | 330,330,0.494310362959892,0,0.2990444617923811 333 | 331,331,0.5317030117275793,1,0.5829727287140869 334 | 332,332,0.3998842946101907,0,0.0856930662514634 335 | 333,333,0.28587695052208323,0,0.02100840336134454 336 | 334,334,0.7823430284252652,1,1.0 337 | 335,335,0.7267980418180482,1,0.9972939408521636 338 | 336,336,0.5499311270027994,1,0.6361526640992773 339 | 337,337,0.6515736527145944,1,0.9097236942646386 340 | 338,338,0.2421452142875878,0,0.02100840336134454 341 | 339,339,0.7040289882283908,1,0.9914985907498935 342 | 340,340,0.5544230162303919,1,0.6521794992687411 343 | 341,341,0.77191303520868,1,1.0 344 | 342,342,0.4271348161280217,0,0.1079835235253724 345 | 343,343,0.4753191420176528,0,0.19828910123919125 346 | 344,344,0.6935409280400007,1,0.985961128630678 347 | 345,345,0.8046844547533891,1,1.0 348 | 346,346,0.6043874216118871,1,0.7773762111638154 349 | 347,347,0.410568492549415,0,0.09168436385443822 350 | 348,348,0.7639212276000431,1,1.0 351 | 349,349,0.3062435783444193,0,0.02100840336134454 352 | 350,350,0.6972395413391461,1,0.989770489862037 353 | 351,351,0.6581279495102272,1,0.927532393751079 354 | 352,352,0.6898603273524162,1,0.9814621333083678 355 | 353,353,0.7633876983451147,1,1.0 356 | 354,354,0.2114839021174608,0,0.02100840336134454 357 | 355,355,0.6725111498209305,1,0.9602553045362665 358 | 356,356,0.5244751736692881,0,0.5263040324483326 359 | 357,357,0.5163303772311271,0,0.4624460721871607 360 | 358,358,0.3620122700569299,0,0.05882288573830025 361 | 359,359,0.4003610409921883,0,0.08596040774700073 362 | 360,360,0.7318029535540426,1,0.9985678284833219 363 | 361,361,0.4134011642315436,0,0.09327281992345632 364 | 362,362,0.5108806537140321,0,0.42030805162886387 365 | 363,363,0.5774828581336978,1,0.7309630040834183 366 | 364,364,0.574680628574773,1,0.7244576427224929 367 | 365,365,0.3545189272592104,0,0.053367043207194646 368 | 366,366,0.6050927348588605,1,0.7802887952068475 369 | 367,367,0.5826672940198051,0,0.7337740918913375 370 | 368,368,0.7260993342930528,1,0.9971161005782807 371 | 369,369,0.4098797407261239,0,0.0912981376565147 372 | 370,370,0.4990650933541054,0,0.3338402106939443 373 | 371,371,0.5500156646390782,1,0.6364542901398158 374 | 372,372,0.6783789235777674,1,0.9674277986276342 375 | 373,373,0.6347104933897906,1,0.8639047534733858 376 | 374,374,0.29995804606302073,0,0.02100840336134454 377 | 375,375,0.4359997725983788,0,0.12482707536210427 378 | 376,376,0.4074171052721354,0,0.08991718402982132 379 | 377,377,0.6540253927795214,0,0.9163853245503228 380 | 378,378,0.5849977183357895,0,0.735037686914182 381 | 379,379,0.6961826576104204,1,0.989190256153318 382 | 380,380,0.4771527431261304,1,0.20169029060281518 383 | 381,381,0.7125052460619228,1,0.9936560313966467 384 | 382,382,0.707912357862159,1,0.9924870150827536 385 | 383,383,0.4157462205022523,0,0.09458783955276862 386 | 384,384,0.7691748369742764,1,1.0 387 | 385,385,0.394630156399684,0,0.08257170604286804 388 | 386,386,0.3153787653172307,0,0.024869406576663887 389 | 387,387,0.7297071785427507,1,0.9980343961261887 390 | 388,388,0.41171066982197,0,0.09232485404323454 391 | 389,389,0.3304293868284473,0,0.035827642864140176 392 | 390,390,0.6727162709753274,1,0.9605060351078418 393 | 391,391,0.797171167854817,1,1.0 394 | 392,392,0.7114897328857692,1,0.9933975553749664 395 | 393,393,0.6448373547180192,1,0.89142047876911 396 | 394,394,0.6351146319438332,1,0.8650028395509991 397 | 395,395,0.6188443065007884,1,0.8207946894783877 398 | 396,396,0.21761857796655354,0,0.02100840336134454 399 | 397,397,0.7486824573944444,1,1.0 400 | 398,398,0.4009708741327863,1,0.0863023793067063 401 | 399,399,0.6352082480635586,1,0.8652572041907552 402 | 400,400,0.4473509404687482,0,0.1463944665919802 403 | 401,401,0.4662791384388993,0,0.18152058824199557 404 | 402,402,0.5920458444782556,1,0.7388592987352204 405 | 403,403,0.741303747275078,1,1.0 406 | 404,404,0.6243480866324989,1,0.835749026985713 407 | 405,405,0.33591360054624125,0,0.03982065469596499 408 | 406,406,0.6375740969559602,1,0.8716854593138955 409 | 407,407,0.6874900527156088,1,0.978564819687915 410 | 408,408,0.5986981032988116,0,0.7538822277380111 411 | 409,409,0.6385790715019224,1,0.8744160786676738 412 | 410,410,0.2236597649587369,0,0.02100840336134454 413 | 411,411,0.4536391512398492,0,0.15807438185665654 414 | 412,412,0.5488658684998552,1,0.6323518752608784 415 | 413,413,0.3481643736365416,0,0.048740343901338926 416 | 414,414,0.3808605815456101,0,0.07254618950744912 417 | 415,415,0.7606064327977141,1,1.0 418 | 416,416,0.3772373825453443,0,0.06990816750072527 419 | 417,417,0.4531143003514949,0,0.1571008237306793 420 | 418,418,0.8001971411593913,1,1.0 421 | 419,419,0.7242135115248929,1,0.9966361068390028 422 | 420,420,0.4821968779274815,0,0.2110467736217883 423 | 421,421,0.5570354200720741,1,0.661500424974016 424 | 422,422,0.2290987528416245,0,0.02100840336134454 425 | 423,423,0.5912995809753968,1,0.7384546621883585 426 | 424,424,0.5966525448747828,1,0.7454351145574749 427 | 425,425,0.38182130964826416,0,0.07324568790019363 428 | 426,426,0.33341992688939065,0,0.0380050309932987 429 | 427,427,0.5780464697800304,0,0.7312686037402011 430 | 428,428,0.7109764970303883,1,0.9932669227399622 431 | 429,429,0.5582476064353044,1,0.6658254450388087 432 | 430,430,0.4527241990680129,0,0.15637721578237443 433 | 431,431,0.6498770169786839,1,0.9051137602166537 434 | 432,432,0.5644732029677622,1,0.6880380608006547 435 | 433,433,0.30226668231450604,0,0.02100840336134454 436 | 434,434,0.6478959594106714,1,0.8997310227553929 437 | 435,435,0.35611315227002915,0,0.05452778559586854 438 | 436,436,0.7173798432868144,1,0.9948967504021408 439 | 437,437,0.33498418528929635,0,0.03914395493395201 440 | 438,438,0.35322377524212945,0,0.05242405345618685 441 | 439,439,0.75819905796931,1,1.0 442 | 440,440,0.5702069910183217,1,0.7084959285990137 443 | 441,441,0.3870578435955279,1,0.07705836608619027 444 | 442,442,0.3445328598813356,0,0.04609626798851961 445 | 443,443,0.5360564258269862,0,0.617104968441753 446 | 444,444,0.20647995979982503,0,0.02100840336134454 447 | 445,445,0.3070259149302987,0,0.02100840336134454 448 | 446,446,0.3384683429831597,0,0.04168074208548119 449 | 447,447,0.6664423966078916,1,0.9501236029036568 450 | 448,448,0.5161602972963789,0,0.4611125879439593 451 | 449,449,0.521168115822964,0,0.5003755798301055 452 | 450,450,0.4783919578030158,0,0.20398893874323262 453 | 451,451,0.5461014137800801,1,0.625 454 | 452,452,0.5347044657784716,0,0.6065051441602971 455 | 453,453,0.7069185188969229,1,0.9922340557436163 456 | 454,454,0.3906378856784473,0,0.07966496588902913 457 | 455,455,0.7134467331012774,1,0.9938956657313951 458 | 456,456,0.4545184300470079,1,0.15970537657668657 459 | 457,457,0.3946567247355791,0,0.08259105023422164 460 | 458,458,0.4845129098753298,1,0.22734539809359286 461 | 459,459,0.7364047111000237,1,0.9997391022896622 462 | 460,460,0.707860254178593,1,0.9924737532628587 463 | 461,461,0.3826076782999379,0,0.0738182365811689 464 | 462,462,0.2599328871895831,0,0.02100840336134454 465 | 463,463,0.3304785798081871,0,0.035863459876432476 466 | 464,464,0.6077948171377191,1,0.7907720887812947 467 | 465,465,0.5950910922740197,1,0.7405104829988168 468 | 466,466,0.36244253948273053,0,0.059136161441345904 469 | 467,467,0.27512275477855325,0,0.02100840336134454 470 | 468,468,0.3675659786814389,0,0.06286649625191323 471 | 469,469,0.34843623817075536,0,0.04893828627879348 472 | 470,470,0.31750363646253066,0,0.02641650814142546 473 | 471,471,0.7747617933589741,1,1.0 474 | 472,472,0.5437362963340535,1,0.625 475 | 473,473,0.5659733983452511,0,0.6933906825637459 476 | 474,474,0.4310098495620201,0,0.1153461458611809 477 | 475,475,0.7185117423739924,1,0.9951848498378422 478 | 476,476,0.7482262565885697,1,1.0 479 | 477,477,0.7137036245192091,1,0.9939610516596824 480 | 478,478,0.5878410414229533,1,0.7365793842251123 481 | 479,479,0.22261274807971565,0,0.02100840336134454 482 | 480,480,0.6377763274335048,1,0.8722349403511274 483 | 481,481,0.4207853382714464,0,0.09741358785726625 484 | 482,482,0.2522317481626723,0,0.02100840336134454 485 | 483,483,0.3437706313698141,0,0.045541295549137044 486 | 484,484,0.6230880483589124,1,0.8323253732124103 487 | 485,485,0.7222382298308416,1,0.996133343343693 488 | 486,486,0.4992846437485827,0,0.3354469096577317 489 | 487,487,0.5457788072662514,0,0.625 490 | 488,488,0.5584044013584156,1,0.6663848814498129 491 | 489,489,0.4613146442579005,0,0.1723118324629343 492 | 490,490,0.5788318143998485,1,0.7316944307220143 493 | 491,491,0.6308289776997342,1,0.853358275547306 494 | 492,492,0.6074812295662466,1,0.7899200390500776 495 | 493,493,0.3373571406987272,0,0.04087168465307416 496 | 494,494,0.3775152936764281,0,0.07011051235665477 497 | 495,495,0.6998138289326369,1,0.9904257168265422 498 | 496,496,0.6770799088236011,1,0.9658399433293618 499 | 497,497,0.3812997246745965,1,0.0728659260823054 500 | 498,498,0.33184183615777285,0,0.036856035843347165 501 | 499,499,0.3206619839352253,0,0.028716075495368178 502 | 500,500,0.6488487913112968,1,0.9023199651710978 503 | 501,501,0.4829184477242702,0,0.21567691217715945 504 | 502,502,0.7591657719915643,1,1.0 505 | 503,503,0.2294545518828077,0,0.02100840336134454 506 | 504,504,0.6610900336898673,1,0.935580681561893 507 | 505,505,0.5790138046030843,0,0.7317931088526183 508 | 506,506,0.37979197199338255,0,0.07176814350027017 509 | 507,507,0.8320343260434734,1,1.0 510 | 508,508,0.34062351667622154,0,0.04324990669117697 511 | 509,509,0.4330421073065784,0,0.1192074664193278 512 | 510,510,0.2480201453634258,0,0.02100840336134454 513 | 511,511,0.6921072921833269,0,0.9842087187496419 514 | 512,512,0.6907670453257547,1,0.9825704632433334 515 | 513,513,0.6141380710615278,1,0.8080073630475689 516 | 514,514,0.28922745976895675,0,0.02100840336134454 517 | 515,515,0.5734627301118094,1,0.7201122421727286 518 | 516,516,0.30798317421279553,0,0.02100840336134454 519 | 517,517,0.37486997912929976,0,0.06818448014326561 520 | 518,518,0.2432468686518746,0,0.02100840336134454 521 | 519,519,0.4522726397183645,0,0.1555396078370213 522 | 520,520,0.6983349347464115,1,0.9900492975985915 523 | 521,521,0.5747646772876773,1,0.7247575243089865 524 | 522,522,0.4424642161176848,1,0.13710961615935988 525 | 523,523,0.6333991424703856,1,0.8603416779561907 526 | 524,524,0.31117342757125616,0,0.021807534038397736 527 | 525,525,0.5283179114862239,0,0.5564323973093939 528 | 526,526,0.7838888415781363,1,1.0 529 | 527,527,0.5806357643328074,1,0.7326725625330811 530 | 528,528,0.3327389844129176,1,0.03750924226126462 531 | 529,529,0.6737558545746908,1,0.9617767738126579 532 | 530,530,0.3946076259173689,0,0.08255530178018058 533 | 531,531,0.4966333990663982,0,0.31604474873812 534 | 532,532,0.783004671067353,1,1.0 535 | 533,533,0.6352897802698946,1,0.865478735593189 536 | 534,534,0.7170795634469966,1,0.9948203209276549 537 | 535,535,0.69125497785779,1,0.983166889306784 538 | 536,536,0.4301838418090792,0,0.11377671859430998 539 | 537,537,0.4862101355869501,0,0.23976592140290368 540 | 538,538,0.6872568210950569,1,0.9782797281881993 541 | 539,539,0.30213115789919537,0,0.02100840336134454 542 | 540,540,0.65232016526055,0,0.9117520457428077 543 | 541,541,0.5353260609163645,1,0.6113786603881699 544 | 542,542,0.5247497947021791,0,0.5284571542775129 545 | 543,543,0.7152671153466476,1,0.9943590030585412 546 | 544,544,0.5302137978029982,1,0.5712967875977787 547 | 545,545,0.5909242140869206,1,0.738251131986255 548 | 546,546,0.4826916681792007,0,0.21401730932862517 549 | 547,547,0.35410748945596865,0,0.053067478656842804 550 | 548,548,0.7132757384917793,1,0.9938521429023175 551 | 549,549,0.3968519518109061,0,0.08399264210656304 552 | 550,550,0.4886490147696849,0,0.2576139633753718 553 | 551,551,0.3171914476313072,0,0.02618920596813592 554 | 552,552,0.4512074629810859,0,0.15356378672060644 555 | 553,553,0.2714135234212829,0,0.02100840336134454 556 | 554,554,0.6542939038879516,1,0.9171148968885774 557 | 555,555,0.596528553462925,1,0.7449230932546003 558 | 556,556,0.5920981273442684,1,0.738887647379208 559 | 557,557,0.39504562031481016,0,0.08287420197285203 560 | 558,558,0.7004773847910889,1,0.990594610034905 561 | 559,559,0.6174278732891839,1,0.8169460945498083 562 | 560,560,0.5059540107213362,1,0.38425422348539584 563 | 561,561,0.5947649978877453,0,0.7403336691711171 564 | 562,562,0.572177728909374,1,0.7155274224186065 565 | 563,563,0.8078040708497988,1,1.0 566 | 564,564,0.4018854184357419,0,0.08681522146119279 567 | 565,565,0.3248124861173849,0,0.03173802270132871 568 | 566,566,0.6859080238811396,1,0.9766310211234454 569 | 567,567,0.475416907047981,1,0.19847044786891857 570 | 568,568,0.369964115278425,0,0.0646125601908564 571 | 569,569,0.5347588250036169,1,0.6069313388805104 572 | 570,570,0.7932302398030007,1,1.0 573 | 571,571,0.34254150568147923,0,0.04464637903623805 574 | 572,572,0.6263852322335401,1,0.8412841614097814 575 | 573,573,0.7733917982710898,1,1.0 576 | 574,574,0.3421923850174037,0,0.04439218709293993 577 | 575,575,0.5747864167457777,1,0.7248350896036754 578 | 576,576,0.4606713308782096,0,0.1711185355051798 579 | 577,577,0.3674154773103718,1,0.06275691741594863 580 | 578,578,0.6863687961235286,1,0.9771942477103858 581 | 579,579,0.3158495695631775,0,0.025212195355744357 582 | 580,580,0.5319188119763651,1,0.5846646756910245 583 | 581,581,0.4521612267827723,0,0.15533294539023595 584 | 582,582,0.7073068080412346,1,0.9923328860056355 585 | 583,583,0.5565918894252263,1,0.6599179299013481 586 | 584,584,0.6504950334310579,1,0.9067929745734611 587 | 585,585,0.3323846685374475,0,0.0372512677263983 588 | 586,586,0.6745842712208517,1,0.9627893918422568 589 | 587,587,0.6605938590172475,1,0.9342325238703836 590 | 588,588,0.20558563748563094,0,0.02100840336134454 591 | 589,589,0.18717157202448212,0,0.02100840336134454 592 | 590,590,0.3525445638143725,0,0.051929525086779116 593 | 591,591,0.3340289851788493,0,0.0384484814278047 594 | 592,592,0.3314335478467145,0,0.036558764412884484 595 | 593,593,0.3698982782625865,0,0.06456462478995065 596 | 594,594,0.3185692502807909,0,0.027192372978833878 597 | 595,595,0.5683951813763765,1,0.7020314827904367 598 | 596,596,0.4382459804898563,0,0.12909490444650962 599 | 597,597,0.5761156413510063,0,0.7295776962379474 600 | 598,598,0.4987115207514976,0,0.3312527194002662 601 | 599,599,0.3240537308572305,0,0.031185579108288987 602 | 600,600,0.6706205607507599,1,0.9579443362718199 603 | 601,601,0.3817997430189702,0,0.07322998541112688 604 | 602,602,0.43029922482588695,0,0.11399594807740765 605 | 603,603,0.6761529864070747,1,0.9647069164215084 606 | 604,604,0.38979714794648135,0,0.07905283151864356 607 | 605,605,0.2701471306728677,0,0.02100840336134454 608 | 606,606,0.6235362902594954,1,0.8335432926252194 609 | 607,607,0.24805735401531284,0,0.02100840336134454 610 | 608,608,0.482330286356839,1,0.21137266912698735 611 | 609,609,0.4948249849582207,0,0.30281053396649604 612 | 610,610,0.6666526828979791,1,0.9506949724133167 613 | 611,611,0.502260816692833,1,0.3572269387802639 614 | 612,612,0.4520969008607098,0,0.15521362574055164 615 | 613,613,0.5686753584804973,1,0.703031140626703 616 | 614,614,0.6303565512851155,1,0.8520746443188564 617 | 615,615,0.7087290096126238,1,0.992694875404525 618 | 616,616,0.7463478638734423,1,1.0 619 | 617,617,0.4128066845471428,0,0.09293945800631215 620 | 618,618,0.7246966673884949,1,0.9967590832891443 621 | 619,619,0.5494783242015489,1,0.6345370864453723 622 | 620,620,0.19737312170525256,0,0.02100840336134454 623 | 621,621,0.6343575019879434,1,0.8629456394764452 624 | 622,622,0.7548068853128171,1,1.0 625 | 623,623,0.3787060047440735,0,0.07097745949366484 626 | 624,624,0.39227359299949294,0,0.08085591121363345 627 | 625,625,0.4551611774604295,0,0.16089762371043823 628 | 626,626,0.6554188487542826,1,0.9201714879597431 629 | 627,627,0.5660130874565785,1,0.6935322913196699 630 | 628,628,0.6743427391165455,1,0.9624941542167444 631 | 629,629,0.6603590538425529,1,0.9335945340258003 632 | 630,630,0.4711503799990011,0,0.1905563675645863 633 | 631,631,0.4023386758899762,0,0.08706939124675198 634 | 632,632,0.4070659445984514,0,0.08972026629070137 635 | 633,633,0.28893803842063503,0,0.02100840336134454 636 | 634,634,0.4670500820438304,0,0.1829506294744765 637 | 635,635,0.322797598968657,0,0.030270999598534944 638 | 636,636,0.6531443835825052,1,0.913991531818064 639 | 637,637,0.31596438948373545,0,0.02529579481501224 640 | 638,638,0.3300718800810369,0,0.03556734508111701 641 | 639,639,0.4186520759937341,0,0.0962173343580633 642 | 640,640,0.703384241672138,1,0.9913344850260432 643 | 641,641,0.3943146128793921,0,0.08234196134749028 644 | 642,642,0.23754160814981776,0,0.02100840336134454 645 | 643,643,0.697973439129828,1,0.9899572870259831 646 | 644,644,0.520072274401002,0,0.4917838122509526 647 | 645,645,0.6939344241464901,1,0.9864421199939669 648 | 646,646,0.3696761987802372,0,0.06440293050773314 649 | 647,647,0.6452060711544534,1,0.8924223193047024 650 | 648,648,0.3045599896283413,0,0.02100840336134454 651 | 649,649,0.6482003709553462,1,0.9005581402766699 652 | 650,650,0.7121711761198753,1,0.9935710014121905 653 | 651,651,0.5581280592476744,1,0.6653989066773217 654 | 652,652,0.7186330846140063,1,0.99521573477385 655 | 653,653,0.7619898329465171,1,1.0 656 | 654,654,0.6328483487223453,1,0.8588451146109976 657 | 655,655,0.3175229713991824,0,0.026430585753056 658 | 656,656,0.3928620014321942,0,0.08128432665377182 659 | 657,657,0.3302038495003549,0,0.03566343095229052 660 | 658,658,0.4134244994924184,1,0.0932859054628051 661 | 659,659,0.6363032640600186,1,0.8682324754398074 662 | 660,660,0.7814302549780741,1,1.0 663 | 661,661,0.3486624128702496,0,0.04910296225562673 664 | 662,662,0.22498688376368686,0,0.02100840336134454 665 | 663,663,0.41441724679556297,0,0.09384260092759557 666 | 664,664,0.42096740221450135,0,0.09751568249008927 667 | 665,665,0.3993909633382017,0,0.0854164245714652 668 | 666,666,0.6163846041733249,1,0.8141114249052785 669 | 667,667,0.4945820857000569,0,0.3010329649153193 670 | 668,668,0.7462152856020178,1,1.0 671 | 669,669,0.5828197612606696,1,0.7338567621781954 672 | 670,670,0.5696728261350769,1,0.7065900551227726 673 | 671,671,0.7022393978064879,1,0.9910430907883445 674 | 672,672,0.5541742301238469,1,0.6512918429352864 675 | 673,673,0.6147265502006644,1,0.8096063214819663 676 | 674,674,0.2798033563949472,0,0.02100840336134454 677 | 675,675,0.3967076619100355,0,0.08391172974026871 678 | 676,676,0.8069450400029591,1,1.0 679 | 677,677,0.3191051970701719,0,0.027582591519056544 680 | 678,678,0.5730656556013678,1,0.7186955002616077 681 | 679,679,0.5150627440916182,1,0.4525073994083998 682 | 680,680,0.7020619256252926,1,0.9909979192391895 683 | 681,681,0.4873311217224629,0,0.24796944691528772 684 | 682,682,0.3526879956512456,0,0.052033956651843113 685 | 683,683,0.6423797987562044,1,0.8847430460925443 686 | 684,684,0.7019664467872226,1,0.9909736172500095 687 | 685,685,0.6711885369477993,1,0.9586386039934675 688 | 686,686,0.5471905190951825,1,0.6263743127254949 689 | 687,687,0.3473669587400252,0,0.048159752538472746 690 | 688,688,0.7851331355261268,1,1.0 691 | 689,689,0.6776626446509075,1,0.9665522525374858 692 | 690,690,0.29178973778492484,0,0.02100840336134454 693 | 691,691,0.6416487785275848,1,0.8827567888362862 694 | 692,692,0.6690480784994799,1,0.9560222070072167 695 | 693,693,0.291396849740926,0,0.02100840336134454 696 | 694,694,0.3824673216852765,0,0.07371604406062587 697 | 695,695,0.32937016671956426,0,0.03505643323502709 698 | 696,696,0.8227292301189068,1,1.0 699 | 697,697,0.5790235483679544,1,0.7317983920848217 700 | 698,698,0.32341803767347604,0,0.030722736021793998 701 | 699,699,0.6107527668453588,1,0.7988091428058332 702 | 700,700,0.4094951637510281,0,0.09108248130849897 703 | 701,701,0.2827582800125875,0,0.02100840336134454 704 | 702,702,0.3427860703824937,0,0.044824444625044146 705 | 703,703,0.463690494275054,1,0.1767188519452783 706 | 704,704,0.7915993390635543,1,1.0 707 | 705,705,0.6582769201165592,1,0.9279371622324586 708 | 706,706,0.463028684327338,0,0.17549124527308285 709 | 707,707,0.2681758753729032,0,0.02100840336134454 710 | 708,708,0.6402833725501971,0,0.8790468401588796 711 | 709,709,0.6186221020644362,1,0.8201909371377575 712 | 710,710,0.5613873641245872,1,0.6770279427872146 713 | 711,711,0.5359154943891183,0,0.6160000182778975 714 | 712,712,0.4946405524976142,0,0.3014608327097493 715 | 713,713,0.656658758663997,1,0.9235404508951838 716 | 714,714,0.3733281279774608,0,0.06706187073943622 717 | 715,715,0.5128736926635589,0,0.43534449544093873 718 | 716,716,0.3800021061584665,0,0.07192114049320801 719 | 717,717,0.5505819848056328,1,0.6384748920519823 720 | 718,718,0.3540877560505496,0,0.0530531109233125 721 | 719,719,0.35242745791791585,0,0.05184426122687326 722 | 720,720,0.6047599242273253,1,0.778914457011839 723 | 721,721,0.32214235160393256,0,0.02979391927038734 724 | 722,722,0.43375636139310936,0,0.12056456002393942 725 | 723,723,0.6603408565328548,1,0.9335450900610274 726 | 724,724,0.695574132205487,1,0.9884464229633887 727 | 725,725,0.2336001202137287,0,0.02100840336134454 728 | 726,726,0.5596547655061129,1,0.6708461179321944 729 | 727,727,0.6379054852943215,1,0.8725858755622287 730 | 728,728,0.37203759519884205,0,0.06612224421874416 731 | 729,729,0.65723736411561,1,0.9251125815035897 732 | 730,730,0.4982948006236314,1,0.3282031061247992 733 | 731,731,0.3632735722504296,0,0.059741229705423574 734 | 732,732,0.4670238037161794,1,0.18290188519302553 735 | 733,733,0.3885402221778319,0,0.07813767399120103 736 | 734,734,0.6113207909177908,1,0.800352522717122 737 | 735,735,0.7695455125709074,1,1.0 738 | 736,736,0.7309118203319689,1,0.9983410105793917 739 | 737,737,0.4784736357520363,0,0.20414044507149925 740 | 738,738,0.6926041093762947,1,0.9848160050099377 741 | 739,739,0.7544378320715903,1,1.0 742 | 740,740,0.4416125359608219,0,0.13549141093540815 743 | 741,741,0.6200359997376229,1,0.8240326427472954 744 | 742,742,0.5766515232346744,1,0.7305122404033804 745 | 743,743,0.6739133140115386,1,0.9619692449145141 746 | 744,744,0.3519704317845052,0,0.05151150418054576 747 | 745,745,0.6909054604760761,1,0.9827396554939173 748 | 746,746,0.4275203117757828,0,0.10871597110676899 749 | 747,747,0.4611930261122089,0,0.17208624013446255 750 | 748,748,0.4309591474284342,0,0.11524981103786369 751 | 749,749,0.6707689396850343,1,0.9581257077891054 752 | 750,750,0.34257974353071663,0,0.044674219706256624 753 | 751,751,0.35404406742313843,0,0.05302130158561238 754 | 752,752,0.38420865543853744,0,0.07498389513969066 755 | 753,753,0.5063610690313213,0,0.3872331302936082 756 | 754,754,0.3974985794869226,0,0.08435524666212933 757 | 755,755,0.39723671295789503,0,0.084208401731271 758 | 756,756,0.5395208200376653,1,0.625 759 | 757,757,0.6402868457604862,1,0.8790562772289758 760 | 758,758,0.624467854108394,1,0.8360744475537726 761 | 759,759,0.7169593470691227,1,0.9947897225545138 762 | 760,760,0.3071081286974897,0,0.02100840336134454 763 | 761,761,0.7746498184874501,1,1.0 764 | 762,762,0.6455019398978786,1,0.8932262251550755 765 | 763,763,0.4248779333178968,1,0.10369541193352427 766 | 764,764,0.3961692177709844,0,0.08360979045591783 767 | 765,765,0.40387492103523337,0,0.08793085992308289 768 | 766,766,0.6495850964153114,1,0.9043205819781464 769 | 767,767,0.7312360767996592,1,0.9984235427648334 770 | 768,768,0.6044133733013337,1,0.7774833784074426 771 | 769,769,0.36884364059793895,0,0.06379675160157322 772 | 770,770,0.5567725382368804,1,0.6605624757886679 773 | 771,771,0.4142270411298751,0,0.09373594072199189 774 | 772,772,0.5010947353074714,0,0.3486934002365837 775 | 773,773,0.7500304900821582,1,1.0 776 | 774,774,0.4436354965454482,0,0.1393350667485817 777 | 775,775,0.6497294101328929,1,0.9047126972131312 778 | 776,776,0.3666254140112086,0,0.06218167869189718 779 | 777,777,0.4474809187810673,0,0.14664142735806168 780 | 778,778,0.3611707737912132,0,0.0582101990856009 781 | 779,779,0.5620194787766721,1,0.6792832961193798 782 | 780,780,0.7612308305193494,1,1.0 783 | 781,781,0.4907573094676618,0,0.27304274415557744 784 | 782,782,0.36549824202150605,0,0.061360993845721246 785 | 783,783,0.6081810424809003,1,0.7918215028199365 786 | 784,784,0.44046955891670336,0,0.13331973720467075 787 | 785,785,0.3982236340147176,0,0.08476183005557775 788 | 786,786,0.567084374936706,1,0.6973545912458674 789 | 787,787,0.7350942219223421,1,0.9994055467661533 790 | 788,788,0.4519735683913567,0,0.1549848534732282 791 | 789,789,0.4734333738650841,0,0.19479114601156666 792 | 790,790,0.4870987696726933,1,0.24626906373680366 793 | 791,791,0.5906107431527505,1,0.7380811628053179 794 | 792,792,0.6120542444974155,1,0.8023453916386963 795 | 793,793,0.4086753073564988,0,0.09062273658275836 796 | 794,794,0.5270780501463768,0,0.5467114648379132 797 | 795,795,0.7101519434401297,1,0.993057051182727 798 | 796,796,0.5471923074054227,1,0.6263806933266184 799 | 797,797,0.6125712625377081,1,0.8037501829070401 800 | 798,798,0.7009053629520703,1,0.9907035422427918 801 | 799,799,0.43139218953696784,0,0.11607259761633855 802 | 800,800,0.5247827175139154,1,0.5287152802625519 803 | 801,801,0.4181121633468279,0,0.09591457158956859 804 | 802,802,0.4362743523849485,0,0.125348781123872 805 | 803,803,0.6346154522942257,1,0.8636465170275012 806 | 804,804,0.7088773124541703,1,0.9927326225548199 807 | 805,805,0.7662509862789938,1,1.0 808 | 806,806,0.5844739807352929,0,0.7347537076316433 809 | 807,807,0.4249757547445103,0,0.10388127412872145 810 | 808,808,0.20913959587009304,0,0.02100840336134454 811 | 809,809,0.3122077615512546,0,0.022560624277531176 812 | 810,810,0.5351022414366394,1,0.6096238399269742 813 | 811,811,0.3461717109953415,0,0.04728950228345464 814 | 812,812,0.20917216596497776,0,0.02100840336134454 815 | 813,813,0.6853029938480848,1,0.9758914605137458 816 | 814,814,0.4293651068644245,0,0.11222110977356184 817 | 815,815,0.639635678995422,1,0.877286990058106 818 | 816,816,0.7527610031205829,1,1.0 819 | 817,817,0.21544910733948,0,0.02100840336134454 820 | 818,818,0.7589330806603437,1,1.0 821 | 819,819,0.7739067689421392,1,1.0 822 | 820,820,0.5537933705623136,1,0.6499329551475124 823 | 821,821,0.6326653102477161,1,0.8583477802224634 824 | 822,822,0.3095773142339854,0,0.02100840336134454 825 | 823,823,0.4935935606526407,1,0.2937988072489538 826 | 824,824,0.4142877044514205,0,0.09376995843815841 827 | 825,825,0.7162980244670083,1,0.9946213977712853 828 | 826,826,0.25287291836378595,0,0.02100840336134454 829 | 827,827,0.7355261478196042,1,0.9995154837814542 830 | 828,828,0.7593982254645268,1,1.0 831 | 829,829,0.3533997411213518,0,0.05255217279541913 832 | 830,830,0.5412090608567424,1,0.625 833 | 831,831,0.6792228514004048,1,0.9684593767988878 834 | 832,832,0.28980495176250765,1,0.02100840336134454 835 | 833,833,0.3854605188024025,0,0.07589536677098806 836 | 834,834,0.4601498482642612,0,0.1701512252697771 837 | 835,835,0.7508342182093075,1,1.0 838 | 836,836,0.5859996469929806,1,0.7355809493817886 839 | 837,837,0.32593154017117776,0,0.03255279694375992 840 | 838,838,0.4590411229917352,0,0.16809462496196673 841 | 839,839,0.6710202891535514,1,0.9584329457046279 842 | 840,840,0.464311549478442,0,0.17787086169386557 843 | 841,841,0.6840767777612393,1,0.9743925909250212 844 | 842,842,0.5428016733443857,1,0.625 845 | 843,843,0.6364354934033506,1,0.8685917561852656 846 | 844,844,0.3313313401809759,0,0.03648434783475707 847 | 845,845,0.7662822335667651,1,1.0 848 | 846,846,0.7135626053076075,1,0.993925158393449 849 | 847,847,0.4898135760406033,0,0.2661363775195424 850 | 848,848,0.7254152420234928,1,0.9969419802887067 851 | 849,849,0.4994141132188797,0,0.33639438444712333 852 | 850,850,0.28917769762832257,0,0.02100840336134454 853 | 851,851,0.651335756564385,1,0.9090773059212112 854 | 852,852,0.6439933367024759,1,0.8891271948870059 855 | 853,853,0.6908506539478699,1,0.9826726625395162 856 | 854,854,0.4943728849334188,1,0.29950200590346593 857 | 855,855,0.2241191675376911,0,0.02100840336134454 858 | 856,856,0.5872340555940672,1,0.7362502663614438 859 | 857,857,0.5366909434383988,0,0.6220798012349653 860 | 858,858,0.4672191492444703,0,0.18326423615910126 861 | 859,859,0.3016860236212906,0,0.02100840336134454 862 | 860,860,0.2791867210230977,0,0.02100840336134454 863 | 861,861,0.3318603710172819,0,0.03686953092525483 864 | 862,862,0.3993021287446473,0,0.08536660946223089 865 | 863,863,0.4611399955093034,1,0.17198787243512226 866 | 864,864,0.6580986024962234,1,0.9274526548912876 867 | 865,865,0.6036629723267896,1,0.7743846050415494 868 | 866,866,0.5498162097822857,1,0.6357426452279318 869 | 867,867,0.7094517780409568,1,0.9928788398397863 870 | 868,868,0.3617119843083274,0,0.05860425010358609 871 | 869,869,0.6420356176735319,1,0.8838078706402643 872 | 870,870,0.2014726434177275,0,0.02100840336134454 873 | 871,871,0.6977939301201523,1,0.9899115970479835 874 | 872,872,0.3192302923179479,0,0.027673672361302738 875 | 873,873,0.6901020389775135,1,0.9817575903717568 876 | 874,874,0.4415723834272614,0,0.13541512051224996 877 | 875,875,0.6522822905100757,1,0.9116491361446688 878 | 876,876,0.4390999839858368,0,0.13071752405004589 879 | 877,877,0.6804276101068696,1,0.969932017882497 880 | 878,878,0.7160585449655527,1,0.9945604436544272 881 | 879,879,0.7128882526171767,1,0.9937535170945225 882 | 880,880,0.530712046513942,0,0.5752032260981392 883 | 881,881,0.7010341625308896,1,0.9907363252765417 884 | 882,882,0.5652444005220407,0,0.6907896549427348 885 | 883,883,0.5935873541282345,0,0.7396951310375268 886 | 884,884,0.3830608715694572,0,0.07414820295017133 887 | 885,885,0.559654667404581,0,0.6708457679108555 888 | 886,886,0.699061270249116,1,0.9902341699523459 889 | 887,887,0.6662295608406076,1,0.9495453062018576 890 | 888,888,0.2863056189070456,0,0.02100840336134454 891 | 889,889,0.6305814607992821,1,0.8526857466345318 892 | 890,890,0.6107167801839712,0,0.7987113633402162 893 | 891,891,0.35422239619587004,0,0.0531511413285753 894 | 892,892,0.5395399705795796,1,0.625 895 | 893,893,0.3160092597384773,0,0.025328464486053682 896 | 894,894,0.2774492373196624,0,0.02100840336134454 897 | 895,895,0.5820782107666068,0,0.7334546811034921 898 | 896,896,0.4109957072995157,1,0.09192392986798395 899 | 897,897,0.25331938369273405,0,0.02100840336134454 900 | 898,898,0.6510833461897445,1,0.9083914809350391 901 | 899,899,0.6596027203555289,1,0.9315394980391913 902 | 900,900,0.5293325450863846,0,0.5643874680850294 903 | 901,901,0.6033992100757147,0,0.7732954014473572 904 | 902,902,0.2021814869561238,0,0.02100840336134454 905 | 903,903,0.5973437998510303,1,0.7482896451112739 906 | 904,904,0.5348120716741778,1,0.6073488107962949 907 | 905,905,0.4851818340192235,0,0.23224067387483271 908 | 906,906,0.4732108782929713,0,0.1943784337974299 909 | 907,907,0.8082247131409576,1,1.0 910 | 908,908,0.4472442093743354,0,0.1461916758927428 911 | 909,909,0.6749822663895712,1,0.9632758826502411 912 | 910,910,0.7953131515921426,1,1.0 913 | 911,911,0.7270058048386435,1,0.9973468222526976 914 | 912,912,0.33360247661924103,0,0.038137943981139406 915 | 913,913,0.6656612568345021,1,0.9480011656819445 916 | 914,914,0.5815224091363218,1,0.7331533161679737 917 | 915,915,0.3780109448638035,0,0.07047139199364383 918 | 916,916,0.572344893839812,0,0.7161238584949432 919 | 917,917,0.709293376731956,1,0.9928385223518985 920 | 918,918,0.6904086409507978,1,0.9821323663827134 921 | 919,919,0.735149738978807,0,0.999419677383291 922 | 920,920,0.6259280762260755,1,0.8400420214504216 923 | 921,921,0.19531048093107,0,0.02100840336134454 924 | 922,922,0.7311046453479486,1,0.9983900898470776 925 | 923,923,0.5536381138105363,1,0.6493790068545767 926 | 924,924,0.7060321252728383,1,0.9920084441976746 927 | 925,925,0.3700513759036221,0,0.06467609394916599 928 | 926,926,0.29912274786878223,0,0.02100840336134454 929 | 927,927,0.6430802261249943,1,0.8866461793975798 930 | 928,928,0.5694303225582728,1,0.7057248145399081 931 | 929,929,0.7256250190556817,1,0.9969953743105602 932 | 930,930,0.3677873923941152,0,0.0630277057925947 933 | 931,931,0.7186657960459415,1,0.9952240607325756 934 | 932,932,0.8313366488768685,1,1.0 935 | 933,933,0.3087671879858524,0,0.02100840336134454 936 | 934,934,0.3354388289162085,0,0.03947497729648938 937 | 935,935,0.7128991223510789,1,0.9937562837406299 938 | 936,936,0.6555250455510891,1,0.9204600355949389 939 | 937,937,0.5873192835510886,0,0.7362964783845337 940 | 938,938,0.20502498561770116,0,0.02100840336134454 941 | 939,939,0.36765880101055176,0,0.06293407944204357 942 | 940,940,0.4220243334613872,0,0.09827352889719683 943 | 941,941,0.3188317096278045,0,0.027383467515248075 944 | 942,942,0.3534216104867645,0,0.05256809570421535 945 | 943,943,0.41257196661347423,0,0.0928078369898013 946 | 944,944,0.6597686532807342,1,0.9319903548880688 947 | 945,945,0.7302494117943386,1,0.99817240939572 948 | 946,946,0.2600635689435391,0,0.02100840336134454 949 | 947,947,0.7137781895506972,1,0.9939800305101277 950 | 948,948,0.6566518640314482,1,0.9235217174684012 951 | 949,949,0.7621320326925763,1,1.0 952 | 950,950,0.7712163155425734,1,1.0 953 | 951,951,0.3844356255012482,0,0.07514915021407509 954 | 952,952,0.6861378083325358,1,0.9769118989637007 955 | 953,953,0.7049547806968856,1,0.9917342303848852 956 | 954,954,0.6330385649817003,1,0.8593619517799438 957 | 955,955,0.7230858626261416,1,0.9963490891931015 958 | 956,956,0.6448991773074232,1,0.8915884571122488 959 | 957,957,0.7389226315414492,1,1.0 960 | 958,958,0.7335961052114013,1,0.9990242348771995 961 | 959,959,0.6559756358012828,1,0.9216843357155202 962 | 960,960,0.7413766710778629,1,1.0 963 | 961,961,0.4227715311597547,0,0.0996932158642816 964 | 962,962,0.3558058352745346,0,0.054304030566929895 965 | 963,963,0.7358757994811743,1,0.9996044797418994 966 | 964,964,0.4169127407587831,0,0.09524198037424444 967 | 965,965,0.3182516886335344,0,0.02696115890146947 968 | 966,966,0.485797628267658,0,0.23674713802044034 969 | 967,967,0.3724544488519798,0,0.06642575200577985 970 | 968,968,0.6308669572923904,1,0.8534614700124517 971 | 969,969,0.4653671798171074,0,0.1798289749733501 972 | 970,970,0.7256235674385211,1,0.9969950048340852 973 | 971,971,0.6154959723793447,1,0.8116969207919413 974 | 972,972,0.36824214635936103,0,0.06335880849158404 975 | 973,973,0.6725390465836044,1,0.9602894042431978 976 | 974,974,0.3815584885299073,0,0.07305432996097852 977 | 975,975,0.7735852761544662,1,1.0 978 | 976,976,0.20833089977137498,0,0.02100840336134454 979 | 977,977,0.4088568211869406,0,0.09072452273305423 980 | 978,978,0.4867216698314453,0,0.24350939698412044 981 | 979,979,0.7640852169138921,1,1.0 982 | 980,980,0.3617258189855779,0,0.058614323020651914 983 | 981,981,0.656312611155994,1,0.9225999324627392 984 | 982,982,0.5669180942896654,1,0.6967613102482815 985 | 983,983,0.5754187674129394,1,0.7270912850257814 986 | 984,984,0.754035115258057,1,1.0 987 | 985,985,0.22971017278785874,0,0.02100840336134454 988 | 986,986,0.41684061843003994,0,0.09520153687649405 989 | 987,987,0.4210730845790528,1,0.09757494519759576 990 | 988,988,0.7368006214909671,1,0.9998398723684716 991 | 989,989,0.4533276326129061,1,0.15749653870656738 992 | 990,990,0.6438104515899752,1,0.888630277199171 993 | 991,991,0.5180138299236915,0,0.4756449109745441 994 | 992,992,0.5087916116957556,0,0.4050201645172771 995 | 993,993,0.28023169587378843,0,0.02100840336134454 996 | 994,994,0.588875613784222,1,0.7371403466547887 997 | 995,995,0.4946774319558748,0,0.3017307214863716 998 | 996,996,0.7643800728375911,1,1.0 999 | 997,997,0.5847245023224662,1,0.7348895446242871 1000 | 998,998,0.4367309194974459,0,0.126216265566916 1001 | 999,999,0.6563152924772884,1,0.922607217888872 1002 | --------------------------------------------------------------------------------