├── betacal ├── README.md ├── version.py ├── __init__.py └── beta_calibration.py ├── tutorial ├── utils │ ├── __init__.py │ ├── utils.py │ └── visualisations.py ├── calmap.py ├── adaboost.py └── Python tutorial.ipynb ├── setup.cfg ├── requirements.txt ├── runtests.sh ├── .gitignore ├── README.md ├── setup.py ├── LICENSE.md └── tests └── test_beta_calibration.py /betacal/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tutorial/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /betacal/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.1.0' 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.12.1 2 | scikit-learn>=0.19.0 3 | coverage>=4.4.2 4 | -------------------------------------------------------------------------------- /runtests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Activating virtual environment" 4 | . ./venv/bin/activate 5 | 6 | # with coverage it tests sklearn as well 7 | # nosetests --with-coverage 8 | nosetests 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python modules. 2 | *.pyc 3 | 4 | # Setuptools distribution folder. 5 | /dist/ 6 | 7 | # Python egg metadata, regenerated from source files by setuptools. 8 | /*.egg-info 9 | 10 | # Jupyter notebook checkpoints 11 | /python/.ipynb_checkpoints 12 | .ipynb_checkpoints 13 | .ipynb_checkpoints 14 | -------------------------------------------------------------------------------- /tutorial/utils/utils.py: -------------------------------------------------------------------------------- 1 | from sklearn.linear_model import LogisticRegression 2 | 3 | 4 | class MyLogisticRegression(LogisticRegression): 5 | def fit(self, x, y, **kwargs): 6 | return super(MyLogisticRegression, self).fit( 7 | x.reshape(-1, 1), y, **kwargs) 8 | 9 | def predict(self, x, **kwargs): 10 | return super(MyLogisticRegression, self).predict_proba( 11 | x.reshape(-1, 1), **kwargs)[:, 1] 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Beta calibration - Python package 2 | 3 | This package provides a BetaCalibration class which allows the user to fit any of our three proposed beta calibration models. For the paper and tutorials, check [https://betacal.github.io/](https://betacal.github.io/). 4 | 5 | ## Dependencies 6 | 7 | * [Numpy] - NumPy is the fundamental package for scientific computing with 8 | Python. 9 | * [Scikit-learn] - Machine Learning in Python. 10 | 11 | ## Usage 12 | 13 | - Install from pip using "pip install betacal" 14 | - Alternatively, download from the repository, cd to the folder and use "pip install ." 15 | - Once installed, import the package using "import betacal" 16 | 17 | ## Unittest 18 | 19 | Create a virtual environment with the necessary dependencies 20 | 21 | ``` 22 | virtualenv venv 23 | . ./venv/bin/activate 24 | pip install -r requirements.txt 25 | ``` 26 | 27 | and then run the script runtests.sh 28 | 29 | ``` 30 | bash runtests.sh 31 | ``` 32 | 33 | ## License 34 | 35 | MIT 36 | 37 | [//]: # (References) 38 | [Numpy]: 39 | [Scikit-learn]: 40 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from distutils.util import convert_path 3 | from os import path 4 | 5 | this_directory = path.abspath(path.dirname(__file__)) 6 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 7 | long_description = f.read() 8 | 9 | main_ns = {} 10 | ver_path = convert_path('betacal/version.py') 11 | with open(ver_path) as ver_file: 12 | exec(ver_file.read(), main_ns) 13 | 14 | setup( 15 | name='betacal', 16 | version=main_ns['__version__'], 17 | description='Beta calibration', 18 | author='Telmo de Menezes e Silva Filho and Miquel Perello Nieto', 19 | author_email='tmfilho@gmail.com', 20 | url = 'https://betacal.github.io/', 21 | download_url = 'https://github.com/betacal/python/archive/refs/tags/{}.tar.gz'.format(main_ns['__version__']), 22 | keywords = ['classifier calibration', 'calibration', 'classification'], 23 | license='MIT', 24 | packages=['betacal'], 25 | install_requires=[ 26 | 'numpy', 27 | 'scikit-learn', 28 | ], 29 | long_description=long_description, 30 | long_description_content_type='text/markdown' 31 | ) 32 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016-2021 Meelis Kull, Telmo de Menezes e Silva Filho, Peter Flach and contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tutorial/calmap.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | from matplotlib import rc 3 | import numpy as np 4 | 5 | 6 | def plot_calibration_map(scores_set, prob, legend_set, alpha=1, **kwargs): 7 | rc('text', usetex=True) 8 | fig_reliability_map = plt.figure('reliability_map') 9 | fig_reliability_map.clf() 10 | ax_reliability_map = plt.subplot(111) 11 | ax = ax_reliability_map 12 | ax.set_ylim([-0.01, 1.01]) 13 | ax.set_xlim([-0.01, 1.01]) 14 | ax.set_xlabel((r'$s$'), fontsize=16) 15 | ax.set_ylabel((r'$\hat{p}$'), fontsize=16) 16 | n_lines = len(legend_set) 17 | bins = np.linspace(0, 1, 11) 18 | hist_tot = np.histogram(prob[0], bins=bins) 19 | hist_pos = np.histogram(prob[0][prob[1] == 1], bins=bins) 20 | centers = (bins[:-1] + bins[1:])/2.0 21 | empirical_p = np.true_divide(hist_pos[0]+alpha, hist_tot[0]+2*alpha) 22 | ax.plot(centers, empirical_p, 'ko', label='empirical') 23 | 24 | for (scores, legend) in zip(scores_set, legend_set): 25 | if legend == 'isotonic': 26 | pr_iso, sc_iso = get_iso_points(scores, prob[2]) 27 | ax.plot(sc_iso, pr_iso, '-', label=legend, linewidth=n_lines, 28 | **kwargs) 29 | elif legend != 'uncalib': 30 | ax.plot(prob[2], scores, '-', label=legend, linewidth=n_lines, 31 | **kwargs) 32 | n_lines -= 1 33 | ax.legend(loc='upper left') 34 | return fig_reliability_map 35 | 36 | 37 | def get_iso_points(probas, scores): 38 | n = len(scores) 39 | pr_iso = [] 40 | sc_iso = [] 41 | for i in np.arange(n - 1): 42 | s = scores[i] 43 | p = probas[i] 44 | if probas[i + 1] == p and scores[i + 1] > s: 45 | pr_iso.append(p) 46 | sc_iso.append(s) 47 | elif probas[i + 1] > p and scores[i + 1] > s: 48 | pr_iso.append(p) 49 | sc_iso.append(s) 50 | sc_iso.append(scores[i + 1]) 51 | pr_iso.append(p) 52 | return pr_iso, sc_iso 53 | -------------------------------------------------------------------------------- /tutorial/adaboost.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import numpy as np 4 | 5 | from sklearn.tree import DecisionTreeClassifier 6 | from sklearn.base import BaseEstimator 7 | from sklearn.base import clone 8 | 9 | 10 | class AdaBoostClassifier(BaseEstimator): 11 | def __init__(self, base_estimator=None, n_estimators=50): 12 | self.base_estimator = base_estimator 13 | if base_estimator is None: 14 | self.base_estimator = DecisionTreeClassifier(max_depth=1) 15 | self.n_estimators = n_estimators 16 | self.alphas = np.zeros(n_estimators) 17 | self.estimators = [] 18 | 19 | def fit(self, X, y): 20 | sample_weights = np.ones(np.alen(X)) * (1.0 / np.alen(X)) 21 | y_changed = y 22 | y_changed[y == 0] = -1 23 | for iboost in np.arange(self.n_estimators): 24 | estimator = clone(self.base_estimator) 25 | estimator.fit(X, y_changed, sample_weights) 26 | self.estimators.append(estimator) 27 | predictions = estimator.predict(X) 28 | incorrect = (predictions != y_changed).astype(float) 29 | error = (sample_weights * incorrect).sum() 30 | if error > 0: 31 | self.alphas[iboost] = 0.5 * np.log((1-error) / error) 32 | a = self.alphas[iboost] 33 | modifier = np.exp(-y_changed * a * predictions) 34 | sample_weights *= modifier 35 | sample_weights /= sample_weights.sum() 36 | else: 37 | self.alphas[iboost] = 1.0 38 | self.alphas = self.alphas[:(iboost + 1)] 39 | self.n_estimators = len(self.estimators) 40 | break 41 | return self 42 | 43 | def predict_proba(self, X): 44 | predictions = np.zeros(np.alen(X)) 45 | for iboost in np.arange(self.n_estimators): 46 | a = self.alphas[iboost] 47 | predictions += a * self.estimators[iboost].predict(X) 48 | probas = 1.0 / (1.0 + np.exp(-2*predictions)).reshape(-1, 1) 49 | probas = np.hstack((1.0 - probas, probas)) 50 | return probas 51 | -------------------------------------------------------------------------------- /betacal/__init__.py: -------------------------------------------------------------------------------- 1 | from .beta_calibration import _BetaCal, _BetaAMCal, _BetaABCal, _BetaACal 2 | from sklearn.base import BaseEstimator, RegressorMixin 3 | from .version import __version__ 4 | 5 | 6 | class BetaCalibration(BaseEstimator, RegressorMixin): 7 | """Wrapper class for the three Beta regression models introduced in 8 | Kull, M., Silva Filho, T.M. and Flach, P. Beta calibration: a well-founded 9 | and easily implemented improvement on logistic calibration for binary 10 | classifiers. AISTATS 2017. 11 | 12 | Parameters 13 | ---------- 14 | parameters : string 15 | Determines which parameters will be calculated by the model. Possible 16 | values are: "abm" (default), "am" and "ab" 17 | 18 | Attributes 19 | ---------- 20 | calibrator_ : 21 | Internal calibrator object. The type depends on the value of parameters. 22 | """ 23 | def __init__(self, parameters="abm"): 24 | if parameters == "abm": 25 | self.calibrator_ = _BetaCal() 26 | elif parameters == "am": 27 | self.calibrator_ = _BetaAMCal() 28 | elif parameters == "ab": 29 | self.calibrator_ = _BetaABCal() 30 | elif parameters == "a": 31 | self.calibrator_ = _BetaACal() 32 | else: 33 | raise ValueError('Unknown parameters', parameters) 34 | self.parameters = parameters 35 | 36 | def fit(self, X, y, sample_weight=None): 37 | """Fit the model using X, y as training data. 38 | 39 | Parameters 40 | ---------- 41 | X : array-like, shape (n_samples,) 42 | Training data. 43 | 44 | y : array-like, shape (n_samples,) 45 | Training target. 46 | 47 | sample_weight : array-like, shape = [n_samples] or None 48 | Sample weights. If None, then samples are equally weighted. 49 | Currently, no sample weighting is done by the models. 50 | 51 | Returns 52 | ------- 53 | self : object 54 | Returns an instance of self. 55 | """ 56 | self.calibrator_.fit(X, y, sample_weight) 57 | return self 58 | 59 | def predict(self, S): 60 | """Predict new values. 61 | 62 | Parameters 63 | ---------- 64 | S : array-like, shape (n_samples,) 65 | Data to predict from. 66 | 67 | Returns 68 | ------- 69 | : array, shape (n_samples,) 70 | The predicted values. 71 | """ 72 | return self.calibrator_.predict(S) 73 | -------------------------------------------------------------------------------- /tests/test_beta_calibration.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from betacal import BetaCalibration 3 | import numpy as np 4 | 5 | # from generate_scores_and_y(0.8, 0.4, 40, 42) 6 | s = [0.43155171, 0.81164906, 0.66544901, 0.53177918, 0.15549116, 0.16259166, 7 | 0.36480309, 0.81877874, 0.94360637, 0.73610746, 0.55979712, 0.85662239, 8 | 0.38039252, 0.7806386 , 0.38548359, 0.7468978 , 0.26136641, 0.21377051, 9 | 0.01270956, 0.05394571, 0.51571459, 0.31820521, 0.65717799, 0.25428535, 10 | 0.45378324, 0.62464611, 0.20519146, 0.87777557, 0.11439908, 0.53848995, 11 | 0.64487573, 0.08061064, 0.46484883, 0.40406019, 0.81670188, 0.9357303 , 12 | 0.40183604, 0.09328503, 0.9462795 , 0.26967112] 13 | 14 | y = [0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15 | 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0] 16 | 17 | 18 | def generate_scores_and_y(accuracy=0.8, prior=0.4, n_samples=40, seed=42): 19 | np.random.seed(seed) 20 | y = np.random.binomial(1, prior, n_samples) 21 | predicted_class = (y == 0).astype(int) 22 | correct = np.random.binomial(1, accuracy, len(y)).astype(bool) 23 | predicted_class[correct] = y[correct] 24 | s = (np.random.rand(len(y))+predicted_class)/2 25 | return s, y 26 | 27 | def map_predictions(s, a, b, m): 28 | c = b * np.log(1. - m) - a * np.log(m) 29 | p = np.array(s) 30 | beta = 1 / (1 + 1 / (np.exp(c) * p ** a / (1 - p) ** b)) 31 | return beta 32 | 33 | 34 | class BetaCalibrationTests(unittest.TestCase): 35 | def test_betacal(self): 36 | bc = BetaCalibration(parameters="abm") 37 | bc.fit(s, y) 38 | 39 | pred = bc.predict(s) 40 | 41 | beta_map = bc.calibrator_.map_ 42 | 43 | pred_abm = map_predictions(s, *beta_map) 44 | 45 | np.testing.assert_allclose(pred, pred_abm, rtol=1e-5, atol=1e-5) 46 | assert(len(bc.calibrator_.lr_.coef_[0]) == 2) 47 | 48 | def test_betacal_am(self): 49 | bc = BetaCalibration(parameters="am") 50 | bc.fit(s, y) 51 | 52 | pred = bc.predict(s) 53 | 54 | beta_map = bc.calibrator_.map_ 55 | 56 | pred_am = map_predictions(s, *beta_map) 57 | 58 | np.testing.assert_allclose(pred, pred_am, rtol=1e-5, atol=1e-5) 59 | assert(len(bc.calibrator_.lr_.coef_[0]) == 1) 60 | 61 | def test_betacal_ab(self): 62 | bc = BetaCalibration(parameters="ab") 63 | bc.fit(s, y) 64 | 65 | pred = bc.predict(s) 66 | 67 | beta_map = bc.calibrator_.map_ 68 | 69 | pred_ab = map_predictions(s, *beta_map) 70 | 71 | np.testing.assert_allclose(pred, pred_ab, rtol=1e-5, atol=1e-5) 72 | assert(len(bc.calibrator_.lr_.coef_[0]) == 2) 73 | 74 | def test_betacal_a(self): 75 | bc = BetaCalibration(parameters="a") 76 | bc.fit(s, y) 77 | 78 | pred = bc.predict(s) 79 | 80 | beta_map = bc.calibrator_.map_ 81 | 82 | pred_a = map_predictions(s, *beta_map) 83 | 84 | np.testing.assert_allclose(pred, pred_a, rtol=1e-5, atol=1e-5) 85 | assert(len(bc.calibrator_.lr_.coef_[0]) == 1) 86 | -------------------------------------------------------------------------------- /tutorial/utils/visualisations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | def plot_reliability_diagram(score, labels, linspace, scores_set, legend_set, 7 | alpha=1, scatter_prop=0.0, fig=None, n_bins=10, 8 | bins_count=True, title=None, **kwargs): 9 | ''' 10 | Parameters 11 | ========== 12 | scores_set : list of array_like of floats 13 | List of scores given by different methods, the first one is always the 14 | original one 15 | labels : array_like of ints 16 | Labels corresponding to the scores 17 | legend_set : list of strings 18 | Description of each array in the scores_set 19 | alpha : float 20 | Laplace regularization when computing the elements in the bins 21 | scatter_prop : float 22 | If original first specifies the proportion of points (score, label) to 23 | show 24 | fig : matplotlib.pyplot.figure 25 | Plots the axis in the given figure 26 | bins_count : bool 27 | If True, show the number of samples in each bin 28 | 29 | Regurns 30 | ======= 31 | fig : matplotlib.pyplot.figure 32 | Figure with the reliability diagram 33 | ''' 34 | if fig is None: 35 | fig = plt.figure() 36 | 37 | ax = fig.add_subplot(111) 38 | if title is not None: 39 | ax.set_title(title) 40 | 41 | n_lines = len(legend_set) 42 | 43 | # Draw the empirical values in a histogram style 44 | # TODO careful that now the min and max depend on the scores 45 | s_min = min(score) 46 | s_max = max(score) 47 | bins = np.linspace(s_min, s_max, n_bins+1) 48 | hist_tot = np.histogram(score, bins=bins) 49 | hist_pos = np.histogram(score[labels == 1], bins=bins) 50 | edges = np.insert(bins, np.arange(len(bins)), bins) 51 | empirical_p = np.true_divide(hist_pos[0]+alpha, hist_tot[0]+2*alpha) 52 | empirical_p = np.insert(empirical_p, np.arange(len(empirical_p)), 53 | empirical_p) 54 | p = plt.plot(edges[1:-1], empirical_p, label='original') 55 | # Draw the centroids of each bin 56 | centroids = [np.mean(np.append( 57 | score[np.where(np.logical_and(score >= bins[i], 58 | score < bins[i+1]))], 59 | bins[i]+0.05)) for i in range(len(hist_tot[1])-1)] 60 | proportion = np.true_divide(hist_pos[0]+alpha, hist_tot[0]+alpha*2) 61 | plt.plot(centroids, proportion, 'o', color=p[-1].get_color(), linewidth=2, 62 | label='centroid') 63 | for (x, y, text) in zip(centroids, proportion, hist_tot[0]): 64 | if y < 0.95: 65 | y += 0.05 66 | else: 67 | y -= 0.05 68 | plt.text(x, y, text, horizontalalignment='center', 69 | verticalalignment='center') 70 | 71 | # Draw the rest of the lines 72 | for (scores, legend) in zip(scores_set, legend_set): 73 | # reliability_diagram(scores, labels, marker='o-', label=legend, 74 | # linewidth=n_lines, alpha=alpha, n_bins=n_bins, 75 | # **kwargs) 76 | plt.plot(linspace, scores, label=legend, linewidth=n_lines) 77 | n_lines -= 1 78 | 79 | # Draw some samples with the labels 80 | if scatter_prop: 81 | n_points = int(scatter_prop*len(labels)) 82 | plt.plot(score[:n_points], labels[:n_points], 'kx', 83 | label='samples ({:d}%)'.format(int(scatter_prop*100)), 84 | markersize=6, markeredgewidth=1, alpha=0.4) 85 | 86 | ax.plot([0, 1], [0, 1], 'r--') 87 | ax.set_ylim([0, 1]) 88 | ax.set_xlim([0, 1]) 89 | ax.set_xlabel(r'$s$') 90 | ax.set_ylabel(r'$\hat p$') 91 | ax.legend(loc='lower right') 92 | ax.grid(True) 93 | 94 | return fig 95 | -------------------------------------------------------------------------------- /betacal/beta_calibration.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | from sklearn.base import BaseEstimator, RegressorMixin 4 | from sklearn.utils import indexable, column_or_1d 5 | from scipy.optimize import minimize_scalar 6 | from sklearn.linear_model import LogisticRegression 7 | 8 | 9 | import warnings 10 | 11 | 12 | def _beta_calibration(df, y, sample_weight=None): 13 | warnings.filterwarnings("ignore") 14 | 15 | df = column_or_1d(df).reshape(-1, 1) 16 | eps = np.finfo(df.dtype).eps 17 | df = np.clip(df, eps, 1-eps) 18 | y = column_or_1d(y) 19 | 20 | x = np.hstack((df, 1. - df)) 21 | x = np.log(x) 22 | x[:, 1] *= -1 23 | 24 | lr = LogisticRegression(C=99999999999) 25 | lr.fit(x, y, sample_weight) 26 | coefs = lr.coef_[0] 27 | 28 | if coefs[0] < 0: 29 | x = x[:, 1].reshape(-1, 1) 30 | lr = LogisticRegression(C=99999999999) 31 | lr.fit(x, y, sample_weight) 32 | coefs = lr.coef_[0] 33 | a = 0 34 | b = coefs[0] 35 | elif coefs[1] < 0: 36 | x = x[:, 0].reshape(-1, 1) 37 | lr = LogisticRegression(C=99999999999) 38 | lr.fit(x, y, sample_weight) 39 | coefs = lr.coef_[0] 40 | a = coefs[0] 41 | b = 0 42 | else: 43 | a = coefs[0] 44 | b = coefs[1] 45 | inter = lr.intercept_[0] 46 | 47 | m = minimize_scalar(lambda mh: np.abs(b*np.log(1.-mh)-a*np.log(mh)-inter), 48 | bounds=[0, 1], method='Bounded').x 49 | map = [a, b, m] 50 | return map, lr 51 | 52 | 53 | class _BetaCal(BaseEstimator, RegressorMixin): 54 | """Beta regression model with three parameters introduced in 55 | Kull, M., Silva Filho, T.M. and Flach, P. Beta calibration: a well-founded 56 | and easily implemented improvement on logistic calibration for binary 57 | classifiers. AISTATS 2017. 58 | 59 | Attributes 60 | ---------- 61 | map_ : array-like, shape (3,) 62 | Array containing the coefficients of the model (a and b) and the 63 | midpoint m. Takes the form map_ = [a, b, m] 64 | 65 | lr_ : sklearn.linear_model.LogisticRegression 66 | Internal logistic regression used to train the model. 67 | """ 68 | def fit(self, X, y, sample_weight=None): 69 | """Fit the model using X, y as training data. 70 | 71 | Parameters 72 | ---------- 73 | X : array-like, shape (n_samples,) 74 | Training data. 75 | 76 | y : array-like, shape (n_samples,) 77 | Training target. 78 | 79 | sample_weight : array-like, shape = [n_samples] or None 80 | Sample weights. If None, then samples are equally weighted. 81 | 82 | Returns 83 | ------- 84 | self : object 85 | Returns an instance of self. 86 | """ 87 | X = column_or_1d(X) 88 | y = column_or_1d(y) 89 | X, y = indexable(X, y) 90 | 91 | self.map_, self.lr_ = _beta_calibration(X, y, sample_weight) 92 | 93 | return self 94 | 95 | def predict(self, S): 96 | """Predict new values. 97 | 98 | Parameters 99 | ---------- 100 | S : array-like, shape (n_samples,) 101 | Data to predict from. 102 | 103 | Returns 104 | ------- 105 | S_ : array, shape (n_samples,) 106 | The predicted values. 107 | """ 108 | df = column_or_1d(S).reshape(-1, 1) 109 | eps = np.finfo(df.dtype).eps 110 | df = np.clip(df, eps, 1-eps) 111 | 112 | x = np.hstack((df, 1. - df)) 113 | x = np.log(x) 114 | x[:, 1] *= -1 115 | if self.map_[0] == 0: 116 | x = x[:, 1].reshape(-1, 1) 117 | elif self.map_[1] == 0: 118 | x = x[:, 0].reshape(-1, 1) 119 | 120 | return self.lr_.predict_proba(x)[:, 1] 121 | 122 | 123 | def _beta_am_calibration(df, y, sample_weight=None): 124 | warnings.filterwarnings("ignore") 125 | 126 | df = column_or_1d(df).reshape(-1, 1) 127 | eps = np.finfo(df.dtype).eps 128 | df = np.clip(df, eps, 1-eps) 129 | y = column_or_1d(y) 130 | 131 | x = np.log(df / (1. - df)) 132 | 133 | lr = LogisticRegression(C=99999999999) 134 | lr.fit(x, y, sample_weight) 135 | coefs = lr.coef_[0] 136 | inter = lr.intercept_[0] 137 | a = coefs[0] 138 | b = a 139 | m = 1.0 / (1.0 + np.exp(inter / a)) 140 | map = [a, b, m] 141 | return map, lr 142 | 143 | 144 | class _BetaAMCal(BaseEstimator, RegressorMixin): 145 | """Beta regression model with two parameters (a and m, fixing a = b) 146 | introduced in Kull, M., Silva Filho, T.M. and Flach, P. Beta calibration: 147 | a well-founded and easily implemented improvement on logistic calibration 148 | for binary classifiers. AISTATS 2017. 149 | 150 | Attributes 151 | ---------- 152 | map_ : array-like, shape (3,) 153 | Array containing the coefficients of the model (a and b) and the 154 | midpoint m. Takes the form map_ = [a, b, m], where a = b 155 | 156 | lr_ : sklearn.linear_model.LogisticRegression 157 | Internal logistic regression used to train the model. 158 | """ 159 | def fit(self, X, y, sample_weight=None): 160 | """Fit the model using X, y as training data. 161 | 162 | Parameters 163 | ---------- 164 | X : array-like, shape (n_samples,) 165 | Training data. 166 | 167 | y : array-like, shape (n_samples,) 168 | Training target. 169 | 170 | sample_weight : array-like, shape = [n_samples] or None 171 | Sample weights. If None, then samples are equally weighted. 172 | 173 | Returns 174 | ------- 175 | self : object 176 | Returns an instance of self. 177 | """ 178 | X = column_or_1d(X) 179 | y = column_or_1d(y) 180 | X, y = indexable(X, y) 181 | 182 | self.map_, self.lr_ = _beta_am_calibration(X, y, sample_weight) 183 | 184 | return self 185 | 186 | def predict(self, S): 187 | """Predict new values. 188 | 189 | Parameters 190 | ---------- 191 | S : array-like, shape (n_samples,) 192 | Data to predict from. 193 | 194 | Returns 195 | ------- 196 | S_ : array, shape (n_samples,) 197 | The predicted values. 198 | """ 199 | df = column_or_1d(S).reshape(-1, 1) 200 | eps = np.finfo(df.dtype).eps 201 | df = np.clip(df, eps, 1-eps) 202 | 203 | x = np.log(df / (1. - df)) 204 | return self.lr_.predict_proba(x)[:, 1] 205 | 206 | 207 | def _beta_ab_calibration(df, y, sample_weight=None): 208 | warnings.filterwarnings("ignore") 209 | 210 | df = column_or_1d(df).reshape(-1, 1) 211 | eps = np.finfo(df.dtype).eps 212 | df = np.clip(df, eps, 1-eps) 213 | y = column_or_1d(y) 214 | 215 | x = np.hstack((df, 1. - df)) 216 | x = np.log(2 * x) 217 | 218 | lr = LogisticRegression(fit_intercept=False, C=99999999999) 219 | lr.fit(x, y, sample_weight) 220 | coefs = lr.coef_[0] 221 | a = coefs[0] 222 | b = -coefs[1] 223 | m = 0.5 224 | map = [a, b, m] 225 | return map, lr 226 | 227 | 228 | class _BetaABCal(BaseEstimator, RegressorMixin): 229 | """Beta regression model with two parameters (a and b, fixing m = 0.5) 230 | introduced in Kull, M., Silva Filho, T.M. and Flach, P. Beta calibration: 231 | a well-founded and easily implemented improvement on logistic calibration 232 | for binary classifiers. AISTATS 2017. 233 | 234 | Attributes 235 | ---------- 236 | map_ : array-like, shape (3,) 237 | Array containing the coefficients of the model (a and b) and the 238 | midpoint m. Takes the form map_ = [a, b, m], where m = 0.5 239 | 240 | lr_ : sklearn.linear_model.LogisticRegression 241 | Internal logistic regression used to train the model. 242 | """ 243 | def fit(self, X, y, sample_weight=None): 244 | """Fit the model using X, y as training data. 245 | 246 | Parameters 247 | ---------- 248 | X : array-like, shape (n_samples,) 249 | Training data. 250 | 251 | y : array-like, shape (n_samples,) 252 | Training target. 253 | 254 | sample_weight : array-like, shape = [n_samples] or None 255 | Sample weights. If None, then samples are equally weighted. 256 | 257 | Returns 258 | ------- 259 | self : object 260 | Returns an instance of self. 261 | """ 262 | X = column_or_1d(X) 263 | y = column_or_1d(y) 264 | X, y = indexable(X, y) 265 | 266 | self.map_, self.lr_ = _beta_ab_calibration(X, y, sample_weight) 267 | 268 | return self 269 | 270 | def predict(self, S): 271 | """Predict new values. 272 | 273 | Parameters 274 | ---------- 275 | S : array-like, shape (n_samples,) 276 | Data to predict from. 277 | 278 | Returns 279 | ------- 280 | S_ : array, shape (n_samples,) 281 | The predicted values. 282 | """ 283 | df = column_or_1d(S).reshape(-1, 1) 284 | eps = np.finfo(df.dtype).eps 285 | df = np.clip(df, eps, 1-eps) 286 | 287 | x = np.hstack((df, 1. - df)) 288 | x = np.log(2 * x) 289 | return self.lr_.predict_proba(x)[:, 1] 290 | 291 | 292 | def _beta_a_calibration(df, y, sample_weight=None): 293 | warnings.filterwarnings("ignore") 294 | 295 | df = column_or_1d(df).reshape(-1, 1) 296 | eps = np.finfo(df.dtype).eps 297 | df = np.clip(df, eps, 1-eps) 298 | y = column_or_1d(y) 299 | 300 | x = np.log(df / (1. - df)) 301 | 302 | lr = LogisticRegression(fit_intercept=False, C=99999999999) 303 | lr.fit(x, y, sample_weight) 304 | coefs = lr.coef_[0] 305 | a = coefs[0] 306 | b = a 307 | m = 0.5 308 | map = [a, b, m] 309 | return map, lr 310 | 311 | 312 | class _BetaACal(BaseEstimator, RegressorMixin): 313 | """Beta regression model with one parameter (a = b, fixing m = 0.5) 314 | introduced in Kull, M., Silva Filho, T.M. and Flach, P. Beta calibration: 315 | a well-founded and easily implemented improvement on logistic calibration 316 | for binary classifiers. AISTATS 2017. 317 | 318 | Attributes 319 | ---------- 320 | map_ : array-like, shape (3,) 321 | Array containing the coefficients of the model (a and b) and the 322 | midpoint m. Takes the form map_ = [a, b, m], where a = b and m = 0.5 323 | 324 | lr_ : sklearn.linear_model.LogisticRegression 325 | Internal logistic regression used to train the model. 326 | """ 327 | def fit(self, X, y, sample_weight=None): 328 | """Fit the model using X, y as training data. 329 | 330 | Parameters 331 | ---------- 332 | X : array-like, shape (n_samples,) 333 | Training data. 334 | 335 | y : array-like, shape (n_samples,) 336 | Training target. 337 | 338 | sample_weight : array-like, shape = [n_samples] or None 339 | Sample weights. If None, then samples are equally weighted. 340 | 341 | Returns 342 | ------- 343 | self : object 344 | Returns an instance of self. 345 | """ 346 | X = column_or_1d(X) 347 | y = column_or_1d(y) 348 | X, y = indexable(X, y) 349 | 350 | self.map_, self.lr_ = _beta_a_calibration(X, y, sample_weight) 351 | 352 | return self 353 | 354 | def predict(self, S): 355 | """Predict new values. 356 | 357 | Parameters 358 | ---------- 359 | S : array-like, shape (n_samples,) 360 | Data to predict from. 361 | 362 | Returns 363 | ------- 364 | S_ : array, shape (n_samples,) 365 | The predicted values. 366 | """ 367 | df = column_or_1d(S).reshape(-1, 1) 368 | eps = np.finfo(df.dtype).eps 369 | df = np.clip(df, eps, 1-eps) 370 | 371 | x = np.log(df / (1. - df)) 372 | return self.lr_.predict_proba(x)[:, 1] 373 | -------------------------------------------------------------------------------- /tutorial/Python tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Beta calibration\n", 8 | "\n", 9 | "## Introduction\n", 10 | "\n", 11 | "Logistic calibration is designed to correct for a specific kind of distortion where classifiers tend to score on too narrow a scale. However, many classifiers including Naive Bayes and standard Adaboost suffer from the opposite distortion where scores tend too much to the extremes. \n", 12 | "\n", 13 | "In this tutorial, we will motivate Beta calibration and our betacal Python package.\n", 14 | "\n", 15 | "## Probability estimation with Adaboost\n", 16 | "\n", 17 | "First, let's train an Adaboost model with 200 decision stumps to estimate class probabilities for the well-known spam dataset. The dataset will be divided into a training set (50%), a test set (25%) and a calibration set (25%). The classifier will be trained on the training set and we'll estimate class probabilities for the test set.\n" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 1, 23 | "metadata": { 24 | "collapsed": false 25 | }, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEACAYAAACwB81wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEhlJREFUeJzt3X+s3Xddx/HnqytFQdZMcbfaDqhOxmbUjUjR4I+jzEkx\ntgt/zIrBjcVoMhWMibqamLWJSdV//BEzEyJiNWAtEmxFzMqcV4NhdtMBk5Za0NZypXfCBIJEbdnb\nP8537Ky0u9977jn33PXzfCTf9PP9nM/3nPf99t7X+dzPOd9zU1VIki5/62ZdgCRpdRj4ktQIA1+S\nGmHgS1IjDHxJaoSBL0mNWDLwk7w0ySNJ/qn797NJ3pTkqiRHkpxIcl+SjSPH7E5yMsnxJLdM90uQ\nJPWR5bwPP8k64BPAK4GfAT5dVb+R5JeAq6rq7iQ3AG8HXgFsAe4Hvql8w78kzdRyl3RuBj5eVWeA\nncD+rn8/cGvX3gEcqKrzVXUKOAlsm0CtkqQVWG7g/wjwjq49V1WLAFV1Fri6698MnBk5ZqHrkyTN\nUO/AT/IchrP3d3ZdFy7RuGQjSWvY+mWM3Q78Y1V9qttfTDJXVYtJNgGPdf0LwDUjx23p+p4miU8Q\nkjSGqso4xy1nSedHgT8Z2T8M3NG1bwcOjfTvSrIhyVbgWuDoxe6wqtyquOeee2Zew1rZPBeeC8/F\nM28r0WuGn+R5DF+w/cmR7l8HDia5EzgN3NaF+LEkB4FjwDngrlpplZKkFesV+FX1BeBrL+h7nOGT\nwMXG7wP2rbg6SdLEeKXtGjAYDGZdwprhuXiK5+IpnovJWNaFVxN94MSVHklapiTUKrxoK0l6FjPw\nJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+S\nGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUiF6Bn2RjkncmOZ7kI0lemeSqJEeSnEhy\nX5KNI+N3JznZjb9leuVLkvrqO8P/beC9VXU98G3AR4G7gfur6jrgAWA3QJIbgNuA64HtwL1JMunC\nJUnLs2TgJ7kS+O6qehtAVZ2vqs8CO4H93bD9wK1dewdwoBt3CjgJbJt04ZKk5VnfY8xW4FNJ3sZw\ndv8w8HPAXFUtAlTV2SRXd+M3Ax8YOX6h6/vyB1//3HHrnogrrljPgw++n5tuummmdUjSaugT+OuB\nlwM/XVUPJ/lNhss5dcG4C/eX9MUv/sLI3vcA37vcu1iR5z//dZw5c8bAl7Rmzc/PMz8/P5H76hP4\nnwDOVNXD3f67GAb+YpK5qlpMsgl4rLt9Abhm5PgtXd9F/Oo4NU/QFTN+fEl6ZoPBgMFg8KX9vXv3\njn1fS67hd8s2Z5K8tOt6NfAR4DBwR9d3O3Coax8GdiXZkGQrcC1wdOwKJUkT0WeGD/Am4O1JngP8\nK/BGhtPjg0nuBE4zfGcOVXUsyUHgGHAOuKuqlr3cI0marF6BX1UfAl5xkZtuvsT4fcC+FdQlSZow\nr7SVpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMM\nfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqRG9Aj/JqSQf\nSvJIkqNd31VJjiQ5keS+JBtHxu9OcjLJ8SS3TKt4SVJ/fWf4TwCDqrqpqrZ1fXcD91fVdcADwG6A\nJDcAtwHXA9uBe5NksmVLkparb+DnImN3Avu79n7g1q69AzhQVeer6hRwEtiGJGmm+gZ+Ae9L8lCS\nn+j65qpqEaCqzgJXd/2bgTMjxy50fZKkGVrfc9yrquqTSb4WOJLkBMMngVEX7vewZ6Q96DZJ0pPm\n5+eZn5+fyH31Cvyq+mT3738m+XOGSzSLSeaqajHJJuCxbvgCcM3I4Vu6vovYM17VktSIwWDAYDD4\n0v7evXvHvq8ll3SSPC/JV3Xt5wO3AI8Ch4E7umG3A4e69mFgV5INSbYC1wJHx65QkjQRfWb4c8C7\nk1Q3/u1VdSTJw8DBJHcCpxm+M4eqOpbkIHAMOAfcVVVjLPdIkiZpycCvqn8DbrxI/+PAzZc4Zh+w\nb8XVSZImxittJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJek\nRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWpE\n78BPsi7JPyU53O1fleRIkhNJ7kuycWTs7iQnkxxPcss0CpckLc9yZvhvBo6N7N8N3F9V1wEPALsB\nktwA3AZcD2wH7k2SyZQrSRpXr8BPsgV4LfD7I907gf1dez9wa9feARyoqvNVdQo4CWybSLWSpLH1\nneH/JvALQI30zVXVIkBVnQWu7vo3A2dGxi10fZKkGVq/1IAkPwQsVtUHkwyeYWg9w22XsGekPeg2\nSdKT5ufnmZ+fn8h9LRn4wKuAHUleC3wl8IIkfwycTTJXVYtJNgGPdeMXgGtGjt/S9V3EnjHLlqQ2\nDAYDBoPBl/b37t079n0tuaRTVb9cVS+qqm8AdgEPVNUbgL8A7uiG3Q4c6tqHgV1JNiTZClwLHB27\nQknSRPSZ4V/KrwEHk9wJnGb4zhyq6liSgwzf0XMOuKuqxljukSRN0rICv6r+Fvjbrv04cPMlxu0D\n9q24OknSxHilrSQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS\n1AgDX5IaYeBLUiMMfElqxEo+D1+SmrJp00tYXDw96zLGZuBLUk/DsJ/133PK2Ee6pCNJjTDwJakR\nBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiOWDPwkz03yD0keSfJoknu6/quSHElyIsl9STaOHLM7yckk\nx5PcMs0vQJLUz5KBX1X/C3xfVd0E3AhsT7INuBu4v6quAx4AdgMkuQG4Dbge2A7cm2T8KwUkSRPR\na0mnqr7QNZ/L8OrcAnYC+7v+/cCtXXsHcKCqzlfVKeAksG1SBUuSxtMr8JOsS/IIcBZ4X1U9BMxV\n1SJAVZ0Fru6GbwbOjBy+0PVJkmao12fpVNUTwE1JrgTeneSb+fIPlBjjAyb2jLQH3SZJesp8t63c\nsj48rao+l2QeeA2wmGSuqhaTbAIe64YtANeMHLal67uIPcssV5JaM+Dpk+G9Y99Tn3fpvPDJd+Ak\n+UrgB4DjwGHgjm7Y7cChrn0Y2JVkQ5KtwLXA0bErlCRNRJ8Z/tcB+5OsY/gE8adV9d4kDwIHk9wJ\nnGb4zhyq6liSg8Ax4BxwV1XN+vNEJal5SwZ+VT0KvPwi/Y8DN1/imH3AvhVXJ0maGK+0laRGGPiS\n1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mN\nMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRSwZ+ki1JHkjykSSPJnlT\n139VkiNJTiS5L8nGkWN2JzmZ5HiSW6b5BUiS+ukzwz8P/HxVfTPwncBPJ3kZcDdwf1VdBzwA7AZI\ncgNwG3A9sB24N0mmUbwkqb8lA7+qzlbVB7v254HjwBZgJ7C/G7YfuLVr7wAOVNX5qjoFnAS2Tbhu\nSdIyLWsNP8lLgBuBB4G5qlqE4ZMCcHU3bDNwZuSwha5PkjRD6/sOTPJVwJ8Bb66qzyepC4ZcuN/D\nnpH2oNskSU+Z77aV6xX4SdYzDPs/rqpDXfdikrmqWkyyCXis618Arhk5fEvXdxF7xihZkloy4OmT\n4b1j31PfJZ0/AI5V1W+P9B0G7ujatwOHRvp3JdmQZCtwLXB07AolSROx5Aw/yauAHwMeTfIIw6Wb\nXwZ+HTiY5E7gNMN35lBVx5IcBI4B54C7qmqM5R5J0iQtGfhV9ffAFZe4+eZLHLMP2LeCuiRJE+aV\ntpLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEv\nSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1YsnAT/LWJItJ\nPjzSd1WSI0lOJLkvycaR23YnOZnkeJJbplW4JGl5+szw3wb84AV9dwP3V9V1wAPAboAkNwC3AdcD\n24F7k2Ry5UqSxrVk4FfV+4H/uqB7J7C/a+8Hbu3aO4ADVXW+qk4BJ4FtkylVkrQS467hX11ViwBV\ndRa4uuvfDJwZGbfQ9UmSZmz9hO6nxjtsz0h70G2SpKfMd9vKjRv4i0nmqmoxySbgsa5/AbhmZNyW\nru8S9oz58JLUigFPnwzvHfue+i7ppNuedBi4o2vfDhwa6d+VZEOSrcC1wNGxq5MkTcySM/wk72D4\n9PI1Sf4duAf4NeCdSe4ETjN8Zw5VdSzJQeAYcA64q6rGXO6RJE3SkoFfVa+/xE03X2L8PmDfSoqS\nJE2eV9pKUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAl\nqREGviQ1wsCXpEZM6k8cStLUbNr0EhYXT8+6jGe9zOrvkySpsf8U7oRceeUO1q17iM985uxM65ib\nezFnz56aaQ3SWpaEWefF0FqoI1RVlh735Zqf4Q/Dfrb/gYuLX9F9Q8+WTzzS5a35wF8b/pdZP+kA\nLC7O/klHa4/LKZcPA1/SMxqG/awnJE5GJsF36UhSI5zha8RzZ/5agq8jPMWlFE2aga8Rs38twdcR\nnrI2llLA5ZTLx9SWdJK8JslHk/xLkl+a1uNIkvqZygw/yTrgd4FXA/8BPJTkUFV9dBqPp8vJc2a+\nrASwbt3zeOKJL8y6DGmiprWksw04WVWnAZIcAHYCBr6WcI61sIzxxBNr4wIbaZKmtaSzGTgzsv+J\nrk+SNCMzfdH2yit/eJYPz//939GZPr4kraZpBf4C8KKR/S1d39N87nPvmdLDL9da+NV5LdQAa6OO\ntVADrI061kINsDbqWAs1wNqpY/mm8uFpSa4ATjB80faTwFHgR6vq+MQfTJLUy1Rm+FX1xSQ/Axxh\n+DrBWw17SZqtmX08siRpdU39s3T6XICV5HeSnEzywSQ3TrumWVnqXCR5fZIPddv7k3zLLOpcDX0v\nzEvyiiTnkrxuNetbTT1/RgZJHknyz0n+ZrVrXC09fkauTHK4y4pHk9wxgzKnLslbkywm+fAzjFl+\nblbV1DaGTygfA14MPAf4IPCyC8ZsB/6ya78SeHCaNc1q63kuvgPY2LVf0/K5GBn318B7gNfNuu4Z\nfl9sBD4CbO72Xzjrumd4LnYD+548D8CngfWzrn0K5+K7gBuBD1/i9rFyc9oz/C9dgFVV54AnL8Aa\ntRP4I4Cq+gdgY5K5Kdc1C0uei6p6sKo+2+0+yOV77UKf7wuAnwX+DHhsNYtbZX3OxeuBd1XVAkBV\nfWqVa1wtfc5FAS/o2i8APl1V51exxlVRVe8H/usZhoyVm9MO/D4XYF04ZuEiYy4Hy70Y7SeAv5pq\nRbOz5LlI8vXArVX1ezyb3we3tD7fFy8FvjrJ3yR5KMkbVq261dXnXPwucEOS/wA+BLx5lWpba8bK\nTT8tcw1K8n3AGxn+Wteq3wJG13Av59Bfynrg5cD3A88HPpDkA1X1sdmWNRM/CDxSVd+f5BuB9yX5\n1qr6/KwLezaYduD3uQBrAbhmiTGXg14XoyX5VuAtwGuq6pl+pXs263Muvh04kOEnqb0Q2J7kXFUd\nXqUaV0ufc/EJ4FNV9T/A/yT5O+DbGK53X076nIs3AvsAqurjSf4NeBnw8KpUuHaMlZvTXtJ5CLg2\nyYuTbAB2ARf+wB4GfhwgyXcAn6mqxSnXNQtLnoskLwLeBbyhqj4+gxpXy5Lnoqq+odu2MlzHv+sy\nDHvo9zNyCPiuJFckeR7DF+kux+ta+pyL08DNAN2a9UuBf13VKldPuPRvtmPl5lRn+HWJC7CS/NTw\n5npLVb03yWuTfAz4b4bP4JedPucC+BXgq4F7u5ntuaraNruqp6PnuXjaIate5Crp+TPy0ST3AR8G\nvgi8paqOzbDsqej5ffGrwB+OvF3xF6vq8RmVPDVJ3gEMgK9J8u/APcAGVpibXnglSY3wj5hLUiMM\nfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGvH/HLG4BAE48ZwAAAAASUVORK5CYII=\n", 30 | "text/plain": [ 31 | "" 32 | ] 33 | }, 34 | "metadata": {}, 35 | "output_type": "display_data" 36 | } 37 | ], 38 | "source": [ 39 | "%matplotlib inline\n", 40 | "\n", 41 | "import numpy as np\n", 42 | "import matplotlib.pyplot as plt\n", 43 | "from sklearn.datasets import load_iris\n", 44 | "from sklearn.naive_bayes import GaussianNB\n", 45 | "from sklearn.model_selection import train_test_split\n", 46 | "from adaboost import AdaBoostClassifier\n", 47 | "\n", 48 | "np.random.seed(42)\n", 49 | "\n", 50 | "data = np.genfromtxt('spambase.data', delimiter=',')\n", 51 | "target = data[:,-1]\n", 52 | "data = data[:,0:-1]\n", 53 | "\n", 54 | "x_train, x_test, y_train, y_test = train_test_split(data, target, test_size=0.5, stratify=target)\n", 55 | "x_cal, x_test, y_cal, y_test = train_test_split(x_test, y_test, test_size=0.5, stratify=y_test)\n", 56 | "\n", 57 | "ada = AdaBoostClassifier(n_estimators=200)\n", 58 | "ada = ada.fit(x_train, y_train)\n", 59 | "probas = ada.predict_proba(x_test)\n", 60 | "\n", 61 | "plt.clf()\n", 62 | "plt.hist(probas[:,1])\n", 63 | "plt.show()" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "## Calibrating the scores\n", 71 | "\n", 72 | "We can clearly see from the histogram that the probabilities produced by the model tend to assume extreme values. Therefore, it might be useful to apply calibration techniques to try and fix these distortions. Two calibration methods have been widely used in machine learning literature: logistic calibration and isotonic regression. The first one is a parametric method that assumes an underlying distribution of the scores composed of two Gaussians of equal variance, (one for the positives and another for the negatives). The second method is a non-parametric approach, therefore it doesn't make any assumption about the distribution of the scores, however, it needs lots of data to produce a good model. Let's see the effect of applying these methods to the previously trained classifier's outputs." 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 2, 78 | "metadata": { 79 | "collapsed": false 80 | }, 81 | "outputs": [ 82 | { 83 | "data": { 84 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEQCAYAAACwSgOGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X98lXX9//HHe2SipuxHqQ0TxsHMygKG/TDT4Rha2g/B\ngbWVZcC0vv1UBOyzHJ99/IFYfrIyYWhlUMIGVpalgznT/JmMyj5lsnPQ2tIKtlEqKPD6/vE+O9tg\nP87Y2bnOj+f9djs3zrmu61zndS6267X39X5f75czM0RERHKCDkBERFKDEoKIiABKCCIiEqWEICIi\nQEAJwTk3dZB1c5xzpc65RcmMSUQk2yU9ITjnSoH6AdZNBczMNgOdzrkpSQ1ORCSLJT0hRE/2rQOs\nngd0Rp+HgZlJCUpERFKuDyEX2NnrdUFQgYiIZJvXBB3AcDnndCediMgwmZkbaptUayF0APnR57nA\njv42MjPMjKuvvjr2PJsfOg46BjoOOg6DHYN4BdVC6JOpnHPjzKwLWA8UA03AJKCxvzfX1NQA0Nzc\nTHNzMyUlJaMZq4hIWuk+N3afK+OV9ITgnJsDFDvnZpvZxujiTcBpZtbinCuOjkTqMLOt/e2j+0vW\n1NQoGYiIHKCkpISSkpLYuXLZsmVxvS/pCcHMNgAbDlh2Wq/nq+Pdl5KBp+OgY9BNx8HTcTi0Y+CG\nc30pFTjnLN1iFhGJVyQSobq6mra2NsaPH09tbS1FRUUj2qdzDoujUzljEsLEiRN59tlnA4go+0yY\nMIHt27cHHYZIxolEIpSVldHa2nOrVigUorGxcURJIesSQvQLBxBR9tGxFhkdlZWVrF279qDlFRUV\nrFmz5pD3G29CSLVhpyIiWautra3f5e3t7Un5/LS7MU1EJFONHz++3+WFhYVDvrezE/74x76Pdeug\nYBjzPaiFkOZaWlo47bTT4l4+nP1Onjx5JKGJyDDV1tYSCoX6LAuFQtTW1sZe798P27ZBfT189avw\noQ/BhAmQlwdnnAFVVXDzzbB5s08Kw6EWQpqbNGkSy5cvj3v5cDg35CVHEUmgoqIiGhsbqa6upr29\nnTe+sZAFC67j8cffxC23wG9/C1u2wK5dB7937Fh461vh7W+Ht73NP97xjmEGEPTt1cN9+JAPNtDy\ncDhsFRUVVlJSYhUVFRYOh/vdbjAj3UdjY6OFQiHLz8+3uXPnWmdnZ2y/xcXFVlVVZXl5eTZr1izb\nsmWLhUIhy8nJsRUrVpiZWUNDg1VVVVl5ebk552z69OkWiUTMzGzLli1WXFwc219ZWZktXrzYTjnl\nFJs+fXoshvr6+lgMVVVVseUrV660vLw8y8nJOWi/kydP7vf7DHSsRWRkdu0ya2w0W7bM7IMfNHv9\n683g4Mcb32j2gQ+YXXWV2bp1Zn/6k9nevQPvN/o7O/T5NZ6NUukxnIQQDoctFAoZEHuEQqFhndBH\nuo/Ozk7Ly8uzpqYm6+rqsksvvdTKy8tj+3bO2caNG62rqyt2wt61a5dt2rTJoiOqrKGhwZxztnr1\nauvq6rKqqqpYEtiyZUvsxN+9v8suu8zWrl0bW97a2mp5eXm2detWi0QiNnnyZNuwYYOZmeXk5NjW\nrVutq6vLysvLbcmSJbH9KiGIjK4XXjBraDD7/OfNpkwxy8k5+OR/7LFm551nVlNj9vOfm/3978P/\nnHgTQkZfMqquru4znhegtbWV6urquIdwjXQf69evp6ysjBkzZgBw3XXXMWnSpNj6vLw8LrjgAgBm\nzpxJXl4eRx99NKWlpTjn2BVtGxYXF/OZz3wGgFtvvZUxY8bE1vXmnOOWW26hpaUltmzDhg1UVVXx\nzne+E4D6+p76RB0dHRxzzDEA5Ofn09nZiYiMjh07oLkZ7r8fmprgT3/qu/41r4HiYjj9dHjPe/xj\nwgRI1tXbjE4IiRjCNdJ9tLa2Ul9fz6ZNmwDfIut9Is/Pz489z83NpWCAIQG9kwj4a43hcHjI7bpj\nmD59euz1lCk9heiuueYaNm/eHPvc/t4vIofmlVfg4Yfhvvv8Y8sW/3d/tyOP9Cf/M8+E978f3vUu\nvywoGZ0QRjKEK1H7CIVClJeXs27dutiyQ7nL98BWSjgcZtKkSQct709ubi7btm2LvW5paYklk6am\nJu6//36OPvpo6urq2LJly7BjE5Ee7e1wzz3+0dgI//lPz7rXvhb2L8pn72EdALyEn9lz037ggehj\nhOzqQ79pNKMTQm1tLY8++uhBt4H3HsI12vuYO3cuS5YsYfPmzRQXF7N48WK2b9/OvffeCzDoHb+9\n123ZsoXVq1dTXl7OlVdeyfTp02OXegZ6T7eqqiqmT5/OvHnzGDduHOXl5SxZsgQzIz8/n6OPPprO\nzk5Wrlx50JA3ERmcGfz+9/Czn/nHb3/bd/3b3gbnnAOzZvlWwFErOkZ00h5NGZ0QDhzCVVhYOOyJ\noka6j3HjxlFfX8/ChQtj85T0voY/2NDO3uvKyspobGxk4cKFTJ8+nfXr1w/5nt7fYfny5ZSWltLV\n1UVVVRXz588HfH9Cfn4+oVCIG264gfLycpqamsjLy4vr+4lko/374fHHYcMG2LgRel+9PeIImDkT\nzjsPPvABOPHE4OIcLs1llAY2bNjA+vXr+1x2ClImH2uRgZj5PoA77/R3AP/1rz3rjj0WPvxh+MhH\noLTUJ4WBuGUu6S2EeOcyyugWgojISLW2wtq1sGYNPPNMz/ITToA5c2D2bHjf+2DMmOBiTBQlBBGR\nA+zaBevXw/e+50cJdTvuOCgvh4sugve+F3IybPIfXTKSYdOxlkxkBg89BHV10NAAL7/slx91lG8F\nVFbC2Wf7ewVGQpeMRERS1M6dcMcdsGpV3xvFzjoLLrnEXxY66qjg4ksmJQQRyUq/+x18+9u+f6C7\nNXD88T4JXHIJZOMIbCUEEcka+/b5ewVuugkefLBn+cyZcOmlfqTQYYcFF1/QlBBEJOO9+CLcfjv8\n7//23DNw9NHw6U/DZz8LJ58cbHypQglBRDLWjh3+stDNN/u+AoCiIvjSl3wyOProYONLNRk2aCr1\nJKLyWLzVz3KiY+BGWi1NJN09/zxcfrm/S7imxieDd7/b31n8zDPwhS8oGfRHLYQkGGnlsXirn3V/\nTiKqpYmko/Z2WL7cjxjavdsvO/dcWLLEzyiqIoCDUwshyRoaGpg8eTJjxoxh3rx5fabCXrVqFfn5\n+Zx00knU1dXFWhbhcJgrr7wytl1VVRX5+fkUFBSwYsUKAGbNmoWZUVBQcND23Z9ZUFDApZdemqRv\nKpI8L7wAX/4yTJrkLw/t3g0XXABPPgm//KUfQqpkEId4quik0oNhltD06xL3GK7elcdaW1vNORer\nnlZeXh4rZ9m7qllXV5cVFxfH3te7Klp9fb1NnjzZdu3aZS0tLZaTk2NdXV1m5qufHbj9YNXSDtVg\nx1okmXbuNFu61OzII3t+Ry+80Ox3vws6soFRk/zfH+KsmKYWQhJ1Vy6bMWMGxxxzDMuXL4/NfLpq\n1apYVbNjjjmGpUuX9ruP7stC27ZtY8qUKX0qng32me985zuZOHEi9fX1KoIjaW/3bvj61/29Atdd\nBy+9BOefDy0tUF9/CMXlBciSS0aJbCOMxI4dO/rUGygqKoqVrAyHw33WDXTSnjNnDosXL6a8vJyC\nggJWrlw56Ge2trb22e+UKVP6VEwTSSf79/sbyU4+Ga64Ajo6oKQEHnkE7r4b9KM9MlmREFJFQUFB\nn8plnZ2d5ObmAj4B9F43UCW0SCRCaWkp27Zt48knn2TlypU0NTUB/RfH6a9a2oYNGxLyfUSS6ZFH\nfLnJykp47jl4+9vhF7/wtYnf856go8sMSghJdOGFF1JXV0dTUxOdnZ0sXLiQefPmATBv3jzq6upo\naWmhs7OT66+/vt99NDQ0UF5eTiQSYf/+/QB0dXUB/Y9mqqqqiu03HA5TXl5OR0fHKH1DkcRra4OP\nf9wng8ce89NL3HYbbN0KH/xgYjuLI5EIlZWVzJgxg8rKSiKRSOJ2ng7i6WhIpQeH0KkcpN6dymZm\nGzZssFAoZDk5OTZv3rxYh7CZWV1dneXl5dnkyZOtrq4u1jHcu5PYzKysrMxycnIsPz/fli5dGlte\nXl5uOTk51tLS0mf77v3m5OTYZZddNuLvlKrHWjLLnj1my5ebHXWUv2A7dqzZV79qtmvX6HxeOBy2\nUChkQOwRCoUsHA4n9HNSuVM58BP8cB/plhDiFQ6HbcuWLbHXDQ0NNmvWrAAjGli6H2tJfU1NZief\n3NN7d8EFZpHI6H5mRUVFn2TQ/aioqEjo56RyQtAloxTR2dkZq3kMsHLlSsrLywOOSiS5/vlPuPhi\nX3fg6afhzW+GX/3K1y2eOHF0P7utra3f5e3t7aP7wSlECSFFTJ06lauuuoqioiIKCgqYPHky8+fP\nDzoskaQwg+9/H97yFl+bYOxY+J//gT/8Ac45JzkxjB8/vt/lhYWFyQkgBahimgybjrUk0vbtsHAh\nNDb617NmwS23JL8eQSQSoaysrM8Iv1AoRGNjI0VFRQn7HFVM68U5NwfoBKaZ2YpB1heZ2epkxyci\nybF/P3znO7B0qZ+euqDAT09dURHMNBNFRUU0NjZSXV1Ne3s7hYWF1NbWJjQZpLqkthCcc1PxJ/qN\nzrkFwBNmtvWA9ZhZi3OuFNjRe310G7UQAqZjLSMVifjppx94wL++6CL45jfh2GODjSsZ1ELoMQ+4\nL/o8DMwEth6wzXJgFjDJzDYnMTYRGWVmfibSyy/3rYJjj4WVK+GjHw06soHlL8+nY3fi7t3JG5uX\nsH0lWrITQi6ws9frgt4roy2DsHNuJ6AeVZEM8sILvlbxPff413Pn+ktGr399sHENpWN3R9L/og9K\nSo0ycs6NAzqAa4E659zEQANKgEQUq+kufBNkDCIjcffdcOqpPhnk5cGdd8K6damfDLJNslsIHUB+\n9HkusOOA9QuB68xsl3MuDFwI3HjgTmpqamLPS0pKKCkpGY1YEyIRxWqSVWBHJNFeftlfHvrud/3r\n0lI/vPSEEwINK+M1NzfT3Nw8/DfGc/daoh7AVGB+9PkiYEr0+bjov1d0P4++nt/PPga7Ey/lbNmy\nxYqLi2OvFy5caHl5eZafn2833HBDbHl9fX1sSou5c+fGprQoKysz55zl5+cPul04HLbi4mK74YYb\nYtNftLS0HBRD9/vz8/NjtRiGK1WPtaSW//s/s1NP9Xcav/a1Zl//utm+fUFHNbS86/OMGmKPvOvz\ngg5pxEjVqSvwfQOlvU/2+NFG3c8XAbP7SwaWpglhqOI2gxXOMespfDPYduFw2JxzduONN5qZWVVV\nVWzqi+4YwuFwQorlpOqxltSwf7/ZbbeZHXGEP8OcdJJZr1lZUl4QU0uMtngTQtLvQ7B+7i0ws9N6\nPT/o3oREcMsSM7B5JJ1LvYvbTJ06NVbcZuXKlbHCOQDLly+nuLiYW2+9tc/7exfY6W+7vLw8Lr/8\ncsDPcrpw4cKeuM1oaGiIFcsBYsV5RBLlxRfhc5+DH/zAv/7kJ+Hb31ZB+3SR9IQQlFQYJTBnzhw6\nOjpiU1AvXbqUK664ot/COd1zGvU21Hb5+fkHvae3cDhMcXFx7LUK5Ugi/fnPcOGF8Mc/whFH+H6D\niy8ONqZDGTKaysNCR1vWJIRU0F3cZv78+Wzfvp2ZM2cybdq0QQvn9BbvdgPpr1hOOBxmzpw5h/iN\nRLyGBn+j2X/+46uZNTT4AjZBy6Yho4mQUsNOM92BxW2cc3R2dvZbOGfu3LkHvX+wAjvQf8W0bs45\nqqqqWLVqlYrlSMLs3QtXXgnl5T4ZzJsHTzyRGslAhk8JIYkWLVpEfn4+kydP5rTTTqO8vJzZs2dT\nVFREfX09CxcupKCggJycnD7DROfMmcOYMWP63a53ZbWhhqdOnDiR5cuXU1paykknncSsWbM0o6oc\nsn/9C849F1asgDFj4Kab4Mc/Vn9BOtNspzJsOtbyu9/BRz4Czz7rp59Yvx7OOivoqA4WxLxBqSje\nuYzUQhCRYdmwwdc3fvZZOO00ePLJ1EwGMnxKCCISFzOoqfEjiV56CT7xCfj1r3XXcSbRKCMRGdLL\nL/tRROvWQU4O3HADfOUrwdQtkNGjhCAig3r+eT899WOP+Q7jO++ED34w6KhkNCghiMiAnnoKzjsP\nnnsOJkzombVUMlPGJIQJEyaMeFZQic+ECROCDkGSYPNmmD0bdu2C97wHfvITOO64oKOS0ZQxCWH7\n9u1BhyCSMX7wA5g/3994duGFcMcdfjoKyWwaZSQiMWZQWwuf+pRPBldc4TuSlQyyQ8a0EERkZPbt\n8zOVrlzpRw9961v+tWQPJQQR4eWX4eMf9/0Ehx/up6C44ILkx5FNBe1TkRKCSJbr7IQPfQgeeghy\nc/1IojPOCCYWzU4aLCUEkSz297/7Cep+/3t/x/GvfgVve1vQUUlQlBBEslQ4DGVl/t+TT4b77oMT\nTww6KgmSRhmJZKGnnoL3vc8ng+nT4cEHlQxECUEk6zzxhJ+d9Pnn4eyzoakJ3vCGoKPyFQUBZsyY\nQWVlZey1JE/G1EMQkaE98ACcf76vbvbhD/t7DMaODToqnwzKyspo/UQr1PhloVCIxsZGioqKAo0t\nE6gegoj08atf+Q7k//zHDzFtaEiNZABQXV1Na2trn2Wtra1UV1cHFFF2UkIQyQI//alvEezeDQsX\n+qkoDjss6Kh6tLW19bu8vb09yZFkNyUEkQy3fr2fj+jVV+GLX4Rbb/U1kFPJ+PHj+11eWFiY5Eiy\nmxKCSAZbswY+9jE/L9HixXDTTalZ1Ka2tpZQKNRnWSgUora2NqCIspM6lUUy1Pe/D5dc0lP68mtf\nS81k0C0SiTDpjknMeGAGhYWF1NbWqkM5QeLtVFZCEMlAt9/up682g2uugauuCjqi+LhlTlNXjIJ4\nE4LuVBbJMHV1vuMY4Prr/aUikXgoIYhkkNWre5LBihW+noFIvJQQRDLE7bfDggX++Y03wuWXBxuP\npB+NMhLJAN0lL8G3DJQM5FCohSCS5tasgU9/2ncgL1+e3MtEKmiTWZQQRNLYunVw8cU+GVx7LVx5\nZXI/XwVtMosuGYmkqbvugooK2L/f32ewdGnQEUm6U0IQSUO/+AXMmwf79vlE8LWvBR2RZIKkXzJy\nzs0BOoFpZrain/VTgUkAZrYhyeGJpLxNm2DOHD830Ve+4m88S+U7kCV9JLWFED3Zm5ltBjqdc1P6\n2WxpNBEUDbBeJGs9+CB85COwZw989rN+eKmSgSRKsi8ZzcO3DgDCwMzeK6Oth8cBzOxGM9ua3PBE\nUtfjj8N558FLL/lRRd/6lpKBJFayE0IusLPX64ID1p8GFDjnpjrnFiUvLJHU9oc/+OI2//43XHSR\nn54iRz2AkmCpOOx0h5m1OOdmOufm9NePUFNTE3teUlJCSUlJEsMTSa5nnoGyMujo8EVu7rgj9eoZ\nSGppbm6mubl52O9L6mynzrnrgEYza4peHioysxt7rV8EtJrZRufcAmCSmS09YB+a7VSyxnPPwfvf\n7/+dORPuvjt1yl6CZidNF6laU3k90RFE0X83ATjnxkWXNfRanws8kdToRFLICy/4JPDcc3D66fCT\nn6RWMpDMk9SEYGYtAM65UqCjV6fxpuj6CH700Rwg38w2JjM+kVTR2QnnnOMvF02Z4u87OOqooKOS\nTKcCOSIp5sUXYdYsePhhePOb/VDTY48NOqr+6ZJRekjVS0YiMog9e2D2bJ8M3vQmaGxM3WQgmUcJ\nQSRF7NsHlZVw333whjf4O5JPPDHoqCSbKCGIpAAzuPRSaGiAY46Be+/1l4tEkkkJQSQFLF3qy1+O\nHQs//zlMnRp0RJKNlBBEArZihS9s85rX+BbC+98fdESSrZQQRAJ02209RW2+/30/V5FIUJQQRAJy\n112wcKF/fvPNvtiNSJCUEEQC0NwMH/uYr3Z29dXw+c8HHZGIEoJI0m3Z4iep27MHPvc5+OQnI1RW\nVjJjxgwqKyuJRCJBhyhZKhVnO5UsEolEqK6upq2tjfHjx1NbW0tRUVHQYY2aZ57pmcZ63jz48pcj\nzJpVRmtra2ybRx99lMbGxow+DpKizCytHj5kyQThcNhCoZABsUcoFLJwOBx0aKOirc1s4kQzMJs1\ny2zPHrOKioo+37/7UVFRkdTYwuGwVVRUWElJiVVUVMT9f0CNfh/TQfS8OeT5VS0ECUx1dXWfv4wB\nWltbqa6uZs2aNQFFNTo6O33LYPt2eNe7YMMGeO1roa2trd/t29vbkxZbJBKhrEytFFEfggQoFU6G\nyfDyy/ChD/mqZ295i5+59HWv8+vGjx/f73sKCwuTFt9giVmyixKCBCYVToajbe9e31fw0ENwwgl+\nSorXv75nfW1tLaFQqM97QqEQtbW1SYsxWxKzDO2QEoJz7grn3Hedc/Odc8ckOijJDqlwMhxNZv4+\ng7vvhvx8nwwOnKyuqKiIxsZGKioqmDFjBhUVFUm/VJMNiVniE1c9BOfc2cBOM9vqnLu+16ppQCmw\n0MxuG6UYD4zF4olZ0kP3KKP29nYKCwszapTRkiV+Soojj/Qzl773vUFH1L/++hBCoVBciUn1ENJD\nvPUQhkwI0WQwycxWR1/3KXzvnMsFlgCPWxIqnCkhSDr4xjfg8sv9/EQ/+xl84ANBRzS4Q03MSgjp\nId6EEM8oo2KgcaCVZtYJLHHOLRhGfCIZ64c/9MkA4HvfS+1kkL88n47dHf7FSdEHsPaOtXG9P29s\n3ugEJoEYMiGY2Qrn3ALnXKeZbQfCzrkngCuBJ81sV3TTHaMYp0hauOce+PSn/fNvfMMXvEllHbs7\n9Be+xMTVqWxmddFkADAPWA9cBmx3zj3hnLsXmOScOxrAOTd7NIIVSWWPPAIXXugrny1eDF/+ctAR\niQzPodyY1go0mtkKAOfcVGAmMAu4yjnX3VIY9f4EkVTxxz/6qatfftm3EK67LuiIRIZv2MNOzawO\nyHPOTYy+bjGzFWY2y8zy8S0Izc4lWeO55+Ccc6Cjw09at2oVuCG770RSzyFNXWFmLYOs2+KcW3zo\nIYmkj3/9C2bNgrY2X+nszjv9yCKRdDQqdyoPljBEMsW//w0f/CA8/TS84x1+eOkRRwQdlcih09QV\nIodgzx6YPRueeAKKiuBXv4Lc3KCjEhkZJQSRYdq3Dz75SX/38XHHwX33wRvfGHRUIiOnhCAyDGa+\n3OX69XDMMb5lMHly0FGJJIYSgsgwXH01fPe7cPjhvs9gypSgIxJJHCUEkTh985tQWwtjxvgWwlln\nBR2RSGIpIYjE4Yc/hC99yT+/7TZ/v4FIplFCEBnC3Xf3zE/09a/DxRcHG4/IaFFCEBlEczOUl/uR\nRUuXwle+EnREIqNHCUFkAE8+6S8N7dkDl14K11wTdEQioyuuimmpRAVy0kOfefYlZeWNzWPn4p1B\nhyGjLJEFchLKOTcH6ASmdc+YOsB2iwZbL6ktnefZ374dzjjDz0/0gQ/AT34Cr31t0FGJjL6kXjKK\nTpVtZrYZ6HTO9TuK2zlXip9SWySpnn8eysp6JqtraFAykOyR7D6EefjWAUAYnfQlhezc6ZPBtm0w\nbZofXXTkkUFHJZI8yU4IuUDvC5YFB27gnJsabUFoRnlJmu6ZS596Ct7yFj8lxbhxQUclklypOMpI\nVbslqV56CT70IXjsMZg4ERob4Q1vCDoqkeRLdqdyB5AffZ4L7Oi9Mto6aIq+HLBHsqamJva8pKSE\nkpKShAYp2WPPHpgzBx54AAoL/QymJ5wQdFQiI9Pc3Exzc/Ow35fUYafRTuViM1vtnFuEr8281Tk3\nzsy6oiOQDH8pqQqYb2ZbD9iHhp2mAbfMpfwoo717Yd482LgRXv96+PWv4ZRTgo5KJPHiHXaa1EtG\n3ZXUoqOIOnqd7DdF128ws43RZbqCK6Nm3z4/BcXGjb6wTWOjkoGIbkyTUZHKLYT9+2HBArj9dnjd\n6+AHP2hn48YraWtrY/z48dTW1lJUVBR0mCIJk7I3pokEqbvAze23+/rHq1f/nSuvPJPW1tbYNo8+\n+iiNjY1KCpJ1UnGUkcioMIPLL4dbbvEFbn76U7j77kV9kgFAa2sr1dXVAUUpEhwlBMkKZrB4Mdx0\nExx2mL8D2d+R3Nbv9u3t7UmOUCR4SgiS8czgv/4LVqyA17wG6uvh/PP9uvHjx/f7nsLCwiRGKJIa\nlBAk4y1bBtde60tf3nknfOQjPetqa2sJhUJ9tg+FQtTW1iY5SpHgqVNZMlpNjU8IOTnwox/5m9B6\nKyoqorGxkerqatrb2yksLNQoI8laGnYqoyIVhp32TgZr18JFFwUajkhgUvLGNJFkMIOrr1YyEBku\nXTKSjNLdgXzttUoGIsOlhCAZo3to6YoVvgN57Vo/V5GIxEcJQTKCGXz5y/DNb/qhpXfeeXAHsogM\nTglB0t7+/fC5z8Gtt/bcdPbhDwcdlUj6UUKQtLZ3L1xyCfzwh346io0bfeUzERk+JQRJW6+8AhUV\nvkVw1FHws5/B2WcHHZVI+lJCkLT00ktQXg733ONrH99zD5x+etBRiaQ3JQRJO11dvgbygw9CQQHc\ndx9MmxZ0VCLpTwlB0so//wnnnAMtLTB+vCqdiSSS7lSWtPHss/D+9/tkMHkyPPSQkoFIIikhSFp4\n6infR/D00/COd/jLRRMnBh2VSGZRQpCU99BDvmXQ3g5nngkPPADHHx90VCKZRwlBUtpPf+orm3V2\nwgUXwL33Qm5u0FGJZCYlBElZ3/kOzJ4Nu3fDwoW+0tnYsUFHJZK5lBAk5ezfD0uWwP/7f/55ba2f\nlmLMmKAjE8lsGnYqKWX3bj8VxY9/7Cepq6uDT30q6KhEsoMSggwqf3k+Hbs7hv2+vLF5w37PP//p\n+wl+8xt43ev8lBTnnDPs3YjIIVJCkEF17O5ISinMp5/2k9KFw3DCCfCLX/jhpSKSPOpDkMBt2gTv\neY9PBtOmwWOPKRmIBEEJQQL1ne/Auef6YaUf/Sj8+tdQWBh0VCLZSQlBAvHqq/DZz/qRRPv2wVVX\nwYYNfhonbA1PAAAMtElEQVRrEQmG+hAk6f7xDz919a9/7Yva3Habr2sgIsFSQpCk+u1v/Uiiv/3N\nXxrauBHe/e6goxIR0CUjSaLvfx/OOMMng9NPhyefVDIQSSVKCDLquqee+PSnYc8eqKqC++/XBHUi\nqUaXjGRUPfsszJnjWwOHH+5HFX3mM0FHJSL9UQshi0UiESorK5kxYwaVlZVEIpGE7v9nP4OpU30y\nKCqCRx5RMhBJZUlvITjn5gCdwDQzW9HP+gXRpyEzW5LU4LJIJBKhrKyM1tbW2LJHH32UxsZGioqK\nRrTvV17xk9PddJN/ff75cMcdkDf82SxEJImS2kJwzk0FzMw2A53OuSkHrC8FGs2sDpjknDs7mfFl\nk+rq6j7JAKC1tZXq6uoR7Tcc9sVsbrrJT053442+paBkIJL6kn3JaB6+dQAQBmYesH5Sr2Xh6GsZ\nBW1tbf0ub29vP+R9rlkDU6bA44/DhAm+zOXll4Nzh7xLEUmiZF8yygV29npd0HtltGXQbRpwZzKC\nykbjx4/vd3nhIcwb0dUFn/scrF3rX194IaxapVaBSLpJyVFG0UtLT5rZ1qBjyVS1tbX8+MQfs//w\n/X2Wr2Uta5etjb0eahrr5ma4+GJ47jk48ki4+WZfz0CtApH0k+yE0AHkR5/nAjsG2K7UzJYOtJOa\nmprY85KSEkpKShIUXuYZrJ7BuKPHcf4fz6e9vZ3CwkJqa2vj7lDevRu++lX4xjf86+Ji30I4+eRE\nRS4ih6q5uZnm5uZhv8+Zjf5c97EP83/5F5vZaufcInwH8lbn3Dgz64pus6D70pFzrjTaAd17H5bM\nmNOdW+YSXs/gscf8TWZ/+pMva/lf/+WTw2GHJfRjRCRBnHOY2ZDt9qR2KptZC8RGE3X0uiS0qdfy\n651z25xzOwCd+VPI7t2weLGfduJPf/KtgYcfhpoaJQORTJD0PgQzW93PstOi/27mgI5mSQ0PPggL\nFvjKZjk5sGgRLFsGRxwRdGQikigp2aksqaOz07cKVq3yr085Bb73PU1KJ5KJNHVFQEZ72oiRMoN1\n63wCWLXKXxL62tegpUXJQCRTqYUQgNGcNiIR/vIXf1/Bpk3+9emnQ10dvPWtwcYlIqNLLYQAjNa0\nESP1n//40UKnnuqTQV6ebx08+KCSgUg2UAshAKMxbcRImMGPfgRXXgndIVxyCVx/PbzhDYGEJCIB\nUEIIQCKnjRipRx/18w09/LB/PX26v9v4ve9NeigiEjBdMgpAbW0toVCoz7JQKERtbW3SYgiHYd48\nf+J/+GE47ji4/XZ/05mSgUh2SuqdyomQKXcqRyIRqquraW9v5zdn/IZXxrwyKp+TNzaPnYt75hP8\nxz/gmmvgu9+FV1+FsWPhK1/xQ0uPOWZUQhCRgMV7p7ISQgoYjeklDrRrF3z96/7x4ot+2Sc+4ZPD\nm940qh8tIgGLNyGoDyHD/fvf8K1v+USwM9pQOP98nwje8Y5gYxOR1KKEkKF27YJbbvEVy3ZE55Q9\n80y49lp43/uCjU1EUpMSQgAOnJJ6qJoDw7FjB3zzm75V0BmtTfe+98F//zfMmKE6BSIyMCWEAHTs\n7kh4n8Gzz/o6xqtX9/QRnHmmn5p65kwlAhEZmhJCAg1WjKa3RLYInnzS9w+sXw/79vll557r7zg+\n44yEfYyIZAElhAQajb/8+7N3L9x1l7809Jvf+GVjxkBFBVxxhS90LyIyXEoII9S7VZDIv/z78/e/\n+0tCq1bB3/7ml40bB/Pnwxe+ACeeOKofLyIZTglhhEa7VbB/PzQ1+SRw112+dQDw5jfDF78In/wk\nvO51o/bxIpJFlBBS1F//Cj/4Adx2G2zf7peNGQOzZ8Nll8HZZ/vKZSIiiZJ1CSHejt94JfIy0Ysv\nwsaNPhE0NflZSAEmTIDPfMYXtj/hhIR9nIhIH1mXEJLV8RuvV16B++6DH/8YfvITeOklv/zww+Gj\nH/WJoLRUrQERGX0ZmxAGagmMdsdvPF59FTZvhvp6nwR29sw9x+mnw8UXw9y5kJsbXIwikn0yNiGk\nWkvgpZd8S+Cuu+Duu6GjV656+9vh4x+Hiy6CFKigKSJZKmMTQipob4df/AJ+/nNobISXX+5Z97a3\nQXm5f6g8pYikAiWEBHr1VXjkEbj3XvjlL6Glpe/6d70LLrjA9w285S3BxCgiMhAlhBEwgz//2Rek\n37QJ7r/fTzfd7Ygj/DxC558P550HA1TOFBFJCUoIw2AGTz8NDzwAzc3+8fzzfbc55RQ45xw/n9CZ\nZ/qkICKSDpQQBrF7t7/s8/DD8NBD/vGvf/Xd5rjj/LDQmTP9v5o+QkTSlRJC1P79sG0bPPEEPP64\nLzbf0uLvE+jt+OP9X/4zZsBZZ/m+AE0tLSKZICsTwt698Je/wNatsGVLz6Orq+92zvnRQO99r59K\n+owzYNIkJQARyUwZnRDM4IUX4Kmn4A9/8P/+/vf+3927D97++OP9SKDej3Hjkh+3iEgQMiIh7N3r\nK4b9+c89D06AgoK+N4D1NnEivPOdMG1az6OwMJlRi4iklrRMCB0dvkbwM8/4RyTi7wHoo8Zvl5vr\nL/uceqq/I/jUU2HcuGdZseKrtLW18Ze/jOcTn6ilsFC3CItIdnNmqTO9Qzycc/bii8ZRR/VdfsIJ\ncPLJvpP35JPhCzsd7QuN44/ve80/EolQVlZGa2trbFkoFKKxsZEizRshIhnIOYeZDdn7mZYJwcz4\n1rf8JZ43vxlCITjyyAO2W+b6ncuosrKStWvXHrS8oqKCNWvWjFbYIiKBiTchpOUlI4DPf/7Q3tfW\n1tbv8vb29hFEIyKS/pKeEJxzc4BOYJqZrRju+gH3u6xv8htomuvxA8wfUageZRHJckm9ZOScmwoU\nmdlG59wC4Akz2xrv+ug2NpKY1YcgItkm3ktGya7DNQ//1z9AGJg5zPUjVlRURGNjIxUVFcyYMYOK\nigolAxERkp8QcoFe9cEoGOb6Ppqbmw8piKKiItasWUNTUxNr1qxJ+2RwqMchk+gYeDoOno7DoR2D\ntK7Uq/90T8dBx6CbjoOn43BoxyDZncodQH70eS6wY5jrAaipqQH8F25ubqakpCTRcYqIpK3uc2P3\nuTJeyU4I64FioAmYBDQCOOfGmVnXQOsP1P0la2pqlAxERA5QUlJCSUlJ7Fy5bNmyuN6X9BvTnHPz\ngQh+NNHq6LInzOy0gdYf8P70upNORCQFZOSdyiIiMjrSulNZREQSRwlB0o5zbo5zrtQ5t2iI7QZd\nL5JJojf2DrQurt+ZtEkIQ32heL9wOovjGCyIPq5PdmzJEv2hNzPbDHQ656YMsF0po3BjYyqJ4+dh\nanSbOcmOLVmGcV6Yn+zYkin6814/wLq4fmcgTRLCUF9oOF84XcVxDEqBRjOrAyY5584OIs4kGPW7\n2dNBnD/zS81sA1CUpb8TU4FwdH0kE49Bt+h3bB1gddy/M2mREEiBKS9SwFDfcVKvZeHo60w05N3s\nzrmp0V+QTK5+PejPQ7RV8DiAmd144JxgGSKe3/vl0X8nZegxiEfcM0CkS0JI6JQXaWrQ72hmdb2G\n6U4DfpuswFJQ/1PdZpahfuZPAwqil40y9TLqUL8TLUDYObeTAW5ylb7SJSFInKLN5Ccz+K+hQe9m\nj7YOmqIvs31M9Y7oSbG7xZBVnHPj8D8v1wJ1zrmJgQYUnLhmgID0SQgJmfIizcX7HUvNbGlyQgrE\nenouh00CNkHslx98/8ns6PTpBRl83Xion4cd+Mso4C+rTE9SXMk01DFYCFxnZjcCC4ALkxhbEPpc\nIu31O9Hv70x/0iUhDHUSiPsLp7GhjgHOuQXRH/7uTuaM0+sv3lKgo1dLaFN0/QYz2xhdNq6fXWSK\noX4eGnqtzwWeSGp0yTHUMTCiJ8noz0TngTvIFNEWYLFzbnavxd2/EwP9zhy8n3S5U3mkU15kgsGO\nQfQ/ez3+r6Y8oLzXpRPJQHH+TnQA0zO11RjHMViEH32Tn6nnhURKm4QgIiKjK10uGYmIyChTQhAR\nEUAJQUREopQQREQEUEIQEZEoJQQREQGUEEREJEoJQUREACUEERGJUkIQEREAXhN0ACLpLjqZ2lz8\nbJsFwCYziwQblcjwqYUgMnJ1wLrojJo7gKyrPSCZQZPbiYyQc24bfrrpdd1TDYukI7UQREbuQqLz\n8TvnMrHugGQJJQSREXDOzTGzrWY218wKgLwsLtUoaU4JQeQQRTuTlxywuNXMtgcQjsiIqQ9BZASi\nJQs78VXq8oD1ZrYr2KhEDo0SgoiIALpkJCIiUUoIIiICKCGIiEiUEoKIiABKCCIiEqWEICIigBKC\niIhEKSGIiAighCAiIlH/H0svyNUPeTDOAAAAAElFTkSuQmCC\n", 85 | "text/plain": [ 86 | "" 87 | ] 88 | }, 89 | "metadata": {}, 90 | "output_type": "display_data" 91 | } 92 | ], 93 | "source": [ 94 | "from sklearn.isotonic import IsotonicRegression\n", 95 | "from sklearn.linear_model import LogisticRegression\n", 96 | "from calmap import plot_calibration_map\n", 97 | "\n", 98 | "cal_probas = ada.predict_proba(x_cal)[:, 1]\n", 99 | "\n", 100 | "lr = LogisticRegression(C=99999999999)\n", 101 | "lr.fit(cal_probas.reshape(-1, 1), y_cal)\n", 102 | "\n", 103 | "iso = IsotonicRegression()\n", 104 | "iso.fit(cal_probas, y_cal)\n", 105 | "\n", 106 | "linspace = np.linspace(0, 1, 100)\n", 107 | "pr = [lr.predict_proba(linspace.reshape(-1, 1))[:, 1], iso.predict(linspace)]\n", 108 | "idx = cal_probas.argsort()\n", 109 | "scores = cal_probas[idx]\n", 110 | "y_c_2 = y_cal[idx]\n", 111 | "methods_text = ['logistic', 'isotonic']\n", 112 | "fig_map = plot_calibration_map(pr, [scores, y_c_2, linspace], methods_text, alpha=0)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "## Calibration map\n", 120 | "\n", 121 | "In the calibration map shown above, we can see that:\n", 122 | "\n", 123 | "* Logistic calibration fit the scores in the first and last bins well, but was either underconfident or overconfident in the rest of the map, due to the limitations of its shape\n", 124 | "* As a non-parametric method, Isotonic regression managed to fit the scores better than Logistic calibration by finding an inverted sigmoid shape\n", 125 | "\n", 126 | "## Fitting a beta calibration model with the betacal package\n", 127 | "\n", 128 | "Our beta calibration is a parametric method which is able to circumvent logistic calibration's shape limitations by fitting three parameters, i.e. two for shape (a and b) and one for location (m). These parameters can be fitted by training a logistic regression model with two features extracted from the classifier's probability outputs. Beta calibration is adequate for calibrating probabilities, because the beta distribution has support [0, 1], while logistic calibration assumes the scores are distributed in two gaussians, which have infinite support.\n", 129 | "\n", 130 | "To make it easy for practitioners to fit a beta calibration model, we have provided a Python package called __betacal__. The package can be installed via pip (_pip install betacal_). Below, we show how to use the package to fit the three-parameter version of beta calibration. For the other two versions, the practitioner can set parameters=\"ab\" (fit shape parameters a and b and fix location parameter m = 0.5) or parameters=\"am\" (fit shape parameter a, setting a = b, and fit location parameter m)." 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 3, 136 | "metadata": { 137 | "collapsed": false 138 | }, 139 | "outputs": [ 140 | { 141 | "data": { 142 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEQCAYAAACwSgOGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4lNX1wPHvDYqisiRxDVYJAXFBWQJoqUtiCFqtK4So\niVorENTa4grYRsBUEcHWqiibv9qKWtncrRgS4tKKIAQpCCqZgEpUZEmQqgjk/P64M5OZZJLMJLPn\nfJ5nHvIu886dIXnP3PUYEUEppZRKiHQBlFJKRQcNCEoppQANCEoppZw0ICillAIiFBCMMf2aODbM\nGJNljLkrnGVSSqm2LuwBwRiTBSxo5Fg/QESkBKg2xvQNa+GUUqoNC3tAcN7sKxo5nAtUO392AEPC\nUiillFJR14fQBdjpsZ0cqYIopVRbc1CkCxAoY4zOpFNKqQCJiGnunGirIewCkpw/dwF2+DpJRBAR\nJk6c6P65LT/0c9DPQD8H/Ry8Hh98wMRzznFv+ytSNQSvSGWM6SwiNcB8IB0oBboDxb6ePGnSJADK\nysooKysjIyMjlGVVSqnY8f33lA0fTtkRR7jvlf4Ke0AwxgwD0o0xV4rIYufupcBAESk3xqQ7RyLt\nEpE1vq7hepOTJk3SYKCUUp7GjyfjnHPI6NnTfa+cPHmyX08Ne0AQkUXAonr7Bnr8PNffa2kwsPRz\n0M/ART8Hq81+DkuXwosvwtq1ZHz0UcBPN4G0L0UDY4zEWpmVUspflZWVFBYWsnXrVrp27UpRURGp\nqanNP7G6Gs44A+bOhaFDvQ4ZYxA/OpXjJiB069aNLVu2RKBEbc+JJ57I5s2bI10MpeJOZWUl2dnZ\nVFTUTdVKS0ujuLi46aAgArm5cOSR8MQTDQ63uYDgfMMRKFHbo5+1UqGRn5/Ps88+22B/Xl4e8+bN\na/yJf/oTvPoqlJVBhw4NDvsbEGJuHoJSSsWrrVu3+txfVVXV+JMWL4ZZs2DFCncw2L4d1q+H5GTo\n3dv/14+2eQhKKdVmde3a1ef+lJQUn/t/eH8N+24s4KVfv8TvHzyOrCw49lg46ijIyIAnnwzs9TUg\nxLjy8nIGDhzo9/5ArtujR4/WFE0pFaCioiLS0tK89qWlpVFUVER1NZSWwrRpcM01cG6PKrYNvoz8\n6se54k/pPPqoPf7NN3XPXb8+sNfXJqMY1717d6ZOner3/kAY02yTo1IqiFJTUykuLuaee+7j008P\nw5izSEm5gqFDj2DTprrzOlPNO1zIk9zEfHIbXKdDBzjlFOjTJ8ACRHyKdYAPW+SGGtvvcDgkLy9P\nMjIyJC8vTxwOh8/zmtLaaxQXF0taWpokJSXJiBEjpLq62n3d9PR0KSgokMTERBk6dKisXr1a0tLS\nJCEhQaZNmyYiIgsXLpSCggLJyckRY4wMGDBAKisrRURk9erVkp6e7r5edna2jBs3Tk455RQZMGCA\nuwwLFixwl6GgoMC9f9asWZKYmCgJCQkNrtujRw+f76exz1op1TJffy2yaJHI7beLnHWWyMEHi9ih\nQw0f7flRSsmQv3KrGGqlVy+RnByRoiKRF18U+ewzkf37va/v/Jtt/v7qz0nR9AgkIDgcDklLSxPA\n/UhLSwvoht7aa1RXV0tiYqKUlpZKTU2NjBkzRnJyctzXNsbI4sWLpaamxn3D3r17tyxdulScI6pk\n4cKFYoyRuXPnSk1NjRQUFLiDwOrVq903ftf1brrpJnn22Wfd+ysqKiQxMVHWrFkjlZWV0qNHD1m0\naJGIiCQkJMiaNWukpqZGcnJyZPz48e7rakBQKjS2bhWZN09k1CiRXr0av/l7Pg46SKR/3wOyInWE\nbOo3XN5/b7/s2ePf6/kbEOK6yaiwsNBrPC9ARUUFhYWFTQ/hCuI15s+fT3Z2NpmZmQBMmTKF7t27\nu48nJiZyxRVXADBkyBASExPp2LEjWVlZGGPYvXs3AOnp6dx4440AzJw5k3bt2rmPeTLG8MQTT1Be\nXu7et2jRIgoKCujjrD8uWFCXn2jXrl106tQJgKSkJKqrq1FKBVdNDSxbBsXFtp1/48bmn9OzJ5x5\npn0MGgRnnC4cevstsOEbePNN0g5tF/RyxnVAaNEQriBfo6KiggULFrB06VLA1sg8b+RJSUnun7t0\n6UJysu8UEJ5BBGxbo8PhaPY8VxkGDBjg3u7bty4R3f33309JSYn7dX09XykVmNpaWL0a/vUvePNN\n+OADOHAAuOZiuOoNv67xmfMxbwfwBjx0G5y3GYZcB99NbTjXwEUmtnyOUFwHhECHcIXiGmlpaeTk\n5PDCCy+497Vklm/9WorD4aB79+4N9vvSpUsXNnn0SJWXl7uDSWlpKcuWLaNjx47MmTOH1atXB1w2\npRTs2WNrAK+8Am+8Adu2+TjpJP+CQX2Fb8MFmyDj1/Ddoa0qZpPiOiAUFRWxfPnyBtPAi4qKwnaN\nESNGMH78eEpKSkhPT2fcuHFs3ryZJUuWALj6RXzyPLZ69Wrmzp1LTk4Od999NwMGDHA39TT2HJeC\nggIGDBhAbm4unTt3Jicnh/HjxyMiJCUl0bFjR6qrq5k1a1aDIW9KqcZ9+60NAIsXQ0kJ7N3r+zxj\nID0dPnRuf3+3+JpQ7Nuf/wxbZ8Lad9l5zDHBKHaj4noegmsIV15eHpmZmeTl5TW/JkiQr9G5c2cW\nLFhAQUEBycnJbNmyxasNv6mhnZ7HsrOzKS4uJjExkfLycubPn9/sczzfw9SpU8nKyqJnz54MHTqU\nkSNHMmrUKHdQyM7O5qGHHmLp0qWUlpb69d6Uaou2bbMTvs4/304CGznS1gjqB4Mjj4Rrr4XnnrPP\nWbmy7pjfweCRR2DGDBttQhwMQNcyigmLFi1i/vz5Xs1OkRTPn7VSvtTUwKJF8M9/2ntzba3v804/\nHS69FC65BAYOhIR6X7nNZPuFza92/r/+1T7KyuCEE1pVfl3LSCmlWmHfPtspPG+ebRby1RxkDAwe\nDFdeCZdfDkEbk/HYYzYYLFvW6mAQCA0ISinlYf16+Nvf4JlnGukYBs4+2642PWwYHHdckAvw8MO2\nmai0FE48McgXb5o2GamA6Wet4s3338P8+TB7Nrz/vu9z+vWD/HwYMQKOP75lr9Nsk9H998Pf/27b\npX72s5a9iK/X1SYjpZRq2ief2Hwyf/+77SeoLyUFrrvOBoLTTgthQUTg3nttR8Xbb4eg2uEfDQhK\nqTalthZef9020xcXNzx+8MFw2WXwm9/YTJTtgj8huGGB7rjDNhGVlcHRR4f4BRunAUEp1Sb873+2\nJvCXv+C1cqhLjx4wejRcf30Y78n798OoUbaqUlYGiYlhemHfNCAopeLatm3w6KO2aWjXLu9jCQl2\niOgtt0BWVsNhoiG1d69NbOCa4nz44WF8cd80ICil4tLmzTB9Ojz1FPz4o/exLl3sF/Obb4Zu3SJQ\nuO++s2NVO3e2Y1oPOSQChWgormcqR4NgZB7zN/tZgvPrTWuzpSkVyyoq4MYb7WqhM2Z4B4Pu3W3f\nwRdfwEMPRSYYHL0Hm98yLQ1eeCFqggFoDSEsWpt5zN/sZ67XCUa2NKViTUUF3HcfPPusc2VRD/37\nw/jx9kt5yDuJm9B9Jyx5Bvj9pXZUUZRlJdQaQpgtXLiQHj160K5dO3Jzc72Wwp49ezZJSUn07NmT\nOXPmuGsWDoeDu+++231eQUEBSUlJJCcnM23aNACGDh2KiJCcnNzgfNdrJicnM2bMmDC9U6XC4/PP\nbWdwr17wj394B4OMDNs8/+GHkJMT2WDAhx/yzt9g+mBg4sSoCwZAfGdMqzsW3EcgPDOPVVRUiDHG\nnT0tJyfHnc7SM6tZTU2NpKenu5/nmRVtwYIF0qNHD9m9e7eUl5dLQkKC1NTUiIjNflb//KaypbVU\nU5+1UuHy7bciY8eKtG/f8G90yBCRd96JdAk9vPaayFFHyaVXIUwK/98PfmZM0xpCGLkyl2VmZtKp\nUyemTp3qXvl09uzZ7qxmnTp1YsKECT6v4WoW2rRpE3379vXKeNbUa/bp04du3bqxYMECTYKjYtr3\n38MDD9gm+EcegZ9+qjuWkQHvvWdrBeecE7Eieps92y6J+sorvHJypAvTNA0IYbRjxw6vfAOpqanu\nlJUOh8PrWGM37WHDhjFu3DhycnJITk5m1qxZTb5mRUWF13X79u3rlTFNqVhRW2vXFzrpJPjDH8Az\ng+yZZ8LSpXYtuF/8InJl9FJbCxMmwLRp8O67cNZZkS5Rs9pEQAh2o1FLJScne2Uuq66upkuXLoAN\nAJ7HGsuEVllZSVZWFps2bWLVqlXMmjXLnb9AfBTOV7a0RYsWtfxNKBUB//mPvZ9edx14ZrXt1csm\np3n/fTuPIGr88ANcdZWtrrz/vp31FgPaRECIFsOHD2fOnDmUlpZSXV3N6NGjyc3NBSA3N5c5c+ZQ\nXl5OdXU1Dz74oM9rLFy4kJycHCorK6l1Lspe41yExddopoKCAvd1HQ4HOTk57Ko/O0epKPXVVzbJ\nzC9+4Z1g5phjYOZMWLcOrrgieP2zlZWV5Ofnk5mZSX5+PpWVlYFfZNs2mz3n4INtteXII4NTuHDw\np6Mhmh60oFM5kjw7lUVEFi1aJGlpaZKQkCC5ubnuDmERkTlz5khiYqL06NFD5syZ4+4Y9uwkFhHJ\nzs6WhIQESUpKkgkTJrj35+TkSEJCgpSXl3ud77puQkKC3HTTTa1+T9H6Wav48dNPIg8/LNKxo3f9\n/JBDRCZMENm9O/iv6XA4JC0tTQD3Iy0tTRwOh/8XWbtWpFs3kcJCkdraBoeZFN2dyhG/wQf6iLWA\n4C+HwyGrV692by9cuFCGDh0awRI1LtY/axXd/v1vkd69GzbWDhsmEsi9OVB5eXlewcD1yMvL8+8C\nr74qctRRIs8+2+gp0R4QtMkoSlRXV5OVleVu/pk1axY5OTkRLpVS4bNrFxQU2Oahdevq9p98sh01\ntHAhBJAOPWBbPTsnPFRVVTX9RBG7RkZBAbz6ql2fKEbpTOUo0a9fP+655x5SU1MxxpCbm8vIkSMj\nXSylwmLRIrvA3Dff1O077DCYPBl+9zto3z70ZejatavP/SkpKY0/6ccf7ay4dets53EY012GgmZM\nUwHTz1oFy1dfwW9/a0cKebrkEnj88fDeXysrK8nOzvYa4ZeWlkZxcTGpvqomVVW2R7tbN5tz87DD\nmn2NZjOmhYi/GdPC3mRkjBlmjMkyxtzVzHH9eqxUnBKxyetPPdU7GKSk2O2XXw7/l+3U1FSKi4vJ\ny8sjMzOTvLy8xoPB8uUwaJDNpPPPf/oVDGJBWGsIxph+QKqILDbGjAJWisiaescRkXJjTBaww/O4\n8xytIUSYftaqNb75BsaMgZde8t4/apRdgdQ5NScqXPzcxbzx2Rte+25cBQ+UwI2XwWu9WnZdrSFY\nuUC182cHMMTHOa5lOrvXDwZKqdi2eDH07u0dDFJTbU752bOjKxgAXsHg4P3w5Ktwx/twzm9aHgwu\n6nlRkEoXfOHuVO4C7PTYTvY86KwZOIwxOwFtMlIqTnz3HYwdC//3f977b7rJ1gqOOCIy5fKXjPwS\nhg+H44+B0n/wSRPrh8WyqBp2aozpDOwCHgDmGGO6RbRAQRCMZDUJrczrpwlzVCQtXw79+nkHg+OP\nhyVLbFrLaA8G51UCAwfanu7FiyFOgwGEv4awC0hy/twF2FHv+GhgiojsNsY4gOHA9PoXmTRpkvvn\njIwMMjIyQlHWoAhGsppwJdhRKphqa2HqVCgs9M5RcNVVNhBEOJ+8T159BgK3vw93/RtY/DQMHRrJ\nogWkrKyMsrKywJ/oz+y1YD2AfsBI5893AX2dP3d2/nun62fn9kgf12hqJl7UWb16taSnp7u3R48e\nLYmJiZKUlCQPPfSQe/+CBQvcS1qMGDHCvaRFdna2GGMkKSmpyfMcDoekp6fLQw895F7+ory8vEEZ\nXM9PSkpy52IIVLR+1ip6fPWVzUngOdO4U6cmJ/FGBddM4k7jkUUnIx+kINc/khnpYrUafs5UDmsN\nQWwfQbpzBNEuqes0XgoMFJHpxpi7jDEVQJKIzA3G67rG/gZDS0YHuL7hL1y4kNLSUrZs2UJFRQXp\n6ekUFBSwfft2RowYQUlJCenp6YwcOZK7776bmTNn8tZbb9GuXTt27NiBw+Fo9DyA1atXc/XVV7Nz\n507GjBnDuHHjWLJkibsMlZWVjB49mmXLltG5c2eys7NZvHgxV155ZdA+H6WWLoW8PLvGm8vgwfDc\nc3DiiZErl7/6fAVrSnrABRfAww8zKIpyHoda2Gcq+7rJi8hAj5+nhbdE4eOZ3KZfv37u5DazZs1y\nJ84BmDp1Kunp6e4bvYtngh1f5yUmJnLHHXcAdpXT0aNHu58rIixcuNCdLAdwJ+dRKhgOHID774dJ\nk+qWiTcG7rnH7jsozHcbX0NGmyRw42qYUgLMmRzTS1C0VJtYuiLcY34bM2zYMHbt2uVegnrChAnc\neeedPhPnuNY08tTceUlJSQ2e48nhcJCenu7e1kQ5Kli+/Rby8+Gtt+r2HXOMTXgfqTwFgQSDw/fC\nk69Dv6+gcNK5zGyDwQDaSECIFq7kNiNHjmTz5s0MGTKE/v37N5k4x5O/5zXGV7Ich8PBsGHDWviO\nlLIJ7K+8Er74om5fRgY8/zwce2zEiuXW7BfCdesgJwd+/nN4/HFmxsms45aIqmGn8a5+chtjDNXV\n1T4T54wYMaLB85tKsAO+M6a5GGMoKChg9uzZmixHBc3//R+cfbZ3MLjnHrs6aTQEgyaJwJw5kJkJ\n48fbN9OGgwGg+RBCzd/kNk0lznElvmnqPIfD4ZWIx/N1PX+ePXt2q5PlROtnrcJn716RMWO8RxF1\n7izy2muRLlmdJnMP1NSI5OaKnHGGyIYN4S1YBODnKCNd7VQFTD/rtm3bNjtp99136/b17g0vvhhd\nqYMbXVl0xQq4+mrIzoa//AU6dIhA6cIrWtcyUkrFsDVr7KRdz2CQm2tnI0dTMPCpttauk3HJJfbf\nmTPbRDAIhHYqK6X8snixTXj//fd22xiYMgXuvjt4Se5D5quv4Prr4YcfYOXKmE9kEypaQ1BKNUkE\nHnwQhg2rCwadOtlskePGxUAweOUVu5jS4MGwbJkGgyZoDUEp1aiffrKpgp9+um5fWpoNBqecErFi\n+aXDTzD9LeDp39scnb/4RaSLFPU0ICilfNq1y84v8Fwj7dxzbdNRcnKjT4sOq1axehasSsF2fHTu\nHOkSxQRtMlJKNbB5s/1C7RkMbrjBzi+I6mBw4IDt2PjlL5mcAfnD0GAQAA0ISikvH34IZ54JGzbU\n7ZsyBZ56Ctq3j1y5mlVZaadIv/UWfPgh/zw90gWKPRoQQmzRokU+Zx37o7WJcZQK1L/+BeedV7dS\nafv2dgmK8eOjuPNYxEarQYPgiitsPk7tOG4R7UMIg5YmuGltYhylAvH00zByZF0ym8REePllOOec\n8Lx+wKuTAkfvgVmvQrdqyM+B9d/dAUV3hKiE8U+/goaBiDBixAgSEhIYOHAglZWV7mNLly6lR48e\nJCcnk5uby+7duwEYOnQoIkKys8F29uzZJCUl0a5dOwYOHMjmzZsj8VZUHBKxTUI33FAXDE48Ef7z\nn/AFAwhsdVKAKz+Gj56EDUfBmaNg/TENz4nmhPZRyZ/1LaLpQYytZbRw4UIxxsjcuXOlpqZGCgoK\n3NnLdu3aJYmJiVJaWio1NTUyZswYycnJcT/XuUyHiIgkJCTImjVrpKamRnJycmT8+PFhfy8u0fpZ\nq8AdOCAydqz3mkR9+ohs3Rr+sjS59pCnHTtErrlGpGdPkfffD33B4gDRmDEtooLV/NKCNXzS09O5\n8cYbAZg5cybt2rVj9+7dLFiwgOzsbHfCmylTptC9e3ePIteV2ZVMB2zeg+rq6ta8C6XYtw9+8xuY\nN69uX2amXZMoagfmvPYajBljx8OuWaOrkwZZ2wkIEVyMzfMmDzaxjcPhoKKiggULFrB06VLA1tZc\nTUb13X///ZSUlLibkOpfU6lA/PADjBhh768uw4bZhDZRmTGyuhrGjoV33rERLCMj0iWKS9qHEAYV\nFRVe25WVlXTv3p20tDRycnLYsWMHO3bsYOfOnV4JbFxcuZiXLVvGkiVLGD58eLiKruLQ7t1w4YXe\nwWDUKHjhhSgNBq+/DqefbmsDa9dqMAghDQhhUF5ezuLFi6murqagoIDs7Gw6derEiBEjWLp0KSUl\nJe5jBQUFDZ6/a9cukpKS6NixI9XV1cyaNYudO3dG4J2oWLdjh01p+c47dfsmTIBZs6Bdu8iVy1Nm\nZib5+flsWbMGfv1ruPVW+Pvf4Ykn4IgjIl28+OZPR0M0PYjBTuUxY8bI0KFDxRgjF1xwgVfym5KS\nEnfCm/rHPBPjZGdnS2JiogwYMEBKSkokKSlJSkpKwv5+RKL3s1ZNq6oSOe007w7khx6KdKksh8Ph\n7lQG5FKQr9q1k5prrxX57rtIFy/moQlyVKjoZx17Pv/c1gxcLZLGwJNP2oXrokF+fj7P9nyWI/8H\nj06DAcBI4Gd5eczz7PVWLaIJcpRSgF3R4bzz6oJBu3a2XzZaggHA1i+/5Kr/wn+fgK1AH+AdoKqq\nKsIla1vazigjpdqgTz+F88+HrVvt9sEHw/z5cPnlkS2Xly++4M+ffcZBG+DSq2Hl3LpDKSkpkStX\nG6Q1BKXi1IYNtmbgCgaHHGKXooiaYFBbazuK+/enW24u6aNh5fF1h9PS0igqKopc+dog7UNQAdPP\nOvqtX29rBq5F6g47zCYOy8qKbLncPv7YjnUFmDMHTj0VM9k2cWe+nUlKSgpFRUWkpqZGsJDxw98+\nBG0yUirOrFtng8G339rtww+HN96wyW0ibu9eu3DSjBkwebKddVxvVd/S0tIIFU7FTUA48cQTdXXQ\nMDnxxBMjXQTViP/+1waD7dvtdseOdknrqMge+fbbtif75JOhvByOP77556iwipuAoKt/qrbOVTPw\nDAZLlsDPfx7ZcrFzJ9x9ty3Mo4/anAUqKmmnslJxwNVn4AoGnTrZdJcRDQYi8MwzcOqp0KGDLaQG\ng6gWN53KSrVVGzbY5X1cHcgdO9pgcOaZoX3dphLa9NwOT74OiT9AwSXwYVf/rysT9e872LRTWak2\nwDXPwDMYLFkS+mAAvhPaHLIPJrwHt6yA+8+FxwbBgQDWSNKENpGlNQSlYlRFhfc8gyOOsMFg8ODw\nvL5rmKj7G/1bb8Ett0CfPvDII9ppHEW0hqBUHNuyxXsG8mGH2aGl4QoGXrZuhdtvh5Ur4bHH4OKL\nI1AIFQzaqaxUjNm61QaDzz+324ceCq++Gt78xwAHHYDb/oOtEZx0kh3mpMEgpoW9hmCMGQZUA/1F\nZJqP4/2A7gAisijMxVMqqm3bBkOGgMNht9u3h5desgEirN55h9Wz4OsjgH//G3r1CnMBVCiEtYbg\nvNmLiJQA1caYvj5Om+AMBKmNHFeqTdq5E7KzYeNGu33QQbBwIVxwQRgL8fXXcO21kJ/P5PNg6LVo\nMIgj4W4yysXWDgAcwBDPg87awwoAEZkuImvCWzylopMr7eXatXY7IQGeew4uuSRMBdi3D/7yF5vK\nsmtX+PhjFp0G6OIAcSXcTUZdAM/cj8n1jg8ExFmTGOKrSUmptub77+2Nf+VKu20MPP005OSEqQBl\nZfDb30JKCrz3ntYI4lg0jjLaISLlxpghxphhvvoRJk2a5P45IyODDE26reLU3r0wbJh3DuQnn7St\nNiH3xRdw552wfDn8+c9w5ZU2GqmoV1ZWRllZWcDPC+s8BGPMFKBYREqdzUOpIjLd4/hdQIWILDbG\njAK6i8iEetfQeQiqTdi/H666ChZ5fCV6+GE7wjOkfvzRBoCHH7bzCsaPt+Na62kwD0FFrWhNoTkf\n5wgi579LAYwxnZ37Fnoc7wKsDGvplIoStbU2XYBnMJg4McTBQMQmTTjtNFixwrZR3Xefz2Cg4lNY\nm4ycTUHpxpgsYJdHp/FSYKCIVBpjqp21hyTtQ1BtkYi98T/9dN2+226zASFkNmyAsWNtM9HMmXY4\nk2pzdOkKpaLM5Mng0U3GjTfapGIhab7ftcu+4LPPwj332M7jgw/266naZBQ7orXJSCnVhEcf9Q4G\nw4fDrFkhCAYHDtgLn3wy/PCDTWl5221+BwMVn6JxlJFSbdI//gG//33d9tChMG8etAtgtVC/lJba\n5qHERLsaXl+d/6ksDQhKRYFXXoHf/KZu++c/h8WL4ZBDgvgimzbBXXfBRx/BtGk6jFQ1oE1GSkVY\nWRmMGGFbccBOBn79dTj88CC9QE2NDQRnnQWDBtnmoWHDNBioBjQgKBVBq1fDpZfaCWgAaWm2FScx\nMQgX37/fzmLr1ct2Hq9bBxMm2OVRlfJBm4yUipBPP7XrE333nd0+7jib+vK444Jw8SVL4I474Oij\n4c03tZ9A+UUDglIRsHWrHer/7bd2u0sXm3AsNbWVF163zjYPVVTA9Ol2ESRtGlJ+0iYjpcJs5047\ngsiV4MaV7ax371Zc9OuvYfRomxjhwgttYLj0Ug0GKiAaEJQKo//9D371K9uvCzanwYwZXzNjRj6Z\nmZnk5+dTWVnp/wW//x7+9Ce73ETHjvDJJ3bsavv2oXkDKq5pk5GKqMrKSgoLC9m6dStdu3alqKiI\n1Fa3m0SnffvsktXvv2+3jYHp07fxpz+dTUVFhfu85cuXU1xc3PTncOCAnaTwxz/aMaorVtgeaaVa\nQQOCipjKykqys7MDvxnGoNpauOEG+Ne/6vb99a/wwQe3e71/gIqKCgoLC5k3b57vixUX236Cww6D\n+fNtQGilthSYVRNEJKYetsgqHuTl5QnQ4JGXlxfpogVVba3I2LEidtk6+ygstMcyMjJ8fgaZmZkN\nL7R2rcgFF4j06CGycKG9cBA4HA5JS0vzev20tDRxOBxNPo9JCJP07zEWOO+bzd5ftQ9BRczWrVt9\n7q+qqgpzSULrwQfhkUfqtgsK7HpyAF27dvX5nJSUlLqNL7+01YshQ+Dii2H9+qBOLCssLGy0lqLa\nFg0IKmJH4x9yAAAbrElEQVT8uhnGuLlz7SKiLsOGwYwZdffyoqIi0uq1/aelpVFUVATV1XYiWZ8+\nNn3lp5/CrbcGvcO4rQRm1bwWBQRjzJ3GmCeNMSONMZ2CXSjVNjR5M4wDL71kawMu559vV5n2XKwu\nNTWV4uJi8vLyyMzMJC8vj+LXXiP1pZfgpJPgm2/s2kP33w+dOzd8kSBoC4FZ+cevfAjGmPOBnSKy\nxhjzoMeh/kAWMFpEngpRGeuXRfwps4oNrs7MqqoqUlJS4qYz85137FwD15IU/fvDsmXQqamvT7W1\n8PzzduRQ794wZUorJyf4x1fnflpaWrOd+5oPIXb4mw+h2YDgDAbdRWSuc9sr8b0xpgswHlghIotb\nV+zmaUBQ0e6jj+C88+yacgA9esC//21XkfBJxI4cGjfOLm86daq9QIhd/NzFvPHZG62+jgaE6Odv\nQPBn2Gk6UNzYQRGpBsYbY0YFUD6l4pLDYScKu4LBscfaJSkaDQYffmiT2H/xBTzwQFiXpA5GMLio\n50VBKImKFs0GBBGZZowZZYypFpHNgMMYsxK4G1glIrudp+4IYTmVinrffAMXXGBXkQDb5L9kSSPr\nE332GfzhD7bqcO+9NhlChLKV6Td85eJXp7KIzHEGA4BcYD5wE7DZGLPSGLME6G6M6QhgjLkyFIVV\nKlrt3g2//KXNQQN2helXX4Uzzqh3YlUVjBljJ5P17WtHDhUUaOpKFRVaMsqoAlggIiNEJAkYDSwF\nhgJbjDGfAVODWEalotqPP8Lll0N5ud1OSIB//hPOOcfjJNcQ0tNPr1tz6J57gpgFR6nWCzggiMgc\nINEY0825XS4i00RkqDNA5AIBrM6lVOw6cADy8uwIIpc5c+Cyy5wb339vO4l79rRrXa9ZY9NXJidH\npLxKNaVFaxmJSHkTx1YbY8a1vEhKxQYRuPlmm/vY5cEHnbmR9+2Dp56CoiIYPBjefRdOPjliZVXK\nHyFZ3K6pgKFUvPjjH2H27Lrt226Du++shef+aTuKu3e3s9MGDoxcIZUKgK52qlQLPPKIHSXqkp8n\nTM94HdP/D7ZHefZsOzVZqRiiAUGpAM2bZ2sDLuPOLOOBintIuGe3TVZz2WWaqUzFJA0ISgXgjTfs\nwqMAA1nBjM5/IH2bg4Si++Cqq7wXKlIqxmhAUMpP771nVyvttX8dRRQy+KCVHDGxkITfRm5SmVLB\npMtfK+WHtWvh1l9uYs6P+ZSQxceJZ3Ng42ccfptOKlPxQwOCUs3Y8t4XrD1rNMV7zuITevHzIzcx\n4oM7SEnrEOmiKRVU2mSkVGO++YY9f5hCp789wxe1ozmJTznQKYm3i+08M6XijQYEperbsQOmTaN2\n9hxeTLiWO2s/ZhvHcOihsORVuwSRUvHIrwQ50UTzIUS/YK2zH26dfoSxy+HWD2DhqXD/ufBlaJKU\nRRVd7TT+BTMfglIBibVgcPhe+O0KuP19+FdPGDQKKpMiXarw0HwGylPYA4IxZhhQDfQXkWlNnHdX\nU8dV9Iv6b54//ABPPglPPgSZmbBqInk9TublEVD5aN1pjz8Ot9wSuWIqFS5hHWVkjOkHiIiUANXG\nGJ+tscaYLGBIOMum2pC9e+1dvkcPO7mguBief57ak07mxhvhxRfrTr3vPg0Gqu0I97DTXGztAMCB\n3vRVOO3bZ9cYOukkePNNm8Fm8WI4/XREYOxY+Mc/6k6/4w67gJ1SbUW4A0IXYKfHdoNF4Y0x/Zw1\nCF0MRgXH/v3w9NPQqxcsXAgvvACvvQb9+7tP+eMf4bHH6p4yapRNW6BLEqm2JBo7lRMjXQAVJw4c\nsKnLJk+G446zQeHccxuc9sAD3iuX5ubargUNBqqtCXdA2AW4xm90AXZ4HnTWDkqdm432SE6aNMn9\nc0ZGBhkZGUEtpIpxtbW2JjBpEnTpYu/u55/v8w7/6KM2173Lr34Fzzyja9Sp2FZWVkZZWVnAzwvr\nPARnp3K6iMw1xtwFFIvIGmNMZxGpcY5AEmxTUgEwUkTW1LuGzkOIcmayvfGGfZSRiO0RnjgROnSw\nNYMLL2z0q/5TT8HIkXXb558Pr79u0xkoFU+ich6CiJQbY9Kdo4h2edzslwIDRWQRgDFmFNAGpgSp\noBCxfQITJ9rtKVPg4oubbPOZN8/2E7gMHgwvv6zBQLVtOlNZBV3YaggidrTQvffCTz/ZJqLLL2+2\n8X/hQttPUFtrt3v33sspp9zKt99+RteuXSkqKiI1NTW0ZVcqjKKyhqBUUIjYuQP33gvffWebhq68\nEhKaHzT36qtw9dV1waBXr5/Ys+cXLFiwyn3O8uXLKS4u1qCg2hxd/lrFDhEoKYFzzoHf/c5OHFi7\nFoYP9ysY/Otf9tT9++32ySfDaaeNZfPmVV7nVVRUUFhYGIp3oFRU04CgYsPbb0NGBowZAwUFsH59\nQCkr33oLrrjCtiwBdO8OS5fCzp0bfJ5fVVUVpIIrFTu0yUhFt/fes53FW7ZAYSHk5cFBgf3alpba\nvPd799rtbt1g2TLo2hW6du3q8zkpKSmtLLhSsUdrCCo6LV8OQ4fCtdfaILBhA1x/fcDBoKwMLrkE\nfvzRbp9wgg0GJ5xgt4uKikhLS/N6TlpaGkVFRUF4E0rFFq0hqOiyYoWtEXz8sV1P4vrroX37Fl2q\nrAwuusguagpw/PG2ttCtW905qampFBcXU1hYSFVVFSkpKTrKSLVZOuxUBV2Lhp2uWmUDwUcf2anD\nN9wAhxzS4jLUDwYpKXafpr5UbZG/w061yUhFVnm5beC/7DI7q3jTJttx3IpgUFqqwUCpltCAoCJj\n7VoYNszOKM7KsoHgt79tVSAAO5ro4os1GCjVEhoQVHitWwc5ObbDePBgGwh+97ugrBnxxhtw6aV1\nHchdu2owUCoQGhBUeGzcaKcIZ2XBoEFQUWEz0Bx2WFAu/8ordtUK19DSE06wUxc0GCjlPw0IKrQ+\n+8wOHT33XOjTxwaCu+6Cww8P2ku88IJtfdq3z25362aDQb3RpEqpZmhAUCGRuhM7UmjwYJuyctMm\nGD8ejjgiqK/zt7/BNdfULUeRlgbvvOM9tFQp5R+dh6CCa8sWZr8CV2wA7j7R1hC6dAnJS82YYfuh\nXU45xS5HoZOMlWoZrSGo4PjyS7jpJujfn22Hw0m3UpexLMhE4P77vYNB3762mUiDgVItpwFBtc5X\nX9lRQn36QKdO8Mkn/DELdgWnr7gBEdsF8cc/1u0780w79+Coo0Lzmkq1FRoQVMts22ZHCZ12ml1f\n6OOPYepUOPLIkL3kgQM2y9nDD9ftO/98mxohMTFkL6tUm6EBQQVmxw6YMME22O/da+cV/PnPcMwx\nIX3ZH3+00xeeeqpu3+WX2xzIHTuG9KWVajM0ICj/1NTYtYZ69YKdO+2SE48/HpZG+5oau6rFiy/W\n7bv+eliwQHMgKxVMGhBU0/bsgQcesDO8Pv/crkY6a1bd+tEh9tVXcN55tsPY5fbb4f/+L+CVsJVS\nzdCAoHz74QfbFNSjh20WevddO+i/e/ewFWHjRjuN4aOP6vY99JDtQ/AjY6ZSKkD6HUt5++knmDvX\njus880w7sL9377AX47337LpEu3bZ7XbtbLF+/euwF0WpNkMDgrL274dnnoHJk22H8SuvQHp6RIqy\ncCHk59etS3TYYTB/vl3FVCkVOhoQ2rraWnu3nTgRjjsO5s2Ds88G4OLnLuaNz94IW1FEYPp0uPvu\nun1HH21HEg0YELZiKNVmaUBoq0TgtdfsDK9DD7UjhoYMAVOXVKk1weCinhcFdP6+fXbm8ezZdft6\n9oQ33wxrt4VSbZoGhLZo2TK45x47guhPf7KN9abx7HoBpcJsgepqyM21yW1czj7bDjMN4Tw3pVQ9\nGhDakpUrbb7iigrbV3D11ba3NoI++wwuuQQ++aRuX16enYDWyuRpSqkA6eC9tmDDBpsw4Ior7L8b\nN9pe2wgHg5ISO5DJMxhMnGj7tjUYKBV+GhDi2eefw29+Y2d2nXWW/TpeUAAHHxzRYonYpasvuKBu\nWOmhh8Jzz9kFUptovVJKhZA2GcWj7dvt7OK//x3GjIFPPw1ZToJA/fgj3HyznePmctxx8PLLMHBg\n5MqllNIaQnzZswfuu8+uN7R3L6xfbyeYRUkw2LrVVlY8g8GAAbZrQ4OBUpGnASEe7NsHTzxhx2lu\n3GjXG5oxA449NtIlc3v7bTvPbcWKun3XXWfTXXbtGrlyKaXqaECIZSJ2Utmpp9o2lzfesA3xUZRd\nXgSmTYOsLPjmG7uvXTt45BF4+mno0CGixVNKedA+hFhVVman9B44ADNn2jtulKmutn3anstWH3UU\nvPACZGZGrlxKKd+0hhBr1q+HX/3K3mlvu802wLcwGFRWVpKfn09mZib5+flUVlYGrZgrV0L//t7B\nYPBgm0ZBg4FS0SnsNQRjzDCgGugvItN8HB/l/DFNRMaHtXDRrKrKDtJ/+WU7y3jRolYN1q+srCQ7\nO5uKigr3vuXLl1NcXExqamqLrysCjz5q8x7v21e3//e/t0tXt2/f4ksrpUIsrDUEY0w/QESkBKg2\nxvStdzwLKBaROUB3Y8z54SxfVNqzxw7OP/10mzj4k09g7NhWz9wqLCz0CgYAFRUVFBYWtvia27fb\ntJZjx9YFg06dbGazRx7RYKBUtAt3k1EutnYA4ACG1Dve3WOfw7ndNh04YNdv6NXLziNYtcp+xQ5S\nNvmtW7f63F9VVdWi6y1dCmecYVfNdklPh9WrYfjwFl1SKRVm4W4y6gLs9NhO9jzorBm49Af+GY5C\nRZ2SEpsnslMn2wg/aFDQX+LTgZ9CRsP9y1iGmez/VOG9e+2CqdOne+///e9h6lRdgkKpWBKVo4yc\nTUurRGRNpMsSVp98AnfeCR9/bGsDV17ZqnUcmsxncLh/12hqGeuPPrJLIq1bV7fv6KPtcNJf/tL/\nciqlokO4A8IuIMn5cxdgRyPnZYnIhMYuMmnSJPfPGRkZZGRkBKl4EbJrl51h/MwzMG6cTRkWhK/W\nzeUzyOiaQdeyrlRVVZGSkkJRUZFfHcr799sawb33enccX3ihDQbHHNPKgiulWqWsrIyysrKAn2dE\nQrvWvdeL2W/+6SIy1xhzF7YDeY0xprOI1DjPGeVqOjLGZDk7oD2vIeEsc0jt3w9z5thO4yuusEHh\n6KODdnlX008w8xmsXw833GCHlbp06GADxE036cJ0SkUjYwwi0uxfZ1g7lUWkHNyjiXZ5NAkt9dj/\noDFmkzFmBxAnd34fyspsr+v8+VBcbCeXBTEYBNv+/TBlip1b4BkMBg2CNWvsgnUaDJSKbWHvQxCR\nuT72DXT+W0K9jua4s2WL7SdYudJ+rR42LOrvpB9+CKNG2Ru/S/v2tslo3Dg4KCp7opRSgdKZyuHy\n4482XWX//tC7t01aM3x4VAeDPXvsYKczz/QOBgMH2uGkf/iDBgOl4okGhHB47TU47TR7F121CiZO\npPLrr0O2bERridjRrqeeCn/5C9TW2v2HHmqHkv7nP/btKKXii36/C6XNm+2A/A0b7PLUF1wAhG7Z\niGBwOODWW+3CqZ6GDLHdHFG0kKpSKsi0hhAKP/1kM5alp9v2lf/+1x0MIDTLRrTW//5nJ5ideqp3\nMDjySJt47a23NBgoFe+0hhBsb79tx1927257Y3184w/2shGtIWJTKIwbZzOauRhj0y/ffz8kJTX+\nfKVU/NCAECw7dtjRQ0uXwl//aucVNNJh3LWRFGEpKSmhLGED770Hd9zhncUMbKVmxgxNa6lUWxPW\niWnBEHUT00Tg+eftnXXECDuSqGPHJp/i1YdwDXBSiItYb2Lap5/ChAmweLH3ecccAw8+aFNbJmhj\nolJxIyonpsWdLVvgoovsXfTll23NoJlgAJCamkpxcTF5eXkhDwaeaxF9+SWMHm37CTyDwSGH2ORr\nn34Kv/61BgOl2iqtIbREba0dcjNxos1adtddcPDBLbpUKJaXqG/bNpvX+PHH7XQIT9dcY/sJunUL\n2csrpSLM3xqC9iEEatMmuPFGu6rbO+/AKadEukSN2r69LhB8/733sfPPtxUb7SdQSrloQPBXba3t\naZ082U7R/d3voF27gC/T5JLUQfLVV/DnP8OTT9rhpJ4GDLBrEg2pn5pIKdXmaUDwx+bNNqn9Dz/Y\nabontbzh31cwaCrnQCAqK+3ySE89ZRPXeDrjDLuo6uWXR/VqGUqpCNKA0BQR+Mc/7HDSu+6yI4ka\nqRUE+s0/mH0Gq1bZpqEFC+qWmXDp3btudW3tLFZKNUUDQmN27rQzszZutCktzzijydMDCQbBqBEc\nOACvvmoHNvnKgzFwoG3ZuuQSDQRKKf9oQPBl2TI7GD8nx2YxO/RQn6f5qhWEcrQQ2ORqf/ub7Sj2\ntR7ekCF2COmQIdo0pJQKjAYET/v326xlc+faXJBDhzZ5ev1gEKy+AF8+/NCuj/f88w2HjrZrB7m5\ntmWrX7+QFUEpFec0ILh8+aUdlH/IIXaZ6mOP9fupoaoVVFfbdYbmzoXy8obHk5LsRLObb4af/Swk\nRVBKtSFtKiA01vGbVQHzFsOjZ8KDg0FmHReB0lkHDkBpqV1hdNGihrUBgL59bRDIy4PDDgt/GZVS\n8alNBYT6wcDUwvj34Lcr4OrhUNaCVATBaCYSsStkP/cczJvnveqoy6GH2i6Nm2+2Gcy0f0ApFWxx\nGRCaGwIqEwV277Ydxz9sg40LWNbICqShtGkTzJ9vA8H69b7P6dPH5jO+5hpITAxv+ZRSbUtcBoSm\ngsFFPS+yacEuvRTOPtvekdu3D1vZPvnENgUtWOCdp9jTUUfZAHDddbaTWGsDSqlwiMuA4OKzs/ed\nd+AXv7CD9G+5JeR329paO0LopZdsnuKNG32f16EDXHYZXH01/PKXLV4rTymlWiyuA0IDzzxjZxs/\n+yxkZ4fsZWpq7Fy2116D11+3q4360r49XHihTaNw2WVwxBEhK5JSSjWr7QSE6dPhscfstN5TTw3q\npQ8csCNVi4vhzTftckcHDvg+t0MHm155+HA7i7hTp6AWRSmlWiz+A0JtrV2H6M034d//huOPb/Ul\nReDjj21sKSmxE5urqxs//6ij4OKL7cJy2dk6VFQpFZ3iOyAcOGBzF2zaBO++2+Js8fv3w0cf2RzE\n775ruyG+/bbp56Sn2+agSy6x6wrpekJKqWgXtwHB1AIjR8Lnn8NbbwX0tXz7dvjgA3j/fftYsQL2\n7Gn6OccdB1lZtjlo6FA4+ujWlV8ppcItLgOCqYXZrwKdKm2vbhPBoLraLguxahWsXGkfvhaNqy85\nGc47DzIy7EJyJ5+sw0OVUrEt/nIqizBzYAKnfgvnrv/OPXRHxFYW1q614/8/+sj+W1Hh3+sefzyc\nc46dunDOOXDaadoMpJSKDW02p7I8NI2BVZDxayiaewQffwzr1tnHd9/5d4327e2EsLPOgp//3D5+\n9jOtASil4lvM1hCqq+Gzz+oen34KRy9/hTsrb+bMsVvZ2hmY1Px7O/hgm1Wsf3/bETxoEJx+elgn\nLyulVEjFdQ3huuvsHDNPvfkvJYzkV7zG1s5n+nxeUpK92aem1rBx4wvs27eKnj338sADE0lNbcHK\ndkopFUdiMiDUT1WQyE5e4VLG8ggrGeTeP2qUnYN26qk2EBx7LGzeXEl2djYVzs4D25n8HsXFxRoU\nlFJtWkw2Gc2eLdx6K/ToAT17wu+qxnNc+x18+8AcTj4Zjn7C1ox8rWWUn5/Ps88+22B/Xl4e8+bN\nC3n5lVIq3OK6yej66+18s4QE4Ntv2XnCVPqOgS9K50Jp08/d6ivZAFBVVRX8giqlVAwJe0AwxgwD\nqoH+IjIt0ONQr8P34Yd5oTd80cX7nMYS13RtJO9BSkqKH6VXSqn4FdYmI2NMPyBVRBYbY0YBK0Vk\njb/HnefUzUPYvh1OOslOKDjhBL/KUFnp3YcAkJaWpn0ISqm45W+TUbinVuViv/0DOIAhAR739vDD\nkJvrdzAASE1Npbi4mLy8PDIzM8nLy9NgoJRShD8gdAF2emwnB3i8zvbtlM2YARMmBFyI1NRU5s2b\nR2lpKfPmzYv5YFBWVhbpIkScfgaWfg6Wfg4t+wxid/GFxx6j7KSTAqodxCv95dfPwEU/B0s/h5Z9\nBuHuVN4FuNag7gLsCPA4AJMmTYJ9+yg75BDKysrIyMgIQVGVUio2lZWVUVZWZu+VAQh3QJgPpGMH\nh3YHigGMMZ1FpKax4/W53uSkSZM0GCilVD0ZGRlkZGS475WTJ0/263lhn5hmjBkJVGJHE8117lsp\nIgMbO17v+bE1k04ppaKAP6OMYm6mslJKqdCI3U5lpZRSQaUBQcUcY8wwY0yWMeauZs5r8rhS8cQ5\nsbexY379zcRMQGjuDfn7hmOZH5/BKOfjwXCXLVycv/QiIiVAtTGmbyPnZdHcxMYY58fvQz/nOcPC\nXbZwCeC+MDLcZQsn5+/7gkaO+fU3AzESEJp7Q4G84Vjlx2eQBRSLyByguzHm/EiUMwwCm80ep/z8\nnZ8gIouA1Db6N9EPcDiPV8bjZ+DifI+NJQT2+28mJgICwV7yIjY19x67e+xzOLfjUbOz2Y0x/Zx/\nIPGc9LTJ3wdnrWAFgIhMr78mWJzw5+9+qvPf7nH6GfjD7xUgYiUgBG/Ji9jV5HsUkTkew3T7Ax+G\nq2BRKDHSBQiD5n7nBwLJzmajeG1Gbe5vohxwGGN20sgkV+UtVgKC8pOzmrwqjr8NNTmb3Vk7cGXF\naOtjqnc4b4quGkObYozpjP19eQCYY4zpFtECRY5fK0BA7ASEoCx5EeP8fY9ZIhL4in+xYz51zWHd\ngaXg/uMH239ypXP59OQ4bjdu7vdhB7YZBWyzyoAwlSucmvsMRgNTRGQ6MAoYHsayRYJXE6nH34TP\nvxlfYiUgNHcT8PsNx7DmPgOMMaOcv/yuTua44/GNNwvY5VETWuo8vkhEFjv3dfZxiXjR3O/DQo/j\nXYCVYS1deDT3GQjOm6Tzd6K6/gXihbMGmG6MudJjt+tvorG/mYbXiZWZyq1d8iIeNPUZOP+z52O/\nNSUCOR5NJyoO+fk3sQsYEK+1Rj8+g7uwo2+S4vW+EEwxExCUUkqFVqw0GSmllAoxDQhKKaUADQhK\nKaWcNCAopZQCNCAopZRy0oCglFIK0ICglFLKSQOCUkopQAOCUkopJw0ISimlADgo0gVQKtY5F1Mb\ngV1tMxlYKiKVkS2VUoHTGoJSrTcHeMG5ouYOoM3lHlDxQRe3U6qVjDGbsMtNv+BaalipWKQ1BKVa\nbzjO9fiNMfGYd0C1ERoQlGoFY8wwEVkjIiNEJBlIbMOpGlWM04CgVAs5O5PH19tdISKbI1AcpVpN\n+xCUagVnysJqbJa6RGC+iOyObKmUahkNCEoppQBtMlJKKeWkAUEppRSgAUEppZSTBgSllFKABgSl\nlFJOGhCUUkoBGhCUUko5aUBQSikFaEBQSinl9P/2F6UkV9OPzQAAAABJRU5ErkJggg==\n", 143 | "text/plain": [ 144 | "" 145 | ] 146 | }, 147 | "metadata": {}, 148 | "output_type": "display_data" 149 | } 150 | ], 151 | "source": [ 152 | "from betacal import BetaCalibration\n", 153 | "\n", 154 | "# Fit three-parameter beta calibration\n", 155 | "bc = BetaCalibration(parameters=\"abm\")\n", 156 | "bc.fit(cal_probas.reshape(-1, 1), y_cal)\n", 157 | "\n", 158 | "pr = [lr.predict_proba(linspace.reshape(-1, 1))[:, 1], iso.predict(linspace), bc.predict(linspace)]\n", 159 | "methods_text = ['logistic', 'isotonic', 'beta']\n", 160 | "fig_map = plot_calibration_map(pr, [scores, y_c_2, linspace], methods_text, alpha=0)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": {}, 166 | "source": [ 167 | "Beta calibration was able to find the inverted sigmoid shape, as isotonic regression did, resulting in a much better fit for the scores than logistic calibration. Now let's see what happens when a classifier outputs probabilities that tend to be concentrated around mean values.\n", 168 | "\n", 169 | "## Probability estimation with logistic regression\n", 170 | "\n", 171 | "Below, we train a logistic regression using the same spam dataset. We can see that its probability outputs are concentrated around 0.5. This distortion is what logistic calibration works on best." 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 4, 177 | "metadata": { 178 | "collapsed": false 179 | }, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD/CAYAAADllv3BAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADj5JREFUeJzt3c9uHNeZxuH3HQuw4gkgWhqD2gQWaXgvysy+YXIwezOJ\nbkBUZi6AQLwyV4kG1gV4Il1ARp5oPYAsotdjGyZm64h0EBiREPNPAmRhjIJvFn0olTrN7upms6v4\n8fcAhKpOFas+lrpeHp6q6nZECACQxz80XQAAYLoIdgBIhmAHgGQIdgBIhmAHgGQujFrB9kqZ/OeI\n+EVpW5N0KOlGRHx8XBsAYPaG9thLqP8kIh5LumH7uu0lSVHaDmwv9bUd2r5++qUDAAYZGuwR8Tgi\n/q3MLkTEtqSb6vXMJWlX0mpf205pAwA0oNYYu+0NST8vs3OS9iuLr0i6NKANANCAWsFexsz/1fal\nU64HAHBCQy+eVsbOt9UbYrkt6UDS5bLKnKTvJEVf296AbfHeBQAwgYjwOOuP6rGv6tXAfiLpPyUt\nlrZFSZ8d0zaoOL4i9NFHHzVeQ1u+OBYcC47F8K9JjLrd8T8k/cz2O71cjoeSZHu53DFzEL3e/MA2\nAMDsDQ32iPiLpPsD2mu1AQBmjydPG9DpdJouoTU6nY6uXr0m241+Xb16relDweuigmNxMp50DGfs\nHdkxq33hbLGt3vX3RquYeDwTOE22FVO+eAoAOGMIdgBIhmAHgGQIdgBIhmAHgGQIdgBIhmAHgGQI\ndgBIhmAHgGQIdgBIhmAHgGQIdgBIhmAHgGQIdgBIhmAHgGQIdgBIhmAHgGQIdgBIhmAHgGQIdgBI\nhmAHgGQIdgBIhmAHgGQIdgBIhmAHgGQIdgBIZmSw214vX3cqbXeOllXa1myv2N44nVIBAHUMDXbb\nK5IeRcQ9SYu23y+Lbtv+WtKTst6SpIiIx5IObV8/zaIBAMcb1WNflLRapnfKvCTdioh3I2KrzN+U\ndFhZb1UAgEZcGLaw9NSP3JD0mzK9WHrzNyLiY0lzkvYr616ZapUAgNqGBvuRMtTyZURsS1JE3C3t\nqyXgAQAtUfeumJWI+FB6cTH1g9K+L2lB0oGky6VtTtLeVKsEANQ2ssdue73SQ1+R9Ll64+iS9I6k\nTyR9IWlZ0pZ64/CPBm1rc3PzxXSn01Gn05m8cgBIqNvtqtvtnmgbjojjF/aC/IF6PfI3Jf00IrbK\nbY77khYqoX9L0m5puz9gWzFsXzi/bEtq+rVh8fpEG9lWRHis75nVi5lgx3EIduB4kwQ7T54CQDIE\nOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAk\nQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7ADQDIEOwAkQ7AD\nQDIEOwAkc6HpAoB2eF22G61gfv5tPX36TaM1IAdHxPAV7PUy+U5E/KK0rUk6lHQjIj4+rq1vOzFq\nXzifeoHa9GujHTVwjqCfbUXEWL2OoT122yuSHkXEN7Yf2H5f0oGkiIjHthdsL5XVj9oWbV+PiO3J\nfgwAwEmMGmNflLRapnfK/E31euaStFuWV9t2Kt8DAJixoT32iLhXmb0h6TeSliXtV9qvSLo0oA0A\n0IBad8WU4ZYvGV4BgPare1fMSkR8WKYPJF0u03OSvlPvqlO1bW/QRjY3N19MdzoddTqd8aoFgOS6\n3a663e6JtlHrrpijIZlyMXVP0nJE3Le9IelRWfWVtv7ePXfF4DjcFfOyBs4R9JvkrpihQzElyO/Y\n/p3tPfXufNmuLDuIiO1BbZP9CACAkxrZY5/ajuix4xj02F/WwDmCflPvsQMAzh6CHQCSIdgBIBmC\nHQCSIdgBIBmCHQCSIdgBIBmCHQCSIdgBIBmCHQCSIdgBIBmCHQCSIdgBIBmCHQCSIdgBIBmCHQCS\nIdgBIJm6H2aNhK5evaZnz37fdBkApoyPxjvH2vGRdFJbPpauDTVwjqAfH40HACDYASAbgh0AkiHY\nASAZgh0AkiHYASAZgh0AkiHYASCZWsFue6lv/k75d73StmZ7xfbGdEsEAIxjZLDbXpH0aV/zbdtf\nS3pS1lmSFBHxWNKh7etTrxQAUMvIYC9h/aSv+VZEvBsRW2X+pqTDMr0jaXV6JQIAxjHpGPti37DL\nnKT9yvIrJysLADCpid7dMSLuSpLt1TJUAwBoibF77LbXbX9QZvclLUg6kHS5tM1J2ptOeQCAcdXt\nsVffMvJz9cbRJekdSZ9I+kLSsqQtSYuSHg3ayObm5ovpTqejTqczVrEAkF2321W32z3RNka+H7vt\nNUm/lrQeEQ9L27pKb70yLHNL0m5puz9gO7wfe8vwfuztq4FzBP0meT92PmjjHCPY21cD5wj68UEb\nAACCHQCyIdgBIBmCHQCSIdgBIBmCHQCSIdgBIBmCHQCSmehNwACchtfLQ2PNmp9/W0+fftN0GTgB\nnjw9x3jylBoG4wnYNuHJUwAAwQ4A2RDsAJAMwQ4AyRDsAJAMwQ4AyRDsAJAMwQ4AyRDsAJAMwQ4A\nyRDsAJAMwQ4AyRDsAJAMwQ4AyRDsAJAMwQ4AyRDsAJAMwQ4AydQKdttLffNrtldsbwxrAwDM3shg\nt70i6dPK/JKkiIjHkg5sL/W1Hdq+fmoVAwCGGhnsJayfVJpuSjos07uSVvvadkobAKABdcfYq5+Q\nPSdpvzJ/RdKlAW0AgAZcaLqA8+rq1Wt69uz3TZcBIKG6wR6V6QNJl8v0nKTvyvJq295UqkusF+ox\ncr3T5dGrADhz6gZ7NQEeSHpP0pakRUmPSvvygLZXbG5uvpjudDrqdDpjFQsA2XW7XXW73RNtwxHD\ne4221yT9WtJ6RDwsbbfUu3C6EBH3j2vr206M2td5Ylvt6LE3XYPUjjqo4SWLc7U9bCsixvrzemSw\nTwvB/iqCvaoNdVDDSwR7m0wS7Dx5CgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzB\nDgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJ\nEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJEOwAkAzBDgDJTBTstu+Uf9crbWu2V2xvTKs4AMD4\nJu2x37b9taQnkmR7SVJExGNJh7avT6tAAMB4Jg32WxHxbkRslfmbkg7L9I6k1RNXBgCYyKTBvtg3\n7DInab+y/MrJygIATOrCJN8UEXclyfaq7ZW63/ftt9/q+fPnk+xyat544w299dZbjdYAtNvrst1o\nBfPzb+vp028areEsGzvYywXTvYh4qF4vfUHSgaTLZZU5SXuDvvdHP3pbFy78oyTptdcu6rXXLk5S\n84n87W/7+uMf/6C5ubmZ7xs4G76XFI1W8OxZs79YmtTtdtXtdk+0DUeM9x9YLozuRMRfbH8i6ZOy\naDki7pfhmUcRsd33fXHp0r/oz3/+7xMVfFI/+MG8dnf/V/Pz843W0esRNXvySG2oQWpHHdTwUhvq\nsMbNpqxsKyLG+k03do89IrZtr9vel/S7owC3vVyGZQ76Qx0AMDtj99gn3lGLeuw//OHr+tOf/tBo\nHT1N90ja0DOT2lEHNbzUhjouqjck1Kw2jPXPpMeeQS/Um37hnt8xRGC05sf5pbM71s9bCgBAMuey\nxw4A9TR/6+ckCHYAOFYbhoTG/8XCUAwAJEOwA0AyBDsAJEOwA0AyBDsAJEOwA0AyBDsAJEOwA0Ay\nBDsAJEOwA0AyBDsAJEOwA0AyBDsAJEOwA0AyBDsAJEOwA0AyBDsAJEOwA0AyBDsAJEOwA0AyBDsA\nJEOwA0AyBDsAJEOwA0AyF6a1Idtrkg4l3YiIj6e1XQDAeKbSY7e9JCki4rGkQ9vXp7FdAMD4pjUU\nc1O93rok7UhandJ2k+o2XUCLdJsuAEhnWsE+J2m/Mn9lSttNqtt0AS3SbboAIB0ungJAMtO6eHog\n6XKZnpO0N2ilv/61q4sX/2lKu5zU/zW8fwA4XdMK9geS3pO0JWlR0qNBKz1//r2eP/9+Srs8KTdd\ngKihqg11UMNLbaijDTVI7amjvqkEe0R8Zfs92yuSDiJie8A6Z+/oAMAZ5IhougacE3WfdbC9wbMQ\nOK9sL0XEV8csq3UOncrFU9trtldsb0yyPJMax2K9fN2ZdW2zVPdZh/JXX/rbZWu8LpbKOmuzrm3W\nxsiLW7OubdbK6//TY5bVfl5o6sE+aufn6WGmGsdiRdKjiLgnadH2+03UOSM861DUPAc+jIjfSlo4\n5+fIkqSdsnw387GQpPJzPjlmce1z6DR67KN2fp5O8FE/62KlbafMZzXyWYfyJ+hjncWrVeMZ+roo\nvfT/kaSIuDvomlUidfLg38u/i8mPxSi1nxc6jWAftfPz9DDT0J81Iu5FxP0ye0PSF7MqrKXebLqA\nGRl1DvxY0pUyHJN9uHLUOfKVpB3b+zrmNmr8PR5QaoHy5+aXyXsjQ591KL31rTLLFX1p7+gC2nkY\nZz+O7UvqvXZ+Keme7WuNFtSsWs8LSacT7KN2Xru4BOr+rCsR8eFsSmrMA70calqU9Jn04sSVetcY\nPrC9rl5vNfNY6qjXxZ56wxJSb5hieUZ1NWHUsbgt6VcRcVfSuqSfzLC2prwyFFk5RwaeQ4OcRrCP\nOoFrF5fAqGMh2+vlRXt0MTWlSu+z/1mHz8ry30bEw9J2acAmMhn1uvivyvI5SZ/PtLrZGnUsQiXo\nyuvjsH8DmZS/zt6z/UGl+egcOe4c+vvtnMZ97OW2pF1JC0djyLY/j4gfH7c8q2HHovwHPVCv1/Km\npJ9WhiOQWM1z5EDScva/5mociw317hS5nD0vpoUHlAAgGS6eAkAyBDsAJEOwA0AyBDsAJEOwA0Ay\nBDsAJEOwA0AyBDsAJPP/ZgdFaV71jYIAAAAASUVORK5CYII=\n", 184 | "text/plain": [ 185 | "" 186 | ] 187 | }, 188 | "metadata": {}, 189 | "output_type": "display_data" 190 | } 191 | ], 192 | "source": [ 193 | "lrc = LogisticRegression(C=0.001)\n", 194 | "lrc = lrc.fit(x_train, y_train)\n", 195 | "probas_lrc = lrc.predict_proba(x_test)\n", 196 | "\n", 197 | "plt.clf()\n", 198 | "plt.hist(probas_lrc[:,1])\n", 199 | "plt.show()" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": {}, 205 | "source": [ 206 | "Now let's train the calibration models." 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 5, 212 | "metadata": { 213 | "collapsed": false 214 | }, 215 | "outputs": [ 216 | { 217 | "data": { 218 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEQCAYAAACwSgOGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VNX5+PHPycpOMgGtIEISENeyBNxabdIQqNBqWRKq\nAbUiBLpoW0XAmhqKiogLtlWW4A4oWbDanyhkIfarVaEkqLUWJRlQiWuSSSQQluT8/riTySRMksk2\nd5bn/XrdV2bOvXN55pKZJ2e55yitNUIIIUSQ2QEIIYTwDpIQhBBCAJIQhBBC2ElCEEIIAZiUEJRS\n49rYN1MplaiUWuzJmIQQItB5PCEopRKB7Fb2jQO01roAsCmlxno0OCGECGAeTwj2L/vSVnbPBmz2\nx2XAJI8EJYQQwuv6ECKASqfnUWYFIoQQgSbE7AA6Sikld9IJIUQHaa1Ve8d4W0KoAiz2xxFAhauD\nGu+uzsjIICMjwyOBeTO5DnINGsl1MLh7HWpr4ZtvoKICKiuhqsr4WVMD1dXG9t13TduRI8Zramvh\n6FFjO3YMTp7s+ffUcRn87GcZvPIKKNVuLgDMSwjNolNKDdRaVwNZQBxQCMQAea5e3PgfXVRURFFR\nEfHx8T0ZqxDCx9TXQ1kZHDwIn38On31m/PzyS2P74gv4+mvjy9yzNH04Sl9q6ccR+lJLX2odZb05\nRh+O0oej9KKO3hyjF3WnbeEcJ5zjhHHC8dN5K6GaP1PP//4HHfn7wOMJQSk1E4hTSs3QWm+zF+cD\nE7XWJUqpOPtIpCqt9T5X52hMCBkZGZIMhAhQWsNXX8EHH8D//gf79xtbaamRCO69t0f+VQZQwyC+\nZRDfYqGSSKqwUMkZIVVEBduwBNmIUDYGUMMAXU3/hhr6NnxHn4bv6FVfy8mgcOpC+nE8pC8nQvpy\nIrQvJ0L7cDKkDyfD+nIytDenQntzKqwP9aG9aAjtxamwAcbjkHAaQsNpCOvFydBwjoeEUR0Wjg4J\noyEkDB0Wjg4NoyE4lNH/Ws/832fws5/B8uXL3Xp3Hk8IWutcILdF2USnxxvdPZckA4NcB7kGjfz1\nOmht/IX/7ruwezcUF8P77xvNPa7Fu3XesDAYPBi+F3WSUX3LiQn7nHOCPud7+gsGn/qCyONfMODo\nV/St/Zpe331NWPU36PBeNEQOQkdFoaKiCBpkISgqEmWJhIizIOJ8GDjQ2AYMMLb+/Y2tXz+CQ0Lo\n1V0Xpg3Di35KR38dlK/NdqqU0r4WsxCiY7SGjz+GoiJ44w1jKy/v2DmGDIHoaBg2DIadrRnd7zCx\n+gBD60qJspXS/1srIeWHUIcOGZnlzDPh7LNh6FDjxWedZWxnnmlsZ5xhZI/w8B55zz1JKeVWp7Lf\nJIQRI0Zw6NAhEyIKPMOHD+fgwYNmhyH8zLFjUFgI27fDq6+COx/nfv3goovgwgth9Gg479wGzg8v\nY1j1fwj/5D/w3/8a7Ukff2z8hT5yJMTGGlt0NAwfDiNGGAkgOLjH36NZAi4h2N+wCREFHrnWorvU\n1cGOHbB1K7zyijF6pzX9+sHEiXDJJcY29vsNjDi+n6A978LevVBSAu+9BxYLXHxxU6Y47zw491yj\nCSdASUIQPUauteiq996DzEzYtMkY2ulK//4QH9+0jTnvOMF7dxvtR//3f0ZngsUCl14KcXEwfjyM\nHQuRkR58J75BEoLoMXKtRWecPAlZWfDYY7Bnj+tjRo+Ga66BqVPhB1doQj9636hC7Nxp9Cifdx78\n6Edw1VVw2WVGu75ol7sJwdumrhAdVFJSwsSJE90u78h5R44c2ZXQhACMG7oefdRotp8z5/RkEBMD\nd91l1Bo+eu8EDybuIH7rIkJHDIWZM43OhFtvhcOHjRc/9JCRNSQZdDtvu1NZdFBMTAyrVq1yu7wj\n3L27UQhXjh+HtWvhvvvg22+b7wsLM77r58+HH/3gFEFFhfDIFnj5ZbjgAvj5z40hRueea0rsZrJa\nraSnp3P48GGGDh3KihUriI6OdnnstC3T2P7J9mZl+p4u1N611j61GSGfrrXysrIynZqaquPj43Vq\naqouKytzeVxbunqOvLw8HRsbqy0Wi05JSdE2m81x3ri4OJ2WlqYjIyP15MmTdXFxsY6NjdVBQUF6\n9erVWmutc3JydFpamk5OTtZKKT1hwgRttVq11loXFxfruLg4x/mSkpL0kiVL9Pnnn68nTJjgiCE7\nO9sRQ1pamqN8/fr1OjIyUgcFBZ123pEjR7p8P61dayG01rq+Xutnn9X6nHO0NgaQNm2DB2u9YoXW\n33yjtf7oI63/8AetzzxT60su0XrNGq3Ly80O31RlZWU6NjZWA44tNja21e8cMjhtc3mc8Zlt//vV\nnYO8aetIQujoxXWlq+ew2Ww6MjJSFxYW6urqar1w4UKdnJzsOLdSSm/btk1XV1c7vrBramp0fn6+\ntveX6JycHK2U0hs3btTV1dU6LS3NkQSKi4sdX/yN51u0aJHevHmzo7y0tFRHRkbqffv2aavVqkeO\nHKlzc3O11loHBQXpffv26erqap2cnKyXLl3qOK8kBNFR77+v9RVXnJ4IzjlH67VrtT5afULrzZu1\nvuoqrb/3Pa2XLdP644/NDttrpKamNvuuadxSU1NdHt9WEmh2nJsJwa+bjNLT0yktbb70QmlpKenp\n6WzatMkj58jKyiIpKYmEhAQAVq5cSUxMjGN/ZGQk06dPB2DSpElERkbSv39/EhMTUUpRU1MDQFxc\nHPPmzQNg3bp1BAcHO/Y5U0rxxBNPUFJS4ijLzc0lLS2NMWPGAJCd3bQ+UVVVFQMGDADAYrFgs9kQ\noqNqa2H5cnjkEWMeoUaDB8Mf/wgLf2Ej/NkNcOFfYdQoo0/gmmsgNNS8oDvIVfNMtxsFZJxevJnN\nbF6+uWf/bfy8D+Hw4cMuy8s7cMtjV89RWlpKdnY2+fn5gFEjc/4it1gsjscRERFERbleAsI5iQBE\nR0dTVlbW7nGNMUyYMMHxfOzYpoXo7rvvPgoKChz/rqvXC9GWvXvh+uuNe78ahYbCHXfAsrRK+m94\nGM5fawwdevllY3iol2qr/b7Hk0EnTR01tdvO5dcJYejQoS7LhwwZ4rFzxMbGkpyczNatWx1lnbnL\nt2UtpaysjJiYmNPKXYmIiODAgQOO5yUlJY5kUlhYyK5du+jfvz+ZmZkUFxd3ODYRmOrrjQE/d98N\np041lV91Fax/sJrztj8CcY/DjBnGTWPDh5sXrBusVitJSUnNPlPvvPMOeXl5zTp1u9Rp24kYYmNj\nT4uhp/j1sNMVK1YQGxvbrCw2NpYVK1Z47BwpKSnk5+dTUFCAzWYjLS2NtLQ0x37dxnh+533FxcVs\n3LiR6upq0tLSmDBhgqOpp7XXNEpLSyMzM9ORCJKTk6mqqqKyshKLxUL//v2x2WysX7+eysrK014v\nREsVFfCTn8DSpU3JoF8/eHJDPUXXree8a0cbw0V374YNG9xKBlarlTlz5pCQkMCcOXOwWq09/C6a\na6t52FOio6PJy8sjNTWVhIQEUlNTPZYMAP/uVNa6aYRQQkJCl0cZdfYcBQUFjpFDU6ZM0dXV1Y7z\nOnfcLlmyxDGySGujw7e6ulrn5OToyZMn65SUFK2U0hMnTmw2Gsi5U7nxfM7lWmudmZnpGE20aNEi\nR3lSUpKOjIzUEyZM0AUFBdpiseiCggLpVBat+uADrWNimncaX3qp1p9v2qX1xRcbHcbFxR06Z3cM\nAOmq+Ph4lx26CQkJWmv3O3C9EW52Ksudyj4gNzeXrKysZs1OZvLnay3aNuGRaez9rnlbesQxeGgn\nJJXCH6ZA7gW0WALLv/Rkk1FPkTuVhRDd6vHHOS0Z/Pwj+M8TUBcCF/0Kci/Er5NBd3bgeiO/7lQW\nQnSd1sbdxunpOIZEXvTMd/xz7K1EfvgmvPokv77ySn7dhX9jzpw5bN58+rDK1NRUt4eId4fGUUbl\n5eUMGTKkzbuE/ZE0GYkOk2sdOLQ2ho8+8oi9IEMx9gv4d8Fogn94Ofz1r0ZvcheZPbrG38lsp6LH\nyLUODFrDb39rNBU1WvBTxb2FMDhzs3HzQTcK9L/Oe5IkBNFj5Fr7N1d35IbUw2OvQfxBuPY6+OQv\n8v/vS9xNCNKHIIRopmUyiKqF7GyoDYXLb4EfXuzfHauBTBKCEMK1DM0IrLzVZzJnLppB8Kr7qfbj\ndYeFDDsVQjjZubPp8YX8h91hVzLo3t8R/NAqv16EXhgkIfSw7lh5zN3Vz4KCgjp0vBDO9u+HlBTj\n8WWfwRshifRdu5qw33dlQKnwJdJk5AFdXXnM3dXPGv+d7lgtTQSWqipjNurqavjBIdi2FdTTT9Nn\njvQXBBKpIXhYTk4OI0eOJDg4mNmzZzebCnvDhg1YLBZGjRpFZmamo2ZRVlbGnXfe6TguLS0Ni8VC\nVFQUq1evBmDy5MlorYmKijrt+MZ/MyoqioULF3ronQpfceoUzJ5tTF99Ce+ybStcPxMskgwCjzsT\nHnnTRgcntzP2de/WEc6TxJWWlmqllGP1tOTkZMdyls6rmlVXV+u4uDiXE9VlZ2frkSNH6pqaGl1S\nUuKYAE9rYzK8lse3tVpaZ7V1rYXvSU83fq/HUqy/5Aw99XrfncRNuIabk9tJDcGDGlcuS0hIYMCA\nAaxatcqxetmGDRscq5oNGDCAZcuWuTxHY7PQgQMHGDt2bLMVz9r6N8eMGcOIESPIzs6WRXCEw65d\ncO+9cD7/5TWupnDWWrYH3rr2wk4SggdVVFQ0W1shOjrasWRlWVlZs32tfWnPnDmTJUuWkJycTFRU\nFOvXr2/z3ywtLW123rFjxzZbMU0Erq+/htRUOEN/yatM45nzHyTlxRlmhyVMFBAJobsbjTorKiqq\n2cplNpuNiIgIwEgAzvtaWwnNarWSmJjIgQMH2Lt3L+vXr6ewsND+Pk8PztVqabm5uZ1/E8IvNDTA\nTTeB7YujvMI1ZPW+gd3n7WbSpASzQxMmCoiE4C1mzZpFZmYmhYWF2Gw2FixYwOzZswGYPXu2Y1Uz\nm83GAw884PIcOTk5JCcnY7VaaWhoAKC6uhpwPZqptdXSRGD7y19gx2v1bCaV/3Eejw38lJdeepyi\noiLHMZ5esUyYTxKCB0VHR5Odnc2CBQuIiooiKCjI8cU/btw4Vq1aRWJiIhMnTmThwoWO2oOzxYsX\nY7FYGDlyJBMnTiQlJYXp06cDRnNScHBws8QQHR3tOO+oUaOYPHkyt9xyi2fesPBKn3wCy5bBAywl\nkioeHj2dL7585rTjPLl0pPAOMrmdl7BardhsNsaNGwcYncEbNmxgx44dJkd2Ol+/1oGsoQESEmDw\nP3NYzWJuvODfqEG/4J//zG86KMP4kfBGgqM5Uvg2WTHNx9hsNhITEx3NP+vXryc5OdnkqIS/WbcO\nvvznftayiNlBOax5Pophw850eeyQIUM8HJ0wmyQELzFu3DjuuusuoqOjiYqKYuTIkdK0I7rVwYOQ\nsbiWXGZyF/eTtDSO8eNhxYoVzUaiNVqxYoXngxSmkiYj0WFyrX2P1vCTKZq5eXM5SSirz3uK4hJF\nr17GfufFaXb9aJfxGh9cTF645rXrISilZgI2YLzWenUb+6O11hs9HZ8Q/mjbNhict5mx7ONSdlPw\ndFMyAGPwQePaxWp51+beEr7Lo01GSqlxGLdQFwA2pdRYF/vL7PutLfcLITru6FF46NZPeZTfM4dN\n/PLXfbjsMrOjEt7I030IszH++gcoAya5OKZxms4YrfU+j0QlhB9btbKB+8tv5GFu5/CgsUjXgGiN\npxNCBFDp9DzKeafWugQoU0pVAhWeDEwIf1RWBsdWriGUk6xmMfffD5GRZkclvJVXjTJSSg0EqoD7\ngUyl1AhTA+oG3bFYTePCN2bGIHzTI/M+ZHH9Sm7gOcZPCObmm82OSHgzT3cqVwEW++MITq8FLABW\naq1rlFJlwCzgoZYnycjIcDyOj48nPj6+J2LtFt2xWI2nFtgR5pu2Zdppi9x3lmqA/yuFP00D68RY\nrEDIvd1yauHlioqKmk1D4i5P1xCygMZpPGOAfHDUDAA0oAC01tto6m9oJiMjw7F5czIA9xa3gdYX\nznFe+Kat46xWKxMmTGD16tWORXb27dt3WgyyWI53665kALBgLygN6+M6/tqpo2RxHF8WHx/f7HvS\nXR6tIWitS5RScUqpRKDKqdM4H5iotX5IKbVYKVUKWLpr2Gl3DqPrzNjsxr/wc3JyKCws5NChQ5SW\nlhIXF0daWhrffvstKSkpFBQUEBcXxy233MKdd97JunXr2LlzJ8HBwVRUVFBWVtbqcQDFxcVcd911\nVFZWsnDhQpYsWeKY+kIphdVqZcGCBezatYuBAweSlJTEtm3bmDFDpjz2Nl29B+DldV9wxavfJ4Fd\nhD94EZ98Amef3U3BCb/l8T4ErfVGrXWB85e91nqi0+PVWutt/ngPQmuL27haOCcrK+u017d3XGRk\nJLfffjtg1EQqK5v677XW5OTkyGI5AeDkSQi94zYymc+HXMStt0oyEO7x+I1pZvCWOy5nzpxJVVWV\nYwrqZcuWcccdd7hcOKdxTiNn7R1nsVhOe42zsrIy4uKa2g9koRz/tPO2VxldW8xMniUiApYuNTsi\n4Su8apSRv2u5uM26desoLCxsc+EcZ+4e1xpZLMf/Hak4zgUbbuPXPE4dvbnrLhlmKtwnCcGDWi5u\no5TCZrO5XDgnJSXltNe3tcAOuF4xrZFSirS0NDZs2CCL5fixd+b8lf/Un89OpnD22fCb35gdkfAl\nkhA8qOXiNsnJycyYMcPlwjnOw0QbF75pa4EdaH946ogRI2SxHD92pOxrxu14gDvsI7UzMqB3b3Nj\nEr5FZjsVHSbXuuc0jojrTL9XyaULeWN3L37PGmJiYP9+CAmIXkLRHq+d7VQI0f2OvvsBZ+/Zxp/5\nHwB//KMkA9Fx8isjhK/Tmq/n/oGH9J+owsLw4TB3rtlBCV8kfQhC+Li6VwuoLzvEetIAuOsuCA01\nOSjhkyQhCOHLtKbq13/kj/V/5hShDBsGN95odlDCV0lCEMKHncx9hcryOrIwhikvXQrh4SYHJXyW\n9CEI4asaGjjyu7tZeup+NEGcdRYyvbXoEqkhCOGjGra8yKeVffl//BSA226j2TrJQnSUJIQelpub\n6/KuY3d0dWEc4RusVitz5swhISHB/RedPMmxJffw+2P3A4p+/SAtrcdCFAFCmow8oLML3HR1YRzh\n/axWK0lJSZSWlhoF8U3l0dHRrb9w0yb2Hx3GLn4MwPz50IFprYRwSf4E9QCtNSkpKQQFBTFx4kSs\nVqtjX35+vmPBmrYWxtmwYQMWi4Xg4GAmTpzIwYMHzXgropulp6c3JYMW5a2qr6cu4wFutxnHBAfD\n737XUxGKQCIJwQNycnKYMmUKNpuNuLg4kpOTAWO20pSUFDIzM7FarVgsFsfcQjt37gSMKa8BFi1a\nxK5du6iqqiI6Opr169eb82ZEtzp8+LDL8vLy8tZflJvLp7VRFNmrE7Nnwznn9EBwIuAETpNRdzW/\ndGIOn7i4OObNmwfAunXrCA4OpqamhuzsbJKSkhxtxytXrmy2YI1zk1HjYjpgrHtgs7lcXVT4mKFD\nh7osHzJkiOsXaM2JjPu5o/Je7KvNsnhxDwUnAk7g1BC07p6tE1quShYdHU1ZWRmlpaVkZ2cTFRVF\nVFQUMTExjiajlu677z4mTJjAlClTmjU5Cd+2YsWKZoseOZe79OqrfPst/ENPAyAxEWSdI9FdAich\nmKhlG7HVaiUmJobY2FiSk5OpqKigoqKCysrKZgvYNGpci3nXrl3s2LGDWbNmeSp00cOio6PJy8sj\nNTW12Sgjlx3KWlO/4j7uPnoXjbWD227zUKAiIEhC8ICSkhK2bduGzWYjLS2NpKQkBgwYQEpKCvn5\n+RQUFDj2pbkYO1hVVYXFYqF///7YbDbWr1/fbL1k4duio6PZtGkThYWFbR9YVMR3h6p4tnam/XUw\ndaoHAhQBQxKCByxYsID169djsVg4dOgQWVlZAAwcOJDs7GzS0tKIiori0KFDZGdnO17XuDDO/Pnz\n0VpjsVhISkriwQcfJD8/v/0vEOFX9IMP8kjInTQQDBiroQUHmxyU8CuyQI7oMLnWPafVBXI+/JDj\nV01iQOVBThBOnz5w+LDceyDc4+4COVJDEMIXrFlD7vd+zQmMmevmzpVkILpf4Aw7FcJXff019dm5\n/K7mY0fRb35jYjzCb0kNQQhvt3Yte6OT+UYPAiAhAS66yOSYhF+SGoIQ3qyuDr12Lbef2uUoktqB\n6ClSQxDCm23ezJdD4niz4nwAhgyBa64xOSbht/ymhjB8+HCZHdRDhg8fbnYIgUFrePRRHg1b4yi6\n5RYI8ZtPrfA2fvOrJbN/Cr/zxhscP65Z/WEiAEFBRkIQoqdIk5EQ3urxx9k+/Fc0TlMxbRoMG2Zu\nSMK/+U0NQQhvMG3LNLZ/sr3rJzp8GF1QwO950lEkK6KJnuY3dyoL4Q0a7zTuiqmjpvLq/gl8/K9v\nGZ3/OGCsd1BWJlNViM5x905lqSEI0QNOm3qiI06cgBEjuH9onqNo/nxJBqLnSR+CEN7mpZc4Ouxc\nnv33hYCRCG6+2eSYRECQhCCEt3niCV4a8mvH02nTjPsPhOhpHm8yUkrNBGzAeK31ahf7xwExAFrr\nXA+HJ4S5/vMf9IEDLK3/uaNIhpoKT/FoDcH+Za+11gWATSnlavG/ZfZEEN3KfiH8V2YmH//wZj7/\nKhSAs86Cq682OSYRMDzdZDQbo3YAUAZMct5prz3sBtBaP6S13ufZ8IQwUV0dbN7MQ5XzHEU33ih3\nJgvP8XRCiACc136MarF/IhCllBqnlFrsubCE8AK5uRy/KI6nd41wFElnsvAkb/zbo0JrXaKUmqSU\nmumqHyEjI8PxOD4+nvj4eA+GJ0QPyczktbN/Q3298fSqq2DUKHNDEr6pqKiIoqKiDr/O0wmhCrDY\nH0cAFS32V2A0JYHRtDQBaDMhCOEXPv4Y/dFH3F3eNJXpvHltHC9EG1r+obx8+XK3XufpJqMs7COI\n7D/zAZRSA+1lOU77I4A9Ho1OCLNkZnI48UY+/CQMgAEDYNYsk2MSAcejCUFrXQKglEoEqpw6jfPt\n+60Yo49mAhat9TZPxieEKU6cgOeeY+3JpvGlv/gF9OljYkwiIHm8D0FrvdFF2UQX++UeBBEYXn6Z\n+tEX8JfXz3UU/fKXJsYjApbcqSyE2Z56ircvmMeRI8bT886DSy81NyQRmLxxlJEQgePwYXj3Xe4/\n1lQhvukmkMX/hBmkhiCEmZ5/nu+mzOK1N4wOg6AgmDvX5JhEwJKEIIRZtIannya7X1OHwZQpMpGd\nMI8kBCHM8vbbaKW4N/8yR9FNN5kXjhCSEIQwyzPPYL3qJqwHjQ6DiAi45pp2XiNED5KEIIQZjh6F\nnBz+Vt3UYXDdddCrl4kxiYAnCUEIM2zbxqkJl5K5faijSJqLhNkkIQhhhmee4V+jf9ns3oOJE9t+\niRA9Te5DEMLTPvsMSkp4oKGpw+DGG+XeA2E+qSEI4WmbN3Pk6lm8XmR0GCgFc+aYHJMQSEIQwrO0\nhuee46V+N6C1UZSYCGefbW5YQoAkBCE8a+9e9PHj3F90haPoxhtNjEcIJ5IQhPCk55+nPPEG/rff\n6DDo1w+mTzc5JiHsOtWprJS6A4gF9gJZWuuabo1KCH908iS88AIbprzjKJo1C/r2NTEmIZy4VUNQ\nSv1YKTXW/vgBYBBQDaQAVUopWexPiPa8/joNI8/lb9tjHEXSXCS8Sbs1BKXUj4EYrXWhvWiP88L3\nSqkIYKlSaoascCZEG557jvfG3EDl28bT4cPhqqvMDUkIZ+7UEOKAf7e2U2tt01ovBaK6LSoh/E1V\nFeTl8dChZEfR3LnGdNdCeIt2fx211quBiUqpEfaiMqXUHqVUglJqgNOhFT0QnxD+ISeH41dOIisv\n0lF0ww0mxiOEC279faK1ztRaH7Q/nQ1kAYuAg/bksAOIUUr1B1BKzeiJYIXwWZs2kfe9uZw6ZTy9\n/HIYNcrckIRoqTOjjEqBPHvNAaXUOGASMBm4SynVWFOQ/gQhAA4dgg8/5L4jVzuKpDNZeKMOt2Bq\nrTOByMYmJK11idZ6tdZ6stbaglGDsHZrlEL4si1bqJyUzDvFYQCEhUFKiskxCeFCp7q07EngYCv7\nioElXQlKCL+hNTz/PC8GN01WdM01EBnZxmuEMEmPjHHQWpf0xHmF8Dn79qGPHeO+XTJVhfB+MuhN\niJ60aRNlV8yh/AtjqorBg2HKFJNjEqIVkhCE6Cn19fDCC6ytTnUUzZkDoaEmxiREGyQhCNFTCgs5\n9b2hPF5wnqNImouEN5OEIERP2bSJPaNSqaszno4ZY2xCeCtJCEL0hNpaePllVlp/4Si66SbzwhHC\nHZIQhOgJr7xC7fcv4x97vgdASAhcf73JMQnRDkkIQvSEzZt5zdJ078HUqXDGGSbGI4QbOrVAjhCi\ndYNqQb/5Jnf3e9FRJp3JwhdIDUGIbjb7P/BV3DT2H+4HQFQU/PSnJgclhBskIQjRzVI/gOcampqL\nrrvOmL9ICG/n8YSglJqplEpUSi1u57g29wvhjWIrIKYK/vx2kqPs5ptNDEiIDvBoQrBPla211gWA\nrXGdZhfHJWJMqS2ET0n9ALZeCLXHje65sWNh3DiTgxLCTZ7uVJ4N7LQ/LsP40t/n4RiE6DbTtkxj\n+yfbjSca9r8Pc2cAu40iqR0IX+LpJqMIoNLp+WnrMCulxtlrEMpjUQnRSY5kAFxy2Pi5u9ZYCCcs\nTO49EL7FGzuVZaZ44XP0PZp3w39N6Xl/hheMJHHttcYIIyF8haebjKoAi/1xBFDhvNNeOyi0P9Wt\nnSQjI8PxOD4+nvj4+G4NUogOO3ECvXUrf2p411EkzUXCLEVFRRQVFXX4dUrrVr93u529UzlOa73R\nPoooT2u9Tyk1UGtdrZSaiZEIooA04Bat9b4W59CejFmItqjlRsumHv8KFXeuYtD/3gRg6FBjKeXg\nYDOjE8J0ghp6AAASK0lEQVSglEJr3W4zvEebjBpXUrOPIqpy+rLPt+/P1Vpvs5cN9GRsQnTJ88/z\nYuhcx9ObbpJkIHyPR2sI3UFqCMKbqOWKAXVQ9fgABh85SKU2usAOHIDYWJODE8LOK2sIQvijWf+F\n/WcnOpJBYqIkA+GbZHI7Ibpozvvw6JGm5qL5800MRogukIQgRBecY4OLv4KfHJsKwKBB8POfmxyU\nEJ0kTUZCdMHc9yDrQjhBOGBMcx0ebnJQQnSSJAQhOktrbngPnnWakUuai4Qvk4QgRGe9/TYNCnYP\nNZ5edRWMHm1uSEJ0hSQEITpJP/OsUTuwD+ZbsMDUcIToMrkPQYjOOHaME2cMJWZ+FYcHQuRjmvJy\n6NXL7MCEOJ3chyBET3rlFT7sFcdh+/30v/ylJAPh+yQhCNEJR9c+yyPf3uh4vmiRicEI0U0kIQjR\nUeXl8M7bbGO6o2jkSBPjEaKbSEIQooNOPr2Jl5jOUfqaHYoQ3UruVBaiI7Tm6N+e5InjT5kdiRDd\nTmoIQnTEW29RZVP8iyvMjkSIbicJQYgO+HbVkzxeNw9QhIWZHY0Q3UsSghDuqqmh946XeI4bAEhJ\nMTkeIbqZJAQh3FS1bit5pxL4mjMBuO02kwMSoptJQhDCTd899iQb9TwArrwSJkwwOSAhupkkBCHc\ncHTPh4R88Rmv8xMA/vAHkwMSogfIXEZCuOHZH0ZzWB3kj5Nc79f3yO+k8F4yl5EQ3aThaB1TSw7y\n5HjX+6eOmurZgIToIXJjmhDteO/ubL46C8osMPBRzeefQ79+ZkclRPeTGoIQ7Qh5ch3r7B3ICxZI\nMhD+SxKCEG3Y+8z7RNR8yv8713j+29+aG48QPUkSghBt+PJPa8lkPvXBxvNhw8yNR4ieJAlBiFa8\n/9Z3XP7ZVp5kntmhCOER0qksRCvevXUzn5BAOUPNDkUIj5AaghAufLxfM7F4HWuRpdBE4JCEIIQL\nL/3h/+jNMQr5MVPlNgMRICQhCNFCWRmc+9oa1vA7NEHcdZfZEQnhGZIQhGjhiTvKuFL/k+e4gfh4\n+MEPzI5ICM+QhCCEk48+grNf+itPMo+j9OXee82OSAjPkVFGQjhZuayGNTzLGN7j6quldiACiyQE\nIexKSsDy8lPsZDKfM4y/rzA7IiE8SxKCEHb33F3PGv7CdbzAjBkQF2d2REJ4lscTglJqJmADxmut\nV7vYP9/+MFZrvdSjwYmA9eabELr973zFmexRl/LUn82OSAjP82inslJqHKC11gWATSk1tsX+RCBP\na50JxCilfuzJ+ERgamiA3/9Os4yVrGIJ118PF15odlRCeJ6nRxnNxqgdAJQBLdefinEqK7M/F6JH\nbd4Mlr076UUdO8Ku4b77zI5ICHN4uskoAqh0eh7lvNNeM2g0HnjRE0GJwFVbC0uXwhbuZ+XlIdRN\nCWbEM2ZHJYQ5vLJT2d60tFdrvc/sWIR/e/BBiC5/k7P5nK2Tyto8VpbKFP7O0wmhCrDYH0cAFa0c\nl6i1XtbaSTIyMhyP4+PjiY+P76bwRCD57DNYvRpyuJ9VLKE+OA0AfY82OTIhuqaoqIiioqIOv05p\n7blffvtf/nFa641KqcUYHcj7lFIDtdbV9mPmNzYdKaUS7R3QzufQnoxZ+K/Zs+GTrGJe4Rpmjill\n9/RegCQE4X+UUmitVXvHebRTWWtdAo7RRFVOTUL5TuUPKKUOKKUqAPlkih6xfTtkZcHd3MvD3M6A\nwU0joK1Wq4mRCWEej9YQuoPUEERX1dYaw0oHH9rDS0xnXN/VfFt7PWQY+2OfjyUvL4/o6GhT4xSi\nu3hlDUEIb3DPPXDoEKxkGStD7uTb2t82219aWkp6erpJ0QlhHkkIIqCUlMCaNZBIPufwKf+M/Q5X\nYxvKy8s9H5wQJpOEIALGyZNwyy1QX69ZyTK2XriC78d95PLYIUOGeDg6IcznlfchCNETli+H4mKY\nwTZCVD2/yE0mJOwS3n33HUpLSx3HxcbGsmKFTHUqAo90KouA8Oab8KMfQVDDST7gYj68ZQ0zM38C\nGKOK0tPT2TxqMwBlN5RJh7LwK+52KktCEH6vpgbGjIGDB+F3PEpq5GuM/2YHQcGKaVumsf2T7c2O\nl/sQhL9xNyFIk5Hwe7feaiSDM/mSu9V9nNj2FkHBxmejZTKQ6SlEIJOEIPzas88aG8AqlvD1T+dx\nfvzo046TWoEQkhCEHysuhoULjcdX8BY/612AZbPrUUVCCBl2KvxURQXMmAF1dRBEPRt7/YY+j6+G\n/v3NDk0IryUJQfid+nq47jrjbmSAxeF/ZfjFA+l10y/MDUwILydNRsKvaA233w55ecbzc9nP8rB7\nCd/yDqh2B1kIEdAkIQi/8sgj8NhjxuMg6skbehPhSzNg5EhT4xLCF0hCEH5jyxa4446m589c9DDD\nBvWGX/3KvKCE8CGSEIRfKCiAm25qej53/IfM+XQ16h97IEi6yoRwh3xShM8rLISf/cyYvA5g/HlH\neer49aiVK2HECFNjE8KXSA1B+LT8fCMZ1NUZz4cO0fzzgoWE9B0D8+aZG5wQPkYSgvBZO3fCtdc6\nJYOhUDx/HX1z98E7MqpIiI6ShCB80ubNcPPNcOKE8fzss+Ffj77LGb+6B956C/r0MTdAIXyQ9CEI\nn6I1rFgBc+Y0JYNzzoE3X/ycYX9IhsxMGDXK3CCF8FFSQxA+4/hxY26iZ55pKrvgAnh9SyXDrp9i\nTGt67bWmxSeEr5OEIHxCWRnMng3//ndT2aRJkP3cMSKSr4Grr25+E4IQosOkyUh4vW3bYPz45sng\n5pth+yuniFj4C4iOhgcfNC9AIfyEJAThtWpqjCaimTOhutooCw2FNWtg4+PHCU1NgVOn4Kmn5OYz\nIbqBNBkJr7R9O6SlweefN5UNHw5ZWXDJhbVwzXQYOBBeesnIEkKILpM/q4RX+fxzSE2FadOaJ4Pp\n06GkBC451waTJxvjTF94AcLCzAtWCD8jCUF4hSNH4E9/gnPPNSapazR4MLz4IuTmQuSXH8Fll8Gl\nl8LGjRAiFVwhupN8ooSpjh6F9euNPuEvv2y+7/rrjamsBw3CyAiLFsGqVfDLX5oSqxD+ThKCMIXN\nZiSChx+Gb75pvm/cOGNdg/h4jJsP7kw3Og9eew3i4swIV4iAIAlBeNT778Pjj8OmTUbtwNmQIXDf\nfXDDDfZBQ+++a4wvHTXKGHM6aJApMQsRKCQhiB73zTfGH/ibNhlzzrU0bBgsW2a0BPXqBXz3HSxf\nbrzgsccgJUUmqhPCAyQhiB7x5Zfwj3/A3/9uzEp66tTpx1x8Mdx2G8ydax8sdOIE/G0D3HsvTJkC\nH3xg9Cp30bQt09j+yfYun0cIfycJwSRWq5X09HQOHz7M0KFDWbFiBdHR0WaH1WknThgtPPn5sGMH\n7N5tTETXUkgIzJgBv/kN/PCH9j/8jx+HZ14wZq0791x4/XUYO7bbYmsvGUwdNbXb/i0hfJnSrj61\nXkwppX0t5pasVitJSUmUlpY6ymJjY8nLy/OZpFBVZSSAt9+Gf/3L+Flb2/rxl19u1ASSk526Ar7+\nGtauhXXrYMwYWLIEEhI6HVN7NQF9j2//3gjRWUoptNbttrtKDcEE6enpzZIBQGlpKenp6WzatMmk\nqFw7eRIOHID//hc+/BD27YPiYjh0qO3XBQXBlVcak49eey3ExNh31NbCC68YCxq8+SY1U6dy74QJ\n7DlyhKFPPsmKESM6nRTbSgZSCxCifR5PCEqpmYANGK+1Xt3R/f7g8OHDLsvLy8s9HAk0NEBFBXz6\nqfEl/+mnYLUaSeDAAWOWUVft/67ExEBiojELaWIiREXZdxw8COteN4aNFhUZ1YXUVA6uXMmk6dOb\nJcd33nmnyzUlqQkI0TkeTQhKqXGA1loXKKVilFJjtdb73N3vL4YOHeqyfMiQIV06r9ZGc7zN1rRV\nVhpf+I3bV18ZLTVffQVffAHl5U2L03dEWBh8//vGd/sVVxg/hw8Hjh0zqhIvOrUn1dYancSzZ8OT\nTzrajO6eM8dnakpCBAKP9iEopR4AdmqtC5VSicA4rfVD7u63H9PlPoSOduhqbWwNDU1bfX3Tz5aP\n6+vhxp3TKDrsnyNbep2EId/BcBvEVkFsJYyqhIu/gmE18HEU7BkCbw+Dfw2D/VGgPThJitQQhGjO\nW/sQIoBKp+dRHdwPGKMSn3gC6uqKCAuLB5q+tBsfuyrTGurr66mtHURDwxOAAhRbtgQRHt6A1kGO\nL37nn52S4cFkYAXaaGEJaoDQeghpgLB6Ywuvh16noPdJ6H0K+pyEfieg7wnofwIi6mBgnfFz0FEY\nfBQG18JZR4zjyvvDZwPggAVKLZB1IfwpwUgGp4I99s6b2K9BoPcVFBUVER8fb3YYppPr0Llr4JOd\nyjU1MOeLB6njAW7FApGl7b8IjO//EGBgz8TVLP0+Zv9ZFePiuNazjEI79rt63HILooHVHGEpvQmi\ngWDqCaaeEGX8DNL1ANQHhaKDQ2gIDUeHhVNTdwxb3VGOcYo6NEeBI8Dg4cO5fPJkiIgwppeOiDCa\neAYPNn6edRYMGkS0UkQDV3XhenXnaKuMjAwy7snoQjT+Qb4IDXIdfCMhVAEW++MIoKKD+wF4660M\nSjiKZhS53ErvOXN6JNjOcK6U9f7yR5y150mCgyE4GEJCIcT+ODRMERJilIWFGlP6N25h4YrQMEVY\nGPTqrQgLV4SHQ+8+il69Fb37GFuffkH07ac4ueUh9B1L6T0giPDeQajGfyQ42Bj4HxR02rS2UxMS\nKCoqOi3+hJgYCjds6NFr1Cg6Opq8vDzS09MpLy9nyJAhPn8/hhDeoKioiKKiIjIyMjr0Ok8nhCwg\nDigEYoA8AKXUQK11dWv7W9q+PYPaWnj44QzuuCMVpVIxztM0w4HzT+fHaWkLyMnJAnSzbfbs2Tz9\n9JMEBTW9JigIx3NvFvFGXwaNdtm61qqe6tjuqOjoaOlAFqKbxcfHEx8f70gIy5cvd+t1Hr8xTSl1\nC/YWX631RnvZHq31xNb2t3i99BgKIUQHudOp7HN3KgshhOgZsmKaEEIIQBKC8EFKqZlKqUSl1OJ2\njmtzvxD+xH5jb2v73PrM+ExCaO8NufuGfZkb12C+fXvA07F5ivPd7IBNKeVyWlT7jY2TPBqch7nx\n+zDOfsxMT8fmKR34XrjF07F5kv33PbuVfW59ZsBHEkJ7b6gjb9hXuXENEoE8rXUmEKOU+rEZcXrA\nbIy5rgDK8PMv/da4+Tu/TGudC0QH6GdiHFBm32/1x2vQyP4eW7shy+3PjE8kBNp/Q4HwJdHee4xx\nKiuzP/dH7d7NrpQaZ/+AePmA4S5p8/fBXivYDaC1fsgf5wTDvc/9KvvPGD+9Bu5wawYI8J2E0C1T\nXvi4Nt+j1jrTaZjueODfngrMC0WaHYAHtPc7PxGIsjcb+WszanufiRKgTClVSSs3uYrmfCUhCDfZ\nq8l7/fivoTbvZrfXDgrtTwN9THWF/UuxscYQUJRSAzF+X+4HMpVSI0wNyDxuzQABvpMQumXKCx/n\n7ntM1Fov80xIpsiiqTksBsgHx4cfjP6TGUqp+Rh/Iftru3F7vw8VGM0oYDSrTPBQXJ7U3jVYAKy0\nz5g8H5jlwdjM0KyJ1Okz4fIz44qvJIT2vgTcfsM+rL1rgFJqfuN04fZOZr/j9BdvIlDlVBPKt+/P\n1Vpvs5f10DSGXqG934ccp/0RwB6PRucZ7V0Djf1L0v47YWt5An9hrwHGKaVmOBU3fiZa+8ycfh5f\nuVO5q1Ne+IO2roH9PzsL46+mSCDZqelE+CE3PxNVwAR/rTW6cQ0WY4y+sfjr90J38pmEIIQQomf5\nSpOREEKIHiYJQQghBCAJQQghhJ0kBCGEEIAkBCGEEHaSEIQQQgCSEIQQQthJQhBCCAFIQhBCCGEn\nCUEIIQQAIWYHIISvs0+mloIx22YUkK+1tpoblRAdJzUEIbouE9hqn1GzAgi4tQeEf5DJ7YToIqXU\nAYzpprc2TjUshC+SGoIQXTcL+3z8Sil/XHdABAhJCEJ0gVJqptZ6n9Y6RWsdBUQG8FKNwsdJQhCi\nk+ydyUtbFJdqrQ+aEI4QXSZ9CEJ0gX3JQhvGKnWRQJbWusbcqIToHEkIQgghAGkyEkIIYScJQQgh\nBCAJQQghhJ0kBCGEEIAkBCGEEHaSEIQQQgCSEIQQQthJQhBCCAFIQhBCCGH3/wE3MEJgD/L4FwAA\nAABJRU5ErkJggg==\n", 219 | "text/plain": [ 220 | "" 221 | ] 222 | }, 223 | "metadata": {}, 224 | "output_type": "display_data" 225 | } 226 | ], 227 | "source": [ 228 | "cal_probas_lrc = lrc.predict_proba(x_cal)[:, 1]\n", 229 | "\n", 230 | "lr = LogisticRegression(C=99999999999)\n", 231 | "lr.fit(cal_probas_lrc.reshape(-1, 1), y_cal)\n", 232 | "\n", 233 | "iso = IsotonicRegression()\n", 234 | "iso.fit(cal_probas_lrc, y_cal)\n", 235 | "\n", 236 | "bc = BetaCalibration(parameters=\"abm\")\n", 237 | "bc.fit(cal_probas_lrc.reshape(-1, 1), y_cal)\n", 238 | "\n", 239 | "pr = [lr.predict_proba(linspace.reshape(-1, 1))[:, 1], iso.predict(linspace), bc.predict(linspace)]\n", 240 | "methods_text = ['logistic', 'isotonic', 'beta']\n", 241 | "idx = cal_probas_lrc.argsort()\n", 242 | "scores = cal_probas_lrc[idx]\n", 243 | "y_c_2 = y_cal[idx]\n", 244 | "fig_map = plot_calibration_map(pr, [scores, y_c_2, linspace], methods_text, alpha=0)" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "Notice how all three methods find sigmoid shapes. This is because the sigmoid shape is part of the beta calibration family, which also contains an identity calibration map, which just keeps the scores with their original values (this is not possible with logistic calibration). In both maps, isotonic regression did a great job fitting the scores, but as a non-parametric method, it needs a lot of data to train (as is the case with the spam dataset), which is not a problem for beta calibration. Therefore, given that beta calibration is able to fit the classifier's output probabilities better than or at least equal to logistic calibration, we argue that a practicioner who wants to use a parametric calibration method should choose beta calibration." 252 | ] 253 | } 254 | ], 255 | "metadata": { 256 | "kernelspec": { 257 | "display_name": "Python 2", 258 | "language": "python", 259 | "name": "python2" 260 | }, 261 | "language_info": { 262 | "codemirror_mode": { 263 | "name": "ipython", 264 | "version": 2 265 | }, 266 | "file_extension": ".py", 267 | "mimetype": "text/x-python", 268 | "name": "python", 269 | "nbconvert_exporter": "python", 270 | "pygments_lexer": "ipython2", 271 | "version": "2.7.13" 272 | } 273 | }, 274 | "nbformat": 4, 275 | "nbformat_minor": 0 276 | } 277 | --------------------------------------------------------------------------------