├── LICENSE ├── README.md ├── adult ├── rf.r ├── test.csv ├── train.csv └── y_and_p.csv ├── get_diagram_data.py ├── isotonic_regression.py ├── load_data.py ├── load_data_adult.py ├── log_loss.py ├── platts_scaling.py └── reliability_diagram.py /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Zygmunt Zając 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | classifier-calibration 2 | ====================== 3 | 4 | Reliability diagrams and calibration with Platt's scaling and isotonic regression. 5 | 6 | adult - a dir containg data, code and results for a random forest and the Adult dataset 7 | get_diagram_data.py - a helper function for reliability diagrams 8 | isotonic_regression.py - calibrate a classifier using isotonic regression 9 | load_data.py - helper for loading data, a Vowpal Wabbit example. You'll need to edit this file. 10 | load_data_adult.py - loads y and p from random forest trained on the Adult dataset 11 | log_loss.py - from log_loss import log_loss 12 | platts_scaling.py - calibrate a classifier using Platt's scaling 13 | reliability_diagram.py - check your classifier's calibration 14 | 15 | See [http://fastml.com/calibrating-a-classifier-with-isotonic-regression/](http://fastml.com/calibrating-a-classifier-with-isotonic-regression/) for description. 16 | -------------------------------------------------------------------------------- /adult/rf.r: -------------------------------------------------------------------------------- 1 | # train a random forest on original a9 data 2 | 3 | library( randomForest ) 4 | library( caTools ) 5 | 6 | ntrees = 100 7 | 8 | train_file = 'train.csv' 9 | validation_file = 'test.csv' 10 | label_index = 1 11 | 12 | output_file = 'y_and_p.csv' 13 | 14 | ### 15 | 16 | train <- read.csv( train_file, header = F ) 17 | validation <- read.csv( validation_file, header = F ) 18 | 19 | x_train = train[, -label_index] 20 | y_train = train[, label_index] 21 | 22 | x_validation = validation[, -label_index] 23 | y_validation = validation[, label_index] 24 | 25 | ### 26 | 27 | rf <- randomForest( x_train, as.factor( y_train ), ntree = ntrees, do.trace = 1 ) # mtry = nvars 28 | 29 | p <- predict( rf, x_validation, type = 'prob' ) 30 | p_binary <- predict( rf, validation[,-1] ) 31 | 32 | probs = p[,2] 33 | 34 | accuracy = sum( p_binary == y_validation ) / length( p_binary ) 35 | cat( "accuracy:", accuracy, "\n" ) 36 | 37 | auc = colAUC( probs, ( y_validation + 1 ) / 2 ) 38 | auc = auc[1] 39 | 40 | cat( "auc:", auc, "\n" ) 41 | 42 | write.table( cbind( y_validation, probs ), quote = F, col.names = F, row.names = F, sep = ',' ) 43 | 44 | ### 45 | 46 | # accuracy: 0.8485351 47 | # auc: 0.8792633 48 | 49 | # accuracy: 0.8491493 50 | # auc: 0.8800499 51 | 52 | # accuracy: 0.8468767 53 | # auc: 0.8793631 -------------------------------------------------------------------------------- /get_diagram_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def get_diagram_data( y, p, n_bins ): 4 | 5 | n_bins = float( n_bins ) # a float to take care of division 6 | 7 | # we'll append because some bins might be empty 8 | mean_predicted_values = np.empty(( 0, )) 9 | true_fractions = np.zeros(( 0, )) 10 | 11 | for b in range( 1, int( n_bins ) + 1 ): 12 | i = np.logical_and( p <= b / n_bins, p > ( b - 1 ) / n_bins ) # indexes for p in the current bin 13 | 14 | # skip bin if empty 15 | if np.sum( i ) == 0: 16 | continue 17 | 18 | mean_predicted_value = np.mean( p[i] ) 19 | # print "***", np.sum( y[i] ), np.sum( i ) 20 | true_fraction = np.sum( y[i] ) / np.sum( i ) # y are 0/1; i are logical and evaluate to 0/1 21 | 22 | print mean_predicted_value, true_fraction 23 | 24 | mean_predicted_values = np.hstack(( mean_predicted_values, mean_predicted_value )) 25 | true_fractions = np.hstack(( true_fractions, true_fraction )) 26 | 27 | return ( mean_predicted_values, true_fractions ) 28 | -------------------------------------------------------------------------------- /isotonic_regression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | "calibrate a classifier's predictions using isotonic regression" 4 | 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | from sklearn.isotonic import IsotonicRegression as IR 9 | from sklearn.metrics import accuracy_score 10 | from sklearn.metrics import roc_auc_score as AUC 11 | 12 | from log_loss import log_loss 13 | from get_diagram_data import get_diagram_data 14 | from load_data_adult import y, p 15 | 16 | ### 17 | 18 | # train/test split (in half) 19 | 20 | train_end = y.shape[0] / 2 21 | test_start = train_end + 1 22 | 23 | y_train = y[0:train_end] 24 | y_test =y[test_start:] 25 | 26 | p_train = p[0:train_end] 27 | p_test =p[test_start:] 28 | 29 | ### 30 | 31 | ir = IR( out_of_bounds = 'clip' ) # out_of_bounds param needs scikit-learn >= 0.15 32 | ir.fit( p_train, y_train ) 33 | p_calibrated = ir.transform( p_test ) 34 | 35 | p_calibrated[np.isnan( p_calibrated )] = 0 36 | 37 | ### 38 | 39 | acc = accuracy_score( y_test, np.round( p_test )) 40 | acc_calibrated = accuracy_score( y_test, np.round( p_calibrated )) 41 | 42 | auc = AUC( y_test, p_test ) 43 | auc_calibrated = AUC( y_test, p_calibrated ) 44 | 45 | ll = log_loss( y_test, p_test ) 46 | ll_calibrated = log_loss( y_test, p_calibrated ) 47 | 48 | print "accuracy - before/after:", acc, "/", acc_calibrated 49 | print "AUC - before/after: ", auc, "/", auc_calibrated 50 | print "log loss - before/after:", ll, "/", ll_calibrated 51 | 52 | """ 53 | accuracy - before/after: 0.847788697789 / 0.845945945946 54 | AUC - before/after: 0.878139845077 / 0.877184085166 55 | log loss - before/after: 0.630525772871 / 0.592161024832 56 | """ 57 | 58 | ### 59 | 60 | print "creating diagrams..." 61 | 62 | n_bins = 10 63 | 64 | # uncalibrated 65 | 66 | mean_predicted_values, true_fractions = get_diagram_data( y_test, p_test, n_bins ) 67 | plt.plot( mean_predicted_values, true_fractions ) 68 | 69 | # calibrated 70 | 71 | mean_predicted_values, true_fractions = get_diagram_data( y_test, p_calibrated, n_bins ) 72 | plt.plot( mean_predicted_values, true_fractions, 'green' ) 73 | 74 | # perfect calibration line 75 | plt.plot( np.linspace( 0, 1 ), np.linspace( 0, 1 ), 'gray' ) 76 | 77 | plt.show() 78 | 79 | 80 | -------------------------------------------------------------------------------- /load_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import numpy as np 4 | from scipy.special import expit as sigmoid 5 | 6 | # this example shows how to load y from a test file in VW format 7 | # and predictions too - VW doesn't output probabilities hence the sigmoid 8 | # we want y like np.array([ 0, 1, 0, 1 ]) 9 | # and p like np.array([ 0.3, 0.98, 0.2, 0.75435832345 ]) 10 | 11 | test_file = 'test_v.vw' 12 | predictions_file = 'p.txt' 13 | 14 | print "loading data..." 15 | 16 | y = np.loadtxt( test_file, usecols = [ 0 ] ) 17 | 18 | # y need to be 0/1 19 | y[y == -1] = 0 20 | 21 | p = sigmoid( np.loadtxt( predictions_file, usecols = [ 0 ] )) 22 | 23 | assert( y.shape[0] == p.shape[0] ) 24 | -------------------------------------------------------------------------------- /load_data_adult.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import numpy as np 4 | from scipy.special import expit as sigmoid 5 | 6 | # predictions from a random forest 7 | input_file = 'adult/y_and_p.csv' 8 | 9 | print "loading data..." 10 | 11 | y_and_p = np.loadtxt( input_file, delimiter = ',' ) 12 | 13 | y = y_and_p[:,0] 14 | p = y_and_p[:,1] 15 | 16 | # y need to be 0/1 17 | y[y == -1] = 0 18 | 19 | -------------------------------------------------------------------------------- /log_loss.py: -------------------------------------------------------------------------------- 1 | # https://www.kaggle.com/wiki/LogarithmicLoss 2 | 3 | import scipy as sp 4 | 5 | def log_loss( act, pred ): 6 | epsilon = 1e-15 7 | pred = sp.maximum(epsilon, pred) 8 | pred = sp.minimum(1-epsilon, pred) 9 | ll = sum(act*sp.log(pred) + sp.subtract(1,act)*sp.log(sp.subtract(1,pred))) 10 | ll = ll * -1.0/len(act) 11 | return ll 12 | -------------------------------------------------------------------------------- /platts_scaling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | "calibrate a classifier's predictions using Platt's scaling (logistic regression)" 4 | 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | from sklearn.linear_model import LogisticRegression as LR 9 | from sklearn.metrics import accuracy_score 10 | from sklearn.metrics import roc_auc_score as AUC 11 | 12 | from log_loss import log_loss 13 | from get_diagram_data import get_diagram_data 14 | from load_data_adult import y, p 15 | 16 | ### 17 | 18 | # train/test split (in half) 19 | 20 | train_end = y.shape[0] / 2 21 | test_start = train_end + 1 22 | 23 | y_train = y[0:train_end] 24 | y_test =y[test_start:] 25 | 26 | p_train = p[0:train_end] 27 | p_test =p[test_start:] 28 | 29 | ### 30 | 31 | lr = LR() # default param values 32 | lr.fit( p_train.reshape( -1, 1 ), y_train ) # LR needs X to be 2-dimensional 33 | p_calibrated = lr.predict_proba( p_test.reshape( -1, 1 ))[:,1] 34 | 35 | ### 36 | 37 | acc = accuracy_score( y_test, np.round( p_test )) 38 | acc_calibrated = accuracy_score( y_test, np.round( p_calibrated )) 39 | 40 | auc = AUC( y_test, p_test ) 41 | auc_calibrated = AUC( y_test, p_calibrated ) 42 | 43 | ll = log_loss( y_test, p_test ) 44 | ll_calibrated = log_loss( y_test, p_calibrated ) 45 | 46 | print "accuracy - before/after:", acc, "/", acc_calibrated 47 | print "AUC - before/after: ", auc, "/", auc_calibrated 48 | print "log loss - before/after:", ll, "/", ll_calibrated 49 | 50 | """ 51 | accuracy - before/after: 0.847788697789 / 0.846805896806 52 | AUC - before/after: 0.878139845077 / 0.878139845077 53 | log loss - before/after: 0.630525772871 / 0.364873617584 54 | """ 55 | 56 | ### 57 | 58 | print "creating diagrams..." 59 | 60 | n_bins = 10 61 | 62 | # uncalibrated 63 | 64 | mean_predicted_values, true_fractions = get_diagram_data( y_test, p_test, n_bins ) 65 | plt.plot( mean_predicted_values, true_fractions ) 66 | 67 | # calibrated 68 | 69 | mean_predicted_values, true_fractions = get_diagram_data( y_test, p_calibrated, n_bins ) 70 | plt.plot( mean_predicted_values, true_fractions, 'green' ) 71 | 72 | # perfect calibration line 73 | plt.plot( np.linspace( 0, 1 ), np.linspace( 0, 1 ), 'gray' ) 74 | 75 | plt.show() 76 | 77 | 78 | -------------------------------------------------------------------------------- /reliability_diagram.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | "plot a reliability diagram showing how good a classifier's calibration is" 4 | 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | from get_diagram_data import get_diagram_data 9 | 10 | #from load_data import y, p 11 | from load_data_adult import y, p 12 | 13 | print "computing..." 14 | 15 | n_bins = 20 16 | 17 | mean_predicted_values, true_fractions = get_diagram_data( y, p, n_bins ) 18 | plt.plot( mean_predicted_values, true_fractions ) 19 | 20 | # perfect calibration line 21 | plt.plot( np.linspace( 0, 1 ), np.linspace( 0, 1 ), 'gray' ) 22 | 23 | plt.show() 24 | --------------------------------------------------------------------------------