├── support ├── __init__.py ├── parameters.py ├── experiments │ ├── __init__.py │ ├── experiment_1.py │ └── experiment_2.py ├── datasets.py ├── evaluation.py └── model.py ├── notebooks ├── utils │ ├── __init__.py │ └── code.py ├── 3.0-experiment-2-feature-engineering.ipynb ├── 4.0-model-evaluation.ipynb └── 3.1-experiment-2-model.ipynb ├── docs ├── 1-CoNVO.docx ├── 2-Problem Model.xlsx └── 3-Vision, Arguments, and Results.docx ├── requirements.txt ├── models └── experiment-1-model.pkl ├── Makefile ├── .gitignore ├── data └── bank-additional │ ├── .Rhistory │ └── bank-additional-names.txt └── Readme.md /support/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /support/parameters.py: -------------------------------------------------------------------------------- 1 | P_TARGETED = .066 2 | AVG_REVENUE = 1083 3 | AVG_COST = -8 -------------------------------------------------------------------------------- /docs/1-CoNVO.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/calvdee/end-to-end-lead-scoring/HEAD/docs/1-CoNVO.docx -------------------------------------------------------------------------------- /docs/2-Problem Model.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/calvdee/end-to-end-lead-scoring/HEAD/docs/2-Problem Model.xlsx -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.15.2 2 | pandas==0.23.4 3 | matplotlib==2.2.2 4 | seaborn==0.9.0 5 | scikit-learn==0.20.0 -------------------------------------------------------------------------------- /models/experiment-1-model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/calvdee/end-to-end-lead-scoring/HEAD/models/experiment-1-model.pkl -------------------------------------------------------------------------------- /notebooks/utils/code.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | 3 | # Adds the root folder to the python path 4 | sys.path.insert(1, os.path.join(sys.path[0], '..')) -------------------------------------------------------------------------------- /docs/3-Vision, Arguments, and Results.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/calvdee/end-to-end-lead-scoring/HEAD/docs/3-Vision, Arguments, and Results.docx -------------------------------------------------------------------------------- /support/experiments/__init__.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import roc_auc_score, make_scorer 2 | 3 | def get_scorer(): 4 | scorer = make_scorer(roc_auc_score) 5 | return scorer -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc 2 | 3 | clean-pyc: 4 | find . -name '*.pyc' -exec rm -f {} + 5 | find . -name '*.pyo' -exec rm -f {} + 6 | find . -name '__pycache__' -exec rm -rf {} + 7 | 8 | clean-notebooks: 9 | find . -name '.ipynb_checkpoints' -exec rm -rf {} + -------------------------------------------------------------------------------- /support/datasets.py: -------------------------------------------------------------------------------- 1 | def get_data(data_path): 2 | import pandas as pd 3 | from sklearn.preprocessing import LabelBinarizer 4 | 5 | data = pd.read_csv(data_path) 6 | 7 | X = data.drop('y', axis=1) 8 | y = data.y.apply(lambda x: 1 if x == 'yes' else 0) 9 | 10 | return X, y -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Internal 2 | # ------------------------------------------ 3 | .DS_STORE 4 | .ipynb_checkpoints/ 5 | ~$* 6 | # ------------------------------------------ 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # celery beat schedule file 92 | celerybeat-schedule 93 | 94 | # SageMath parsed files 95 | *.sage.py 96 | 97 | # Environments 98 | .env 99 | .venv 100 | env/ 101 | venv/ 102 | ENV/ 103 | env.bak/ 104 | venv.bak/ 105 | 106 | # Spyder project settings 107 | .spyderproject 108 | .spyproject 109 | 110 | # Rope project settings 111 | .ropeproject 112 | 113 | # mkdocs documentation 114 | /site 115 | 116 | # mypy 117 | .mypy_cache/ 118 | .dmypy.json 119 | dmypy.json 120 | 121 | # Pyre type checker 122 | .pyre/ 123 | -------------------------------------------------------------------------------- /support/experiments/experiment_1.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.preprocessing import OneHotEncoder, FunctionTransformer, RobustScaler 3 | from sklearn.compose import ColumnTransformer 4 | from sklearn.pipeline import Pipeline, FeatureUnion 5 | 6 | NUMERIC_FEATURES = [ 7 | 'age', 8 | 'campaign', 9 | 'previous', 10 | 'emp.var.rate', 11 | 'cons.price.idx', 12 | 'cons.conf.idx', 13 | 'euribor3m', 14 | 'nr.employed' 15 | ] 16 | 17 | CATEGORICAL_FEATURES = [ 18 | 'job', 19 | 'marital', 20 | 'education', 21 | 'default', 22 | 'housing', 23 | 'loan', 24 | 'contact', 25 | 'month', 26 | 'day_of_week', 27 | 'poutcome' 28 | ] 29 | 30 | def get_categorical_pipeline(): 31 | # Create the transformers for categorical features 32 | cat_ct = ColumnTransformer([('categoricals', 'passthrough', CATEGORICAL_FEATURES)]) 33 | 34 | # Create the pipeline to transform categorical features 35 | cat_pipeline = Pipeline([ 36 | ('cat_ct', cat_ct), 37 | ('ohe', OneHotEncoder(handle_unknown='ignore')) 38 | ]) 39 | 40 | return cat_pipeline 41 | 42 | def get_numeric_pipeline(): 43 | # Create the transformers for numeric features 44 | num_ct = ColumnTransformer([('numerics', 'passthrough', NUMERIC_FEATURES)]) 45 | 46 | # Create the pipeline to transform numeric features 47 | num_pipeline = Pipeline([ 48 | ('num_union', num_ct), 49 | ('scaler', RobustScaler()) 50 | ]) 51 | 52 | return num_pipeline 53 | 54 | def get_pipeline(): 55 | # Create the categorical and numeric pipelines 56 | cat_pipeline = get_categorical_pipeline() 57 | num_pipeline = get_numeric_pipeline() 58 | 59 | # Create the feature union of categorical and numeric attributes 60 | ft_union = FeatureUnion([ 61 | ('cat_pipeline', cat_pipeline), 62 | ('num_pipeline', num_pipeline) 63 | ]) 64 | 65 | pipeline = Pipeline([ 66 | ('ft_union', ft_union) 67 | ]) 68 | 69 | return pipeline 70 | 71 | def baseline_model_predictions(X, y, n_targeted): 72 | # Get all of the instances where the previous campaign was a success 73 | success = X[X.poutcome == 'success'] 74 | 75 | # Calcuate how many more instances we need 76 | n_rest = n_targeted - len(success) 77 | 78 | # Randomly choose from the remaining instances 79 | rest = X[~(X.index.isin(success.index))].sample(n=n_rest, random_state=1) 80 | 81 | # Combine the targeted and random groups 82 | baseline_targets = pd.concat([success, rest], axis=0) 83 | baseline_ys = y.loc[baseline_targets.index] 84 | 85 | return baseline_ys -------------------------------------------------------------------------------- /support/evaluation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .model import build_tuned_model 3 | 4 | def evaluate_model(features, target, name, model, param_grid, scorer, pipeline=None, cv_folds=5): 5 | tuned_model = build_tuned_model(name, model, features, target, param_grid, scorer, pipeline=pipeline) 6 | results = tuned_model.results 7 | best_result = results.query('rank_test_score == 1') 8 | test_mean = best_result['mean_test_score'].values[0] 9 | test_std = best_result['std_test_score'].values[0] 10 | return (tuned_model, name, test_mean, test_std) 11 | 12 | def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None, 13 | n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring=None): 14 | """ 15 | Generate a simple plot of the test and training learning curve. 16 | 17 | Parameters 18 | ---------- 19 | estimator : object type that implements the "fit" and "predict" methods 20 | An object of that type which is cloned for each validation. 21 | 22 | title : string 23 | Title for the chart. 24 | 25 | X : array-like, shape (n_samples, n_features) 26 | Training vector, where n_samples is the number of samples and 27 | n_features is the number of features. 28 | 29 | y : array-like, shape (n_samples) or (n_samples, n_features), optional 30 | Target relative to X for classification or regression; 31 | None for unsupervised learning. 32 | 33 | ylim : tuple, shape (ymin, ymax), optional 34 | Defines minimum and maximum yvalues plotted. 35 | 36 | cv : int, cross-validation generator or an iterable, optional 37 | Determines the cross-validation splitting strategy. 38 | Possible inputs for cv are: 39 | - None, to use the default 3-fold cross-validation, 40 | - integer, to specify the number of folds. 41 | - An object to be used as a cross-validation generator. 42 | - An iterable yielding train/test splits. 43 | 44 | For integer/None inputs, if ``y`` is binary or multiclass, 45 | :class:`StratifiedKFold` used. If the estimator is not a classifier 46 | or if ``y`` is neither binary nor multiclass, :class:`KFold` is used. 47 | 48 | Refer :ref:`User Guide ` for the various 49 | cross-validators that can be used here. 50 | 51 | n_jobs : integer, optional 52 | Number of jobs to run in parallel (default 1). 53 | """ 54 | import matplotlib.pyplot as plt 55 | from sklearn.model_selection import learning_curve 56 | import numpy as np 57 | 58 | plt.figure() 59 | plt.title(title) 60 | if ylim is not None: 61 | plt.ylim(*ylim) 62 | plt.xlabel("Training examples") 63 | plt.ylabel("Score") 64 | train_sizes, train_scores, test_scores = learning_curve( 65 | estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes, scoring=scoring) 66 | train_scores_mean = np.mean(train_scores, axis=1) 67 | train_scores_std = np.std(train_scores, axis=1) 68 | test_scores_mean = np.mean(test_scores, axis=1) 69 | test_scores_std = np.std(test_scores, axis=1) 70 | plt.grid() 71 | 72 | plt.fill_between(train_sizes, train_scores_mean - train_scores_std, 73 | train_scores_mean + train_scores_std, alpha=0.1, 74 | color="r") 75 | plt.fill_between(train_sizes, test_scores_mean - test_scores_std, 76 | test_scores_mean + test_scores_std, alpha=0.1, color="g") 77 | plt.plot(train_sizes, train_scores_mean, 'o-', color="r", 78 | label="Training score") 79 | plt.plot(train_sizes, test_scores_mean, 'o-', color="g", 80 | label="Cross-validation score") 81 | 82 | plt.legend(loc="best"); 83 | return plt -------------------------------------------------------------------------------- /support/experiments/experiment_2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.preprocessing import OneHotEncoder, FunctionTransformer, KBinsDiscretizer 3 | from sklearn.compose import ColumnTransformer 4 | from sklearn.pipeline import Pipeline, FeatureUnion 5 | from sklearn.base import clone 6 | 7 | 8 | NUMERIC_FEATURES = [ 9 | 'age', 10 | 'campaign', 11 | 'previous', 12 | 'emp.var.rate', 13 | 'cons.price.idx', 14 | 'cons.conf.idx', 15 | 'euribor3m', 16 | 'nr.employed', 17 | 'campaign_to_previous' 18 | ] 19 | 20 | CATEGORICAL_FEATURES = [ 21 | 'job', 22 | 'marital', 23 | 'education', 24 | 'default', 25 | 'housing', 26 | 'loan', 27 | 'contact', 28 | 'month', 29 | 'day_of_week', 30 | 'poutcome' 31 | ] 32 | 33 | NEW_CATEGORICAL_FEATURES = [ 34 | 'pcontacted_last_campaign', 35 | 'pcampaign', 36 | 'previous', 37 | 'campaign_gte10' 38 | ] 39 | 40 | def ft_pcontacted_last_campaign(X): 41 | pcontacted = ~(X == 999) 42 | return pcontacted.values.reshape(-1,1) 43 | 44 | def ft_pcampaign(X): 45 | pcampaign = ~(X == 'nonexistent') 46 | return pcampaign.values.reshape(-1,1) 47 | 48 | def ft_previous(X): 49 | previous = X.astype(str) 50 | return previous.values.reshape(-1,1) 51 | 52 | def ft_campaign_gte10(X): 53 | campaign_gte10 = X >= 10 54 | return campaign_gte10.values.reshape(-1,1) 55 | 56 | def ft_campaign_to_previous(X): 57 | ratio = lambda x: 0 if x.previous == 0 else x.campaign / x.previous 58 | campaign_to_previous = X[['campaign', 'previous']].apply(ratio, axis=1) 59 | return campaign_to_previous.values.reshape(-1,1) 60 | 61 | def get_categorical_ct(): 62 | # Create the transformers for categorical features 63 | add_pcontacted_last_campaign = FunctionTransformer(ft_pcontacted_last_campaign, validate=False) 64 | add_pcampaign = FunctionTransformer(ft_pcampaign, validate=False) 65 | add_previous = FunctionTransformer(ft_previous, validate=False) 66 | add_campaign_gte10 = FunctionTransformer(ft_campaign_gte10, validate=False) 67 | 68 | cat_features = [ 69 | ('categoricals', 'passthrough', CATEGORICAL_FEATURES), 70 | ('pcontacted_last_campaign', add_pcontacted_last_campaign, 'pdays'), 71 | ('pcampaign', add_pcampaign, 'poutcome'), 72 | ('previous', add_previous, 'previous'), 73 | ('campaign_gte10', add_campaign_gte10, 'campaign') 74 | ] 75 | cat_ct = ColumnTransformer(cat_features) 76 | 77 | return cat_ct 78 | 79 | def get_categorical_pipeline(): 80 | cat_cts = get_categorical_ct() 81 | 82 | # Create the pipeline to transform categorical features 83 | cat_pipeline = Pipeline([ 84 | ('cat_ct', cat_cts), 85 | ('ohe', OneHotEncoder(handle_unknown='ignore')) 86 | ]) 87 | 88 | return cat_pipeline 89 | 90 | def get_numeric_pipeline(): 91 | binning_pipeline = Pipeline([ 92 | ('log', FunctionTransformer(np.log, validate=True)), 93 | ('kbins', KBinsDiscretizer()) 94 | ]) 95 | 96 | # Create the transformers for numeric features 97 | # num_ct = ColumnTransformer([('numerics', 'passthrough', numerics)]) 98 | 99 | # new_num_features = [ 100 | # ('num_ct', num_ct), 101 | # ('ft_campaign_to_previous', FunctionTransformer(ft_campaign_to_previous, validate=False)) 102 | # ] 103 | # num_union = FeatureUnion(new_num_features) 104 | 105 | # # Create the pipeline to transform numeric features 106 | # num_pipeline = Pipeline([ 107 | # ('num_union', num_union), 108 | # ('scaler', RobustScaler()) 109 | # ]) 110 | 111 | age_campaign_ct = ColumnTransformer([ 112 | ('age_pipeline', clone(binning_pipeline), ['age']), 113 | ('campaign_pipeline', clone(binning_pipeline), ['campaign']) 114 | ]) 115 | 116 | return age_campaign_ct 117 | 118 | def get_pipeline(): 119 | # Create the categorical and numeric pipelines 120 | cat_pipeline = get_categorical_pipeline() 121 | num_pipeline = get_numeric_pipeline() 122 | 123 | # Create the feature union of categorical and numeric attributes 124 | ft_union = FeatureUnion([ 125 | ('cat_pipeline', cat_pipeline), 126 | # ('num_pipeline', num_pipeline) 127 | ]) 128 | 129 | pipeline = Pipeline([ 130 | ('ft_union', ft_union) 131 | ]) 132 | 133 | return pipeline -------------------------------------------------------------------------------- /data/bank-additional/.Rhistory: -------------------------------------------------------------------------------- 1 | ?wireframe 2 | librar(lattice) 3 | library(lattice) 4 | ?wireframe 5 | wireframe(volcano, shade = TRUE,# 6 | aspect = c(61/87, 0.4),# 7 | light.source = c(10,0,10)) 8 | library(rminer) 9 | help(package=rminer) 10 | ls() 11 | class(x) 12 | x 13 | class(y) 14 | y 15 | x=1 16 | y="hello" 17 | y 18 | x=c(1.1,2.3,-1,4,2e-2) 19 | x 20 | x=c(1,2,3,4,5) 21 | x 22 | x=1:10 23 | x 24 | x=10:1 25 | x 26 | x=rnorm(18,25,1) 27 | x 28 | x=round(rnorm(18,25,1)) 29 | x 30 | x=round(rnorm(18,35,10)) 31 | x 32 | x=c(18,x) 33 | x 34 | x=c(98,x) 35 | x 36 | summary(x) 37 | lenght(x) 38 | length(x) 39 | x 40 | x[3] 41 | x[3]=30 42 | x 43 | z[]=1 44 | z=vector(length=5); print(z) 45 | x 46 | z 47 | z[]=1; print(z) 48 | z 49 | z[c(1,3,5)]=2 50 | z 51 | sum(z) 52 | min(z) 53 | max(z) 54 | sort(z) 55 | plot(z,type="b",lwd=2,col="blue") 56 | plot(z,type="b",lwd=20,col="blue") 57 | a=rep(3,10) 58 | a 59 | rep(10,1000) 60 | rep(10,5,3) 61 | ?rep 62 | rep(10,5,times=3) 63 | rep(x=10,5,times=3) 64 | rep(x=10,times=3) 65 | rep(x=10,5,each=3) 66 | rep(c(1,2,3),5,each=3) 67 | rep(c(1,2,3),5) 68 | c=seq(2,20,2) 69 | seq(2,20,2) 70 | seq(1,20,2) 71 | seq(1,20,1) 72 | seq(1,20,-1) 73 | seq(20,1,-1) 74 | seq(20,1,lenght=3) 75 | seq(20,1,length=3) 76 | x 77 | x^2+1 78 | m=matrix(ncol=3,nrow=2) 79 | m 80 | m[,]=0 81 | m 82 | m[1,] 83 | m[1,]=1:3 84 | m 85 | m[2,]=4:6 86 | m 87 | m[,1]=c(-1,-1) 88 | m 89 | m[,3]=c(10,10) 90 | m 91 | class(m) 92 | sum(m) 93 | summary(m) 94 | print(m) 95 | plot(m) 96 | m=matrix(ncol=10,nrow=4); m[,]=0! 97 | m=matrix(ncol=10,nrow=4); m[,]=0 98 | m 99 | m[c(1,3),seq(1,10,2)]=1 100 | m 101 | seq(1,10,2) 102 | m 103 | m[1,] 104 | m[3,] 105 | m[c(1,3),] 106 | m[,1] 107 | m 108 | m[,3] 109 | m[,c(1,3)] 110 | m[c(1,3),seq(1,10,2)] 111 | m[c(1,3),seq(1,10,2)]=3 112 | m 113 | seq(1,10,2) 114 | ?seq 115 | y=factor(c("a","a","a","b","b","c")) 116 | y 117 | y+1 118 | summary(y) 119 | plot(y) 120 | l=list(a="ola",b=1:3) 121 | l 122 | l=list(a="ola",b=1:3,f=y) 123 | l 124 | l$b 125 | l$a 126 | l$f 127 | m=matrix(ncol=3,nrow=3); m[,]=1 128 | m 129 | d=data.frame(m); names(d)=c("dia","mes","ano")! 130 | d=data.frame(m); names(d)=c("dia","mes","ano") 131 | class(d) 132 | class(m) 133 | d 134 | d[1,] 135 | d[,1] 136 | d$dia 137 | d$mes=factor(c("Jan","Feb","Apr")) 138 | d$ano=c(1999,1999,2000); print(d) 139 | d 140 | plot(d$mes) 141 | plot(d$ano) 142 | plot(d$ano,type="b") 143 | d 144 | summary(d) 145 | d[2,1]=2 146 | d 147 | d[2,]=c(31,"Jan",2009) 148 | d 149 | edit(d) 150 | names(d) 151 | names(d)=c("Dia","Mes","Ano") 152 | names(d) 153 | d 154 | d2=rbind(d,c(21,"Feb",2008)) 155 | d 156 | d2 157 | d 158 | d2 159 | d2=rbind(d2,d2) 160 | d2 161 | sample(1:10,3) 162 | c(sample(1:50,5),sample(1:11,2)) 163 | for(i in 1:1000) c(sample(1:50,5),sample(1:11,2)) 164 | for(i in 1:1000) print(c(sample(1:50,5),sample(1:11,2))) 165 | c(sort(sample(1:50,5)),sort(sample(1:11,2))) 166 | cat("numeros:",c(sort(sample(1:50,5)),"estrelas:",sort(sample(1:11,2)))) 167 | set.seed(1) 168 | cat("numeros:",c(sort(sample(1:50,5)),"estrelas:",sort(sample(1:11,2)))) 169 | set.seed(1) 170 | cat("numeros:",c(sort(sample(1:50,5)),"estrelas:",sort(sample(1:11,2)))) 171 | x=rnorm(100,10) 172 | y=rnorm(100,8) 173 | x 174 | y 175 | t.test(x,y) 176 | cor(x,y) 177 | d2 178 | d2[d2$Mes==Jan] 179 | d2[,d2$Mes==Jan] 180 | I=whic(d2$Mes==Jan) 181 | I=which(d2$Mes==Jan) 182 | I=which(d2$Mes=="Jan") 183 | I 184 | d2[I,] 185 | library(foreign) 186 | help(package=foreign) 187 | q("no") 188 | 2509+2013+724 189 | 2507+2006+604 190 | 2507+2006+588 191 | 2507+1961+588 192 | 2507+1950+574 193 | 2507+1896+574 194 | 2507+1863+574 195 | setwd("/Users/pcortez/R/rminer") 196 | source("popular2.R") 197 | RStudio_CRAN_data_folder <- download_RStudio_CRAN_data(START = '2013-04-02', END = '2013-04-05') 198 | if(packageVersion("installr") %in% c("0.8","0.9","0.9.2")) install.packages('installr') 199 | require(installr) 200 | RStudio_CRAN_data_folder <- download_RStudio_CRAN_data(START = '2013-04-02', END = '2013-04-05') 201 | my_RStudio_CRAN_data <- read_RStudio_CRAN_data(RStudio_CRAN_data_folder) 202 | summary(my_RStudio_CRAN_data) 203 | NROW(my_RStudio_CRAN_data) 204 | which(my_RStudio_CRAN_data$package=="rminer") 205 | cat("rminer installs:",length(which(my_RStudio_CRAN_data$package=="rminer")),"\n") 206 | 1+1+1+1+2+1 207 | 160-7 208 | 6+2+1+1+1+1+1 209 | 141-13 210 | 49-23 211 | 45-6 212 | 37-2 213 | 33-5 214 | 29-7 215 | 28-2 216 | 27-2 217 | 22-4 218 | 18-12 219 | 17-12 220 | 15-3 221 | 15-2 222 | 14-6 223 | q("n") 224 | q() 225 | mean(3,4,3.5) 226 | mean(3,4.5,3.5) 227 | mean(c(3,4,3.5)) 228 | mean(c(3,4.5,4)) 229 | 70+90+45 230 | mean(c(3,4.5,4)) 231 | 4.3/6 232 | setwd("/Users/pcortez/DOUTORAMENTO/SergioMoro/DSS-article/UCI/bank-additional") 233 | d=read.table("bank-additional.csv",header=TRUE,sep=";") 234 | nrow(d) 235 | ncol(d) 236 | setwd("../bank") 237 | d=read.table("bank-full.csv",header=TRUE,sep=";") 238 | ncol(d) 239 | nrow(d) 240 | d=read.table("bank.csv",header=TRUE,sep=";") 241 | nrow(d) 242 | ncol(d) 243 | table(d$y) 244 | summary(d) 245 | setwd("../bank-additional/") 246 | d=read.table("bank-additional-full.csv",header=TRUE,sep=";") 247 | summary(d) 248 | table(d$emp.var.rate) 249 | d=read.table("bank-additional-full.csv",header=TRUE,sep=";") 250 | summary(d) 251 | q("n") 252 | q("no") 253 | -------------------------------------------------------------------------------- /data/bank-additional/bank-additional-names.txt: -------------------------------------------------------------------------------- 1 | Citation Request: 2 | This dataset is publicly available for research. The details are described in [Moro et al., 2014]. 3 | Please include this citation if you plan to use this database: 4 | 5 | [Moro et al., 2014] S. Moro, P. Cortez and P. Rita. A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems, In press, http://dx.doi.org/10.1016/j.dss.2014.03.001 6 | 7 | Available at: [pdf] http://dx.doi.org/10.1016/j.dss.2014.03.001 8 | [bib] http://www3.dsi.uminho.pt/pcortez/bib/2014-dss.txt 9 | 10 | 1. Title: Bank Marketing (with social/economic context) 11 | 12 | 2. Sources 13 | Created by: Sérgio Moro (ISCTE-IUL), Paulo Cortez (Univ. Minho) and Paulo Rita (ISCTE-IUL) @ 2014 14 | 15 | 3. Past Usage: 16 | 17 | The full dataset (bank-additional-full.csv) was described and analyzed in: 18 | 19 | S. Moro, P. Cortez and P. Rita. A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems (2014), doi:10.1016/j.dss.2014.03.001. 20 | 21 | 4. Relevant Information: 22 | 23 | This dataset is based on "Bank Marketing" UCI dataset (please check the description at: http://archive.ics.uci.edu/ml/datasets/Bank+Marketing). 24 | The data is enriched by the addition of five new social and economic features/attributes (national wide indicators from a ~10M population country), published by the Banco de Portugal and publicly available at: https://www.bportugal.pt/estatisticasweb. 25 | This dataset is almost identical to the one used in [Moro et al., 2014] (it does not include all attributes due to privacy concerns). 26 | Using the rminer package and R tool (http://cran.r-project.org/web/packages/rminer/), we found that the addition of the five new social and economic attributes (made available here) lead to substantial improvement in the prediction of a success, even when the duration of the call is not included. Note: the file can be read in R using: d=read.table("bank-additional-full.csv",header=TRUE,sep=";") 27 | 28 | The zip file includes two datasets: 29 | 1) bank-additional-full.csv with all examples, ordered by date (from May 2008 to November 2010). 30 | 2) bank-additional.csv with 10% of the examples (4119), randomly selected from bank-additional-full.csv. 31 | The smallest dataset is provided to test more computationally demanding machine learning algorithms (e.g., SVM). 32 | 33 | The binary classification goal is to predict if the client will subscribe a bank term deposit (variable y). 34 | 35 | 5. Number of Instances: 41188 for bank-additional-full.csv 36 | 37 | 6. Number of Attributes: 20 + output attribute. 38 | 39 | 7. Attribute information: 40 | 41 | For more information, read [Moro et al., 2014]. 42 | 43 | Input variables: 44 | # bank client data: 45 | 1 - age (numeric) 46 | 2 - job : type of job (categorical: "admin.","blue-collar","entrepreneur","housemaid","management","retired","self-employed","services","student","technician","unemployed","unknown") 47 | 3 - marital : marital status (categorical: "divorced","married","single","unknown"; note: "divorced" means divorced or widowed) 48 | 4 - education (categorical: "basic.4y","basic.6y","basic.9y","high.school","illiterate","professional.course","university.degree","unknown") 49 | 5 - default: has credit in default? (categorical: "no","yes","unknown") 50 | 6 - housing: has housing loan? (categorical: "no","yes","unknown") 51 | 7 - loan: has personal loan? (categorical: "no","yes","unknown") 52 | # related with the last contact of the current campaign: 53 | 8 - contact: contact communication type (categorical: "cellular","telephone") 54 | 9 - month: last contact month of year (categorical: "jan", "feb", "mar", ..., "nov", "dec") 55 | 10 - day_of_week: last contact day of the week (categorical: "mon","tue","wed","thu","fri") 56 | 11 - duration: last contact duration, in seconds (numeric). Important note: this attribute highly affects the output target (e.g., if duration=0 then y="no"). Yet, the duration is not known before a call is performed. Also, after the end of the call y is obviously known. Thus, this input should only be included for benchmark purposes and should be discarded if the intention is to have a realistic predictive model. 57 | # other attributes: 58 | 12 - campaign: number of contacts performed during this campaign and for this client (numeric, includes last contact) 59 | 60 | 13 - pdays: number of days that passed by after the client was last contacted from a previous campaign (numeric; 999 means client was not previously contacted) 61 | 62 | 14 - previous: number of contacts performed before this campaign and for this client (numeric) 63 | 64 | 15 - poutcome: outcome of the previous marketing campaign (categorical: "failure","nonexistent","success") 65 | 66 | # social and economic context attributes 67 | 16 - emp.var.rate: employment variation rate - quarterly indicator (numeric) 68 | 17 - cons.price.idx: consumer price index - monthly indicator (numeric) 69 | 18 - cons.conf.idx: consumer confidence index - monthly indicator (numeric) 70 | 19 - euribor3m: euribor 3 month rate - daily indicator (numeric) 71 | 20 - nr.employed: number of employees - quarterly indicator (numeric) 72 | 73 | Output variable (desired target): 74 | 21 - y - has the client subscribed a term deposit? (binary: "yes","no") 75 | 76 | 8. Missing Attribute Values: There are several missing values in some categorical attributes, all coded with the "unknown" label. These missing values can be treated as a possible class label or using deletion or imputation techniques. 77 | 78 | * Number of employees at the extremes is predictive of the outcome 79 | * Employment variability rate at the extremes is predictive of the outcome 80 | * euribor3m at the extremes is predictive of outcome -------------------------------------------------------------------------------- /support/model.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.pipeline import make_pipeline 3 | from sklearn.base import clone 4 | from sklearn.model_selection import GridSearchCV 5 | from sklearn.pipeline import Pipeline 6 | 7 | def build_tuned_model(name, base_model, X_train, y_train, hparams, scorer=None, cv_folds=5, pipeline=None): 8 | from time import time 9 | start = time() 10 | print('==> Starting {}-fold cross validation for {} model, {} examples'.format(str(cv_folds), name, len(X_train))) 11 | model = TunedModel(hparams, name=name, model=base_model, pipeline=pipeline) 12 | model.train(X_train, y_train, scorer, cv_folds) 13 | elapsed = time() - start 14 | print("==> Elapsed seconds: {:.3f}".format(elapsed)) 15 | 16 | print('Best {} model: {}'.format(model.name, model.model)) 17 | print('Best {} score: {:.3f}'.format( 18 | model.name, 19 | model.results.sort_values('mean_test_score', ascending=False 20 | ).head(1).mean_test_score.values[0])) 21 | 22 | return model 23 | 24 | # ============================================================================================================ 25 | # Model 26 | # ============================================================================================================ 27 | class Model(object): 28 | def __init__(self, name, model, pipeline=None): 29 | self.name = name 30 | self.model = model 31 | self.pipeline = pipeline 32 | 33 | def train(self, X, y): 34 | """ Fits the model and builds the full pipeline """ 35 | if self.pipeline is None: 36 | X_transformed = X 37 | self.model_pipeline = make_pipeline(self.model) 38 | else: 39 | X_transformed = self.pipeline.fit_transform(X) 40 | self.model_pipeline = make_pipeline(self.pipeline, self.model) 41 | 42 | self.model.fit(X_transformed, y) 43 | 44 | return self 45 | 46 | def predict(self, X): 47 | """ Fits the model and builds the full pipeline 48 | TODO: Make sure the model was fitted 49 | """ 50 | # if self.pipeline is None: 51 | # X_transformed = X 52 | # else: 53 | # X_transformed = self.pipeline.fit_transform(X) 54 | 55 | preds = self.model_pipeline.predict(X) 56 | 57 | return preds 58 | 59 | def get_model_pipeline(self): 60 | """ Useful for cross validation to refit the pipeline on every round """ 61 | full_pipeline = clone(self.pipeline) 62 | full_pipeline.steps.append((self.name, self.model)) 63 | return full_pipeline 64 | 65 | def score(self, X, y, scorer): 66 | """ Scores the model using the scorer 67 | 68 | Postcondititions: 69 | - score should not be 0 70 | - model.predictions should have elements 71 | """ 72 | score = 0 73 | 74 | if self.pipeline is None: 75 | model.predictions = self.model.predict(X) 76 | score = scorer(self.model, X, y) 77 | else: 78 | model_predictions = self.model_pipeline.predict(X) 79 | score = scorer(self.model_pipeline, X, y) 80 | 81 | return score 82 | 83 | def save(self, file_path): 84 | from joblib import dump 85 | dump(self, file_path) 86 | 87 | @staticmethod 88 | def load(file_path): 89 | from joblib import load 90 | model = load(file_path) 91 | return model 92 | 93 | 94 | def score_cv(self, X, y, scorer, k=5): 95 | """ Scores the model using the scorer 96 | 97 | Postcondititions: 98 | - score should not be 0 99 | - model.predictions should have elements 100 | """ 101 | from sklearn.model_selection import cross_val_score 102 | 103 | score = 0 104 | 105 | if self.pipeline is None: 106 | score = cross_val_score(self.model, X, y, scoring=scorer, cv=k, n_jobs=-1) 107 | else: 108 | score = scorer(self.model_pipeline, X, y) 109 | 110 | return (score.mean(), score.std()) 111 | 112 | # ============================================================================================================ 113 | # TunedModel 114 | # ============================================================================================================ 115 | class TunedModel(Model): 116 | """ A class used to optimize the hyperparameters for a machine learning algorithm 117 | 118 | Parameters 119 | ---------- 120 | name : string 121 | The name of a model 122 | 123 | param_grid : dict 124 | A dict of (parameter, values) pairs to optimize 125 | 126 | pipeline : object 127 | A pipeline to apply to the data before fitting the model 128 | """ 129 | 130 | def __init__(self, param_grid, **kwargs): 131 | Model.__init__(self, **kwargs) 132 | self.param_grid = param_grid 133 | 134 | def train(self, X, y, scorer, cv_folds=5): 135 | """ Tunes a model using the parameter grid that this class was initialized with. 136 | 137 | Parameters 138 | ---------- 139 | X : array-like, matrix 140 | Input data 141 | 142 | y : array-like 143 | Targets for input data 144 | 145 | cv_folds : int, optional, default: 5 146 | The number of cross-validation folds to use in the optimization process. 147 | """ 148 | if not self.pipeline: 149 | # Setup 150 | grid_search = GridSearchCV( 151 | self.model, 152 | self.param_grid, 153 | cv=cv_folds, 154 | scoring=scorer, 155 | return_train_score=True, 156 | n_jobs=-1) 157 | 158 | # Run it 159 | grid_search.fit(X, y) 160 | 161 | # Save the model 162 | self.model = grid_search.best_estimator_ 163 | else: 164 | # Setup 165 | grid_search = GridSearchCV( 166 | self.get_model_pipeline(), 167 | self.param_grid, 168 | cv=cv_folds, 169 | scoring=scorer, 170 | return_train_score=True, 171 | n_jobs=-1) 172 | 173 | # Run it 174 | grid_search.fit(X, y) 175 | 176 | # Save the model and pipeline 177 | self.model = grid_search.best_estimator_.steps[-1][1] 178 | self.pipeline = Pipeline(grid_search.best_estimator_.steps[:-1]) 179 | 180 | self.results = pd.DataFrame(grid_search.cv_results_) -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # End-to-End Applied Data Science: Lead Scoring 2 | 3 | ## Motivation 4 | The purpose of this project is to provide an end-to-end example of working an enterprise-grade data science problem, from business understanding to model evaluation, by applying two different frameworks: 5 | 6 | * [CoNVO](https://www.oreilly.com/library/view/thinking-with-data/9781491949757/ch01.html) to scope the problem and formulate a data science solution 7 | * [CRISP-DM](https://en.wikipedia.org/wiki/Cross-industry_standard_process_for_data_mining) to iterate on a model-based solution 8 | 9 | Since data and machine learning engineers are typically responsible for deployment, maintenance, and optimization of infrastructure and models, this project omits the Deployment phase of CRISP-DM. 10 | 11 | This project is not without flaws - the argument could be stronger, costs and benefits could be more accurate, and the resulting model could perform better, but its level of detail is reflective of the effort required to to create a reproducible data science solution that is (1) well aligned with stakeholder expectations and (2) poised to deliver business value. We could continue to iterate and improve our results ad infinitum in the pursuit of the perfect solution but in the real world, we are constrained by budgets and so by time. It's clear that even if we had unlimited budget to refine the analysis and model, our early and fast efforts will likely be satisfactory enough to deploy a data solution, allowing us to move on and pursue a different project. From both business and engineering perspectives, it's generally best to start with the [simplest thing that could possibly work](http://www.agilenutshell.com/simplest_thing), especially if it's relatively cheap to build. 12 | 13 | ## Overview 14 | This project uses the Bank Marketing Dataset published to the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/bank+marketing) to demonstrate the scoping and implementation of a real-world data science project. The dataset's description on the UCI Machine Learning website states: 15 | 16 | > The classification goal is to predict if the client will subscribe a term deposit (variable y). 17 | 18 | With a hypothetical business context and feedback from stakeholders regarding various aspects of a marketing problem, we can arrive at a more complete and operational description of our problem, based on the [CoNVO](https://www.oreilly.com/library/view/thinking-with-data/9781491949757/ch01.html) framework: 19 | 20 | ### Context 21 | This Portuguese bank offers term deposits as a financial product offering to its existing customers as a means to provide capital for loan products. It is currently running a telemarketing campaign in which existing customers are pitched on the term deposit product during inbound customer service and outbound sales calls. The decision maker for this campaign is the director of marketing. 22 | 23 | ### Needs 24 | A tactical change in product strategy as a result of changing market conditions (lowered interest rates) requires additional budget resources to jumpstart a new campaign for a different financial product – mortgages. The bank figures that if it can improve its conversion rate with its current campaign, then it will be in a better position to spend the remaining budget more efficiently, leading to higher revenues and more funds available for the mortgage campaign. 25 | 26 | The director of marketing is also interested in quantifying the effect of this modelling effort to ensure that data science efforts are making an impact and, if successful, making it easier to get buy in for allocate resources to data-driven efforts in the future. 27 | 28 | ### Vision 29 | We propose to create a lead scoring model that will rank customers according to a model score that indicates how likely customers are to invest in a term deposit. By focussing on the customers who are most likely to convert, the bank will earn more per marketing dollar spent on customer specialists who are pitching to customers. To this end, we will deliver a ranked list of contacts in an Excel spreadsheet that the director of marketing can hand off to her management team for operationalization. 30 | 31 | Quantifying the impact of the campaign empirically will require the design prior to and execution of a controlled experiment when the campaign is deployed. Two weeks prior to model deployment, we’ll deliver a report outlining the experimental design and two weeks following the conclusion of the campaign, we’ll deliver a report summarizing the results. 32 | 33 | ### Outcome 34 | If the bank observes a significant increase in ROI as a result of the development and deployment of the lead scoring model, the engineering team will be tasked with integrating the model into a CRM system, making the deployment of future models faster and cheaper. 35 | 36 | ## Key Insights 37 | 38 | * The conversion rate of the previous marketing campaign was 25%. 39 | 40 | * The conversion rate of the current marketing campaign is 11%. 41 | 42 | * A simple "domain-driven" model that chooses customers to target based on whether they converted in the last marketing campaign and chooses remaining customers randomly improved the current conversion rate by 250% - bringing the conversion rate of targeted customers to 39%. 43 | 44 | * A Naive Bayes model using only categorical features from the original training set improved the baseline conversion rate by 354% - bringing the conversion rate of targeted customers to 51%. 45 | 46 | * With the Naive Bayes model, targeting 6.6% of the population identifies ~30% of the respondents. 47 | 48 | ## Project Structure 49 | 50 | ### data 51 | 52 | The data retrieved from the UCI Machine Learning Repository as well as training and test sets. 53 | 54 | * `bank_additional/` 55 | * [Original dataset and supporting documentation](https://archive.ics.uci.edu/ml/datasets/bank+marketing) retrieved from the UCI Machine Learning Repository on December 1, 2018. 56 | 57 | * `train.csv` 58 | * The dataset used for exploratory analysis to train machine learning models 59 | 60 | * `test.csv` 61 | * The dataset used to evaluate the optimal model 62 | 63 | ### docs 64 | 65 | Project documentation including the project context and scope, financial models, and an articulation of the solution. All of the documentation can also be found in [this Google Drive folder](https://drive.google.com/drive/folders/1CALQVFCMiGfsMvMqmVnLJII_EXaI5Obn?usp=sharing). 66 | 67 | * `1-CoNVO.docx` ([view online in Google Docs](https://docs.google.com/document/d/1LkOGwPcyZN6hDQ-KPCfb6p9m6VegMKE3FnxdYroSYyU/edit?usp=sharing)) 68 | * A document describing the scope of the project - its Context, Needs, Vision and Outcome 69 | 70 | * `2-Problem Model.xlsx` ([view online in Google Sheets](https://drive.google.com/open?id=1rDIC9r9fduUbbMj7KvcDUYNujHfReoDB)) 71 | * A spreadsheet model of the problem dynamics including costs, benefits, campaign conversion rates, and what-if scenarios 72 | 73 | * `3-Vision, Arguments, and Results.docx` ([view online in Google Docs](https://drive.google.com/open?id=1pO5jlEivQn1SbJtorNDzHBStk1m3i0d2)) 74 | * A document articulating the refinement of the problem, establishing definitions, values, and an operational measure of success / effectiveness 75 | 76 | ### models 77 | 78 | Serialized model objects with optimized hyperparameters. 79 | 80 | * `experiment-1-model.pk` 81 | * The scikit-learn implementation of Naive Bayes which was the optimal model of the project 82 | 83 | ### notebooks 84 | 85 | Exploratory analysis and experiment code. 86 | 87 | * `1.0-exploratory-analysis.ipynb` 88 | * Exploratory analysis of existing marketing data 89 | 90 | * `2.0-experiment-1-baseline-vs-ml.ipynb` 91 | * The first experiment designed to evaluate the difference in performance between the baseline domain-driven model and a set of machine learning models 92 | 93 | * `3.0-experiment-2-feature-engineering` 94 | * Prototyping code for new features derived from existing variables included in the marketing dataset 95 | 96 | * `3.1-experiment-2-model.ipynb` 97 | * The second experiment designed to evaluate the effectiveness of derived features with the best performing model from experiment #1 98 | 99 | * `4.0-model-evaluation.ipynb` 100 | * Evaluation of the best performing model on the held-out test set 101 | 102 | ### support 103 | 104 | A supporting library that encapsulates ML model objects and provides tools for evaluation and hyperparameter tuning. Most of this code will be moved to a standalone library, except for: 105 | 106 | * `experiments/` 107 | * Code modules for experiment code that includes pipelines and derived features 108 | 109 | * `parameters.py` 110 | * Values for e.g. costs, benefits, and proportion of targeted customers from the Problem Model workbook 111 | 112 | ## Dependencies 113 | 114 | * numpy==1.15.2 115 | * pandas==0.23.4 116 | * matplotlib==2.2.2 117 | * seaborn==0.9.0 118 | * scikit-learn==0.20.0 119 | 120 | ## License 121 | The MIT License (MIT) 122 | 123 | Copyright (c) 2018 Calvin De Lima 124 | 125 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 126 | 127 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 128 | 129 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /notebooks/3.0-experiment-2-feature-engineering.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Overview\n", 8 | "\n", 9 | "The purpose of this notebook is to improve the performance of the baseline model by creating derived features for the Naive Bayes classifier. Since the BernoulliNB classifier can only handle categorial attributes, we'll derive categorical features from numeric ones and bin some of the informative attributes using feature selection to identify the best binning strategy.\n", 10 | "\n", 11 | "### Existing Features\n", 12 | "\n", 13 | "#### Categorical Attributes\n", 14 | "* Most categorical attributes are informative of the outcome of the marketing campaign except for `marital` and `day_of_week`\n", 15 | "* Several attributes contain missing values but there are not any a-priori reasons to treat the missing values differently from regular values. \n", 16 | "* An additional indicator that encodes whether a customer was contacted prior to the current campaign may improve predictive performance as `poutcome` values of 'success' and 'failure' have higher response rates than the 'nonexistent' value\n", 17 | "\n", 18 | "#### Numeric Attributes\n", 19 | "* `age` outliers include people over the age of ~62\n", 20 | " * Accross all ages, customers who were contacted more than ten times were not likely to respond to the campaign\n", 21 | " * Younger and older customers tend to convert more than middle-aged customers\n", 22 | "* `pdays` uses a very large number (999) to encode missing values. Non-missing instances are seven times more likely to convert than missing instances\n", 23 | "* `previous` values are finite - can be converted to a categorical variable / binned\n", 24 | "* Socioeconomic distributions are sparse\n", 25 | "* Strong correlation between\n", 26 | " * `emp.var.rate` and `cons.price.idx`\n", 27 | " * `emp.var.rate` and `euribor3m`\n", 28 | " * `emp.var.rate` and `nr.employed`\n", 29 | " * `nr.employed` and `euribor3m`\n", 30 | "\n", 31 | "\n", 32 | "### New Features\n", 33 | "* Create an indicator of whether a customer was previously contacted as part of a previous campaign (`pdays`)\n", 34 | "* Create an indicator of whether a customer was part of a previous campaign (`poutcome`)\n", 35 | "* Transform `previous` to a categorical variable\n", 36 | "* Apply binning to:\n", 37 | " * log(`age`)\n", 38 | " * `campaign`" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 89, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "from sklearn.preprocessing import OneHotEncoder, FunctionTransformer, KBinsDiscretizer\n", 48 | "from sklearn.compose import ColumnTransformer\n", 49 | "from sklearn.pipeline import Pipeline, FeatureUnion\n", 50 | "from sklearn.base import clone\n", 51 | "from utils import code\n", 52 | "\n", 53 | "import pandas as pd\n", 54 | "import numpy as np\n", 55 | "\n", 56 | "%matplotlib inline" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 2, 62 | "metadata": {}, 63 | "outputs": [ 64 | { 65 | "data": { 66 | "text/html": [ 67 | "
\n", 68 | "\n", 81 | "\n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | "
agejobmaritaleducationdefaulthousingloancontactmonthday_of_week...campaignpdayspreviouspoutcomeemp.var.ratecons.price.idxcons.conf.idxeuribor3mnr.employedy
025blue-collarsinglehigh.schoolnoyesnocellularjulmon...29990nonexistent1.493.918-42.74.9625228.1no
140admin.marriedhigh.schoolnononotelephonejunthu...19990nonexistent1.494.465-41.84.9585228.1no
251technicianmarrieduniversity.degreenoyesnocellularaugwed...59990nonexistent1.493.444-36.14.9645228.1yes
337blue-collarmarriedhigh.schoolunknownyesnocellularjultue...19990nonexistent1.493.918-42.74.9615228.1no
454housemaidmarrieduniversity.degreeunknownyesnocellularaugthu...19990nonexistent1.493.444-36.14.9635228.1no
\n", 231 | "

5 rows × 21 columns

\n", 232 | "
" 233 | ], 234 | "text/plain": [ 235 | " age job marital education default housing loan \\\n", 236 | "0 25 blue-collar single high.school no yes no \n", 237 | "1 40 admin. married high.school no no no \n", 238 | "2 51 technician married university.degree no yes no \n", 239 | "3 37 blue-collar married high.school unknown yes no \n", 240 | "4 54 housemaid married university.degree unknown yes no \n", 241 | "\n", 242 | " contact month day_of_week ... campaign pdays previous poutcome \\\n", 243 | "0 cellular jul mon ... 2 999 0 nonexistent \n", 244 | "1 telephone jun thu ... 1 999 0 nonexistent \n", 245 | "2 cellular aug wed ... 5 999 0 nonexistent \n", 246 | "3 cellular jul tue ... 1 999 0 nonexistent \n", 247 | "4 cellular aug thu ... 1 999 0 nonexistent \n", 248 | "\n", 249 | " emp.var.rate cons.price.idx cons.conf.idx euribor3m nr.employed y \n", 250 | "0 1.4 93.918 -42.7 4.962 5228.1 no \n", 251 | "1 1.4 94.465 -41.8 4.958 5228.1 no \n", 252 | "2 1.4 93.444 -36.1 4.964 5228.1 yes \n", 253 | "3 1.4 93.918 -42.7 4.961 5228.1 no \n", 254 | "4 1.4 93.444 -36.1 4.963 5228.1 no \n", 255 | "\n", 256 | "[5 rows x 21 columns]" 257 | ] 258 | }, 259 | "execution_count": 2, 260 | "metadata": {}, 261 | "output_type": "execute_result" 262 | } 263 | ], 264 | "source": [ 265 | "bank = pd.read_csv('../data/train.csv')\n", 266 | "bank.head()" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 3, 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "X = bank.drop('y', axis=1)\n", 276 | "y = bank.y" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "## Computations" 284 | ] 285 | }, 286 | { 287 | "cell_type": "markdown", 288 | "metadata": {}, 289 | "source": [ 290 | "### Was the customer contacted about the last campaign?" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": 7, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "def ft_pcontacted_last_campaign(X):\n", 300 | " pcontacted = ~(X == 999)\n", 301 | " return pcontacted.values.reshape(-1,1)" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 8, 307 | "metadata": {}, 308 | "outputs": [ 309 | { 310 | "data": { 311 | "text/plain": [ 312 | "True" 313 | ] 314 | }, 315 | "execution_count": 8, 316 | "metadata": {}, 317 | "output_type": "execute_result" 318 | } 319 | ], 320 | "source": [ 321 | "add_pcontacted_last_campaign = FunctionTransformer(ft_pcontacted_last_campaign, validate=False)\n", 322 | "add_pcontacted_last_campaign_ct = ColumnTransformer([('add_pcontacted', add_pcontacted_last_campaign, 'pdays')])\n", 323 | "\n", 324 | "pcontacted_last_campaign = add_pcontacted_last_campaign_ct.fit_transform(X)\n", 325 | "\n", 326 | "# Make sure the resulting values are correct\n", 327 | "all(X.loc[np.where(pcontacted_last_campaign.ravel() == True)[0], ['pdays']] == '999')" 328 | ] 329 | }, 330 | { 331 | "cell_type": "markdown", 332 | "metadata": {}, 333 | "source": [ 334 | "### Was the customer part of the last campaign?" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 9, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [ 343 | "def ft_pcampaign(X):\n", 344 | " pcampaign = ~(X == 'nonexistent')\n", 345 | " return pcampaign.values.reshape(-1,1)" 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "execution_count": 10, 351 | "metadata": {}, 352 | "outputs": [ 353 | { 354 | "data": { 355 | "text/plain": [ 356 | "True" 357 | ] 358 | }, 359 | "execution_count": 10, 360 | "metadata": {}, 361 | "output_type": "execute_result" 362 | } 363 | ], 364 | "source": [ 365 | "add_pcampaign = FunctionTransformer(ft_pcampaign, validate=False)\n", 366 | "add_pcampaign_ct = ColumnTransformer([('add_poutcome', add_pcampaign, 'poutcome')])\n", 367 | "\n", 368 | "pcampaign = add_pcampaign_ct.fit_transform(X)\n", 369 | "\n", 370 | "# Make sure the resulting values are correct\n", 371 | "all(X.loc[np.where(pcampaign.ravel() == True)[0], ['poutcome']] != 'nonexistent')" 372 | ] 373 | }, 374 | { 375 | "cell_type": "markdown", 376 | "metadata": {}, 377 | "source": [ 378 | "### Was the customer contacted more than ten times?" 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": 11, 384 | "metadata": {}, 385 | "outputs": [], 386 | "source": [ 387 | "def ft_campaign_gte10(X):\n", 388 | " campaign_gte10 = X >= 10\n", 389 | " return campaign_gte10.values.reshape(-1,1)" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": 12, 395 | "metadata": {}, 396 | "outputs": [ 397 | { 398 | "data": { 399 | "text/plain": [ 400 | "True" 401 | ] 402 | }, 403 | "execution_count": 12, 404 | "metadata": {}, 405 | "output_type": "execute_result" 406 | } 407 | ], 408 | "source": [ 409 | "add_campaign_gte10 = FunctionTransformer(ft_campaign_gte10, validate=False)\n", 410 | "add_campaign_gte10_ct = ColumnTransformer([('campaign_gte10', add_campaign_gte10, 'campaign')])\n", 411 | "\n", 412 | "campaign_gte10 = add_campaign_gte10_ct.fit_transform(X)\n", 413 | "\n", 414 | "# Make sure the resulting values are correct\n", 415 | "all(X.loc[np.where(campaign_gte10.ravel() == True)[0], ['campaign']] >= 10)" 416 | ] 417 | }, 418 | { 419 | "cell_type": "markdown", 420 | "metadata": {}, 421 | "source": [ 422 | "### Transform previous to categorical" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": 13, 428 | "metadata": {}, 429 | "outputs": [], 430 | "source": [ 431 | "def ft_previous(X):\n", 432 | " previous = X.astype(str)\n", 433 | " return previous.values.reshape(-1,1)" 434 | ] 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": 14, 439 | "metadata": {}, 440 | "outputs": [ 441 | { 442 | "data": { 443 | "text/plain": [ 444 | "True" 445 | ] 446 | }, 447 | "execution_count": 14, 448 | "metadata": {}, 449 | "output_type": "execute_result" 450 | } 451 | ], 452 | "source": [ 453 | "add_previous = FunctionTransformer(ft_previous, validate=False)\n", 454 | "add_previous_ct = ColumnTransformer([('add_previous', add_previous, 'previous')])\n", 455 | "\n", 456 | "previous = add_previous_ct.fit_transform(X)\n", 457 | "\n", 458 | "# Make sure the resulting values are correct\n", 459 | "all(previous.astype(int).ravel() == X.previous)" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "### Apply binning as a hyperparameter" 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "Apply a log transform to `age` to transform the distribution" 474 | ] 475 | }, 476 | { 477 | "cell_type": "code", 478 | "execution_count": 30, 479 | "metadata": {}, 480 | "outputs": [ 481 | { 482 | "data": { 483 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmAAAAEICAYAAAD4JEh6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+YVdV97/H3J+DvUQHRKQWSIZUm0ZD4Yy7S+tzcqURETcQ+1RZrI+bSy70J+dXQJJjee2n8cYtPa42axJQEGkytSKg+0GiiVJ2bpjfg74hCLBMkOkLEhB8RbUzHfu8few0eDmdmzpk5Z585Zz6v5znP7L322nvtxT7s8917r7W2IgIzMzMzy89b6r0DZmZmZiONAzAzMzOznDkAMzMzM8uZAzAzMzOznDkAMzMzM8uZAzAzMzOznDkAMzMzG4Yk7Zf09nrvh9WGPA6YmZmZWb58B8zMzMwsZw7ArGYkLZb0Y0mvSNos6XdT+ihJN0j6maTnJH1MUkganZYfL2m5pJ2SXpR0raRR9a2NmTUbSZMl3SXpZUk/l/QlSb8h6cE0/zNJt0saU7DOdkmfkfSUpFfTuapV0nfSue6fJI1NedvSuW2BpB3pnLaoYFvTJf1A0t607EuSDi9YHpJOTtMnSPpHSb+Q9Eg6L36/KO//kLRV0h5JX5akfP4lbTAcgFkt/Rj4z8DxwBeAv5M0AfhvwPnAacAZwMVF660EeoCTgdOBWcAf57TPZjYCpIu6bwM/AdqAicAqQMBfAL8OvAuYDPx50eq/B5wL/CbwQeA7wOeB8WS/q58oyv87wFSyc9liSe9P6W8Af5LW+y1gJvDRPnb5y8CrwK8B89Kn2AeA/wS8F/h94Ly+6m/15wDMaiYivhUROyLiPyLiTmArMJ3sxHBTRHRHxB5gae86klrJgrNPRcSrEbELuBGYW4cqmFnzmk4WZH0mnWt+GRHfj4iuiFgfEa9HxMvAXwP/pWjdWyLipYh4EfhnYGNEPBERrwN3k104FvpCKmMT8LfAZQAR8VhEbIiInojYDvxNibJ6g8XfA5ZExGsRsZnsQrXY0ojYGxHPAw+RXeTaMDW63jtgzUvSFcCnya4uAVrIrvR+HXihIGvh9NuAw4CdBXfP31KUx8xsqCYDP4mInsJESScBN5PdvT+W7Pyzp2jdlwqm/63EfEtR/sLz10+Aaams3yQL8NqBo8l+kx8rsa8npmV9nTd7/bRg+rUS+2HDiO+AWU1IehvwNeBjwAkRMQZ4muz2/k5gUkH2yQXTLwCvA+MjYkz6HBcRp+a062Y2MrwAvLW37WmBvwACeE9EHAf8Edl5aygKz3FvBXak6VuBHwFTU1mf76Osl8maZfR13rQG5ADMauUYspPYywCSPgy8Oy1bDXxS0sTUuPVzvStFxE7gfuAGScdJektqFHvIbXkzsyF4mOxicKmkYyQdKelssrte+4G9kiYCn6lCWf9L0tGSTgU+DNyZ0o8FfgHsl/RO4COlVo6IN4C7gD9P23kncEUV9svqyAGY1URqo3AD8AOy2/PTgH9Ji79GFmQ9BTwB3Et2dfdGWn4FcDiwmezW/xpgQl77bmbNLwU1HyTr7PM80A38AVmHoTOAfcA9ZIHPUP1foAt4APiriLg/pf8p8IfAK2TnxTtLrw5kTxOOJ3vM+E3gDrKnBdagPBCr1Z2k84GvRsTb6r0vZmbVIqkNeA44rLitWRW2fT3waxFRqjekNQDfAbPcSTpK0gWSRqdb/EvIeg6ZmVkJkt4p6T3KTAfm4/NmQ3MAZvUgstv8e8geQW4B/ndd98jMbHg7luxx6Ktk7WhvANbWdY9sSPwI0szMzCxnvgNmZmZmlrNhPRDr+PHj48QTT+SYY46pS/mvvvpq3cqud/muu+teTY899tjPIuLEqm+4iYwfPz7a2trKylvv72gtNWvdmrVe4LoVquhcFxHD9nPmmWfGQw89FPVSz7LrXb7rXj/NWHfg0RgG55Th/DnzzDPL/ves93e0lpq1bs1arwjXrVAl5zo/gjQzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMxtRJK2QtEvS0yWW/amkkDQ+zUvSzZK6JD0l6YyCvPMkbU2feQXpZ0ralNa5WZLyqZmZNRIHYGY20nwDmF2cKGkycC7wfEHy+cDU9FkA3JryjgOWAGcB04ElksamdW5NeXvXO6QsM7Nh/SoiG5y2xfcMeRuLpvVwZdF2ti+9cMjbNau3iPiepLYSi24EPgusLUibA9yWRrjeIGmMpAlAB7A+InYDSFoPzJbUCRwXET9I6bcBFwPfqWYdNr2475D/n9Xg/+Nm+XEAZmYjnqSLgBcj4odFTwwnAi8UzHentP7Su0uk91XuArK7ZbS2ttLZ2VnW/rYelV0kVVu55dfS/v37h8V+VFuz1gtct8FyAGZmI5qko4E/A2aVWlwiLQaRXlJELAOWAbS3t0dHR8dAuwvALbev5YZN1T99b7+8vPJrqbOzk3L/HRpJs9YLXLfBchswMxvpfgOYAvxQ0nZgEvC4pF8ju4M1uSDvJGDHAOmTSqSbmR3EAZiZjWgRsSkiToqItohoIwuizoiInwLrgCtSb8gZwL6I2AncB8ySNDY1vp8F3JeWvSJpRur9eAUHtykzMwMcgJnZCCPpDuAHwDskdUua30/2e4FtQBfwNeCjAKnx/TXAI+lzdW+DfOAjwNfTOj+myg3wzaw5uA2YmY0oEXHZAMvbCqYDWNhHvhXAihLpjwLvHtpemlmzK+sOmKQ/kfSMpKcl3SHpSElTJG1MgxDeKenwlPeINN+VlrcVbOeqlP6spPNqUyUzMzOz4W3AAEzSROATQHtEvBsYBcwFrgdujIipwB6g9zb+fGBPRJxMNq7O9Wk7p6T1TiUbmPArkkZVtzpmZmZmw1+5bcBGA0dJGg0cDewEzgHWpOUryQYbhGzgwpVpeg0wMzVGnQOsiojXI+I5svYR04deBTMzM7PGMmAbsIh4UdJfkb2e49+A+4HHgL0R0TsSYOFggwcGKIyIHkn7gBNS+oaCTZccoLB4YMJ6DvBW78HlBlt+NQZoLDXQY57/Fj7u9Sm/3nU3MxspBgzAUhfrOWTj5OwFvkX2frRivYMNDmmAwuKBCVtaWuo2wFu9B5cbbPnVeEXJomk9hwz0mOcgjfX8t2/U497oZZuZjSTlPIJ8P/BcRLwcEf8O3AX8NjAmPZKEgwcbPDBAYVp+PLCbvgcuNDMzMxtRygnAngdmSDo6teWaCWwGHgIuSXnm8eZgg+vSPGn5g6kr9zpgbuolOQWYCjxcnWqYmZmZNY5y2oBtlLQGeBzoAZ4ge0R4D7BK0rUpbXlaZTnwTUldZHe+5qbtPCNpNVnw1gMsjIg3qlwfMzMzs2GvrIFYI2IJsKQoeRslejFGxC+BS/vYznXAdRXuo5mZmVlT8auIzMzMzHLmAMzMzMwsZw7AzMzMzHLmAMzMzMwsZw7AzMzMzHLmAMzMzMwsZw7AzMzMzHLmAMzMzMwsZw7AzMzMzHLmAMzMzMwsZw7AzMzMzHLmAMzMRhRJKyTtkvR0QdpfSvqRpKck3S1pTMGyqyR1SXpW0nkF6bNTWpekxQXpUyRtlLRV0p2SDs+vdmbWKByAmdlI8w1gdlHaeuDdEfEe4F+BqwAknQLMBU5N63xF0ihJo4AvA+cDpwCXpbwA1wM3RsRUYA8wv7bVMbNGNLreO2CNo23xPTXZ7valF9Zku2alRMT3JLUVpd1fMLsBuCRNzwFWRcTrwHOSuoDpaVlXRGwDkLQKmCNpC3AO8Icpz0rgz4Fbq18TM2tkDsDMzA72X4E70/REsoCsV3dKA3ihKP0s4ARgb0T0lMh/CEkLgAUAra2tdHZ2lrWDrUfBomk9A2esULnl19L+/fuHxX5UW7PWC1y3wRowAJP0Dt48GQG8HfjfwG0pvQ3YDvx+ROyRJOAm4ALgNeDKiHg8bWse8D/Tdq6NiJXVqYaZ2dBJ+jOgB7i9N6lEtqB0843oJ39JEbEMWAbQ3t4eHR0dZe3nLbev5YZN1b9+3n55eeXXUmdnJ+X+OzSSZq0XuG6DNWAbsIh4NiJOi4jTgDPJgqq7gcXAA6mdwwNpHrI2EVPTZwHp1rukccASsqvE6cASSWOrWx0zs8FJF4gfAC6PiN6gqRuYXJBtErCjn/SfAWMkjS5KNzM7SKWN8GcCP46In5C1jei9g7USuDhNzwFui8wGspPRBOA8YH1E7I6IPWSNXosbwpqZ5U7SbOBzwEUR8VrBonXAXElHSJpCdmH5MPAIMDX1eDycrKH+uhS4PcSbbcjmAWvzqoeZNY5KA7C5wB1pujUidgKkvyel9Ikc2jZiYj/pZma5kXQH8APgHZK6Jc0HvgQcC6yX9KSkrwJExDPAamAz8F1gYUS8kdp4fQy4D9gCrE55IQvkPp0a7J8ALM+xembWIMpuRJCu8i4idc/uL2uJtLLbRhQ3Sq1n4756NywcbPnVaJxbq0a+pZSqo497fcqvd93zEBGXlUjuM0iKiOuA60qk3wvcWyJ9G2/2lDQzK6mSVpznA49HxEtp/iVJEyJiZ3rEuCul99dmoqMovbO4kOJGqS0tLXVr3FfvhoWDLf/KKgwXsWhaT00a+ZZSquFvPf/tG/W4N3rZZmYjSSWPIC/jzcePkLWNmJemC9s5rAOuUGYGsC89orwPmCVpbGp8PyulmZmZmY0oZd3ikHQ0cC7w3wuSlwKrU/uJ54FLU/q9ZENQdJH1mPwwQETslnQNWeNVgKsjYveQa2BmZmbWYMoKwFKvoBOK0n5O1iuyOG8AC/vYzgpgReW7ac2s1Aj7i6b1DPlRqkfYNzOz4crvgjQzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5yVFYBJGiNpjaQfSdoi6bckjZO0XtLW9HdsyitJN0vqkvSUpDMKtjMv5d8qaV6tKmVmZmY2nJV7B+wm4LsR8U7gvcAWYDHwQERMBR5I8wDnA1PTZwFwK4CkccAS4CxgOrCkN2gzM8uLpBWSdkl6uiCtaheUks6UtCmtc7Mk5VtDM2sEAwZgko4D3gcsB4iIX0XEXmAOsDJlWwlcnKbnALdFZgMwRtIE4DxgfUTsjog9wHpgdlVrY2Y2sG9w6LmnmheUt6a8vev5PGdmhxhdRp63Ay8DfyvpvcBjwCeB1ojYCRAROyWdlPJPBF4oWL87pfWVfhBJC8hOXrS2trJ//346OzsrqVPV1LPsoZS/aFrPkMtuPao626ln+YM9do163Bu97LxExPcktRUlzwE60vRKoBP4HAUXlMCG1BxjQsq7PiJ2A0haD8yW1AkcFxE/SOm3kV2cfqd2NTKzRlROADYaOAP4eERslHQTb14dllLqdnv0k35wQsQyYBlAe3t7tLS00NHRUcZuVl9nZ2fdyh5K+VcuvmfIZS+a1sMNm8r5etRGNcrffnnHoNZr1OPe6GXXWbUuKCem6eL0koovOMsNfmt1gTQcgu9mvQho1nqB6zZY5fzCdQPdEbExza8hC8BekjQhnawmALsK8k8uWH8SsCOldxSldw5+183Maq7SC8qyLjQPLCi64Cw3+L3l9rU1uUAa7EVLNTXrRUCz1gtct8EasA1YRPwUeEHSO1LSTGAzsA7obXg6D1ibptcBV6TGqzOAfenK8j5glqSxqa3ErJRmZlZvL6ULSSq4oOwrfVKJdDOzg5TbC/LjwO2SngJOA/4PsBQ4V9JW4Nw0D3AvsA3oAr4GfBQgtZW4Bngkfa7ubT9hZlZnVbmgTMtekTQj9X68omBbZmYHlHUPOyKeBNpLLJpZIm8AC/vYzgpgRSU7aGZWTZLuIGsOMV5SN1lvxqXAaknzgeeBS1P2e4ELyC4oXwM+DNkFpaTeC0o4+ILyI2Q9LY8ia3zvBvhmdoj6tbI2M6uDiLisj0VVuaCMiEeBdw9lH82s+flVRGZmZmY5cwBmZmZmljMHYGZmZmY5cwBmZmZmljMHYGZmZmY5cwBmZmZmljMHYGZmZmY5cwBmZmZmljMHYGZmZmY580j41rTaFt8zqPUWTevhyn7W3b70wsHukpmZGeA7YGZmZma5cwBmZmZmljMHYGZmZmY5cwBmZmZmljMHYGZmZmY5KysAk7Rd0iZJT0p6NKWNk7Re0tb0d2xKl6SbJXVJekrSGQXbmZfyb5U0rzZVMjMzMxveKhmG4nci4mcF84uBByJiqaTFaf5zwPnA1PQ5C7gVOEvSOGAJ0A4E8JikdRGxpwr1MDOzIRrs0C0D8dAtZocayiPIOcDKNL0SuLgg/bbIbADGSJoAnAesj4jdKehaD8weQvlmZmZmDancO2AB3C8pgL+JiGVAa0TsBIiInZJOSnknAi8UrNud0vpKP4ikBcACgNbWVvbv309nZ2f5NaqiepY9lPIXTesZctmtR1VnO41Y/kBl1/o7MZK/82ZmI0W5AdjZEbEjBVnrJf2on7wqkRb9pB+ckAV3ywDa29ujpaWFjo6OMnezujo7O+tW9lDK728U93ItmtbDDZvq96KEepY/UNnbL++oafn1/N7V+ztvZjZSlPUIMiJ2pL+7gLuB6cBL6dEi6e+ulL0bmFyw+iRgRz/pZmbDgqQ/kfSMpKcl3SHpSElTJG1MnYfulHR4yntEmu9Ky9sKtnNVSn9W0nn1qo+ZDV8DBmCSjpF0bO80MAt4GlgH9PZknAesTdPrgCtSb8gZwL70qPI+YJaksanH5KyUZmZWd5ImAp8A2iPi3cAoYC5wPXBjREwF9gDz0yrzgT0RcTJwY8qHpFPSeqeStXP9iqRRedbFzIa/cu6AtQLfl/RD4GHgnoj4LrAUOFfSVuDcNA9wL7AN6AK+BnwUICJ2A9cAj6TP1SnNzGy4GA0cJWk0cDSwEzgHWJOWF3c46u2ItAaYKUkpfVVEvB4Rz5GdC6fntP9m1iAGbGQTEduA95ZI/zkws0R6AAv72NYKYEXlu2lmVlsR8aKkvwKeB/4NuB94DNgbEb29Mgo7Dx3oWBQRPZL2ASek9A0Fmy7Z4QgO7XRUbgeIeneSqVQlHTuatSNIs9YLXLfBql8razOzYSQ1jZgDTAH2At8iG9ewWG/noSF1OIJDOx2V2wHiltvX1rWTTKUq6bjSrB1BmrVe4LoNll9FZGaWeT/wXES8HBH/DtwF/DbZWIa90U5h56EDHYvS8uOB3bjDkZmVwQGYmVnmeWCGpKNTW66ZwGbgIeCSlKe4w1FvR6RLgAdTE4x1wNzUS3IK2VtBHs6pDmbWIBrnHraZWQ1FxEZJa4DHgR7gCbLHg/cAqyRdm9KWp1WWA9+U1EV252tu2s4zklaTBW89wMKIeCPXypjZsOcAzMwsiYglZO+sLbSNEr0YI+KXwKV9bOc64Lqq76CZNQ0/gjQzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5yVHYBJGiXpCUnfTvNTJG2UtFXSnZIOT+lHpPmutLytYBtXpfRnJZ1X7cqYmZmZNYJK7oB9EthSMH89cGNETAX2APNT+nxgT0ScDNyY8iHpFLKX1Z4KzAa+ImnU0HbfzMzMrPGUFYBJmgRcCHw9zQs4B1iTsqwELk7Tc9I8afnMlH8OsCoiXo+I54AuSrzg1szMzKzZjS4z3xeBzwLHpvkTgL0R0ZPmu4GJaXoi8AJARPRI2pfyTwQ2FGyzcJ0DJC0AFgC0trayf/9+Ojs7y61PVdWz7KGUv2haz8CZBtB6VHW204jlD1R2rb8TI/k7b2Y2UgwYgEn6ALArIh6T1NGbXCJrDLCsv3XeTIhYBiwDaG9vj5aWFjo6Ooqz5aKzs7NuZQ+l/CsX3zPkshdN6+GGTeXG59VXz/IHKnv75R01Lb+e37t6f+fNzEaKcn7hzgYuknQBcCRwHNkdsTGSRqe7YJOAHSl/NzAZ6JY0Gjge2F2Q3qtwHTMzM7MRY8A2YBFxVURMiog2skb0D0bE5cBDwCUp2zxgbZpel+ZJyx+MiEjpc1MvySnAVODhqtXEzMzMrEEM5RnP54BVkq4FngCWp/TlwDcldZHd+ZoLEBHPSFoNbAZ6gIUR8cYQyjczMzNrSBUFYBHRCXSm6W2U6MUYEb8ELu1j/euA6yrdSTMzM7Nm4pHwzczMzHLmAMzMLJE0RtIaST+StEXSb0kaJ2l9euvHekljU15Jujm93eMpSWcUbGdeyr9V0ry+SzSzkcoBmJnZm24CvhsR7wTeS/b2j8XAA+mtHw+keYDzyToTTSUbu/BWAEnjgCXAWWTNNJb0Bm1mZr0cgJmZAZKOA95H6lAUEb+KiL0c/HaP4rd+3BaZDWRD80wAzgPWR8TuiNgDrCd7/ZqZ2QEOwMzMMm8HXgb+VtITkr4u6RigNSJ2AqS/J6X8B976kfS+3aOvdDOzA+o31LmZ2fAyGjgD+HhEbJR0E28+bixlSG/9gENfvVbua6Dq/aqwSlXyeqtmfR1Ws9YLXLfBcgBmZpbpBrojYmOaX0MWgL0kaUJE7EyPGHcV5C/1do9uoKMovbNUgcWvXiv3NVC33L62rq8Kq1Qlr+9q1tdhNWu9wHUbLD+CNDMDIuKnwAuS3pGSZpINHF34do/it35ckXpDzgD2pUeU9wGzJI1Nje9npTQzswMa5xLKzKz2Pg7cLulwYBvwYbIL1dWS5gPP8+ZA0/cCFwBdwGspLxGxW9I1wCMp39URsTu/KphZI3AAZmaWRMSTQHuJRTNL5A1gYR/bWQGsqO7emVkz8SNIMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPLmQMwMzMzs5w5ADMzMzPL2YABmKQjJT0s6YeSnpH0hZQ+RdJGSVsl3ZnGzUHSEWm+Ky1vK9jWVSn9WUnn1apSZmZmZsNZOXfAXgfOiYj3AqcBs9Ooz9cDN0bEVGAPMD/lnw/siYiTgRtTPiSdAswFTgVmA1+RNKqalTEzMzNrBAMGYJHZn2YPS58AziF7VxrASuDiND0nzZOWz5SklL4qIl6PiOfIRo+eXpVamJmZmTWQskbCT3eqHgNOBr4M/BjYGxE9KUs3MDFNTwReAIiIHkn7gBNS+oaCzRauU1jWAmABQGtra13fsl7vN7wPtvxF03oGzjSA1qOqs51GLH+gsmv9nRjJ33kzs5GirAAsIt4ATpM0BrgbeFepbOmv+ljWV3pxWcuAZQDt7e3R0tJSt7es1/sN74Mt/8rF9wy57EXTerhhU/3eVFXP8gcqe/vlHTUtv57fu3p/583MRoqKekFGxF6gE5gBjJHU+ys1CdiRpruByQBp+fHA7sL0EuuYmZmZjRjl9II8Md35QtJRwPuBLcBDwCUp2zxgbZpel+ZJyx9ML61dB8xNvSSnAFOBh6tVETMzM7NGUc4zngnAytQO7C3A6oj4tqTNwCpJ1wJPAMtT/uXANyV1kd35mgsQEc9IWg1sBnqAhenRppmZmdmIMmAAFhFPAaeXSN9GiV6MEfFL4NI+tnUdcF3lu2lmZmbWPDwSvpmZmVnOHICZmZmZ5cwBmJmZmVnOHICZmZmZ5cwBmJmZmVnO6jfUuZmZjQhtFbydY9G0nrLf5rF96YWD3SWzunMAViflnJAqORGZWXWkMQ8fBV6MiA+kgaNXAeOAx4EPRcSvJB0B3AacCfwc+IOI2J62cRUwH3gD+ERE3Jd/TcxsOPMjSDOzg32S7G0fva4HboyIqcAessCK9HdPRJwM3JjyIekUsgGoTwVmA19JQZ2Z2QEOwMzMEkmTgAuBr6d5AecAa1KWlcDFaXpOmictn5nyzwFWRcTrEfEc0EWJQavNbGTzI0gzszd9EfgscGyaPwHYGxE9ab4bmJimJwIvAEREj6R9Kf9EYEPBNgvXOYikBcACgNbWVjo7O8vaydajsiYKzaiSupX77zUc7N+/v6H2txKu2+A4ADMzAyR9ANgVEY9J6uhNLpE1BljW3zoHJ0YsA5YBtLe3R0dHR6lsh7jl9rXcsKk5T9+LpvWUXbftl3fUdmeqqLOzk3KPb6Nx3QanOf8Hm5lV7mzgIkkXAEcCx5HdERsjaXS6CzYJ2JHydwOTgW5Jo4Hjgd0F6b0K1zEzA9wGzMwMgIi4KiImRUQbWSP6ByPicuAh4JKUbR6wNk2vS/Ok5Q9GRKT0uZKOSD0opwIP51QNM2sQvgNmZta/zwGrJF0LPAEsT+nLgW9K6iK78zUXICKekbQa2Az0AAsj4o38d9vMhjMHYGZmRSKiE+hM09so0YsxIn4JXNrH+tcB19VuD82s0Q34CFLSZEkPSdoi6RlJn0zp4yStl7Q1/R2b0iXpZkldkp6SdEbBtual/FslzeurTDMzM7NmVk4bsB5gUUS8C5gBLEwDDS4GHkiDEz6Q5gHOJ2vzMJWse/WtkAVswBLgLLKrySW9QZuZmZnZSDJgABYROyPi8TT9CtkI0RM5eBDC4sEJb4vMBrIeRBOA84D1EbE7IvYA68lGiTYzMzMbUSpqAyapDTgd2Ai0RsROyII0SSelbAcGJ0x6ByHsK93MzKxilbzkuxJ+ybfloewATFIL8A/ApyLiF9kbN0pnLZFW9uCExSND13OE3VqWXc5Iz/Uc7breI20P57rX+vvYrN95MzN7U1kBmKTDyIKv2yPirpT8kqQJ6e7XBGBXSu9rEMJuoKMovbO4rOKRoVtaWuo2wm4tR8C9sowrt0pGhK62epZd7/IHKrvWo2/Xc1TpZh7R2sxsOCmnF6TIxrvZEhF/XbCocBDC4sEJr0i9IWcA+9KjyvuAWZLGpsb3s1KamZmZ2YhSzi2Gs4EPAZskPZnSPg8sBVZLmg88z5vj4dwLXAB0Aa8BHwaIiN2SrgEeSfmujojdVamFmZmZWQMZMACLiO9Tuv0WwMwS+QNY2Me2VgArKtnXWRrWAAAK2ElEQVRBMzMzs2bjd0GamZmZ5cwBmJmZmVnOHICZmZmZ5cwBmJmZmVnOHICZmZmZ5cwBmJmZmVnOHICZmZmZ5cwBmJmZmVnOHICZmZmZ5ax+b1s2a1BtZbxIfTC2L72wJts1M7Phx3fAzMzMzHLmAMzMDJA0WdJDkrZIekbSJ1P6OEnrJW1Nf8emdEm6WVKXpKcknVGwrXkp/1ZJ8+pVJzMbvhyAmZlleoBFEfEuYAawUNIpwGLggYiYCjyQ5gHOB6amzwLgVsgCNmAJcBYwHVjSG7SZmfVyAGZmBkTEzoh4PE2/AmwBJgJzgJUp20rg4jQ9B7gtMhuAMZImAOcB6yNid0TsAdYDs3Osipk1ADfCNzMrIqkNOB3YCLRGxE7IgjRJJ6VsE4EXClbrTml9pZcqZwHZ3TNaW1vp7Owsa/9aj4JF03rKq0yDGQ51K/c4VGL//v012e5w4LoNjgMwM7MCklqAfwA+FRG/kNRn1hJp0U/6oYkRy4BlAO3t7dHR0VHWPt5y+1pu2NScp+9F03rqXrftl3dUfZudnZ2Ue3wbjes2OAM+gpS0QtIuSU8XpLlRqpk1HUmHkQVft0fEXSn5pfRokfR3V0rvBiYXrD4J2NFPupnZAeW0AfsGh7ZfcKNUM2sqym51LQe2RMRfFyxaB/ReNM4D1hakX5EuPGcA+9KjyvuAWZLGpvPcrJRmZnbAgAFYRHwP2F2U7EapZtZszgY+BJwj6cn0uQBYCpwraStwbpoHuBfYBnQBXwM+ChARu4FrgEfS5+qUZmZ2wGAftOfWKLWejftqWXY5jUzr2Ri13g1hR2Lde79rzfqdH+4i4vuUbr8FMLNE/gAW9rGtFcCK6u2dmTWbard0rHqj1JaWlro17qtl47sry3idTT0bo9a7IexIrHtvw996Nmht5sa0ZmbDyWDHAXOjVDMzM7NBGmwA5kapZmZmZoM04HMWSXcAHcB4Sd1kvRmXAqslzQeeBy5N2e8FLiBrlPoa8GHIGqVK6m2UCm6UamZmZiPYgAFYRFzWxyI3SjUzMzMbhOYcSrmK2spoLG9mZs2jFuf9RdN66Kj6Vq2R+WXcZmZmZjlzAGZmZmaWMwdgZmZmZjlzAGZmZmaWMwdgZmZmZjlzL0gzM7Mc1KpX/falF9Zku1ZbvgNmZmZmljMHYGZmZmY5cwBmZmZmljMHYGZmZmY5cyN8s2Git4Huomk9XFnFxrpuoGtmNvz4DpiZmZlZzhyAmZmZmeXMjyDNzMwamMcXa0y+A2ZmZmaWs9zvgEmaDdwEjAK+HhFLq7HdWlwBLJrWg28Smtlg1OpcZ5aXcn9XK+045DtrmVyjC0mjgC8D5wLdwCOS1kXE5jz3w2wkqeTipJITqU+iffO5zswGkvftnelAV0RsA5C0CpgD+KRkZs3E5zqzPrjNWkYRkV9h0iXA7Ij44zT/IeCsiPhYQZ4FwII0+w7g58DPctvJg42vY9n1Lt91r59mrPvbIuLEGmx3WCrnXJfSi893z5ZZRL2/o7XUrHVr1nqB61ao7HNd3nfAVCLtoAgwIpYByw6sID0aEe213rFS6ll2vct33V13G5IBz3Vw6Pmu7I038XFq1ro1a73AdRusvHtBdgOTC+YnATty3gczs1rzuc7M+pV3APYIMFXSFEmHA3OBdTnvg5lZrflcZ2b9yvURZET0SPoYcB9Z1+wVEfHMAKtVfHu+iupZdr3Ld91HZvn1rntTGOS5rhLNfJyatW7NWi9w3QYl10b4ZmZmZuaR8M3MzMxy5wDMzMzMLGd1CcAkTZb0kKQtkp6R9MkSeT4j6cn0eVrSG5LGpWXbJW1Kyx4dRPlHSnpY0g9T+V8okecISXdK6pK0UVJbwbKrUvqzks6rQdmflrRZ0lOSHpD0toJlbxT8u1TcqLfM8q+U9HJBOX9csGyepK3pM68GZd9YUO6/StpbrboXbGeUpCckfbvEspoc9zLLrtlxL7P8mhx3qx5Js9P3r0vS4nrvT7VIWiFpl6Sn670v1VbO712jKuec3sj6O19WRUTk/gEmAGek6WOBfwVO6Sf/B4EHC+a3A+OHUL6AljR9GLARmFGU56PAV9P0XODONH0K8EPgCGAK8GNgVJXL/h3g6DT9kd6y0/z+If7bl1P+lcCXSqw7DtiW/o5N02OrWXZR/o+TNV6uSt0LtvNp4O+Bb5dYVpPjXmbZNTvuZZZfk+PuT3U+ZI35fwy8HTg8fR/7PG820gd4H3AG8HS996UGdavo966RPpWe0xvt09/5shqfutwBi4idEfF4mn4F2AJM7GeVy4A7qlh+RMT+NHtY+hT3RpgDrEzTa4CZkpTSV0XE6xHxHNBF9tqRqpUdEQ9FxGtpdgPZGEJVUWbd+3IesD4idkfEHmA9MLuGZVf1uANImgRcCHy9jyw1Oe7llF3L415O+f0Y0nG3qjnweqOI+BXQ+3qjhhcR3wN213s/amEQv3cNY4i/J8PaEM6XZat7G7D0iOd0ssi51PKjyU72/1CQHMD9kh5T9iqPwZQ7StKTwC6yH5fi8icCL0DWpRzYB5xQmJ50U+F/pjLKLjQf+E7B/JGSHpW0QdLFlZRbYfm/lx6FrZHUO6BkbnVPj9+mAA8WJA+57sAXgc8C/9HH8pod9zLKLlT1415m+TU57lYVPg4NbqDfu0ZU4e9ZI6nkfD0odQ3AJLWQBVafiohf9JHtg8C/RETh1dHZEXEGcD6wUNL7Ki07It6IiNPI7jJMl/Tu4t0rtVo/6dUsO9sB6Y+AduAvC5LfGtlrEf4Q+KKk36ik7DLL/0egLSLeA/wTb94Ryq3uZI//1kTEGwVpQ6q7pA8AuyLisf6yldrtftKrWXZv3qof9zLLr9lxt6rwcWhgZf7eNZwKzukNo5Lz9VDULQCTdBjZl/H2iLirn6xzKXoMFRE70t9dwN1U+CioaFt7gU4OfaRy4FUikkYDx5PdIq/aK0b6KRtJ7wf+DLgoIl4vWKe37tvSuqcPpuz+yo+InxeU+TXgzDSdS92T/o77YOt+NnCRpO1kj2/OkfR3RXlqddzLKbuWx33A8vM47jYkPg4NqoLfu4ZVxjm9kZR1vh6yWjQsG+hDdiV3G/DFAfL1/vgdU5B2DHBswfT/A2ZXWP6JwJg0fRTwz8AHivIs5ODG2KvT9Kkc3Bh7G5U1wi+n7NPJGttOLUofCxyRpscDW6mwMWeZ5U8omP5dYEOaHgc8l/ZjbJoeV82y07J3kHW0UDXrXlRGB6UbotfkuJdZds2Oe5nl1+S4+1OdD9mbS7al719vI/xT671fVaxfG83ZCL+s37tG/JR7Tm/kT1/ny2p8cn0VUYGzgQ8Bm9KzY4DPA28FiIivprTfBe6PiFcL1m0F7s7aRTMa+PuI+G6F5U8AVkoaRXYXcHVEfFvS1cCjEbEOWA58U1IXWRA4N+3bM5JWA5uBHmBhHPyYrBpl/yXQAnwr1fP5iLgIeBfwN5L+I627NCI216Dun5B0UarfbrLecUTEbknXkL3nDuDqOPjRcDXKhqzx/apI3/6kGnUvKafjXk7ZtTzu5ZRfq+NuVRC1f71R3Ui6g+yHbrykbmBJRCyv715VTcnfu4i4t477VC0lz+l13qeG4VcRmZmZmeWs7r0gzczMzEYaB2BmZmZmOXMAZmZmZpYzB2BmZmZmOXMAZmZmZpYzB2BmZmZmOXMAZmZmZpaz/w8Y3rRhT0gvlQAAAABJRU5ErkJggg==\n", 484 | "text/plain": [ 485 | "
" 486 | ] 487 | }, 488 | "metadata": {}, 489 | "output_type": "display_data" 490 | } 491 | ], 492 | "source": [ 493 | "np.log(X[['age', 'campaign']]).hist(figsize=(10,4));" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": 69, 499 | "metadata": {}, 500 | "outputs": [ 501 | { 502 | "data": { 503 | "text/plain": [ 504 | "<32950x10 sparse matrix of type ''\n", 505 | "\twith 65900 stored elements in Compressed Sparse Row format>" 506 | ] 507 | }, 508 | "execution_count": 69, 509 | "metadata": {}, 510 | "output_type": "execute_result" 511 | } 512 | ], 513 | "source": [ 514 | "binning_pipeline = Pipeline([\n", 515 | " ('log', FunctionTransformer(np.log, validate=True)),\n", 516 | " ('kbins', KBinsDiscretizer())\n", 517 | "])\n", 518 | "\n", 519 | "age_campaign_ct = ColumnTransformer([\n", 520 | " ('age_pipeline', clone(binning_pipeline), ['age']),\n", 521 | " ('campaign_pipeline', clone(binning_pipeline), ['campaign'])\n", 522 | "])\n", 523 | "age_campaign_ct.fit_transform(X)" 524 | ] 525 | }, 526 | { 527 | "cell_type": "markdown", 528 | "metadata": {}, 529 | "source": [ 530 | "Apply a log transform to `campaign` to transform the distribution" 531 | ] 532 | }, 533 | { 534 | "cell_type": "markdown", 535 | "metadata": {}, 536 | "source": [ 537 | "## Pipeline" 538 | ] 539 | }, 540 | { 541 | "cell_type": "markdown", 542 | "metadata": {}, 543 | "source": [ 544 | "* Apply column transformers (categorical)\n", 545 | "* Apply column transformers (numeric)" 546 | ] 547 | }, 548 | { 549 | "cell_type": "code", 550 | "execution_count": 70, 551 | "metadata": {}, 552 | "outputs": [ 553 | { 554 | "data": { 555 | "text/plain": [ 556 | "Pipeline(memory=None,\n", 557 | " steps=[('cat_ct', ColumnTransformer(n_jobs=None, remainder='drop', sparse_threshold=0.3,\n", 558 | " transformer_weights=None,\n", 559 | " transformers=[('pcontacted_last_campaign', FunctionTransformer(accept_sparse=False, check_inverse=True,\n", 560 | " func=, handle_unknown='ignore',\n", 562 | " n_values=None, sparse=True))])" 563 | ] 564 | }, 565 | "execution_count": 70, 566 | "metadata": {}, 567 | "output_type": "execute_result" 568 | } 569 | ], 570 | "source": [ 571 | "# Create the transformers for categorical features\n", 572 | "new_cat_features = [\n", 573 | " ('pcontacted_last_campaign', add_pcontacted_last_campaign, 'pdays'),\n", 574 | " ('poutcome', add_pcampaign, 'poutcome'),\n", 575 | " ('previous', add_previous, 'previous'),\n", 576 | " ('campaign_gte10', add_campaign_gte10, 'campaign')\n", 577 | "]\n", 578 | "cat_ct = ColumnTransformer(new_cat_features)\n", 579 | "\n", 580 | "# Create the pipeline to transform categorical features\n", 581 | "cat_pipeline = Pipeline([\n", 582 | " ('cat_ct', cat_ct),\n", 583 | " ('ohe', OneHotEncoder(handle_unknown='ignore'))\n", 584 | "])\n", 585 | "cat_pipeline.fit(X)" 586 | ] 587 | }, 588 | { 589 | "cell_type": "code", 590 | "execution_count": 71, 591 | "metadata": {}, 592 | "outputs": [ 593 | { 594 | "data": { 595 | "text/plain": [ 596 | "(32950, 4)" 597 | ] 598 | }, 599 | "execution_count": 71, 600 | "metadata": {}, 601 | "output_type": "execute_result" 602 | } 603 | ], 604 | "source": [ 605 | "cat_ct.fit_transform(X).shape" 606 | ] 607 | }, 608 | { 609 | "cell_type": "code", 610 | "execution_count": 72, 611 | "metadata": {}, 612 | "outputs": [ 613 | { 614 | "data": { 615 | "text/plain": [ 616 | "(32950, 24)" 617 | ] 618 | }, 619 | "execution_count": 72, 620 | "metadata": {}, 621 | "output_type": "execute_result" 622 | } 623 | ], 624 | "source": [ 625 | "# Create the feature union of categorical and numeric attributes\n", 626 | "ft_union = FeatureUnion([\n", 627 | " ('cat_pipeline', cat_pipeline),\n", 628 | " ('num_pipeline', age_campaign_ct)\n", 629 | "])\n", 630 | "\n", 631 | "ft_union.fit(X)\n", 632 | "features = ft_union.transform(X)\n", 633 | "features.shape" 634 | ] 635 | }, 636 | { 637 | "cell_type": "markdown", 638 | "metadata": {}, 639 | "source": [ 640 | "Create categorical feature names post-transformation" 641 | ] 642 | }, 643 | { 644 | "cell_type": "code", 645 | "execution_count": 75, 646 | "metadata": {}, 647 | "outputs": [ 648 | { 649 | "data": { 650 | "text/plain": [ 651 | "['pcontacted_last_campaign_False',\n", 652 | " 'pcontacted_last_campaign_True',\n", 653 | " 'previous_False',\n", 654 | " 'previous_True',\n", 655 | " 'poutcome_0',\n", 656 | " 'poutcome_1',\n", 657 | " 'poutcome_2',\n", 658 | " 'poutcome_3',\n", 659 | " 'poutcome_4',\n", 660 | " 'poutcome_5',\n", 661 | " 'poutcome_6',\n", 662 | " 'poutcome_7',\n", 663 | " 'campaign_gte10_False',\n", 664 | " 'campaign_gte10_True']" 665 | ] 666 | }, 667 | "execution_count": 75, 668 | "metadata": {}, 669 | "output_type": "execute_result" 670 | } 671 | ], 672 | "source": [ 673 | "# Add column names\n", 674 | "feature_names = [\n", 675 | " # Don't incclude the last entry in the `named_transformers_` list since\n", 676 | " # it's the `remainder` parameter for the ColumnTransformer\n", 677 | " ['%s_%s' % (name, value) for value in values] for \n", 678 | " name, values in list(zip(list(cat_ct.named_transformers_.keys())[1:], cat_pipeline.named_steps['ohe'].categories_))]\n", 679 | "\n", 680 | "cat_feature_names = [name for names in feature_names for name in names]\n", 681 | "cat_feature_names" 682 | ] 683 | }, 684 | { 685 | "cell_type": "markdown", 686 | "metadata": {}, 687 | "source": [ 688 | "Create numeric feature names post-transformation" 689 | ] 690 | }, 691 | { 692 | "cell_type": "code", 693 | "execution_count": 87, 694 | "metadata": {}, 695 | "outputs": [], 696 | "source": [ 697 | "# TODO\n", 698 | "\n", 699 | "# numeric_feature_names = age_campaign_ct.transformers_[0][2]\n", 700 | "# p = age_campaign_ct.named_transformers_['age_pipeline']\n", 701 | "# p2 = p.named_steps['kbins']\n", 702 | "# p2.n_bins_" 703 | ] 704 | }, 705 | { 706 | "cell_type": "code", 707 | "execution_count": 85, 708 | "metadata": {}, 709 | "outputs": [], 710 | "source": [ 711 | "# features_df = pd.DataFrame(features.todense(), columns=cat_feature_names + numeric_feature_names)\n", 712 | "# features_df.head()" 713 | ] 714 | }, 715 | { 716 | "cell_type": "markdown", 717 | "metadata": {}, 718 | "source": [ 719 | "Try the file we created as code" 720 | ] 721 | }, 722 | { 723 | "cell_type": "code", 724 | "execution_count": 90, 725 | "metadata": {}, 726 | "outputs": [], 727 | "source": [ 728 | "%load_ext autoreload\n", 729 | "%autoreload 2\n", 730 | "\n", 731 | "from support.experiments import experiment_2" 732 | ] 733 | }, 734 | { 735 | "cell_type": "code", 736 | "execution_count": 94, 737 | "metadata": {}, 738 | "outputs": [], 739 | "source": [ 740 | "pipeline = experiment_2.get_pipeline()\n", 741 | "assert features.shape == pipeline.fit_transform(X).shape" 742 | ] 743 | } 744 | ], 745 | "metadata": { 746 | "kernelspec": { 747 | "display_name": "Python 3", 748 | "language": "python", 749 | "name": "python3" 750 | }, 751 | "language_info": { 752 | "codemirror_mode": { 753 | "name": "ipython", 754 | "version": 3 755 | }, 756 | "file_extension": ".py", 757 | "mimetype": "text/x-python", 758 | "name": "python", 759 | "nbconvert_exporter": "python", 760 | "pygments_lexer": "ipython3", 761 | "version": "3.5.5" 762 | } 763 | }, 764 | "nbformat": 4, 765 | "nbformat_minor": 2 766 | } 767 | -------------------------------------------------------------------------------- /notebooks/4.0-model-evaluation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Model Evaluation" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Use the best model from the experiments to estimate generalization performance on the test set." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 62, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "The autoreload extension is already loaded. To reload it, use:\n", 27 | " %reload_ext autoreload\n" 28 | ] 29 | } 30 | ], 31 | "source": [ 32 | "%load_ext autoreload\n", 33 | "%autoreload 2\n", 34 | "\n", 35 | "from utils import code" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 140, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "import pandas as pd\n", 45 | "import numpy as np\n", 46 | "import scikitplot as skplt\n", 47 | "import matplotlib.pyplot as plt\n", 48 | "\n", 49 | "from sklearn.metrics import classification_report, confusion_matrix\n", 50 | "\n", 51 | "from support.model import Model\n", 52 | "from support.datasets import get_data\n", 53 | "from support.experiments import get_scorer\n", 54 | "from support.experiments.experiment_1 import CATEGORICAL_FEATURES, baseline_model_predictions\n", 55 | "from support.parameters import P_TARGETED, AVG_COST, AVG_REVENUE\n", 56 | "\n", 57 | "%matplotlib inline" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "## Model" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "Load the best performing model" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 84, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "tuned_model = Model.load('../models/experiment-1-model.pkl')\n", 81 | "model = Model(tuned_model.name, tuned_model.model, tuned_model.pipeline)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "Load the training and test sets" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 85, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "X_train, y_train = get_data('../data/train.csv')\n", 98 | "X_test, y_test = get_data('../data/test.csv')" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 128, 104 | "metadata": {}, 105 | "outputs": [ 106 | { 107 | "name": "stdout", 108 | "output_type": "stream", 109 | "text": [ 110 | "Number of training instances: 32,950\n", 111 | "Number of test instances: 8,238\n" 112 | ] 113 | } 114 | ], 115 | "source": [ 116 | "print('Number of training instances: {:,}'.format(len(X_train)))\n", 117 | "print('Number of test instances: {:,}'.format(len(X_test)))" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "Build the model using the training set" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 86, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "text/plain": [ 135 | "" 136 | ] 137 | }, 138 | "execution_count": 86, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "model.train(X_train, y_train)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "## Model Performance" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 95, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "name": "stdout", 161 | "output_type": "stream", 162 | "text": [ 163 | "Number of targeted customers: 543\n" 164 | ] 165 | } 166 | ], 167 | "source": [ 168 | "n_targeted = int(len(X_test) * P_TARGETED)\n", 169 | "print('Number of targeted customers:', n_targeted)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "Predict and score the test set" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 96, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "data": { 186 | "text/plain": [ 187 | "0.6560871090617483" 188 | ] 189 | }, 190 | "execution_count": 96, 191 | "metadata": {}, 192 | "output_type": "execute_result" 193 | } 194 | ], 195 | "source": [ 196 | "scorer = get_scorer()\n", 197 | "model.score(X_test, y_test, scorer)" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "We're off by about 1.2% from our cross validation results which is well within two standard deviations of the results." 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "## Financial Performance" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "Evaluate financial performance on baseline" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 126, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "name": "stdout", 228 | "output_type": "stream", 229 | "text": [ 230 | "Baseline profit: $227,418\n", 231 | "Baseline conversion rate: 0.39\n", 232 | "Lift over random model: 3.5\n" 233 | ] 234 | } 235 | ], 236 | "source": [ 237 | "baseline_preds = baseline_model_predictions(X_test, y_test, n_targeted)\n", 238 | "baseline_outcomes = baseline_preds.apply(lambda x: AVG_COST if x == 0 else AVG_COST + AVG_REVENUE)\n", 239 | "baseline_profit = sum(baseline_outcomes)\n", 240 | "baseline_rate = baseline_preds.sum() / len(baseline_preds)\n", 241 | "base_rate = y_test.sum() / len(y_test)\n", 242 | "\n", 243 | "print('Baseline profit: ${:,}'.format(baseline_profit))\n", 244 | "print('Baseline conversion rate: {:.2f}'.format(baseline_rate))\n", 245 | "print('Lift over random model: {:.1f}'.format(baseline_rate / base_rate))" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 98, 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "X_test_trans = model.pipeline.transform(X_test)\n", 255 | "preds = model.model.predict(X_test_trans)\n", 256 | "probs = model.model.predict_proba(X_test_trans)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 111, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "name": "stdout", 266 | "output_type": "stream", 267 | "text": [ 268 | "Model profit: $296,730\n", 269 | "Model lift: 1.3 or $69,312\n", 270 | "Targeted conversion rate: 0.51\n", 271 | "Conversion rate lift: 4.54\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "# Create a dataframe of probabilities and actual / predicted outcomes\n", 277 | "probs_df = pd.DataFrame(np.hstack([probs, y_test.values.reshape(-1,1), preds.reshape(-1,1)]), columns=['p_no', 'p_yes', 'actual', 'predicted'])\n", 278 | "\n", 279 | "# Sort customers by the probability that they will convert\n", 280 | "model_targets = probs_df.sort_values('p_yes', ascending=False)\n", 281 | "\n", 282 | "# Take the top N\n", 283 | "model_targets = model_targets.head(n_targeted)\n", 284 | "\n", 285 | "# Calculate financial outcomes\n", 286 | "model_outcomes = model_targets.actual.apply(lambda x: AVG_COST if x == 0 else AVG_COST + AVG_REVENUE)\n", 287 | "model_profit = sum(model_outcomes)\n", 288 | "model_conv_rate = model_targets.actual.sum() / len(model_targets)\n", 289 | "\n", 290 | "print('Model profit: ${:,}'.format(model_profit))\n", 291 | "print('Model lift: {:.1f} or ${:,}'.format(model_profit / baseline_profit, model_profit - baseline_profit))\n", 292 | "print('Targeted conversion rate: {:.2f}'.format(model_conv_rate))\n", 293 | "print('Conversion rate lift: {:.2f}'.format(model_conv_rate / base_rate))" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "Our model produces the expected lift (see 2.0-experiment-1-baseline)." 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 112, 306 | "metadata": {}, 307 | "outputs": [ 308 | { 309 | "name": "stdout", 310 | "output_type": "stream", 311 | "text": [ 312 | " precision recall f1-score support\n", 313 | "\n", 314 | " 0.0 0.00 0.00 0.00 265\n", 315 | " 1.0 0.51 1.00 0.68 278\n", 316 | "\n", 317 | " micro avg 0.51 0.51 0.51 543\n", 318 | " macro avg 0.26 0.50 0.34 543\n", 319 | "weighted avg 0.26 0.51 0.35 543\n", 320 | "\n" 321 | ] 322 | }, 323 | { 324 | "name": "stderr", 325 | "output_type": "stream", 326 | "text": [ 327 | "/anaconda3/envs/py35-ds/lib/python3.5/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", 328 | " 'precision', 'predicted', average, warn_for)\n" 329 | ] 330 | } 331 | ], 332 | "source": [ 333 | "print(classification_report(model_targets.actual, model_targets.predicted))" 334 | ] 335 | }, 336 | { 337 | "cell_type": "code", 338 | "execution_count": 148, 339 | "metadata": {}, 340 | "outputs": [ 341 | { 342 | "name": "stdout", 343 | "output_type": "stream", 344 | "text": [ 345 | "Cost of false positives: $2,224\n" 346 | ] 347 | } 348 | ], 349 | "source": [ 350 | "print('Cost of false positives: ${:,}'.format(278*8))" 351 | ] 352 | }, 353 | { 354 | "cell_type": "markdown", 355 | "metadata": {}, 356 | "source": [ 357 | "Given the neglible cost of false negatives, we can afford to miss all of the true negatives in the targeted sample - customers who did not end up converting." 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "execution_count": 147, 363 | "metadata": {}, 364 | "outputs": [ 365 | { 366 | "name": "stderr", 367 | "output_type": "stream", 368 | "text": [ 369 | "/anaconda3/envs/py35-ds/lib/python3.5/site-packages/matplotlib/cbook/deprecation.py:107: MatplotlibDeprecationWarning: Passing one of 'on', 'true', 'off', 'false' as a boolean is deprecated; use an actual boolean (True/False) instead.\n", 370 | " warnings.warn(message, mplDeprecation, stacklevel=1)\n" 371 | ] 372 | }, 373 | { 374 | "data": { 375 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl4TFcfwPHvyb6KSBDETgVBLLUVVa1aS+1Ut1fstPbaa2vRFm1VSkPU27ellipKS6vE2oXGEoktCA0isSSSkG3mvH9MJJlkJgkyk+18nieP3HPPvfObm5hfzj33nCOklCiKoiiKMRYFHYCiKIpSuKlEoSiKouRIJQpFURQlRypRKIqiKDlSiUJRFEXJkUoUiqIoSo5UolCKJCHEXCHEt09xfIgQon0+hpQvhBCDhRC/FnQcipKZShTKYxFCvCaEOC6EiBdC3BRC/CKEaFPQceVECLFOCPFB5jIpZX0pZaAJXstGCPG+EOK8ECJBCHE97Rq9nJfjpZTfSSnzVPcJ4porhLiYFle4EGKtEKJafr+WUvyoRKHkmRBiIvAZsBAoD1QBvgR6FmRchcwWdNfjTcAVqA58DnQryKDQxdUDeA1wARoB/wAvPu6JhBBW+RuaUuhJKdWX+sr1C92HSzzQL4c664APMm23ByIybYcDU4DTQAIQgC7h/ALEAXsBV0PHZjr+pbTv5wLfZtq3GYgEYoGDQP208uFACpCcFv9Pmc8FVAQeAmUynasxcBuwTtseApwF7gF7gKpG3v9LaefyzOVaTgMupb3nUKBXpn1vA4czbUtgJHAx7fX9AJG2rxZwIO093wY25hJX5RxiSr+2Wa8vUC0tDl/gWtr13Q2MzXKOU0DvtO+9gN+Au8B5oH9B/w6rryf/Ui0KJa9aAXbAj095nj5AR+AZ4BV0SWIG4I6uhfvuE573F6A2UA4IAr4DkFL6p33/sZTSSUr5SuaDpJQ3gD/S4nrkNWCLlDJFCPFqWny9gbLAIWCDkRheAv6SUkbkEusloC265DsP+FYIUSGH+t2BZ9G1AvoDndLKFwC/omu5eAJf5BDX31LKf3OJKzfPA3XTXn89MOjRDiFEPaAqsEsI4YguSaxH9/MYBHwphKj/lK+vFBCVKJS8cgNuSylTn/I8X0gpb0kpr6P70P1LSnlCSpmELgk1fpKTSinXSinj0s4zF2gkhHDJ4+HpH3pCCAEMTCsDGAEsklKeTXvvCwEfIURVA+dxR9eqIe1cZYQQMUKIWCFEYqZYN0spb0gptVLKjehaC81ziG+xlDJGSnkN2A/4pJWnoPtwriilTJRSHjZyvBtwM8crkDdzpZQJUsqH6H5Wma/DYGBr2vXvDoRLKb+WUqZKKYOAH4C++RCDUgBUolDy6g7gng/3p29l+v6hgW2nxz2hEMJSCLFYCHFJCHEf3W0U0H1w58UWoJUQoiLQDt1tlkNp+6oCn6d94Megu5UigEoGznMHSG8ZSCnvSilLA00B20zxvimEOJnpnN65xBqZ6fsHZFyj99Ji+TvtKa4hRo7Xi+sppLdIpJRxwC50SZW0f79L+74q0OLR+0t7j4MBj3yIQSkAKlEoefUHkAi8mkOdBMAh0/bTfDDonUsIYYnu1o8hr6HrQH4J3e2cao8OS/s3xymSpZQx6G7h9E871wYp5aNj/gVGSClLZ/qyl1IeNXCq34FnhRCexl4r7S/w1cBYwC0tkZzJFGueSSkjpZTDpJQV0bV8vhRC1DJQdS/QPKe4yNvPLut13AAMEkK0AuzRtXZAd80OZLlmTlLKUXl5X0rhoxKFkidSyljgfcBPCPGqEMJBCGEthOgihPg4rdpJoGvaLRcPYPxTvOQFwE4I0U0IYQ3MItNf5Vk4A0no/nJ2QHd7KLNbQI1cXm89uieV+pBx2wlgFTD90f11IYSLEKKfoRNIKX9F92G5TQjRIu2RVGugZaZqjug+cKPTzvcfdC2KxyaE6Jfpw/9e2nk1BuLai67P4EchRFMhhJUQwlkIMTJTK+QkMDDtZ9qMvN0m+hld62E+uo50bVr5TuAZIcQbaeezFkI8K4So+yTvUyl4KlEoeSalXAZMRPehHY3uL8exwLa0Kv9D9+RLOLq/0Dc+xWvFAqOBNcB1dH/xGusk/ga4mlYvFPgzy/4AoF7abZBtWQ9OswNdZ/gtKeWpTHH8CHwEfJ92W+sM0CWH0Huj+6D8FogBrqC77dI57XyhwFJ0LbRbQAPgSA7ny8mzwF9CiPi0+MdJKa8YqdsX3Qf7RnRPSZ0BmqFrbQDMBmqiSzjz0E+WBqX1R2xF15Jbn6k8DngZ3e2oG+hunX2E8USvFHIio4WtKIqiKNmpFoWiKIqSI5MlirTpAaKEEGeM7BdCiOVCiDAhxGkhRBNTxaIoiqI8OVO2KNaRdl/WiC7o7gnXRjd6dqUJY1EURVGekMkShZTyILpnzo3pCXwjdf4ESucyOlVRFEUpAAU5uVclMg3gQfdESyUMjCAVQgxH1+rAzs6uaZUqVfI1EId/dWE8qFw5X89ralqtFgsL1c0E6lpkpq5FhuJ1LSRCSuwSoxBSw6OhN5aaB0aPiE2SXI2RSCBVy20ppbGxSDkqyERhaICRwUew0ubr8QeoU6eOPH/+fP5G0r697t/AwPw9r4kFBgbS/lHsJZy6FhnUtchQZK5FUjwkJ8DtC/Dgjq4s5QHsmQGO5eB21s88y0zfZ5/MIDpBy/g9iawP1s2482xFC47d0F590vAKMlFEAJn/hPdE98y1oihK8RN7HS4HQsTfEHUO7l4CGye4Z2zoS5qH9/L8ElJKZgd7suLXy8QmpGJrZ8e892cy+b3pWFk9+cd9QSaKHcBYIcT3QAsgVkqZHxOXKYqiFA4x/8KvM+F2GESFZN+fEP3Yp0yVFlgJLTNSfLkqywEgrRywqdQQt9jzLPtxJAAvvPACq1evpmbNmk/1FsCEiUIIsQHdmgLuQogIYA5gDSClXIVulGhXIAzdRGf/MVUsiqIoZqXVwMc1IDHm8Y5zLKdLHjU7oLV2JCouiWtR91gQ34Ob0o3bZEyIbGkhaFmjDAOerUKn+uWxtbIkNbUNwXt/oH///vj6+qKbDPnpmSxRSCkH5bJfAmNM9fqKoihmdfE32DMTLG3gVrDxerYu8MzLUNYLPJ8FF0/dMS6eIARarWT7qet88XsYl28n6B0qBLSq4Ua/Zp508CrP7RvXmDx5DM38/KhYsSJWVlbs2bMn3xLEI2pJQ0VRlKdxcAkcWqrrfDbGwQ3aT4eaHcDN+K2gAxei+Xj3OUJu3Ncrt7Gy4PUWVfnPc9WoXMYBjUbDZ599xuzZs3n48CEuLi6sW7cOIN+TBKhEoSiK8niS4uBWKOz/AK4czL3+uFPgWi3HKhduxfHRL+f4/VyUXrmzrRWvt6rKm62qUsHFHoAzZ84wZMgQjh07BsDgwYNZsmTJE72VvFKJQlEUJS8u/Ao/+ELS/ZzrVWoG3T8FG0coU0N3v8iIM9djWXngErtO6z/HY2tlgW+b6oxsX5NSdtYAJCUlsWjRIhYuXEhKSgqenp6sWrWKbt26PfVby41KFIqiKMbE3YKg/8KBj0CbyyrAbSdB63fA3jXHalJKAs9Hs2J/GP9c1X/0VQjo19STyZ3qUM7ZTm9faGgoCxYsQKvVMmrUKBYvXkypUqWe6G09LpUoFEVREm5D1Fm4EQSX9sOtEN0tptSHxo9xcAPnitB5IVRvl6eXCbkRywc7z/LH5TvZ9nWsV553OtSioWfp9LLk5GRsbGwAaNy4MZ988glNmzbl+eeff7z395RUolAUpeRIjIXIYJBauLAH/lkHyfGPd45nOkPftbpbS3kUGZvIst/Os+m4/tpb1paCLt4VeKt1NZpW1W+J7Nu3j2HDhrF8+fL020sTJ058vFjziUoUiqIUbxHH4V64rn/hSdmXAe8+0GEW2JfOvX6apFQNK/aFsfrQZRJTtOnllhaC15pXYcwLtfBw0b/FFBMTw5QpU1izZg0AX375pVn6IXKiEoWiKMVDxD8QfhBungYbBwjbR/u4GxD4GOcoUxOs7aFSU6jRHio0AucKuvM9puPhd5m+NZiLUfotlg5e5ZjR1Yta5ZyzHbNjxw5GjRrFjRs3sLGxYfbs2UydOvWxXzu/qUShKErRdfcKhB+CHe883nFOHuBeGx7cBa+u8OwwcC6fLyHdup/IR7+cY+uJ63rl9SqU4r3OdWhfp1y2Y+7evcvo0aPZuFG3zHzLli0JCAigXr16+RLT01KJQlGUoiUlEbaNgpCtj3dcjfZgZQ8DvgXL/P/oS0zR8M0f4Xy+9yIJyZr0cgcbS6Z18eL1FlWxsDD8qKyVlRVHjhzBwcGBhQsXMnbsWCwtLQ3WLQgqUSiKUvhd2g+h28DSFv7+Kue61dpCmerg+Sznzp/Hq9sYKFXRpOHtCYlk7o4QbsYm6pV38fZgZre6eLpmv3X177//4ubmhoODA6VKlWLjxo1UqFCB6tWrmzTWJ6EShaIohcfRL+B6EFjZgpRw4Rfdk0q5caul61fo9ZXeALfI+4F4mTBJHL10m49+OcepCP0Ya5dzYs4r9WlT2z3bMVqtFn9/f9577z2GDx+ePqq6devWJovzaalEoShKwXkYo1us5+wOXZJ4XCOPgId3/seVi8QUDR/tPsfXR8L1yt2dbBj7Qi1ea1EVG6vsK+tdvHiRoUOHcvCgbuqPq1evFolV+FSiUBTFvO5c0j2qeuPE4x9bqRk06AtO5cCru67lYWb/XL3Le1tOcyk6Y2ZXa0tB78aezOhaFxcH62zHpKamsmzZMubMmUNiYiLlypXDz8+PPn36mGQSv/ymEoWiKKYlJZzeCGF7IXhz3o5pPhwqNtZ9LyygVkdwdDNdjHmQkJTKZ3svsObwFWSmRZtfqFOWxX0aUr6UncHj7t+/T4cOHfjnn38AePPNN1m2bBlubgX7fh6HShSKopjOxjd0t5VyU7oKWFhB07eh9bs5TqRnblJKdgXf5MNdZ/U6qx1sLJndvR4Dn62cY6ugVKlSVKlShejoaL766is6d+5sjrDzlUoUiqLkLynht9m59zlU8IEuH0Pl5oUqMWT2z9V7zN8Zyql/9Veqa1vbncV9GlKptL3B4/7880+cnJzw9tb1n/j7+2Nra4uzc/ZBdkWBShSKojyd5AT4ezU8uA2nNkJClPG6tTqCzyCo1wsKcQdu7MMUFv9ylg1//6tX7uZow7QuXvRt6mmwFZGQkMDMmTNZvnw5TZs25Y8//sDKygp39+xPPxUlKlEoivJkHqdTuu9a3VxJRcCRsNtM3nxK7zaTjZUFb7SsyrsdahvsrAbYu3cvw4YNIzw8HEtLSzp27IhGo8HKquh/zBb9d6AoinlpNTC/TN7qDvpeN9tqIb21lFlCUiqf7DnPuqPheuUv1S3PnFfqUbmM4fme7t27x+TJk1m7di0APj4+BAQE0KRJE1OHbDYqUSiKkjutBnZN1E3LnZNmQ8DFExoOBJdKZgktP+wNvcWcHSFcj8lYf6KMow0fvupNlwYVjB6XmppKixYtuHjxIra2tsyZM4fJkydjbW241VFUqUShKIphKYlw8VfY9EbO9So0gn7rdMt+FjG345P4YGco207e0Ct/0asci/o0yLbKXFZWVlaMGzeO9evXExAQgJeXlynDLTAqUSiKoqNJgT9W6Dqko8/m7Zip4bku/VkYpWq0/BAUwaJfzhHzICW9vIyjDTO71qV3k0oGO6ullHz77bdoNBrefvttAEaNGsWoUaMK/ejqp6EShaKUVFLC3cuwdZhuDQdtSu7HAHRbCo3fBCsb08ZnImeuxzJ58ynORcbplfduXInZ3evh6mj4fV29epWRI0eye/dunJyc6NSpExUqVCjWCeIRlSgUpSS5cwl+ngzX/oKUhNzrP+LVXXd7ybLo3nt/kJzKl/svserAJVK1GUOrK5W254NXvXnBK/s6EaCbxG/lypVMmzaN+Ph4XF1d+fTTT/Hw8DBX6AVOJQpFKSnuhcMXeXwSp80E8HpFN41GEf+LWUrJjlM3WPjzWW7dT0ovt7O2YNTztRjatjqOtoY/Cs+fP8/QoUM5fPgwAH379uWLL74oUUkCVKJQlOIr4bZuDYe988HeBWKuGa/rWA56rYJKTYpkn4MxMYlahqw7xv7z0XrlTau6sqRfI6q7O+Z4vK+vL0eOHKF8+fJ8+eWX9O7d25ThFloqUShKcRIfRa2LqyGwp355koE1Hfr9F2q9BLZO5onNzH4OvsmsIw+JT8l45LWssy1TXq5Dn6aeWBpZbU5Kmd6R7efnx/Lly1myZAmursUngT4ulSgUpShLfgABHeHWmfQiz9yOcakCE4JNGlZBuhOfxOztZ/g5OFKv/O3W1Zj08jM42xnuZ0lMTGTBggVcvnyZDRs2ANCoUSMCAgJMHnNhpxKFohQ1UsJfX8GRzyHuRu71S1eBOt2gxQiwLVXg03WbyqO+iPk/hXInITm9vKKLHUv6N6J1TePzLR05cgRfX1/Onz+PEIJp06bRqFEjc4RdJKhEoShFzQJ30KbmXMexrG5Nh5ajwLZozlj6OKLjkpjxYzC/hd7SK29byQq/Ye0oZaQVERcXx4wZM/Dz80NKiZeXFwEBASpJZKEShaIUFWe2wpb/GN8/bB+Ub0Dg4aO0b9/ebGEVpEetiDk7QvQGznmUsmNxnwZwM9RoktizZw/Dhw/n2rVrWFlZMXXqVGbNmoWdXc6jsUsilSgUpbD7ezXsmQGa5Oz7OszWLQ3qWs3sYRW0m7EPmbsjhD0h+q2IN1pWZWoXL5xsrQi8GWr0+D179nDt2jWaNGlCQEAAPj4+pg65yFKJQlEKI60WNr4O53cZr/PaJnimk/liKiSklGwNus7cn0KIS8y4BVeptD2L+zSgbe2yRo+Njo6mbFnd/gULFlCjRg1GjhxZLKYCNyV1dRSlsNGk6PohjHmmCwz4X5EeJf2kjPVFvNaiCjO61sXJyMC5mzdvMnbsWI4fP86ZM2dwdnbG0dGRsWPHmiPsIs+kiUII0Rn4HLAE1kgpF2fZXwX4L1A6rc40KeXPpoxJUQq1lIfwoZFRv20n6b5sch4kVlz9GhLJtK3B3M30RFOVMg4s7tPA6BNNUkrWrVvHhAkTiImJwcnJiRMnTtCuXTtzhV0smCxRCCEsAT+gIxABHBNC7JBSZr5pOAvYJKVcKYSoB/wMVDNVTIpSaCXGwoGPdbO3ZjXiEFRoaP6YComEpFQW7Azl+2P6y5IOTmtFGJt+Izw8nPfee4/jx48D0KVLF1atWkWVKlVMHnNxY8oWRXMgTEp5GUAI8T3QE8icKCRQKu17FyAPD4UrSjETfR78mhveN/MWWJfcp3D2n49ixtZgvWVJPUrZsaRfI9rUNn577ptvvmH06NEkJCRQpkwZPv/8cwYPHmxw6nAld6ZMFJWAzH8CRAAtstSZC/wqhHgHcAReMnQiIcRwYDhA2bJlCQwMzNdAfWJiADiZz+c1tfj4+Hy/FkVVUboWQqvB7c7fVLq+E4cHN7FNvmOw3uHnviP1yJ+Pff6idC2MuZ8s2Xw+mUPX9ceLNPew5M16FqReP0PgdePHR0REkJCQQNu2bZkwYQKurq4cOHDAxFEXX6ZMFIZSt8yyPQhYJ6VcKoRoBfxPCOEtpdTqHSSlP+APUKdOHZnvz4iXLg1Q5J49DwwMLHIxm0qRuRZ/roLdU3Ou0/0zaPw6bZ6ws7rIXAsDNFrJ+r+v8UngOe5neqLJzdGG2d3r0dOnosFWQUpKCocOHaJDhw6A7v9yx44dSUhIKLLXojAxZaKIACpn2vYk+60lX6AzgJTyDyGEHeAORJkwLkUxvxsnwL99znVKV4Fxp6GE3h65ducBkzaf5Fj4Pb3yrg08WNDTGzcnW4PHBQUFMWTIEIKDg/nrr79o1qwZAM8++2yRb1kVFqZMFMeA2kKI6sB1YCDwWpY614AXgXVCiLqAHRCNohQX4UdgXVfj+6s+By1GQuUW4FzefHEVIlqtZMOxayzcdZaEZE16eVU3B+a8Uo8OXoavy8OHD5k3bx5LlixBo9FQvXp1kpMNDEpUnprJEoWUMlUIMRbYg+7R17VSyhAhxHzguJRyBzAJWC2EmIDuttTbUsqst6cUpejQauHmSTi8DM7+ZLzeyx9A63fMF1chdeV2Au9tOaXXirC0EIxuX5MxL9TCztrS4HGHDh1i6NChXLhwASEEEyZMYMGCBTg6lsxHh03NpOMo0sZE/Jyl7P1M34cCz5kyBkUxm9Qk+MDwcprpqrSCIbvNE08hlpii4ctA3bKkyakZXZI1yzry6QAfGnqWNnrsmjVrGDZsGAD16tUjICCAli1bmjzmkkyNzFaUp5XyUDfld+Ai43Wqt4NXlkOZ6uaLq5A6Fn6XqVtOc/l2xprdVhaC4e1q8O6LtY22Ih7p2rUr7u7ujB49mhkzZmBra7jvQsk/KlEoypOKuQa/zYGQrYb3V24Bz0+Fam3Bysa8sRVCcYkpfLT7HN/+qb8ka0NPFxb2aoB3JReDx925c4cVK1Ywa9YsLC0tqVixIpcvX8bZufhPn15YqEShKI8j5aFu9PS+D3Ku9/5dsMj5L+OS5OCFaKb+cFpv4JyTrRXTunjxWvMqWBhYllRKyebNmxk7dizR0dE4OzszceJEAJUkzEwlCkXJq+QEWFgx5zrtp+taESX0EdesYh4ks2DnWX4IitArf9GrHPNf9aZSaXuDx924cYMxY8awbds2AJ5//nl69Ohh8ngVw1SiUJS8kNJ4kihdVfcEU/Nh5o2pkNsbeosZPwYTFZeUXlbG0YY5r9SjRyPDA+eklKxdu5ZJkyYRGxuLs7MzS5YsYejQoVhYWJgzfCUTlSgUJSdSQtB/4adx2fc9Nx5emqtaD1ncT0xh7o4Qtgbpz7HRub4HH/Tyxt3IwDmALVu2MHToUAC6devGqlWr8PT0NGm8Su5UolAUQ6SEv/3hl/cM759+HWydzBtTEfDX5TtM3HSK6zEP08vcnWxZ1LsBHevlPqCwd+/e9OjRg4EDBzJw4EA1iV8hoRKFomR15xJ80cT4/tF/qiSRRWKKhiV7zhNw5AqZh8y+6lOROa/Ux9XR8FNfISEhTJgwgbVr1+Lp6YmlpSXbt283U9RKXqlEoSiPBP0PduSw4lnrd3QjqhU95yLvM27DSc7fiksvc7G35sNe3nRvaLhfJzk5mY8++ogFCxaQkpLC7Nmz+frrr80VsvKYVKJQlPs3YZmX8f2t34WO81VfRBZarWTd0XAW7z6nN7q6bW13PunbCA8Xw+toHDt2DF9fX4KDgwEYMWIEH330kVliVp6MShRKyXXiW9g+xvj+Cj4weDM45TItRwkUdT+RyVtOc/BCxhyedtYWzOxWj9dbVDHYt/DgwQPmzJnDsmXL0Gq11KxZk9WrV/PCCy+YM3TlCahEoZQ8dy/D8sbG9zd6DbovA2vDz/iXZFJKdpy6wfvbQ4h9mJJeXr9iKT4f6EOtcsYHwl24cIFPP/0UgMmTJzNv3jwcHBxMHrPy9FSiUEqW+CjjScK2FEwIAbtShveXcFFxiczdEcLPwZHpZULA8HY1mNSxDjZW2cc5PHz4EHt7XcL18fHh888/59lnn6V5cyNLvyqFkkoUSskgJUSFwsrW2fc1/Q90/1T1QRih1Uo2Hf+XD38+S1ymVec8Xe35uG9DWtc0vHb1rl27GDFiBH5+fvTs2ROAMWNyuNWnFFoqUSjF2x9+sGeG8f0zI9UtphyERcUz48dg/r5yV698UPPKzOxWDyfb7B8h0dHRjB8/nvXr1wOwbt269EShFE0qUSjFU9jv8G3vnOvMiVGtCCNSNVpWH7rCp79dIFmT8URTNTcH5vf0pt0zZbMdI6Vk48aNvPPOO9y+fRt7e3s+/PBD3n33XXOGrpiAShRK8ZOalHOS8GwOgzaoJGHEhVtxTNp0iuDrselllhaCYW1rMO7F2tjbZJ8VNzo6Gl9fX376SbeqX4cOHVi9ejU1atQwW9yK6ahEoRQfQf+D4wFw40T2fS++D20nmT+mIiRVo2XN4Sss+1W/FdGgkgsf9WlIvYrGO/nt7e05ffo0Li4uLF26lCFDhqjpN4oRlSiUou/+DdoHGrkHXq4+jD5q3niKoCu3E5i46SQnrsWkl9lYWTDuxdqMaFcDK8vsTzSFhYXh4eGBk5MTTk5ObNmyhYoVK1KxYi5TsStFjpq3Vym6tFpYXBWW1TVeZ/h+88VTBGm1km//vErXzw/pJYmGni7seqcNY16olS1JaDQalixZQoMGDZg5c2Z6ebNmzVSSKKZUi0Ipem5f1M3s+re/4f3efXTzMlXwUf0QObh4K46pP5wmKFOCsLYUjH/pGYa3q4G1gVbEmTNnGDJkCMeOHQMgJiYGrVar1ooo5lSiUIqWGyfAv73hfY7lYNxJsHE0a0hFTXKqlhX7w/hyfxip2oypXmuXc+LTAT4G165OTk5m4cKFLFy4kJSUFDw9Pfnqq6/o2rWrOUNXCohKFErRkBQHK5pD3A2Du/9+9guad3vTzEEVPWeuxzJ58ynORWbM9GptKRjergbvdKiNnXX2J5piY2N57rnnCAkJAWDUqFEsXryYUqXUCPaSQiUKpfBKTYK1neFGkPE6gzZCnc48CAw0W1hFUapGy6oDl/hs70W9VkTTqq582MsbLw/jH/ouLi7Ur1+f5ORk1qxZQ7t27cwRslKIqEShFE5bh8Ppjcb3V2sLb+80XzxFWFhUPO9tOaXXF2FnbcGUTl78p3U1LCyy9+Ps27ePMmXK4OPjA8CqVauws7NLn7dJKVlUolAKn1VtIDLY+P7xwVC6ivniKaKSU7UEHL7Cp3sv6K0X0bhKaT7t70M19+x9OTExMUyZMoU1a9bg4+PD33//jbW1Na6uruYMXSlkVKJQCoeYa7p5mf5aZXj/8ED1FNNjOHvzPhM3neLszfvpZVYWgvEv1Wbk8zUNjovYsWMHo0aN4saNG9jY2NC3b19zhqwUYipRKAXr2l+w9mXj+9/cDjXamyuaIi9Fo2XHpWR++u0wKZqMvgjvSqX4uE8jg6Oro6KiePfdd9m4UXerr1XeK0EWAAAgAElEQVSrVgQEBFC3bg7jU5QSRSUKpWDcvQLLfXKuM+IQVGhonniKgYw5mjIWFLK1smBix2cY0qa6wXERqamptGrVisuXL+Pg4MCiRYsYM2YMlpbZn35SSi6VKBTzun8TVjwLyXGG9zuWgxemQ5O3wEJ9WOVFikbLin1hfBkYpteKaFS5NEv7NaJWOSejx1pZWfHee++xZcsW/P39qV69ujlCVooYlSgU87l8AL7pYXhf1TYw4H/gUMa8MRVxYVFxTNioP9OrlQVM6eSFb5vq2foitFot/v7+WFhYMHz4cACGDx/O8OHD1SR+ilEqUSjm8WVriArJXm5XWvcUk1p+9LEkpWr46sBl/PaHkZTliaa+lZMY/HzNbMdcvHiRoUOHcvDgQRwcHOjRowceHh4qQSi5UolCMR0p4VYIrHrO8P6pV8G+tHljKgZOXLvH5M2nuBSdkF5mY2nB5E7P4NumBocOHtCrn5qayrJly5gzZw6JiYmUL18ePz8/PDw8zB26UkSpRKHkv7hIuHoUtvzH8P56r0L//5o3pmIgISmVhT+fZcPf18g0uBrvSqVY0q+RwdHVp06dYsiQIQQF6Ua3v/XWWyxbtowyZdQtPiXvTJoohBCdgc8BS2CNlHKxgTr9gbmABE5JKV8zZUyKCdy/CQc/gZirELY357qj/4JyXuaJqxgJunaPCRtPcvXOg/QyRxtL3uvsxestq2JpYHS1lJIxY8YQFBRElSpV8Pf3p1OnTuYMWykmTJYohBCWgB/QEYgAjgkhdkgpQzPVqQ1MB56TUt4TQpQzVTxKPouPhpPfwdEv4MHt3OtXaARdPlFJ4jGlarR8sS+MFfvD0GRqRrSt7c7CXg2oXMYh2zEajQYAIQSrVq3C39+fDz/8EGdnZ7PFrRQvpmxRNAfCpJSXAYQQ3wM9gdBMdYYBflLKewBSyigTxqPkh2t/wto8/lXqXAE8GsCA78DKxrRxFUMHLkQz/6cQvb4IZ1sr5r9an1d9KmXrhI6Pj2fWrFkEBQXRoUMHhBB4e3uzfPlyc4euFDOmTBSVgH8zbUcALbLUeQZACHEE3e2puVLK3VlPJIQYDgwHKFu2LIH5PFOoT4xusrSTRWwG0vj4+Hy/FjnxOvspHreMv55WWHG1an/inWoQU9objVXaBHKHTb8UqbmvhSklpEi+P5fMoeupeuXPuFowrIE1rrFhHDgQprfv+PHjLF26lMjISCwsLPj666+pUaOGOcMulIrT70VBMmWiMPTMncyybQXUBtoDnsAhIYS3lDJG7yAp/QF/gDp16sj27dvnb6SldU/e5Pt5TSwwMNA8MV/8Db4zMu+Pg5uuc7rlaCzca1FQw7XMdi1MLPB8FPO2nCYqLiNJONtaMbZDLYa2rZGtL+LevXtMmjSJr7/+GgAfHx9Gjx7NkCFDzBp3YVVcfi8KmikTRQRQOdO2J5B11ZkI4E8pZQpwRQhxHl3iOGbCuJS80mpgcRVIjs++z7sv9F4NagnMfBGXmMLiX87x3V/X9Mq7NazAvB71cXeyzXbMtm3bGDVqFJGRkdja2jJ37lwmTZrEkSNHzBW2UkKYMlEcA2oLIaoD14GBQNYnmrYBg4B1Qgh3dLeiLpswJiWvTn0PP44wvK/zR9BypHnjKcYOX7zN1B9Ocz3mYXqZu5MNH7zqTWfvCkaPO3r0KJGRkbRp04Y1a9ZQp04dc4SrlEAmSxRSylQhxFhgD7r+h7VSyhAhxHzguJRyR9q+l4UQoYAGmCKlvGOqmJQ8+ssffpmSvbx6O3hju2pF5JO4xBQ+3HWW74/9q1feqX55FvZqgFuWVoSUkuvXr+Pp6QnA3Llz8fLy4u2338ZC/UwUEzLpOAop5c/Az1nK3s/0vQQmpn0pBS10B2x6w/C+CSHg4mneeIqx/eejmPXjGb1WhKuDNXN71KdHo4rZnmi6evUqI0aM4MyZM4SEhODi4oKDg4Pqi1DMQo3MVnRuXzScJBoOhN5fmT+eYup+Ygrzfwplyz8ReuWd63uw4FVvyjrrtyK0Wi1ffvkl06ZNIyEhAVdXV0JCQmjdurU5w1ZKOJUoSrpja+DGCTjxbfZ9z42HjvPMH1Mxtf98FDO2BnMzNjG9zNXBmgWvetO9YcVs9c+fP8/QoUM5fPgwAH379mXFihWUL1/ebDErCqhEUXJJCUtqQ0J09n2lq8DYf9QguXwS+zCFRT9n74vo6VOR2d3rGXyiyd/fn3fffZekpCQ8PDzw8/Ojd+/e5gpZUfSoRFESabWwwB2kxvD+d0+pDut8IKXk27+u8cnuc9xPzBgXUcZR90RT1wbGn2iqUqUKSUlJ/Oc//2Hp0qW4urqaI2RFMUglipJmiy+c2ZK9vMUoqN0Rar1o/piKoesxD5n2w2kOXdSfB6tbgwoseNWbMo76rbXExET27dtH165dAejcuTPBwcF4e3ubLWZFMUYlipIg6H9w6wz8tcrw/tF/Qrm65o2pmJJSsvmfCObtCCEhOaPFVtXNgamdvQy2Io4cOYKvry8XLlzg6NGjtGzZEkAlCaXQUImiOLscCN/0zLnOuFPgWs0c0RR7d+KTmPFjMHtCbqWXCQHD29VgwkvPYGetvwZ4XFwcM2bMwM/PDyklXl5eWFqqdcKVwifPiUIIUQmomvkYKeVBUwSlPKXr/8B/exieeuORXl9B3VfAxtF8cRVju07fZM6OM9yOT04vq+7uyJJ+DWlaNfsiQXv27GH48OFcu3YNKysrpk2bxqxZs7C1zd6xrSgFLU+JQgjxETAA3RThj9rTElCJojB5eA+WN9b9a4jPYKjYGJ7pDKUrG66jPJbI2ETm7Dij14oAeKNlVaZ39cLBJvt/sZUrVzJ69GgAmjZtSkBAAI0aNTJLvIryJPLaongVqCOlTDJlMMpT+O8rcMVI3m7yJvT4wrzxlAA/nbrBrG1niH2Ykl5WvpQti/s05IU6xtfg6tWrFx988AHjxo1j4sSJWFmpO8BK4ZbX39DLgDWgEkUhYqFJgq+7wdXDhitUbwc9/XTjIpR8E/sghTk7zrDtpP5kyAOaVWZm97qUsrPWK7958yafffYZH374IVZWVnh4eHDp0iXs7OzMGbaiPLG8JooHwEkhxO9kShZSyndNEpWSs8T7sG0U7c7tNLzf0hbeDVJzM5nA3tBbTP8xmOi4jL+ZKpW2Z0m/RrSq6aZXV0rJunXrmDhxIjExMbi7uzNlim6yRZUklKIkr4liR9qXUpCizsLK54wPlAMYeVi3/KiSr2IfpDDvpxC2nriuV96niSdze9TDOUsr4sqVK4wYMYLffvsNgC5dujBw4ECzxaso+SlPiUJK+V9TB6LkIux3+DaHKRz+sxuqtjJfPCVI4Pkopv5wmlv3M1oRZZ1t+eBVbzrV99Crq9Fo8PPzY/r06Tx48AA3Nzc+//xzXnvttWwzwipKUZFjohBCbJJS9hdCBJN9GVOklA1NFpmSIWQbbH7L8L4eK6CJkanBlacS+zCFBTuzz/Tao1FF5vWoj6tj9rmwtmzZwrhx4wAYMGAAy5cvp1w54x3bilIU5NaiGJf2b3dTB6IYEXXWQJIQMPkCgcdDad+kfUFEVez9fvYWM388Q+T9jJle3Z1sWNirAS9naUVk1q9fP7Zu3cprr71Gz565DHZUlCIix0QhpbyZ9u9V84SjpNOkwu5pcGy1frlXdxj4XdpGqNnDKu4SklL5YFcoG/7Wn+m1W4MKzO9ZP9uqc//88w/jxo3ju+++o2rVqlhYWLBx40ZzhqwoJpenKUKFEC2FEMeEEPFCiGQhhEYIcd/UwZVoCytkTxLlvTMlCSW//XP1Ll2XH9JLEm6ONqwc3AS/wU30ksTDhw+ZOnUqzZs358iRI8yfP78gQlYUs8jrU08rgIHAZqAZ8CZQy1RBlWiaFPisAWiS9cs9GsIINRDeFBJTNCz99TxrDl9BZuqJ69rAgw9fbZCtL+LgwYMMHTqUixcvYmFhwcSJE1WiUIq1PA8JlVKGCSEspZQa4GshxFETxlUyPVonIqvOi6HlKPPHUwIcC7/L9K3BhEVlzIvlbGvFvJ716dW4kt6TSvfv32fatGmsXLkSgPr16xMQEECLFi3MHreimFOeB9wJIWyAU0KIj4GbgJpNLj9dPQpfd8le/tZOqN7W/PEUc3fik/ho9zk2Hdd/oqltbXcW9W6Ap6tDtmPCw8NZvXo11tbWzJgxgxkzZmBjo1YBVIq/vCaKN9D1Z4wBJgCeQB9TBVWiaDXwuQ/EXsu+790TUKaG+WMqxqSUbA26ztyfQojLtOqcg40ls7rVY1DzytlaEaVKlQKgYcOGrFq1iubNm9OggRrUqJQcuY2j6Al4Sin90rYPAOXQjan4AwgzeYTF1bld8P1rxvdPuQyObsb3K48tKi6RWT+e4ddQ/ZleO9Yrz7we9alY2j69TErJpk2beOedd1i5ciV9+uj+LvL19TVrzIpSGOTWongPXSf2I7ZAU8AJ+BowsKamkquv2sHNU4b3VWqqG2VtpW5p5BcpJT+dvsn7288Q8yBjptcqZRyY26MeHbzK69W/ceMGo0aNYscO3aw1mzdvTk8UilIS5ZYobKSUmR8oPyylvAvcFUKoPoonsX6A4SRhZQdDfwcPtfxlfoq6n8isbdlbEa+3rML0LnVxtM34LyClJCAggMmTJxMbG0upUqX45JNPGDp0qLnDVpRCJbdE4Zp5Q0o5NtNm2fwPp5g78Alc2K1fVu9V3Wpz1mo20fz0qC9i/s5QvfUiKpW2Z3GfBrStrf/rGxkZyeDBg9m3bx8A3bt3Z+XKlXh6qhl4FSW3RPGXEGKYlFJv5JcQYgTwt+nCKoY0qbD/A/0yNZGfSUTHJTF962n2no3SKx/UvAozunplm+kVoFSpUoSHh+Pu7s7y5csZOHCgmsRPUdLkligmANuEEK8BQWllTdH1VbxqysCKlZSH8GGW+YF6r1ZJwgR2nr7B+9tDuJuQMWDR09Wej/s0pHUt/TEqISEhVK5cmVKlSuHg4MDWrVupWLEiZcuqxrKiZJbjFB5SyigpZWtgARCe9jVfStlKSnkrp2OVNJrU7EnC/Rlo2L9g4immElIk474/wdj1J/SSxJutqrJnfDu9JJGcnMz8+fNp3Lgx06ZNSy9v1KiRShKKYkBe16PYB+wzcSzFi5Tw92r4ZUr2fcMPmD+eYuzopdvMPvKQu4kP0ssquNjxcd+G2foijh07hq+vL8HBwYCuL0Or1WJhkadpzxSlRFKruue3+Cj4axUcWmp4/4ybYJN91K/y+BJTNCz+5Rzrjobrlfdt6smcV/RXnXvw4AFz5sxh2bJlaLVaatasyZo1a2jfvr15g1aUIkglivwUdwuWPmN8/5TLKknkk5AbsYz7/qTeHE2uDtYs6t2Azt4V9OrGxMTQrFkzLl26hIWFBZMnT2bevHk4OKifhaLkhUoU+en4WsPlLcdA54XmjaWY0molqw9dZsmv50nRZEz12qisJauHtaNcqeyPGZcuXZoWLVrg4OBAQEAAzz77rDlDVpQiTyWK/CIlHFisX9ZxAbR+B9Rjlvni2p0HTNx0kuNX76WX2Vtb8v4r9fBIuKSXJHbu3EmFChVo2rQpACtXrsTOzk5N4qcoT0AlivwQFwlL6+iXTboAzuUN11cei5SSTcf/Zf5PoSQka9LLG3m68OkAH2qUdSIw8DIA0dHRjBs3jg0bNtCgQQOOHz+OjY1N+sR+iqI8PpM+6iGE6CyEOC+ECBNCTMuhXl8hhBRCNDNlPCazysA04CpJ5IvrMQ8Zsu4YU38ITk8SVhaCcS/WZsuo1tQo6wToksn69eupW7cuGzZswMHBgSFDhmBpaVmQ4StKsWCyFoUQwhLwAzoCEcAxIcQOKWVolnrOwLvAX6aKxaR2vAMJ+iOAGXe6YGIpRqSUbPkngvk/hRKXlDEdeI2yjnw2wIeGnqXTyyIiIpg5cyZ//PEHAC+++CL+/v7UqKGmaFeU/GDKW0/NgTAp5WUAIcT3QE8gNEu9BcDHwGQTxmIae2ZC0Df6ZXNiVJ/EU4qOS2LaD6f5/Zx+An6rVVWmdamLvU1GKyElJYXnnnuOa9eu4eLiwtKlSxkyZIiafkNR8pEpE0UlIPPMsxGA3pqRQojGQGUp5U4hhNFEIYQYDgwHKFu2LIGBgfkaqE9MDAAnH+O8VinxtPljhV5ZSL0pRB8w32C6+Pj4fL8WBe1kVCoBZ5KIy7RkeDkHwbAGttR2uc1fRw9lO6Z///4cOnSIyZMn4+7uzgEz/gwKo+L4e/Gk1LXIH6ZMFIb+pEt/nlEIYQF8Cryd24mklP6AP0CdOnVkvg+SKq27jZHn8145CP8drF/W5RPqtxiev3HlIjAwsNgMGHuYrOGDXaF8F6S/0t+braoyrYsXDja6X9XU1FQ+++wz7OzsGDtWN5nx888/T2BgIC+88ILZ4y6MitPvxdNS1yJ/mDJRRACVM217AjcybTsD3kBg2m0CD2CHEKKHlPK4CeN6On/4wZ4Z+mXOFcHMSaI4OR0Rw/iNJ7kcnZBeVr6ULUv6NdKbguP06dP4+vpy/Phx7O3t6devH+XLl0cIoW41KYoJmTJRHANqCyGqA9fRrZSXvvanlDIWSJ+pTQgRCEwu1Eli05sQul2/rFQl3drWymNL1Wjx23+J5fsuotFmDJ7rXN+DRb0b4OqoG/OQlJTEwoULWbhwIampqVSuXJmvvvqK8uXVk2WKYg4mSxRSylQhxFhgD2AJrJVShggh5gPHpZQ7TPXaJnHq++xJwqs7DPyuYOIp4sJvJzBh00lOXItJL3OwsWTuK/Xp18wzvYXw559/4uvrS2io7hmI0aNHs2jRIjUuQlHMyKQD7qSUPwM/Zyl730jd9qaM5alotfDjCP2y5iOg68cFE08RJqXk+2P/smBnKA8yDZ5rXq0Mn/RrSFU3R726U6ZMITQ0lNq1axMQEEDbtgbGrCiKYlJqZHZezHfV3379B6j1UsHEUoRF3U/kvR9OE3g+Or3MykIw8eVnGNGuJpYWulZESkoK1tbWCCHw9/fnm2++4f3338fe3r6gQleUEk0litz8Nkd/28JKJYkn8EvwTab/GEzMg4z1q2uWdeTzgY3xruQC6GZ5nTx5MpGRkfz0008IIahbty6LFi0qqLAVRUElipyd+xmOfKZfNiWsYGIpou4lJDN7+xl2nr6pV+7bpjqTX66TPnhu+/btjBo1ips3b2JjY0NoaCj169cviJAVRclCJQpjUhLh+0H6Zb6/gb2r4fpKNvvPRzF1y2mi4pLSyyq62LGkfyNa19Q98Hbr1i3effddNm3aBECrVq0ICAigbt26BRKzoijZqURhzOLK+ttN3oLKzQsmliImLjGF+T+FsvmfCL3yvk09md29Hi72upXn1q9fzzvvvMPdu3dxdHRk0aJFjB49Wk3kpyiFjEoUhtwKAU2mOSQsrKHH8oKLpwg5dDGaaT8Ecz3mYXqZu5MNi3o3pGM9/XEPISEh3L17l44dO+Lv70+1atXMHK2iKHmhEoUh37yqvz09wnA9Jd2D5FQW7DzLhr/1p+Do1rAC83vUx83JFq1WS3h4ePqsrrNnz6Zhw4b0799fjaxWlEJMJYqsUhL1pw1vOQassy+vqWQ4HRHD+O9Pcvl2xhQcpR2s+eBVb7o3rAjAhQsXGDp0KBcvXiQ0NBRXV1fs7OwYMGBAQYWtKEoemXThoiJp1yT97ZcXFEwcRYBGK/HbH0bvL4/qJYku3h78Or4d3RtWJDU1lY8//phGjRpx6NAhpJRcvHixAKNWFOVxqRZFVie/zfjeowFYqI5VQy5HxzNly2n+ybR+taONJfN6etOnSSWEEJw6dYohQ4YQFBQEwNtvv83SpUspU6ZMQYWtKMoTUIkis6tH9bcHqHmcstJoJd/8Ec5Hu8+RmKJNL29SpTSfDvBJn4Jj+fLlTJo0idTUVKpWrYq/vz8vv/xyAUWtKMrTUIniESnh6y4Z25Y24Fq14OIphK7cTmDK5lMcz9SKsLIQjO1Qi7Ev1MLKMuNOZr169dBoNLzzzjssXLgQJyengghZUZR8oBLFIzHh+tsvzS2AIAonrVby9dFwPtmj34rw8nBmaf9G1K/oQnx8PHv27KFPnz4AvPTSS1y4cIFatWoVVNiKouQTlSgeib0OZMxcSqsxBRZKYXIz9iGTNp3i6KU76WVWFoLRL+haETZWFvz6668MHz6ca9eucfDgQdq0aQOgkoSiFBMqUSTcgfDD+mXjThdMLIXMT6duMPPHYO4npqaXeXk4s6RfI7wruXDv3j1GTJzIunXrAGjcuLFaJ0JRiiGVKD6pkb2shPdNxD5IYfb2M+w4lbFyrRAw6vmajH/pGWysLNi6dStjxowhMjISW1tb5s6dy6RJk7C2ti7AyBVFMYWSnSiu/Zm9bGak+eMoRI6G3WbS5lPcjE1ML6tU2p5PB/jQvLrusdbPP/+c8ePHA9CmTRvWrFlDnTp1CiReRVFMr+QOuNNqYW0n/bL374F1yVwcJylVw8KfzzI44C+9JNGniSe/jG+bniQABg0aRLVq1fDz8+PAgQMqSShKMVdyWxSb3tDfdq0OFiUzb4beuM+kzac4e/N+epmrgzWL+zSkU30PwsPDmbl0KcuWLcPa2ppy5cpx4cIFdZtJUUqIkpkoEm7DuZ36ZS6VCiaWAqTVStYeucLHu8+TrMl47LXdM2VZ0rch7k42fPHFF0yfPp2EhAQ8PT2ZOnUqgEoSilKClMxE8VlD/W3PZgUTRwG6dT+RyZtPceji7fQyGysLZnTx4s1W1bhw4Ty9hw7lyJEjAPTr14+33367gKJVFKUglbxEcecSpGRMYEcFH7AqWVNc7wmJZNoPp7mXaf3qBpVc+GygD1VK27J48SLmzZtHcnIyHh4efPnll/Tq1asAI1YUpSCVvETx3x76276/woZOhusWM/FJqXy4S3/NCCFg5PM1mZD22Ov333/PzJkzAfD19eWTTz7B1VUt/6ooJVnJShSJsXA/0yJETd8GK9sCC8ec/rl6lwkbT3Ht7oP0Mo9Sdiwb0IhWNdzSFw7q378/u3fv5vXXX+ell14qqHAVRSlEStZjPms66m93XVowcZhRikbLst8u0G/VH3pJomsDD3aPb4v25jmaNm3K5cuXAbCwsGDdunUqSSiKkq7kJIrE+3D7fMZ2/V5gWbwbVJej4+m76g+W/34RrdSVOdtZsax/Ixa/UptZ702kbdu2nDhxgsWLFxdssIqiFFrF+5Mys4AsrYk+AQUThxlo09aMWPTLOZJSMx57bVG9DMsG+BD85wEa9BrBtWvXsLKyYvr06en9EoqiKFmVjESh1UD0uYztmh2K7cp1kbGJTNmi/9irtaVg8st16F2/NJPHjeSbb74BoGnTpqxdu5aGDRsaO52iKEoJSRT7P9TfHrSxYOIwISkl205eZ+6OUGIfZjz2mnnNiJCQEDZs2ICdnR3z589nwoQJWFmVjF8BpehJSUkhIiKCxMTE3Csb4eLiwtmzZ/MxqsLPzs4OT0/PfB0UW/w/JbQaOJSp07pKK7CyKbh4TOBeQjKztp1hV/DN9DIhYHi7GrzVxI0K5XRTf9evX5+1a9fSokULateuXVDhKkqeRERE4OzsTLVq1dKfyntccXFxODs753NkhZeUkjt37hAREUH16tXz7bzFvzM76zrYPf0KJg4T2X8uik6fHdRLEp6u9mwY2gKPyD+p71WHjRszWlCvv/66ShJKkZCYmIibm9sTJ4mSSAiBm5vbU7XCDCn+LYpTG/S33WoWTBz57GGyhm9Ckti3+5he+aDmlXm9nj3jRw5i7969APzyyy8MHDiwIMJUlKeiksTjM8U1K/4tipPfZXzf/bOCiyMfhdyI5ZUVh9n3b8bKc+5ONvi/3hiPf/fTvKkPe/fuxc3NjW+//TZ9BTpFUZQnUbwTRWSw/natoj2ITKuVrDl0mV5+RwmLik8v71S/POv612Tu8L6MHz+eBw8eMHDgQEJDQxk8eLD6q0xRnlBkZCQDBw6kZs2a1KtXj65du3LhwgXCw8Px9vY2yWsmJSUxYMAAatWqRYsWLQgPDzfJ6zwOkyYKIURnIcR5IUSYEGKagf0ThRChQojTQojfhRD5uwbpmiyJoXTlfD29OUXdT+Str//mg11n06cEt7GExb0bsOr1ptSsXIHbt29TsWJFtm/fzoYNGyhXrlwBR60oRZeUkl69etG+fXsuXbpEaGgoCxcu5NatWyZ93YCAAFxdXQkLC2PChAnpU/sXJJP1UQghLAE/oCMQARwTQuyQUoZmqnYCaCalfCCEGAV8DAzIlwASYyE1U4dOt6I7Xceu0zeZtS0422yvrbRn6PzMcwghsLe3Z9u2bVSsWJHSpUsXYLSKkv+qTdtlsnOHL+5msHz//v1YW1szcuTI9DIfHx/dMZn+yg8PD+eNN94gIUE3K/WKFSto3bo1N2/eZMCAAdy/f5/U1FRWrlxJ69at8fX15fjx4wghGDJkCBMmTNB73e3btzN37lwA+vbty9ixY5FSFuidAVN2ZjcHwqSUlwGEEN8DPYH0RCGl3J+p/p/A6/n26llvOzXzzbdTm0t8UipztofwQ1DGRIZCgG/LSsQcXs/sZUsJDwnC398fgHr16hVUqIpS7Jw5c4amTZvmWq9cuXL89ttv2NnZcfHiRQYNGsTx48dZv349nTp1YubMmWg0Gh48eMDJkye5fv06Z86cASAmJibb+a5fv07lyrq7H1ZWVri4uHDnzh3c3d3z9w0+BlMmikrAv5m2I4AWOdT3BX4xtEMIMRwYDlC2bFkCAwNzffHaF77i0Zp1Nz1e5PyBA0br+qT9sE7m4bzmEnZPw1enk4h+KNPLytgJnrc4x6oxI7h+/ToWFhbExMSwf//+Et8PER8fn6ffi5KguFwLFxcX4uLiTFU2sMQAABiMSURBVP46xl4jMTGR5ORkg/vj4+PRarXExcURGxvL5MmTCQ4OxtLSkrCwMOLi4qhfvz6jR48mPj6e7t2707BhQ8qWLUtYWBgjRoygU6dOvPjii9nOr9FoiI+PTy/XarXEx8dja5v3ma4TExPz9XfAlInC0CeXNFCGEOJ1oBnwvKH9Ukp/wB+gTp06sn379rm/emDP9G8rNOlCheY5HJN2qyZP5zWxVI2WFfvD+OJYGBptxuXqWseFpD/+x2erda2HR7+Eo0ePLqhQC5XAwMBC8fMrDIrLtTh79mz6YDljt4dy8zQD7po2bcrOnTsNHu/k5ISFhQXOzs4sXboUT09P1q9fj1arxc7ODmdnZzp37szhw4fZtWsXI0eOZMqUKbz55psEBwezZ88evv76a3bu3MnatWv1zl2lShXu3buHl5cXqampxMXFUbVq1cf6Y9DOzo7GjRs/0fs2xJSd2RFA5t5jT+BG1kpCiJeAmUAPKWVSvrxyXKT+ds0O+XJaU7tyO4E+q/7gs70X05OEs50VH3atxs45r7F2tT/W1tbMmTOHoKAgdatJUUyoQ4cOJCUlsXr16vSyY8eOcSDL3YnY2FgqVKiAhYUF//vf/9BoNABcvXqVcuXKMWzYMHx9fQkKCuL27dtotVr69OnDggULCAoKyva6PXr04L//b+/O46oq8weOf74giBuWaxoVOtpmKKE09JuyxUZbXCqdbKbUHCebFoumtExfk1PTMmOLLWbhWjM1IfgyNTXTlEjDDQEFNTVTA00RCQRFLvD8/jiHVZYrcrlc+L5fL16ec+5zz3nO87re732ec873+fhjAKKjo7n11lvdPmLgyh7FFqCniHQD0oD7gT+VLSAi1wIfAbcbY47V2ZET/lt+vYE/ZGeMIWprKtOWpXAqv7Bk+3WB7XhrZB8CLmzJ6ltvZefOncydO5egoCA31lappkFEWLx4MeHh4bz++uv4+fkRGBjIjBnln8d67LHHGD58OFFRUdxyyy20atUKsHp206dPx8fHh9atW/PJJ5+QlpbG2LFjKSqy7lx87bXXzjruuHHjGDVqFD169KBdu3Z8/vnnrj/ZGrgsUBhjCkTkCWAV4A3MM8akiMhLwFZjzFJgOtAaiLIj5iFjzNAqd+qstS+XLvd96Lx350pZpxxMXrydFTtKe0HNvKB/s32M630dARe2BGDmzJn4+fnh7d04s94q1RB17dqVhQsXVvpa8QXpnj17sn379pLtxV/+Y8aMYcyYMWe9r7JeRFl+fn5ERUXVtsou4dIUHsaYFcCKCtv+Xma57p+Ay80ov97nT5WXawA27s/gb5GJHM4qvY03wPcUPpvmM//rlWyMvoqEhASaN29e8itFKaXqW+PL9fTFX8uvX1rdjVbu4SgsYsaaPXwQ8yPGvl5tjKHXyS1s+HQG2dnZ+Pv7Ex4eXqepgpVSqjYaV6AwBvZ+Xbre61731aUKP/xykkmLtpP0c+n90y3y0mkeN4eVmzcAMHjwYGbNmkVAQIC7qqmUUiUaV6DYUWFc745/u6celSgsMsxdv583Vu0pScEBEHZZW7575REOp6XRoUMH3nvvPUaOHOn2uxyUUqpY4woUKYtLl7sEQ+uO7qtLGQeO5/Lcou1s+ulEybbi6UkfvrE7/231Kl9//TUzZsxw69OXSilVmcYVKH4oc9081P0pO4wx/GfjQV5dsYs8h9WLMIUO/FKWce9ve/DITXcCMHr0aEaPHu3OqiqlVJUaT5rxzAPl1/v80S3VKHYsO4+H5m/h70tSSoKE45c95Ec/x56V85jx+kukp6e7tY5Kqeq5I814bGwsISEhNGvWjOjoaJcc41w1nh7F4cQyKwLe7rtbaMWOI0xZXJrttciRh/e2KH6JjaKoqIgePXowe/ZsOnZsGENjSqmzFacZHzNmTMlDb4mJiRw9erQkaZ8rXHrppSxYsIA33njDZcc4V40nUOxdXbr8uyfdUoWs0w5eXJLMF4mlmUryDm4nP2YWmb/8jJeXFxMnTmTatGm0bNnSLXVUyiNNa1urtzmV5WlaVqWb3ZVmPDAwEAAvr4Yz4NM4AoUxkFgmbUeX4HqvQtyPGTyzsPzDcxf5Nyd//5ck/PIzQUFBzJs3j379+tV73ZRS585dacYbosYRKOLnl18PvKHeDp17poBXV+zis82HSh+eK8hnxHXd+fuQqzn2hwVERkYyadIkfH19661eSqn64XA4eOKJJ0hMTMTb25s9e/YAEBoayp///GccDgd33303wcHBdO/enf379zNhwgTuuusuBg4c6ObaO6dxBIqkyPLrretnCtCEQ5k8HZnIgYxTABSeyiInZjbdWhfxxvQYRIS2PXsyderUeqmPUo1WFcNDNTmfNOO9evVy6mLy22+/TefOnUlKSipJMw7Qv39/YmNjWb58OaNGjSpJM56UlMSqVauYOXMmCxcuPCvNeEPUcAbBaquosPxsdvfOcfkh8wuKeHv1HkZ8GMeBjFMYY8jdGUP6/Mf4dUcMPyRtZffu3S6vh1LKddyVZrwh8vwexeFEcOSWrvf+g0sPt/NwNs9EJbHrSDYABdnpZK2ZRc7ezQAMGDCAiIgIunfv7tJ6KKVcy11pxrds2cI999xDZmYmy5Yt48UXXyQlJcX1J1wNzw8U828vXe45yGWHKZ557v21+yiwJxU6mfQ12TFzKMg7Rdu2bXnrrbcYO3aspt9QqpFwR5rx0NBQUlNTa1tll/DsQHEmBwodpeuXhrnkMAczcgmPTCThUOkdCs2beRF0EXyZd4phw4bxwQcf0LVrV5ccXyml3MmzA8W+1ZSbhvt3T9Xp7o0xRMenMm1pCrn5hZiiQgoyDxMW0pvpI3oT0HYAq+69jcGDB2svQinVaHl2oEgqM0XgtaPAq+5mf/v1VD5TFiezfMcRAPKP/cSJle/ie+YEs17dTaeOrQEYMmRInR1TKaUaIs8OFHu+Kl3udHWd7TZ2TzrPRCWRfvIMpsBBVlwk2RujMEWFdLrkElJ/PkSnjprlVSnVNHhuoDh1ovx67/vOe5f5BUVMX7Wb2d/9BMCZtN1krHwXR8YhwLq74bXXXsPf3/+8j6WUUp7CcwNFWoU7B1qd3y/8045C/jBrA8lp1m2vWZsW8eu3C8AYevbsydy5c7nxxhvP6xhKKeWJPPeBuyMJpcshZ9+C5ixjDMdOnmFHWlZJkAC4LjSUZt7ePP/88yQlJWmQUKoJ8vb2Jjg4mD59+hASEsL3339fp/t/6KGHSp7+/stf/sLOnTvrdP91xXN7FN+WmeY0ILRWu8g67eCFxTsYlZ6DKSokJ2Ud7XoP4Lk7rmTs/91J2oTBLk0nrJRq2Fq0aEFiojWFwapVq5g8efJZT2bXlTlzXJ9VorY8s0dRcAYK80vXL645w2NFWw+c4M53vmP59iMUns4m/8heMr58kxdCihh3Qze8vESDhFINiIhU+RcREVFSLiIiomS7v7//WWVrKzs7mwsvvBCAnJwcBgwYQEhICEFBQSxZsgSA3Nxc7rrrLvr06cM111xDZKSVhy4+Pp6bbrqJvn37MmjQII4cOXLW/m+++Wa2bt0KQOvWrZkyZQp9+vQhLCyMo0ePApCens7w4cMJDQ0lNDSUDRs21Pp8zoVn9igObSy/3tn5O54KCouYue5H3vlmD46cTE6s/gjHceti9W/Dwgjr9Zu6rKlSyoOdPn2a4OBg8vLyOHLkCGvXrgXAz8+PxYsX4+/vz/HjxwkLC2Po0KF89dVXdO3aleXLlwNWHiiHw8GECRNYsmQJHTt2JDIykilTplSbDDA3N5ewsDBeeeUVJk2axOzZs5k6dSpPPfUUTz/9NDfccAOHDh1i0KBB7Nq1y+Xt4JmBIvHT0uVOvZx+W9qvpwn/PIHNP50gN2Udmd/MpijvJF5eXnTv3p0N69fj7V13z2IopeqOMabmQsD48eMZP348cH7ZY6H80FNcXByjR48mOTkZYwwvvPACsbGxeHl5kZaWxtGjRwkKCuLZZ5/lueeeY/Dgwdx4440kJyeTnJzM73//ewAKCwvp0qVLtcf19fVl8ODBAPTt25fVq62J2dasWVPuOkZ2dvZ5n6MzPDNQbC+TVrzfWKfesjghlb8vSeFkXgEntywmc50VzfvfMoDrTuVYqYE1SCilqnD99ddz/Phx0tPTWbFiBenp6cTHx+Pj40NgYCB5eXlcfvnlxMfHs2LFCiZPnszAgQO555576NWrF3FxcU4fy8fHp2SYzNvbm4KCAgCKioqIi4ujRYsWLjnHqnjcNQoxReU3XDW02vLZeQ7CP0/g6cgkTuZZje3f+zY6XdKNefPmE/PN6pL88UopVZXdu3dTWFhI+/btycrKolOnTvj4+LBu3ToOHjwIwOHDh2nZsiUPPvggzz77LNu2beOKK64gPT29JFA4HI5aZ4MdOHAg77//fsl6cW/H1TyuR9GsILf8hjadqywbfzCT8MgE9u/dS9amRbQf9BiXdWzL2yP/j+A379NhJqVUtYqvUYA19PXxxx/j7e3NAw88wJAhQ+jXrx/BwcFceeWVAOzYsYOJEyfi5eWFj48Ps2bNwtfXl+joaJ588kmysrIoKCggPDycXr2cHzYv9u677/L444/Tu3dvCgoK6N+/Px9++GGdnnNlPC5QeBWVudupZ+XTCBYWGWau28eM1bvJ3LiIX9d/BoUOQoKuYuU/36RVc487baWUGxRPQlRRhw4dKh1KCgwMZNCgs6c7CA4OJjY29qztCxYsKFmOiYkpWc7JySlZHjFiBCNGjCg5bvGdVPXJ474xvYrOUDJiFnR22o60X0/z9OeJrN+0lYyV75B/9EcABgy9j4XvvahBQimlzpHHfWs2KzgFWJlbuTik3Gtfbj/McwvjSV37X7I3RoMpomW7i4iI+IgHhld/LUMppVTlPC5QlPBtAxd2A6wL1i8t20l0fCq5u9aTHbcQEPrfM5ol89/jgraaxE8pT2SM0blezpGztxGfC88NFM18wcuL+IMnePKzbaRlnQGg5ZU3ctHRXfxz4qOMG36HmyuplKotPz8/MjIyaN++vQYLJxljyMjIqPM7OT02UBQFjeTdNXv415yFZKyJoOO9U/FpdzFDgy/mlX8swt/Px91VVEqdh4CAAFJTU0lPT6/1PvLy8prc7e9+fn4EBATU6T49NlD8I74Fb0Y+Tm7yGgDyEpbx4ZyPGNy7i/76UKoR8PHxoVu3bue1j5iYGK699to6qlHT5dIH7kTkdhH5QUT2icjzlbzeXEQi7dc3iUigM/tdtNPBq6+/ZQUJbx+C7n6UbSs/Y0ifrhoklFKqjrksUIiINzATuAO4GvijiFTM3jcOyDTG9ADeBv5V035/zCxiRNRpCnKz8AvoxdS5y0hYNJPLOrg214lSSjVVruxRXAfsM8bsN8bkA58DwyqUGQZ8bC9HAwOkhi5BVh608PXmN3c/xfrvvuXlMYPw9tJehFJKuYorr1FcDPxcZj0V+G1VZYwxBSKSBbQHjpctJCLjgfH26pnT+YXJP37xDv2+eKdua+x5w1YdqNBWTZi2RSlti1LaFqWuqO0bXRkoKvvWrXiDrzNlMMZEABEAIrLVGNPv/Kvn+bQtSmlblNK2KKVtUUpEttb2va4cekoFyk4RFwAcrqqMiDQD2gInXFgnpZRS58iVgWIL0FNEuomIL3A/sLRCmaXAGHt5BLDWuOKxQqWUUrXmsqEn+5rDE8AqwBuYZ4xJEZGXgK3GmKXAXOA/IrIPqydxvxO7jqi5SJOhbVFK26KUtkUpbYtStW4L0R/wSimlquNxM9wppZSqXxoolFJKVavBBgpXpf/wRE60xd9EZKeIbBeRb0TkMnfUsz7U1BZlyo0QESMijfbWSGfaQkTusz8bKSLyWX3Xsb448X/kUhFZJyIJ9v+TO91RT1cTkXkickxEkqt4XUTkXbudtotISGXlzmKMaXB/WBe/fwS6A75AEnB1hTKPAR/ay/cDke6utxvb4hagpb38aFNuC7tcGyAW2Aj0c3e93fi56AkkABfa653cXW83tkUE8Ki9fDVwwN31dlFb9AdCgOQqXr8TWIn1DFsYsMmZ/TbUHoVL0n94qBrbwhizzhhzyl7diPXMSmPkzOcC4GXg30BefVaunjnTFg8DM40xmQDGmGP1XMf64kxbGKB4BrO2nP1MV6NgjIml+mfRhgGfGMtG4AIR6VLTfhtqoKgs/cfFVZUxxhQAxek/Ghtn2qKscVi/GBqjGttCRK4FLjHGfFmfFXMDZz4XlwOXi8gGEdkoIrfXW+3qlzNtMQ14UERSgRXAhPqpWoNzrt8nQMOdj6LO0n80Ak6fp4g8CPQDbnJpjdyn2rYQES+sLMQP1VeF3MiZz0UzrOGnm7F6md+JyDXGmF9dXLf65kxb/BFYYIx5U0Sux3p+6xpjTJHrq9eg1Op7s6H2KDT9Ryln2gIRuQ2YAgw1xpypp7rVt5raog1wDRAjIgewxmCXNtIL2s7+H1lijHEYY34CfsAKHI2NM20xDlgIYIyJA/ywEgY2NU59n1TUUAOFpv8oVWNb2MMtH2EFicY6Dg01tIUxJssY08EYE2iMCcS6XjPUGFPrZGgNmDP/R77AutEBEemANRS1v15rWT+caYtDwAAAEbkKK1DUfo5Vz7UUGG3f/RQGZBljjtT0pgY59GRcl/7D4zjZFtOB1kCUfT3/kDFmqNsq7SJOtkWT4GRbrAIGishOoBCYaIzJcF+tXcPJtngGmC0iT2MNtTzUGH9Yisj/sIYaO9jXY14EfACMMR9iXZ+5E9gHnALGOrXfRthWSiml6lBDHXpSSinVQGigUEopVS0NFEoppaqlgUIppVS1NFAopZSqlgYK5RYiUigiiSKSLCJRItLSTfUId9ex7eNPtzO7TndjHQKryjaqFGigUO5z2hgTbIy5BsgH/ursG0XEuw7rEQ64LVAAjwAhxpiJbqyDUtXSQKEagu+AHmDlqxKRzXZv46PioCAiOSLykohsAq4XkVAR+V5EkuzybUTE2/6FvsXOtf+I/d6bRSRGRKJFZLeIfGo/mfok0BVYJyLr7LKzRGSr/Sv/H8UVFJE77feut/P5f2lvb2XPAbDFnuvgrGy29rGm272nHSIy0t6+FGgFbCreVuY9N9ltkGjvt42ItBZrvpFt9n6G2WUD7brNsY/xqYjcJlYywL0icp1dbpqI/EdE1trbH66krpW2oWri3J0/Xf+a5h+QY//bDFiCNY/GVcAywMd+7QNgtL1sgPvsZV+sVBSh9rq/vZ/xwFR7W3NgK9AN60nVLKy8Nl5AHHCDXe4A0KFMvdrZ/3oDMUBvrHQPPwPd7Nf+B3xpL78KPGgvXwDsAVpVONfhwGp7n52x0kl0KdsOlbTPMuB39nJr+/yaAf72tg5YT9cKEAgUAEH2+cUD8+zXhgFf2O+ZhjVXQwv7/T9jBcpA7PkLqmpDd39e9M+9f9qjUO7SQkQSsb6IDmGlZBkA9AW22K8NwJqMBqwUFIvs5SuAI8aYLQDGmGxjpZofiJXHJhHYhJV2vjgJ3mZjTKqxsoUmYn05VuY+EdmGNeFPL6xJbq4E9hsrsR5YgaLYQOB5+5gxWEHl0gr7vAH4nzGm0BhzFPgWCK2hfTYAb9m9ngvs8xPgVRHZDqzBSg/d2S7/kzFmh31+KcA3xhgD7KhwrkuMMaeNMceBdVhzOZRVXRuqJqpB5npSTcJpY0xw2Q1iJar62BgzuZLyecaYwuKiVJ4aWYAJxphVFfZ7M1A2o24hlXz2RaQb8CxWTyVTRBZgffFXNyGWAMONMT/UUOacGGNeF5HlWHl5NoqVHTgM6Aj0NcY4xMqQ62e/pez5FZVZL6L8uVZst8rS95/Vhqpp0x6Faki+AUaISCcAEWknlc//vRvoKiKhdrk2YqWaXwU8KiI+9vbLRaRVDcc8iZWeHKwhrFwgS0Q6A3eUOV53KZ2Xvez1hFXABDvIFWfyrSgWGGmP/3fEmq5yc3WVEpHf2D2Ef2H1uq7ESqV/zA4StwC1mRt9mIj4iUh7rCG5LRVer00bqkZOexSqwTDG7BSRqcDXYk1C5AAeBw5WKJdvX/x9T0RaAKeB24A5WMMs2+wv7nTg7hoOGwGsFJEjxphbRCQBa+hmP9bwD8aY0yLyGPCViByn/Jf8y8AMYLt9zAPA4ArHWAxcj3V9wACTjDG/1FCvcDsYFAI7sWYtbAMsE5GtWMNnu2vYR2U2A8uxhsdeNsYcLhMAoXZtqBo5zR6rlBNEpLUxJsf+8pwJ7DXGvO3uep0LEZmGdfH8DXfXRXkWHXpSyjkP2xd4U7CGgD5yc32Uqjfao1BKKVUt7VEopZSqlgYKpZRS1dJAoZRSqloaKJRSSlVLA4VSSqlq/T8pFRuZug05igAAAABJRU5ErkJggg==\n", 376 | "text/plain": [ 377 | "
" 378 | ] 379 | }, 380 | "metadata": {}, 381 | "output_type": "display_data" 382 | } 383 | ], 384 | "source": [ 385 | "skplt.metrics.plot_cumulative_gain(y_test, probs);\n", 386 | "plt.axvline(.066, color='red');" 387 | ] 388 | } 389 | ], 390 | "metadata": { 391 | "kernelspec": { 392 | "display_name": "Python 3", 393 | "language": "python", 394 | "name": "python3" 395 | }, 396 | "language_info": { 397 | "codemirror_mode": { 398 | "name": "ipython", 399 | "version": 3 400 | }, 401 | "file_extension": ".py", 402 | "mimetype": "text/x-python", 403 | "name": "python", 404 | "nbconvert_exporter": "python", 405 | "pygments_lexer": "ipython3", 406 | "version": "3.5.5" 407 | } 408 | }, 409 | "nbformat": 4, 410 | "nbformat_minor": 2 411 | } 412 | -------------------------------------------------------------------------------- /notebooks/3.1-experiment-2-model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Experiment #2 - Derived Features" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Overview\n", 15 | "\n", 16 | "The purpose of this experiment is to determine whether derived features improve model performance. As part of the feature engineering process, we added the following categorical features:\n", 17 | "\n", 18 | "* `pcontacted_last_campaign` - whether the customer was previously contacted as part of a campaign\n", 19 | "* `pcampaign` - whether the customer was part of the the previous campaign\n", 20 | "* `previous` - the original `previous` attribute converted to a discrete value\n", 21 | "* `campaign_gte10` - whether the customer was contacted 10 or more times as part of this campaign" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 232, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "The autoreload extension is already loaded. To reload it, use:\n", 34 | " %reload_ext autoreload\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "%load_ext autoreload\n", 40 | "%autoreload 2\n", 41 | "\n", 42 | "from utils import code" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 233, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "import pandas as pd\n", 52 | "import numpy as np\n", 53 | "import scikitplot as skplt\n", 54 | "\n", 55 | "# Preprocessing\n", 56 | "from sklearn.compose import ColumnTransformer\n", 57 | "from sklearn.pipeline import Pipeline, FeatureUnion\n", 58 | "from sklearn.feature_selection import mutual_info_classif, chi2\n", 59 | "\n", 60 | "# Model evaluation\n", 61 | "from sklearn.model_selection import train_test_split\n", 62 | "from support.evaluation import plot_learning_curve, evaluate_model\n", 63 | "\n", 64 | "# Support\n", 65 | "from support.model import Model, build_tuned_model\n", 66 | "from support.datasets import get_data\n", 67 | "from support.experiments import experiment_1, experiment_2, get_scorer\n", 68 | "from support import parameters as params\n", 69 | "\n", 70 | "# Algos\n", 71 | "from sklearn.naive_bayes import BernoulliNB\n", 72 | "from sklearn.ensemble import ExtraTreesClassifier\n", 73 | "\n", 74 | "%matplotlib inline" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "## Data" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 234, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "data": { 91 | "text/html": [ 92 | "
\n", 93 | "\n", 106 | "\n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | "
agejobmaritaleducationdefaulthousingloancontactmonthday_of_weekdurationcampaignpdayspreviouspoutcomeemp.var.ratecons.price.idxcons.conf.idxeuribor3mnr.employed
025blue-collarsinglehigh.schoolnoyesnocellularjulmon61929990nonexistent1.493.918-42.74.9625228.1
140admin.marriedhigh.schoolnononotelephonejunthu9719990nonexistent1.494.465-41.84.9585228.1
251technicianmarrieduniversity.degreenoyesnocellularaugwed51259990nonexistent1.493.444-36.14.9645228.1
337blue-collarmarriedhigh.schoolunknownyesnocellularjultue42319990nonexistent1.493.918-42.74.9615228.1
454housemaidmarrieduniversity.degreeunknownyesnocellularaugthu29719990nonexistent1.493.444-36.14.9635228.1
\n", 250 | "
" 251 | ], 252 | "text/plain": [ 253 | " age job marital education default housing loan \\\n", 254 | "0 25 blue-collar single high.school no yes no \n", 255 | "1 40 admin. married high.school no no no \n", 256 | "2 51 technician married university.degree no yes no \n", 257 | "3 37 blue-collar married high.school unknown yes no \n", 258 | "4 54 housemaid married university.degree unknown yes no \n", 259 | "\n", 260 | " contact month day_of_week duration campaign pdays previous \\\n", 261 | "0 cellular jul mon 619 2 999 0 \n", 262 | "1 telephone jun thu 97 1 999 0 \n", 263 | "2 cellular aug wed 512 5 999 0 \n", 264 | "3 cellular jul tue 423 1 999 0 \n", 265 | "4 cellular aug thu 297 1 999 0 \n", 266 | "\n", 267 | " poutcome emp.var.rate cons.price.idx cons.conf.idx euribor3m \\\n", 268 | "0 nonexistent 1.4 93.918 -42.7 4.962 \n", 269 | "1 nonexistent 1.4 94.465 -41.8 4.958 \n", 270 | "2 nonexistent 1.4 93.444 -36.1 4.964 \n", 271 | "3 nonexistent 1.4 93.918 -42.7 4.961 \n", 272 | "4 nonexistent 1.4 93.444 -36.1 4.963 \n", 273 | "\n", 274 | " nr.employed \n", 275 | "0 5228.1 \n", 276 | "1 5228.1 \n", 277 | "2 5228.1 \n", 278 | "3 5228.1 \n", 279 | "4 5228.1 " 280 | ] 281 | }, 282 | "execution_count": 234, 283 | "metadata": {}, 284 | "output_type": "execute_result" 285 | } 286 | ], 287 | "source": [ 288 | "X, y = get_data('../data/train.csv')\n", 289 | "X.head()" 290 | ] 291 | }, 292 | { 293 | "cell_type": "markdown", 294 | "metadata": {}, 295 | "source": [ 296 | "Create a validation set and train the model then score on the test set. The performance will be biased since we are using less training data." 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 235, 302 | "metadata": {}, 303 | "outputs": [], 304 | "source": [ 305 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, stratify=y, random_state=1)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "metadata": {}, 311 | "source": [ 312 | "## ML Models" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": 236, 318 | "metadata": {}, 319 | "outputs": [], 320 | "source": [ 321 | "scorer = get_scorer()" 322 | ] 323 | }, 324 | { 325 | "cell_type": "markdown", 326 | "metadata": {}, 327 | "source": [ 328 | "Apply feature computations" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": 242, 334 | "metadata": {}, 335 | "outputs": [ 336 | { 337 | "name": "stdout", 338 | "output_type": "stream", 339 | "text": [ 340 | "Instances: 32,950, Features: 61\n" 341 | ] 342 | } 343 | ], 344 | "source": [ 345 | "pipeline_1 = experiment_1.get_pipeline()\n", 346 | "features = pipeline_1.fit_transform(X)\n", 347 | "ps = features.shape\n", 348 | "print('Instances: {:,}, Features: {}'.format(ps[0], ps[1]))" 349 | ] 350 | }, 351 | { 352 | "cell_type": "markdown", 353 | "metadata": {}, 354 | "source": [ 355 | "### Original Model\n", 356 | "\n", 357 | "Make sure we still get the same results ~.67 AUC" 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "execution_count": 245, 363 | "metadata": {}, 364 | "outputs": [ 365 | { 366 | "name": "stdout", 367 | "output_type": "stream", 368 | "text": [ 369 | "==> Starting 5-fold cross validation for nb model, 26360 examples\n", 370 | "==> Elapsed seconds: 3.567\n", 371 | "Best nb model: BernoulliNB(alpha=0.1, binarize=0.0, class_prior=None, fit_prior=False)\n", 372 | "Best nb score: 0.711\n" 373 | ] 374 | } 375 | ], 376 | "source": [ 377 | "# Naive Bayes\n", 378 | "param_grid = [{\n", 379 | " 'nb__alpha': [0, 0.01, 0.1, 1],\n", 380 | " 'nb__fit_prior': [True, False]\n", 381 | "}]\n", 382 | "\n", 383 | "tuned_model = build_tuned_model('nb', BernoulliNB(), X_train, y_train, param_grid, scorer, pipeline=pipeline_1, cv_folds=5)" 384 | ] 385 | }, 386 | { 387 | "cell_type": "markdown", 388 | "metadata": {}, 389 | "source": [ 390 | "### New Model\n", 391 | "\n", 392 | "How does the performance change with the new features?" 393 | ] 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": 243, 398 | "metadata": {}, 399 | "outputs": [ 400 | { 401 | "name": "stdout", 402 | "output_type": "stream", 403 | "text": [ 404 | "Instances: 32,950, Features: 67\n" 405 | ] 406 | } 407 | ], 408 | "source": [ 409 | "pipeline_2 = experiment_2.get_pipeline()\n", 410 | "features = pipeline_2.fit_transform(X)\n", 411 | "ps = features.shape\n", 412 | "print('Instances: {:,}, Features: {}'.format(ps[0], ps[1]))" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 246, 418 | "metadata": {}, 419 | "outputs": [ 420 | { 421 | "name": "stdout", 422 | "output_type": "stream", 423 | "text": [ 424 | "==> Starting 5-fold cross validation for nb model, 26360 examples\n", 425 | "==> Elapsed seconds: 4.290\n", 426 | "Best nb model: BernoulliNB(alpha=0, binarize=0.0, class_prior=None, fit_prior=False)\n", 427 | "Best nb score: 0.658\n" 428 | ] 429 | }, 430 | { 431 | "name": "stderr", 432 | "output_type": "stream", 433 | "text": [ 434 | "/anaconda3/envs/py35-ds/lib/python3.5/site-packages/sklearn/naive_bayes.py:480: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n", 435 | " 'setting alpha = %.1e' % _ALPHA_MIN)\n" 436 | ] 437 | } 438 | ], 439 | "source": [ 440 | "tuned_model = build_tuned_model('nb', BernoulliNB(), X_train, y_train, param_grid, scorer, pipeline=pipeline_2, cv_folds=5)" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": 247, 446 | "metadata": {}, 447 | "outputs": [], 448 | "source": [ 449 | "n_targeted_test = int(params.P_TARGETED * len(X_test))\n", 450 | "revenue = params.AVG_REVENUE\n", 451 | "cost = params.AVG_COST\n", 452 | "\n", 453 | "probs = tuned_model.model.predict_proba(tuned_model.pipeline.transform(X_test))\n", 454 | "preds = tuned_model.model.predict(tuned_model.pipeline.transform(X_test))\n", 455 | "\n", 456 | "# Create a dataframe of probabilities and actual / predicted outcomes\n", 457 | "probs_df = pd.DataFrame(np.hstack([probs, y_test.values.reshape(-1,1), preds.reshape(-1,1)]), columns=['p_no', 'p_yes', 'actual', 'predicted'])\n", 458 | "\n", 459 | "# Sort customers by the probability that they will convert\n", 460 | "model_targets = probs_df.sort_values('p_yes', ascending=False)\n", 461 | "\n", 462 | "# Take the top 6.6%\n", 463 | "model_targets = model_targets.head(n_targeted_test)\n", 464 | "\n", 465 | "# Calculate financial outcomes\n", 466 | "model_outcomes = model_targets.actual.apply(lambda x: cost if x == 0 else cost + revenue)" 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": 248, 472 | "metadata": {}, 473 | "outputs": [ 474 | { 475 | "name": "stdout", 476 | "output_type": "stream", 477 | "text": [ 478 | " precision recall f1-score support\n", 479 | "\n", 480 | " 0.0 0.00 0.00 0.00 228\n", 481 | " 1.0 0.47 1.00 0.64 206\n", 482 | "\n", 483 | " micro avg 0.47 0.47 0.47 434\n", 484 | " macro avg 0.24 0.50 0.32 434\n", 485 | "weighted avg 0.23 0.47 0.31 434\n", 486 | "\n", 487 | "Expected profit: $219,626\n" 488 | ] 489 | }, 490 | { 491 | "name": "stderr", 492 | "output_type": "stream", 493 | "text": [ 494 | "/anaconda3/envs/py35-ds/lib/python3.5/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", 495 | " 'precision', 'predicted', average, warn_for)\n" 496 | ] 497 | } 498 | ], 499 | "source": [ 500 | "print(classification_report(model_targets.actual, model_targets.predicted))\n", 501 | "print('Expected profit: ${:,}'.format(sum(model_outcomes)))" 502 | ] 503 | }, 504 | { 505 | "cell_type": "markdown", 506 | "metadata": {}, 507 | "source": [ 508 | "The score is lower than the original model, is it a high bias or high variance problem?" 509 | ] 510 | }, 511 | { 512 | "cell_type": "code", 513 | "execution_count": 151, 514 | "metadata": {}, 515 | "outputs": [ 516 | { 517 | "name": "stderr", 518 | "output_type": "stream", 519 | "text": [ 520 | "/anaconda3/envs/py35-ds/lib/python3.5/site-packages/sklearn/model_selection/_split.py:1943: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n", 521 | " warnings.warn(CV_WARNING, FutureWarning)\n" 522 | ] 523 | }, 524 | { 525 | "data": { 526 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEWCAYAAACqitpwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXl8XGXZ97/XLNm3bqRLugFt6b4XZG2FAgoPYAVkVVBBfUUfH15AsD4iaH3qo7IIvCqIgFqtiEirsoiFAAVrVxDaUrq3aZO2SbNNktmv94971mSSpu1MMm3v7+czn5lzzn3Ouc9kcn7nWu7rFlXFYrFYLJZ04ujtDlgsFovl+MOKi8VisVjSjhUXi8VisaQdKy4Wi8ViSTtWXCwWi8WSdqy4WCwWiyXtWHGxWCKIyAgRURFxdaPtTSKyvCf6dTSIiEdETu7tflhOPKy4WI4KEdkhIvtEpDBh3RdFpLIHzusXkf7t1r8bEYgRmTx/VxyOSGUaVS1S1W2ZOLaIjBaRP4pIrYg0isi/ReR2EXFm4nyWYwsrLpZ04AL+sxfOux24NrogIhOB/F7oR6/QmzdxETkF+BewG5ioqqXAVcAMoPgIjtfrQmxJL1ZcLOngR8AdIlKWaqOInCYir4rIQRHZJCJXR9aPFJEGEXFEln8pIvsT9vutiHyji/P+BvhswvLngF+3O3epiPxaRA6IyE4R+XbC+Zwi8uPIk/c24JIU+z4pItUiskdEvn+0N3QRcYjI3SKyVUTqRORZEembsP2PIlITsQTeFJHxCdueFpGficiLItICzImse0xE/iYizSLyr8iNP7qPisipCft31fbCyN+nUUT+n4i8ISJf7ORS7gPeUdXbVbUaQFU3qep1qtogIrNFpKrdte8QkQsin78rIs9F/sZNwLdEpK3ddzE18rdxR5Y/LyIbRaReRF4RkeFH/pewZBorLpZ0sBqoBO5ovyHiLnsV+B1wEsbS+H8iMl5VtwNNwNRI83MAj4iMjSyfC7zRxXlXACUiMjZy0/8M8Nt2bR4BSoGTgfMwYnRzZNstwKWR888Army37zNAEDg10uZCoLObbXf5OnBFpC+DgXrgsYTtLwGjMN/VWmBRu/2vAxZgrINozOdazM2+D7Alsr0zUraNuBefA+4B+gGbgDO7OM4FkfZHw+WRY5RhHlD+CXw6Yft1wHOqGhCRK4BvAfOAAcBbwO+P8vyWDGLFxZIuvgN8TUQGtFt/KbBDVZ9S1aCqrgX+RPxG/gZwnogMjCw/F1keCZQA7x3ivFHrZS7wIbAnuiFBcO5R1WZV3QH8BLgx0uRq4CFV3a2qB4H/Sdi3HPgE8A1VbVHV/cCDwDXd+zo65UvAfFWtUlUf8F3gyqhbSFV/FelrdNtkESlN2H+Jqr6tqmFV9UbWPa+qK1U1iBGjKV2cv7O2nwTWq+rzkW0/BWq6OE4/oPpwLjwF/1TVFyLX0oZ5ALkWQEQE813/LtL2S8D/qOrGSP9+AEyx1kv2Yv2clrSgqh+IyF+Bu4GNCZuGA6eLSEPCOhdGFMCIy2VAFfAmxgK6EfACb6lq+BCn/k1kv5G0c4kB/YEcYGfCup3AkMjnwZiYQeK2xH67gWpznwPMw1hi+yNhOPBnEUm8rhBQLiI1GEviKszTebRNf6Ax8jnV+RNFoBUo6uL8nbVN+i5UVdu7tdpRBwzqYnt3aH8tzwGPiMhgjPWmGAsFzPf2sIj8JKG9YP6WO7FkHVZcLOnkXowrJ/EGsBt4Q1XndrLPGxiXSFXk83Lg5xhx6colBoCq7hSR7Zgn7y+021wLBDA3pg2RdcOIWzfVwNCE9sPa9dsH9I88KaeL3cDnVfXt9htE5EaMq+gCYAfGnVePuYlGyVQZ82qgIqEvkricgn9gXFhPdbK9BShIOJ4TI5iJJF1LJFbzd4xFORb4vcbLtu8GFqhqezehJUuxbjFL2lDVLcAfMHGFKH8FRovIjSLijrxmRuMqqroZaANuAN5U1SZgH+bGdUhxifAF4OOq2tKuPyHgWWCBiBRHXCi3E4/LPAt8XUQqRKQPxuqK7lsN/B34iYiURALxp4jIeYfxleSKSF7Cy4ERzgVRd46IDBCRyyPtizGCVoe5Mf/gMM51tPwNmCgiV0RcdF8FBnbR/l7gTBH5UdSlKSKnRgL0ZcBHQJ6IXBIJyH8byO1GP36HcXN+mrhLDMz3dk80wSGSbHHVYV6jpQex4mJJN/cDsTEvqtqMCYRfA+zFuGV+SPKN5g2gTlV3JSwLsK47J1TVraq6upPNX8M8RW/DWEW/A34V2fYE8AomrrMWeL7dvp/FuNU2YCyI5zg8V5AHI5zR18eBh4GlwN9FpBmTlHB6pP2vMS6ePZFzrjiMcx0VqlqLccf9L0bcxmESNXydtN8KfAwYAawXkUZMLG010KyqjcD/AX6JuZ4WjHV6KJZiXGL7VDUWb1PVP2N+N4sj2WUfYGJilixF7GRhFoulPRErqwq4XlVf7+3+WI49rOVisVgAEJGLRKRMRHIxab9CD1pPluMLKy4WiyXKx4CtmESI/wCuiKQIWyyHjXWLWSwWiyXtWMvFYrFYLGnnhBjn0r9/fx0xYkTajtfS0kJhYeGhG/Yw2dovyN6+ZWu/IHv7lq39guztW7b2C7ru25o1a2pVtf34pO6hqsf9a/r06ZpOXn/99bQeL11ka79Us7dv2dov1eztW7b2SzV7+5at/VLtum/Aaj3C+651i1ksFosl7VhxsVgsFkvaseJisVgslrRzQgT0LZbjlUAgQFVVFV6v99CN00RpaSkbN248dMNeIFv7lq39AtO37du3U1FRgdvtTttxrbhYLMcwVVVVFBcXM2LECBKmBsgozc3NFBcf9kzGPUK29i1b+wXQ1NSE3++nqqqKkSNHpu24GXWLicjFkWlTt4jI3Z20uVpENojIehH5XWTdHBF5N+HljcxEF52qdXvCtq4mRrJYjmu8Xi/9+vXrMWGxHH+ICP369Uu79ZsxyyUyf8NjmBkCq4BVIrJUVTcktBmFmVb1LFWtF5GTANQUypsSadMXMx3r3xMOf6eqHu0UqxbLcYEVFsvRkonfUCYtl1nAFlXdpqp+YDFmIqREbgEeU9V6ADVTybbnSuAlVW3NYF8tFovFkkYyKS5DSJ7GtIr49LJRRmMmknpbRFaIyMUpjnMN8Pt26xaIyL9F5MFIBVeLxdIL1NXVMWXKFKZMmcLAgQMZMmRIbNnv93frGDfffDObNm3qss1jjz3GokV2EsruoKqENUwoHCIQChA+5EzhmSFjhSsjs8RdpKpfjCzfCMxS1a8ltPkrZhraqzFTqr4FTFDVhsj2QcC/gcGqGkhYV4OZxOlxYKuq3p/i/LcCtwKUl5dPX7x4cdquzePxUFTU1TTlvUO29guyt2/Z2i/oXt9KS0s59dRTu31M17PPknvffUhVFVpRge/eewleffVh9SsUCuF0Ojus/8EPfkBRURFf//rXk9bHRmw7Mj/yobO+ZYruXlsm+6VobMJoTTELtiBdur2ifduyZQuNjY1J2+bMmbNGVWccWccyVHIFU777lYTle4B72rX5OXBTwvIyYGbC8n8Cj3dxjtnAXw/VF1v+pffJ1r5la79Uu9e3DRs2dP+Av/2takGBKsRfBQVm/WHQ1NSUcv29996rP/rRj1RVdfPmzTp+/Hj90pe+pFOmTNGqqiq95ZZbdPr06Tpu3Di97777YvudddZZum7dOg0EAlpaWqrf/OY3ddKkSXrGGWfovn37VFV1/vz5+uCDD8baf/Ob39SZM2fq6NGj9e2331ZVVY/Ho5dddplOmjRJr7nmGp0+fbquW7euQz/vuOMOHTt2rE6cOFHvuusuVVWtrq7Wyy67TCdOnKiTJk3SFStWqKrqD3/4Qx0/fryOHz9ef/rTn3Z6bS+++KKeccYZOnXqVL366qvV4/F06zs7XMLhsAZDQQ2EAuoL+LQt0GZefvPuDXjVF/TFXm2BNg2Ggl0eM9q3VL8ljqL8SyZTkVcBo0RkJGaa02uA69q1eQG4FnhaRPpj3GTbErZfixGlGCIySFWrxUjxFZjpTi0Wyze+Ae++2/n2FSvA127W4tZW+MIX4IknUu8zZQo89NARdWfDhg089dRT/PznPwdg4cKF9O3bl2AwyJw5c7jyyisZN25c0j6NjY2cd955LFy4kNtvv51f/epX3H13x0RTVWXlypUsXbqU+++/n5dffplHHnmEk046iSVLlvDee+8xbdq0Dvvt27ePF198kfXr1yMiNDQ0APDVr36VuXPncttttxEMBmltbWXlypUsWrSIlStXEgqFmDVrFueddx4FBQVJ17Z//34WLlzIsmXLKCgoYMGCBTz88MN861vfOqLvLfEaFePiir5ihokYi8QhjthytpExO1VVg8BtmDnKNwLPqup6EblfRC6LNHsFqBORDcDrmCywOgARGQEMxcynnsgiEXkfeB/oD3w/U9dgsRxXtBeWQ60/Sk455RRmzpwZW/7973/PtGnTmDZtGhs3bmTDhg0d9snPz+cTn/gEANOnT2fHjh0pjz1v3rwObZYvX86VV14JwOTJkxk/fnyH/fr27YvD4eCWW27hz3/+c6wacGVlJV/60pcAcLlclJSU8NZbb/HpT3+agoICiouLueKKK1i+fHmHa3vnnXfYsGEDZ555JlOmTGHRokWd9rsztF2cxBf04Qv68Af9BMNB43oTBw5H5CWOrM8SzOggSlV9EXix3brvJHxW4PbIq/2+O+iYAICqfjztHbVYjgcOZWGMGAE7d3ZcP3w4VFamvTuJZdw3b97Mww8/zMqVKykrK+OGG25IOa4iJycn9tnpdBIMBlMeOzc3t0Mb7Ub82O12s3r1al599VUWL17Mz372M/7+dzPKof3NuqvjJV6bqnLxxRfzm9/85pDnj7ZXNCYoYQ0nxU0QcOBAHNktHofC1hazWE4UFiyAgoLkdQUFZn2GaWpqori4mJKSEqqrq3nllVfSfo6zzz6b559/HoD3338/pWXU3NxMU1MTl156KQ8++CDr1q0DYM6cOTH3XSgUoqmpiXPPPZc///nPtLW14fF4WLJkCeecc06HY5555pm88cYbbNtmPPotLS1s3rw5tj0agwiGg/iDfnwhH/6Qn0A4QEhDAB2skmx0cx0utvyLxXKicP315n3+fNi1C4YNM8ISXZ9Bpk2bxrhx45gwYQInn3wyZ511VtrP8bWvfY3rrruOSZMmMW3aNCZMmEBpaWlSm8bGRubNm4fP5yMcDvPAAw8A8Oijj3LLLbfwi1/8ApfLxS9+8QtmzZrFtddeG3N/feUrX2HixIls2bIl6Zjl5eU8+eSTfOYzn4mlX3/v+99j+MnDY3ESxYhLUpzkOCdjqcjZxIwZM3T16tVpO15lZSWzZ89O2/HSRbb2C7K3b9naL+he3zZu3MjYsWN7pkMRsrVOVjAYpL6+ngEDBrB582YuvPBCNm/ejMuVmWfolO6t6P1UklOAvS1e8grzMtKPrghrGLfDjdPReRp09O+Z6rckIkecimwtF0t6UDWZR/X1kJcHhYWQmws9MLbBYgEzLujCCy8kHDY3+agVkg46jZOAiZUcJ3GSdGLFxXJ0hMPg8UBtLQQCkJNjso/q6kDEiExxsREfiyWDlJWV8eabb6bFqoqKiKKEw2HChGMiAtmfBpwNWHGxHBnBIDQ1wcGDRmDy8swrEVXw+6G62rxv3w6lpSaInJtrxMdi6WWiAfcwhxhPYn+uh4UVF8vh4fNBY6NxfzkckJ/fuetLxIhI1D3mdBoxOnDALBcVGasmLw8y5Be3WBLpTpzECkl6sP/RlkOjCl6vcXW1tBghKCo6fMvD5YqLiCq0tUFzs/mcmxu3anJyrFVjOWo6FRIB1IxrEcTGSTKEFRdL54TDJkhfW2ssFrfbWBrpQCTZjRYImPOEw8bCKS42ApaXZ5YtlkMQFY9gONh5nMRh4yQ9hU3lsXQkFDKur+3bYc8eIwRR91WmcLvjwf+8PJMksGcPbNlixmQ0NhqBs4kBWUdNTQ3XXHMNp5xyCuPGjeOTn/wkH330UUbPqWoC7cFwEH/IjzfoxR/0x8aTKAnlUiReLmX0KaOpra0F4Lyzz0t57C9+/os8/6fnuzz/r5/5NXv37o0tf/nWL7Nxw8b0XeBxgBUXS5yo9bBtG+zbZ9xTxcXmxt+TRGM50ZhMOAz798OOHUbw9u83FlW4d+apOJZZ9P4iRjw0Asd9DkY8NIJF7x/dHCmqyqc+9Slmz57N1q1b2bBhAz/4wQ/Yt29fUrtQKHRU5+hQdysyyr193a1ozKQ7dbfeWN6+bGH3+c0zv6F6b3Vs+eeP/5yx43p2vFF36Kx8Tk9gxcVi4ik1NUZU6utN3KOoKHvcUTk5casmJ8fEaaqqjFVTVWWy1ro5MdWJzKL3F3HrX25lZ+NOFGVn405u/cutRyUwr7/+Om63my9/+cuxdVOmTOGcc86hsrKSOXPmcN111zFx4kQAHnjgASZMmMCECRN4KFILraWlhUsuuYTJkyczYcIEfv/73xMMB7nzrjsZO24skyZN4vb/ezuBsJn4KiogTzz+BPPvnh8Tkl8/82vuvPNOAK6cdyVnzDqDKZOm8Msnfpmy731L+wJGvP7z6//J5ImTufw/Lmf//viEuAu+t4AzzziTqZOn8pUvfwVV5fk/Pc+aNWv43Gc/x8zpM2lra2Pux+eyZvUaAP6w+A9MmzKNqZOn8q27v5V0vu98+zvMmDaDc848p4MAA7z5xpvMnD6TmdNnMmvGLJqbmwH48Y9+zLQp05gxbQbz75kPwHvvvsc5Z57D9KnTuerTV1FfXw/A3I/P5b/n/zcXzLmARx95lAMHDvDpT3+amTNnMnPmTN5+++3D/TMfETbmcqISDajX1pr3Iw3S9zRRqwbMNQQCRhhVjYVVUnLCDuD8xsvf4N2azkvur6hagS+UXAG5NdDKF5Z8gSfWpC65P2XgFB66uPOCmB988AHTp0/vdPvKlSv54IMPGDlyJGvWrOGpp57iX//6F6rK6aefztnnnM22bdsYOGggf1ryJ1BTomV/7X6WLFnC++vfj5XGb182Zd6n53Hu2efyPz/8HwD++Owfuf2/TA3cx3/5OH379qWtrY0zzziTT837FP369UvZxyUvLOGjTR+x9t217Nu3jykTp3DTzTcB8JWvfoX5/21u5jd/7mb+9te/Me/T8/jZYz9j4f8uZPqM5Gvfu3cv8++Zzz9X/pM+ffpwycWXsGTJEi664CJaWlqYdfos7v/+/dzzzXv41S9/xT3zk2YU4cEHHuThnz7MmWedicfjIS8vj5dfepm/LPkLy99ZTkFBAQcPHgTg8zd/ngcfepBzzzuX++69j+9/7/v85IGfANDQ2MA/Xv8HYQ1z840381//9V+cffbZ7Nq1i4suuoiNGzPvwjux/vssxpXU1GTcS7t3m/hKcbG5YWe7sLRHxFgyUfeZ2w0NDSZGs2UL7N1rYjeBQG/3NCtoLyyHWp8OZs2axYgRIwhrmDffepPLr7gcd54bV56Lyy6/jDfefIOx48eybNkyvn3Pt3nn7Xfo06cPZaVl5OXl8eVbv8wLf36BgvYFN4EBAwYwcuRI/rXiX9TV1fHRRx9xxhlnAPDYI48ZC+Gsc6jaXcWWzVs67B/lrbfe4jPXfAan08ngwYOZPWd2bNsblW9w9sfOZtqUaVS+XnnIuMrq1as597xzGTBgAC6Xi2uuu4blb5oy/Tk5OVxy6SUATJs+jZ0pKlSfeeaZ3HXHXTz6yKM0NDTgcrl4bdlrfPamz8a+g759+9LY2EhjQyPnnncuADd89gaWv7U8dpyrrroq9nnZsmXcdtttTJkyhcsuu4ympqaYRZRJrOVyohAMGndSXZ0RlPz8zAboewOnM171V9UkAOzdaz7n5Bz3Azi7sjAARjw0gp2NHW9ow0uHU3lT5RGdc/z48Tz33HOx5cT031A4RH5BvhEvhWAoGMvoisZFRIQxY8awYuUKXn7pZePOmXsB8/97Pm//821eW/Yaf3z2j/zssZ/x4isvcsYsIx6XXnop9953L1ddfRXP/fE5xpw2hssvvxwR4Y3KN3ht2Wu8ufxNCgoKmPvxuSnL+yeSKkbj9Xr5+m1f551/vcPQoUP53n3fO+RxuqrV6Ha7Y+fpbDqBO795J5/45Cd4+aWXOfesc3nxlRdR1cOeuyVxSoBwOMw///lP8qMWfw9hLZfjHb/fBMC3bTMusLw885SfoUGLbcE22oJt+EN+QuEjD+IeNdEBnFGrxuk0wrpzp7FqamrMmJ0TiAXnL6DAnWwBFLgLWHD+kZXcV1Vmz5mN1+fl57/4eSzQ/s6Kd3jt9dc6lJM/99xz+cvSv9DW1kZLSwtLlizh7LPPZu/evRQUFHDd9dfxjf/7DdatW4fH46GxsZFPfPIT/PiBH/Pee+/hdDpZtWYVq9as4t777gXgik9dwdKlS3l28bNcdbV5Wm9saqSsTxkFBQV8+OGH/Otf/+ryOs455xye/cOzhEIhqqureaPSBPqjQtK/f388Hk+snD9AUXFRyqf/WbNm8dabb1FbW0soFOIPi//AOed1LNPfGVu3bmXCxAnccdcdTJs+jU2bNnHB3At45qlnaG1tBeDgwYOUlpZS1qcsZq387re/SzkdAMDcuXN59NFHY8vvdjVbaRqxlsvxSlubCc43N5sba2FhRp/WA+EA+1traQ60RHzjGivm53a4CYSDHPTWk+PMwSkOnOLE5XD1XPnx9gM4W1vj6c07d5pYzXE+gPP6iaa0/vxl89nVuIthpcNYcP6C2PquSLRIVBVf0Beb4GrxHxdz5/+9k//93/8lLy+P4cOH8+MHfszePXuTjjF12lRu/OyNnPUxU27/5s/fzJSpU/j7K3/nnrvvweFw4Ha7eeTRR2hububKT12J1+dFVfnRT36Usl99+vRh7NixbNy4kZmzZuJt8XLRRRfxxC+eYPrU6YwePZrTTz+9y2u7/IrLef3115k2ZRqjRo3inHPNTbqsrIzPf+HzTJsyjeHDhzNjRrw48Gc/91lu++pt5Ofn8+byN2PrBw0axPcWfI8LL7gwNonYZZddhrela4snyiM/fYQ3Kt/A6XQyduxYLr74YnJzc/n3e//mY6d/jJycHC6++GK+t+B7PPmrJ7nt/9xGa1srI0eO5IknU8fNHnr4Ib7+ta8zadIkgsEg5557bmzumkxiS+4fAdlapr2yspLZM2ea8irRQY8Zdn2pKk3+ZvZ5D+DEQb4rv8P2kIb4cPsBTh5eFhnYJihhRCQmPjkON3muXFMevAfFp3L9emaPHm0svOgAzmhSQC8P4OytkvtdlkgBfG0+8grysnKa3d4qbX8obMl9y7FJKGQC136/Sc3NzU3fSPou8If87Gs7QGvAS6E7P6UQiAgucSEiFLg6+nyj4tMW8tISbO1UfHKdOeQ6c2Li43K4cIozPeLjdsfH8oTDxtqrrzcWTH6+EZv8fGPVHGccqkRKrN5WQuZd4jwlFktnZFRcRORi4GHACfxSVRemaHM18F3MT/k9Vb1OROYADyY0Ow24RlVfEJGRwGKgL7AWuFFVT8xBDoFAvDKxanwkfYZRVRr8jexvq8XtcFOcU3jonTohKj6uTn6KUfFpDbbhCbQQJkz03pfodstz5ZLjcB+9+CSmOkM8ZhUOx8vfRK2aYyzVuX313+gyEJ/cKrHWltUPy1GQMXERESfwGDAXqAJWichSVd2Q0GYUcA9wlqrWi8hJAKr6OjAl0qYvsAX4e2S3HwIPqupiEfk58AXgZ5m6jqzE5zNP1k1Nh65MnGa8QS81bQfwhfwUugoy7rbqjvgENUhLoJVmDacUnxyHm9wE8XGJE6fDiVO64fLKyYlbLKFQXMwdDhOjiaZx93QVgwRSZROlFJJIjCSlkFhOaDIRHsmk5TIL2KKq2wBEZDFwObAhoc0twGOqWg+gqvs7HAWuBF5S1VYx/0EfB66LbHsGY/Uc/+ISHfRYV2eC0S5XxoP0iYQ1zEFvPbXeg+Q6cyl2H7m1kk5EBLe46ezWHtYwIQ3FxCekIQSTcKAovpCf3Z495DhzUopPkng6nckDOP3+5AGcianOPST2eXl51NbW0qdvHxCS5yNJKNgYrQBsrRFLe1SVuro68tIcn82kuAwBdicsVwHt0zZGA4jI2xjX2XdV9eV2ba4BHoh87gc0qGo0Qbwqcp4OiMitwK0A5eXlVFZWHtlVpMDj8aT1eIckHDbjVKKur04ExeP1Url+fdpPrxomkFAMEA4/hdfrC7J+W03a+3a0+P0hNm7fT/ShPj5LVBTBES3NLrHn/djTfxKJtc6i89cchcik+p3FMrYiMySqKGVFZezevbvHhEPDmrUWT7b2rTf71eF32o7otNAtLS0pB3YeKZkUl1RX1P4/1wWMAmYDFcBbIjJBVRsARGQQMBF45TCOaVaqPg48DiZbLJ3ZXT2SLZboggmFzNPwIVwvlevXM3v8+PR1IRyizlfPQV8D+U6TyXWkrN9Ww/iTB6atb+niUP2KWj6hSOFEjZkE5gbvFjdup4scZw65jkjCAYIzEMQZUhxIfK6a/PxuD+BUVSrfqGTmmTPxBr20BlrxBr0x15bDYeJKboe7x4Pr61etZ/zM9P3O0km29q23+uXxexhSPITCLuKimbqfZVJcqoChCcsVwN4UbVaoagDYLiKbMGKzKrL9auDPke0AtUCZiLgi1kuqYx7b+P3xmR7B3JB6IR22JdBKTet+woQpdhWesNlB0XLtbjC2dTsS3W5N4eaY+KiGY/Eid6uQ07SXPEcuLqcLZ3EpzpIyXPmFiNtNWMMEQgEC4QDegJfWYGushPye5j2xtOwCd8EJ+3ewHHtkUlxWAaMi2V17MO6t69q1eQG4FnhaRPpj3GTbErZfiwn4A6CqKiKvY+Iwi4HPAUsydgU9iddrrJQeGvTYGcFwkAPeOhp9zRS48nA5bLZ6V3RXfDwCTdqGhkJQewCtDiIIzrx8QsWFUFAIzvj4nkJ3IQ6Hg6Kcop6+JIslLWTszqGqQRG5DePScgK/UtX1InI/sFpVl0a2XSgiG4AnbD3fAAAgAElEQVQQcKeq1gGIyAiM5dN+0oVvAotF5PvAOuDJTF1DxomOFK+ri1cm7oFU4tRdUTyBFmra9iMIJfamlhZi4pPoUsyNfwz7fTgaWqCxDcrKzENFL2aeWSzpIqOPpar6IvBiu3XfSfiswO2RV/t9d5AiWB/JPpuV7r72KOGwGfRYW2vGqvTQoMfOiJVuCbZQ6MzvcjSvJb04cnIhJ9f8JhobjfWanw99+nQSTbRYjg2sz6MnCQbjQfpw2AzE68XKxO1Lt5S4rbXSayQO3gwEoHqvib/V15vim9aasRxjWHHpCXw+M89IQ0OPD3rsDH/IT03rftqCvk5Lt1h6iWg5GofH/GYO1kF+gbFm8vKO28KaluMLKy6ZQtUE6evqTGn3LJnpMZ2lWyw9QNSa8fth7x5wRQZrFhVlbNoEiyUd2F9nugmHTZC+tjZembgX4ymJ9HTpFksaiZahCYeMW7WuFoqKTVFNa81YshArLukiWpm4ttbEVqKTcmUB0dItdZH5VLKldIvlCHBEZtuMlgPyNBtrpk8fk2nWi1MEWCyJWHE5WgKB+KBHVePG6OHpRLuiLdhGdct+ggQpcp+4gyGPO0TiySChkJnD58CBuMvseJvC2nLMYcXlSPF6kysTFxRklWsiFA5R6z1Ivb+RfGcueQ5rrRy3RAfdqhrruaHBpLf36WN+l8fY1ACW4wMrLodD1BXh95upcbMkSN8eW7rlBCXRmgkGYV8NiMMMziwqOi4nO7NkL1ZcukP7QY+QNfGUZJTq1n22dIvFPPi4iuKDM+sPQl5+PJ3ZWjOWDGPvPoeiocGISihkYilZmJmjqjT7PfhCfloCrbZ0iyVOqsGZTlc8NmMHZ1oyhBWXQ1Fba/zXWZqFk1i6xSGOlPPUWyxAfHBmOGwHZ1oyjhWX7pCFLoTk0i3OSOkWT293y3IskGjN+Hx2cKYlI2TfXdNySHwhH7s9e6hpPUCBM598V++nneYvfYmTZl/CoDEzOGn2JeQvfam3u2TpDrm5UFgEbpepJrFzB+zfbxJXMjCvuuXEwT6iHEOENUyDr5ED3rqsKt2Sv/QlSr/9fRxeLwCuvTWUfvv7ALRd9one7JqluzgS0plbW02KvdsNffuadOYsdQtbshcrLscI0dIt/iwp3SINjbg3bMK9cRPFP/15TFiiOLxeShb8GN+Zswj379dLvbQcNu0HZ+7fbz7bwZmWw8SKS5YTLd1S6z1InjOPop4u3aKKc28N7g0f4t74Ea6NRlBce2viTTrZ1VnfwMAzLyQwcjhjTzuN/PPPxjdzGuFBnc9Zb8ki2g/ObGwwc8+EwiYpIAtjkZbswYpLFtMabKMmUrql2F2U+cGQwSCubTtiFol5fYSjsQkAdTgIjhyOf9pkWm74DMGxowmMHUP/eTckiU2UUP9+tNx8PTmr1lL+5nLcL71iTlMxGP+MqfhmTcc/cxqhYRU2UymbaT84Mxgwg4ij1owdnGlJgRWXLKQnSrdISyuuTZuNgGwwIuL+aAvi9wOgubkExpxK28UXEBg3hsDYMQRHn4oWdEx1br79tqSYC0A4L4+mu//LxFxu+RzrN+9hcsBD7uq15KxcS+4bb1Pwwt/M9Z7UH/+MafhmTsU/azrBU0bap+JsxeUyf5vcXDs409IlGRUXEbkYeBhwAr9U1YUp2lwNfBfjXXlPVa+LrB8G/BIYGtn2SVXdISJPA+cBjZFD3KSq72byOnqSlkAr1a37ANJWusVRdzDm1nJv2IRr4yZcO3YhkWygcFkpgbFjaLnhaiMk404jOGJYt1NSo0H74gcexVm9j9Cgcppvvy05mO90Ehw1huC4MbR89lpQxbV1Bzmr15Kzai25K9eS/+LfAQiVleKfMRX/zGn4Z00jcNpoG1DONhLTmf1+OzjT0oGMiYuIOIHHgLlAFbBKRJaq6oaENqOAe4CzVLVeRE5KOMSvgQWq+qqIFAHhhG13qupzmep7bxAMBzngrTu60i3hMM5du+OWyIYPcW/chHN/bfw8QwYRGDuGtksvjgjJGMIDy4/aLdV22ScOLzNMhOCpIwmeOpLWaz5tYju795Czei25q9aRs3IN+f+oNJdVVIh/2hT8M6fimzmNwIRxkGNvXllDbK6ZsCnmerDOpDeXltrBmScwmbRcZgFbVHUbgIgsBi4HNiS0uQV4TFXrAVR1f6TtOMClqq9G1h+3owOjpVv2eQ8gSPdLt/gDuLZsTRCRj5iz4UNcrW3muE4nwVNH4vvY6RG3lomPaGlJBq/mKBAhNKyCtmEVtM27DABHzT5yV68jZ5VxpZW8+TYA4bxcAlMm4Zs5Df/MqQQmT0CzaJqDE5ZodXBIHpxZVmYSA+zgzBMK0QwNlBKRK4GLVfWLkeUbgdNV9baENi8AHwFnYVxn31XVl0XkCuCLgB8YCfwDuFtVQxG32McAH7Asst6X4vy3ArcClJeXT1+8ePGRXYjP18GP7PF6KUpDSqaiBMNBwhruMrXY2dJK8bZtFG/dRvHWrRRv20bRzl04gkEAgnl5eE4eSf3wEbSNHkXzKSfjGTGccBYFWr2+IHm5R3dzcTc00OeD9ZR9sJ4+739A8bZtiCphl4vG0aNpmDiB+onjaRg7jlBhQY/1K1Nka98Ou1/hiNPB4QSnI6NxGW+Ll7zC7EuX7q1+hTWM2+Hu8v7i8XgoKkr9UDtnzpw1qjrjSM6dSXG5CrionbjMUtWvJbT5KxAArgYqgLeACcAFwJPAVGAX8AfgRVV9UkQGATVADvA4sFVV7++qLzNmzNDVq1cf2YVs2WJ8ywmmfeX69cweP/7IjoexVhr9TexrO4BLXPER9qo49tcaS+RDEx9xb/gQ1+49sX1D/foaS+S00QTGnUZg7GhCw4eC08n6bTWMPzk703wz0TdpaiZnzbvkrF5H7qq1uD/YgARDqMNBYNwYE7OZOQ3f9Clon7Ie61e6yNa+HVG/VM2DWigI7pz4XDNpjqWtX7We8TOP/H8zU/RWvzx+D0OKh1DYxYDryspKZs+enXKbiByxuGTysagKE4yPUgHsTdFmhaoGgO0isgkYFVm/LsGl9gJwBvCkqlZH9vWJyFPAHRm8hrTjC/moaT2A19dK6Z5acj7cHE/73bAJ58H6WNvgsAoC40+j9crLCYyNxEcG9Lc+7AhaUoxvzjn45pxDMyCtbbjX/dtkpK1aR+GiP1L01CIAAqNPwT9zuslImzGV8EkDerfzJxpdDc4sLjbZZ5bjikyKyypglIiMBPYA1wDXtWvzAnAt8LSI9AdGA9uABqCPiAxQ1QPAx4HVACIySFWrxaRRXQF8kMFrSA9eL+GNG2h7bw3+f6+jfNNWcjZtwdFmUnfV7SJ46in45pxjRGTsGAJjR6GdmKqW1GhBPv6zTsd/1ulmhd9Pzr/Xm5jNqrXk//kvFC56FoDgiGH4Zkxl0IhTcV4yh9CQQb3Y8xOMzgZn9u1rvAQ2nTk1qvFXOJz8nvgKBs36cBh8zZA7AHqhVFTGxEVVgyJyG/AKJp7yK1VdLyL3A6tVdWlk24UisgEIYbLA6gBE5A5gWURE1gBPRA69SEQGAAK8C3w5U9dwRNTXw/r15vXBB7B+PbplC45QiEIgv6iQwNgxtF51RcS9NYbgqSfb7KdMkJNjUppnTIWvfAGCQdwbNpnU51Vryf/7a0xoWgI//gnBwQPxzzAJAr5Z0wmNGGYtxEyTaM0EAlBTY4TleBucqWoGUwSDnQtCKGTWJ72HIJywLRzC3PaSDt5xncMR/+22NkB5IPPXmIKMRgtV9UXgxXbrvpPwWYHbI6/2+74KTEqx/uPp72kKFi2C+fNh1y4YPBjuvhvmzUvsCOzZkyQifPCBWRdtUl5OYNwYms6bSXj8OHT8OEIVQ+yTWW/hchGYNJ7ApPG0fOFGCIfZ+dq/GFu9i9xVa8h9ewUFS83PNdS/nxGaGWasTXDUKfbvlkkS55qJDs7Mz4eyPh1inj1GoggkWgqJ28LhuGgkWgyhUHwdgN9nqhokkSAMIh1fUZFwu+PrDhdf7yWEZF8qSjawaBHcequpDgtGMO64A1auhPx8Jq9cCTt2mAmXwPzRTz4ZZsyAz30OJkygdcwp1OSHCBKk0Flg57HPRhwOPCePpPWCj9F642fMWJvtO8ldZWI2OavWkP/SPwAIl5bgi1hB/pnTCIwbY1NrM0E6BmdGb/DthSFxuYOlEDRWQuK6LokIQ1QEon2PioDLabbn5ETaNBtX4AmE/e9Ixfz5cWGJ4vPBb34Dubk4hw+HSy6B8ePNa9y4WH5/UukWR2ZKt1gyhAihk0fQevIIWj8zz4jNnupYzCZ39Tryl70BQLiwAP+0yTFXmn/S+OPHjZMtxAZnhpIHZ0YtnGAQNBwvpBkOm//T7dtJ6S5KJFEIEi0FlzNuKViOCisuqdi1K/V6EfjoI9Zu2pQyFdnjb6GmzWTBmJkhLcc0IoQqBtNWMZi2T10KgGPfgVjqc87qtZQ8+BhgarH5p0yI1EibRmDKxJR12CxHgMMZH5zp94PXm+w2cjhMkkD08wlmIWQrVlxSMWxYCv8oJvaSwhWSltItlmOCcPkAvJdciPeSCwFwHKw3Y21WrSNn9VqKfvYkxY89gbqcBCaMi1QRmIZ/+mS0uDh2nPylL3Vdi82SGmsdHjPYu2AqFixIjrmA8QPffXdSsyMu3WI5bgj37YN37hy8c+cAIB4POWvei7nRip5ehDzxjBnYedoo/LOmgyoFf3geh9cUlrAzd1qOR6y4pOL66817F9li/pCffW0HaA16KXDm4XTYqr0W0KIifOedhe+8s8zAzrY23O99QO5K40Yr/P2fEF+HakVm5s7/eQD/xHGEBpXbGR8txzxWXDrj+uvNq135F1UlpCG2N+/CJS6Ke3pmSMsxhebn4z9jJv4zZpoV/gCDJp6BpKi65Kw7SPlF5gEm1KeMon79cA+vIDR4IKFB0Vc5ocEDTaUGOw2BJYux4nIYREu3BMPBrJjH3nIMkuMmNGhg6pk7+/Wl6a7/xFldg7N6H/6tO8nfVUXuitU4WlqS2qrTSah8AKGB5R3FJ/KuZaU268nSa1hx6QZhDdPga+KAt44cRw4OcVhhsRwxnc7cec/tSTGXxAKR0tyMc68Rnaj4RD/nvPcBzleWIYFg0nnC+XmEBpYTHjSQ4OCBhAeWm/dB5QQHmXc7VYElU1hxOQTekI9qTy2BcICiNM0MaTmx6dbMne3Q4mKCY4oJjhmVukE4jKPuoBGgmn3thKiGvDeW4zxQ12G3UFlpkvgYKyhu/YROGmAHi1qOCPurOQR7Wmtw5BVQZGMrljRy2DN3HgqHg/CA/oQH9CcweULqNn4/zpr9yeJTU4Nzbw2uqj04V67B0Zw8L586HIRPGmBEJyI+Q9355E0YFXPHhfuUWfebpQNWXA6BqpLjsEUlLccBOTmEhlUQGlbRaRPxeOIut701MfFx1uzD/cEG8l59ndMCyYUQNTc32dpJSDwIDYzEf7o5eZvl+MGKi8ViiaFFRQRHFZlCnakIh9m8bhPjcjQuPglxoNzlK3AcqEXaTUIYLi3pUnxC5QO6rBtmB50ee1hxsVgs3cfhwN+nD4GTBxKYOC51G38A5/4DEdGpTk5C2FtDztr3cDQ2Je2iIoQH9O8Y8xk0ENfmrRT94ik76PQYw4qLxWJJLzluQhWDCVUMxsxU3hFpaW0nPnEBcm/8iLzX3ko52DSKw+ul9NvfI/ftFYSLi9DiIsLFxQz2hsg7eYhZLioiXFIc2VZkZ7vsYay4WCyWHkcLCwieOpLgqSM7aaA46htwVtfQ/1M3pKxvLF4fOStW4Wj24PCYcUClXZ3T7Y6LTVFEkEqKjTgVFcVFqiRxuThpuVsl/y2AFReLxZKNiBDu24dw3z6EBncy6HTwQPZX/s0shMNISwtbPtjOmD75OJo9iKfZvDd5Isvm3dHUjDR7cHg8uA7Umm3NzTha2w7ZrXBeLlpSHBenBKsptlyUIFoRIctr8CL9CtCiwh6prBCNUQ2q3ocOGQILF8bLWvUQVlwsFktW09mg0+bbb4s3cjjQ4mK85ScRjAw8PWyCQcTTEheb5ogoxd6b48seD9LUbIRqT7VZbvbE4kLtOSnhc7iwwFhECeITLi5CS4qSRSuViJUUowUFXc6Kmr/0paTvS6qqTCFe6FGByai4iMjFwMOAE/ilqi5M0eZq4LuY2X3eU9XrIuuHAb8Ehka2fVJVd4jISGAx0BdYC9yoqv5MXofFYuk9jmTQ6RHhcqFlpYTKunKuHQJ/IC40zREBavZQvX0PFXnOyPrmJNFyHDyIa+eu+D6Brue8VxG0qDDutitOFKBi8pf8LUmIAVPhff7840NcRMQJPAbMBaqAVSKyVFU3JLQZBdwDnKWq9SKSKPC/Bhao6qsiUgRE5x39IfCgqi4WkZ8DXwB+lqnrsFgsvU/aB51mihw34b59oG8fQgmr926roU93LSqfL+66S7SSEq2mdu49x779uDZvM9s8LamP29kkiBkik5bLLGCLqm4DEJHFwOXAhoQ2twCPqWo9gKruj7QdB7hU9dXIek9kvQAfB66L7P8Mxuqx4mKxWI4PcnMJD8iFAf2TBKq7nDT7kpQxKoYNO+quHQ6ZFJchwO6E5Srg9HZtRgOIyNsY19l3VfXlyPoGEXkeGAn8A7gb6AM0qGow4ZhDUp1cRG4FbgUoLy+nsrLyiC7C19qGw+EDJDYlt9cXZP22FH+8XiZb+wXZ27ds7Rdkb9+ytV+QvX3ryX7V3XAD4x5+BGdCKncoN5dNN9zA/hT3QY/Hc8T3x67IpLikyh5sP4uFCxgFzAYqgLdEZEJk/TmYJPldwB+Am4Cl3TimWan6OPA4wIwZM3T27NmH238AtmxdRX5QzIjjUAhCIdbvqWd8eXFCKyV2ue3n9hYBh4A4Ml5/KbGKbraRrX3L1n5B9vYtW/sF2du3Hu3XydfSdFJZLEalQ4bgXLiQcddfT6phr5WVlRzp/bErMikuVZhgfJQKYG+KNitUNQBsF5FNGLGpAtYluNReAM4AfgWUiYgrYr2kOmZ66dMHXPnJwlDbCiNGQDgcf6maVygEwWDyeyAYb5dEF6LkcJhNPSBKFovl+CIao/I0HmDIKVMp7HPSoXdKM5kUl1XAqEh21x7gGuKxkigvANcCT4tIf4w7bBvQAPQRkQGqegATZ1mtqioirwNXYjLGPgcsyeA1dI7Tefj56qrJgnQoUQqGuhAlMK66BFGKHseKksVi6WUyJi6qGhSR24BXMPGUX6nqehG5HyMUSyPbLhSRDUAIuFNV6wBE5A5gWSSIvwZ4InLobwKLReT7wDrgyUxdQ9oRyawoSb1Zd7iiZC0li8WSZjI6zkVVXwRebLfuOwmfFbg98mq/76vApBTrt2Ey0U4cuitKO/ZDRUI59bRZShBz4VlRslgs3aDb4iIiZwOjVPUpERkAFKnq9sx1zXLUpNtSin5OJUqhkGljDmDExukEtwscmS93YbFYsotuiYuI3AvMAMYATwFu4LfAWZnrmqXXOBpRiopOIABeL7S1QcBrRKmlJSI47h6pr2Q5TlA1v6lgMG5VR39HDmsxZyvdtVw+hUkLXgugqntFpLjrXSwnHCJmvnWXC/LyoDjyEwmHoXYDlJfHBcebUCTQ6bKCYzGEQubBJGoto8byzc2FwkLzrgo+n/kdtbWBhollXapCOGSt5Sygu+Lij2RqKYCI2AnlLd3H4TBjfYqKzAvMzSMQME+jUcFpSShb4XQakXI67ZPp8UiiezUcJhbTy8kxIpKXZ/7+brd5b0/0d5Ro1QQCsM9j3LSBhNpaiQ89XRR8tKSX7orLsyLyC8wYk1uAzxPP3rJYDh+HwzyFRp9Iwdwo2gtOW1vyE2z0JmEF59igffJIVEScEWukpMQISlRIDvfvKmL2c7shP9/E+IYNi4tXMAh+v/k9+XxmOUrU9et0WtHJAN0SF1X9sYjMBZowcZfvROt+WSxpQyJPrjk5UFBg1rUXHG+iKwQrONlEOJLcEQzG/z7iMH/P4mJjjfRUzM3h6PhbgrjQBYNGbKKik5ghaa3mtHBIcYlUN35FVS8ArKBYepbOBCfxBtHWZm4S0RuEdYNknkAgVg4pRjTWVlYWd2dlm+hHLZVEixniv6dE0YlZzRGi13MMxAaf3/4SC997lL2t+6goHsL/zF3I9ROzbLIwVQ2JSKuIlKpqY090ymLpkvaukLIysz7qd/f74y61cPTmJ/EbnhWc7pP4pB91aYUVnI54gD3q0jqWv9eocEDn8Zyo6HhTxXOcWZNE8Pz2l7hr5fdpC5l+7m6u4ta/mMnCelJguhtz8QLvi8irQCzqqqpfz0ivLJYjIXqDyM+H0siET9EbQyAQt3CifvdoyRybXWSsvqiQhEPEsq/cbmMx5ubGBf3gJhicshj58UX7h5goUVdtKBSP53i9EPQSE2CHI27l9LDoLnzv0ZiwRGkNtDJ/2fysFJe/RV4Wy7FFouCUlJh10XTXQABqmk1h0WDCP+PxPBYnOhg2enOM4nAYl1ZRUdwasVZeaqKuWkh+kEn8XhOtnOjg4+g4rzTEc9qCbVS11LC7Za95ecx7laeaPa2pS/vvaszCycJU9RkRySEy/wqwKVLJ2GI59oj63fPyTHbR8OFxwQkG4y417zEuOIkZU4mxg9xcIyKJ6b7H0nVlK9EMSOiYRFC7EQYOjCQQeJOzIKPZcwl/B2/Ix56WGnZ79rC7pZqqlr3sigpISzUHvHVJp85xuKkoHMzQwkEUuPJpDbbRnmGlWThZmIjMxsz6uANjLw8Vkc+p6puZ65rlWOL5jc+zcPlC9jbvZXDxYO4++27mjZ3X293qPokVCY61sTgd0n0jRAPsJSXJ1kg2BdhPBJxOM86rsDCWROAP+tjTsJvd9TvY3bCT3fU7qWqqionJPl+yeLgdLoYUDGRo0RDmDjmHisLBDCsaHBGUwZyU3w+HGCuzfcwFoMBdwILzF/TcNdN9t9hPgAtVdROAiIwGfg9Mz1THLMcOz298nrtevYu2yNPSnuY93PXqXQDHlsC0J9VYnETBiWaqtbYSqzjtSHgCzcRNPByZHyhxvIYI5OV2FBFrjfQagVCAak81uxp3UdVUxdoda2mrbTMC0ribGk8NmjDPoVOcDCkZQkVJBXMGjjOiUTCQoTn9GeoeQLmrGKe4iI33ij7YpHBbzhv5CYDszxaL4I4KC4CqfiQi7gz1yXKMsXD5wpiwRGkLtvHdyu8ybsA4KkoqOtnzGKQ7gz+9bclprNHg7uFYDdHq1W1tCWMwFNw5xs+fl5e96b4nAMFwkBpPDbsbd7O7aXfsvaqpil2Nu6j2VBPW+NgZBw4GFQ9iWOkwzh52NkNLhjK0dGjsfWDRQFyOLm7H7QeFtrWZ92AA40xSU0YpIjrzRn6CeSOPjcnCVovIk8BvIsvXY+ZYsZzAqCrLdy9nT/OelNvr2uo4/9fnA1DiKmHERyOoKKmgoqSCoSVDY+9DS4dSlFPUk11PL90Zi9PamjAWJ1I1OioMUXFqX09LxASLEwcf2gB7jxAKh9jXsi8uHu0EZG/zXoLhuPUoCAOLBjK0dChnVJwR+11XlFQwrHQY9R/WM/n0yUfeofaDQqPp9x0GhUbHfEUy/gK9Fxrvrrh8Bfgq8HWMTL4J/L9MdcqS3TT5mnhuw3M8894zbDm4BQcOwnScA2ZAwQDum3Mfe5r28N6W92jNbWVz3WZe2/4a3mByqmRZXpn5hywZSkVpXHyiAlSce4zVSU01FifV4E+fzwhHtJ5WojVSux769u3tKzkuCWuY/S37jVg0VrGraRdVjVUxEdnTvIdAOPnGPLBwIBWlFUwfNJ3Lx1zOsNJhsd/nkJIh5DhzOj1fs6M5MxdyqEGhrUXxJIMeprvi4gIeVtUHIDZqv3d6bOk1Pqz9kKfffZo/bfwTrYFWpg6cykMXP0RYw8xfNj/JNZbvyuc7532Hy8dcDsB61jN+5njAWDx1bXVJT4LRf/LNBzfz+o7XO7jZynLLkkQn8cmwoqSCktySnvsijpTOBn9aDsnhJoyoKrWttbGYx+6m3Umf9zTtwRfyJe0zoGAAFSUVTB44mUtHX0pFaQXDSoZRUVrBkOIh5LnyMn2Z6SOWSh5MXfizJ7rQzXbLgAsAT2Q5H/g7cGYmOmXJHgKhAC9ueZFfv/trVuxZQZ4zj8tOu4ybJt/E5IFxM9/tcHf7n19E6F/Qn/4F/Zk6aGqH7arKwbaDMXdE7ImyaTdb67dSuaMypfgMKRmSZPkkfj4mxMeSks4SRjx+D5PKJyW5rDbu2kj9+nqqGqvwthtI2C+/H0NLhjJuwDguPuXi5N9JSQX57vxUp7ccId0VlzxVjQoLquoRkYKudgAQkYuBhwEn8EtVXZiizdXAdzHpNu+p6nWR9SHg/UizXap6WWT908B5QLQUzU2q+m43r8PSTaqbq1n0/iIWvb+I/S37GV46nP8+97+5evzV9M3v6KqZN3Ze2jLDRIR+Bf3oV9CPKQOndNjeXnz2NO2J3Vy2N2znzV1v0hpoTdqnNLc0HutJsID8Hj8V3gpK80rT0nfL4aGqtAXbaPI10exrNu/+5thys7+Zh1Y8lDJh5J5l9yStK8sro7+rP6MHjeb8kecnBc0rSioozLEzhfQk3RWXFhGZpqprAURkBtBxlE4CEdfZY8BcoApYJSJLVXVDQptRwD3AWapaLyKJKQ1tqtrxzmK4U1Wf62bfLd1EVfln1T95+t2neXnLy4Q1zJyRc7hp8k3MGTknlkff23RHfOq99clut8jnHQ07OorPOijJLUlONEi4KQ0tGWrFJwWqSkugJS4M/oggJHxOFIlmf3NKAQlp6NAn64SnLn8q9ncqzi1m/aq4+9XSu3RXXL4B/FFE9mIsjMHAZw6xzyxgi6puAxCRxcDlwJxQWHMAAB9CSURBVIaENrcAj6lqPYCq7j+MvlvShMfviQXoP6r7iLK8Mm6Zdgs3Tr6REWUjert7h42I0De/L33z+ya57qJExaeqqYp31r4DA4j54nc27GT5ruW0BFqS9inOKU4pOtG4T2luKXIMpQOHNYzH74nf+FPc9DsTi4PNB/Gu9NLsb05Kt02FQxyU5JRQnFtMcW4xJTklDC4eHPtcnFtMSW5J0nL7bR9/5uMpMxKHFA/hwlMuzNRXZDlKRBPLQrTfKDIT2K2qNZFxLV8C5mEE4juqerCLfa8ELlbVL0aWbwROV9XbEtq8AHwEnIVxnX1XVV+ObAsC7wJBYKGqvhBZ/zTwMcCHiQXdrarJkTnT7lbgVoDy8vLpixcv7tYX0h5fyNfhid3b4iWvMPuCe4fbr50tO1lavZR/7P8HbaE2RhWN4rJBlzF7wGxynenN1ziWvjNVpTnYTI23hv2+/dR4a9jn28c+r3nV+GpoCyUb7gXOAgbmDaQ8t5zyvHJOyj3JLOeVU55bTrGrOEl8lu1fxlM7nuKA7wADcgdw84ibOf+k8w/ZN4CQhmgLtdESbKEl1GLeEz+HWmgNtnZYl/jeFmpLGsSXCqc4KXQWUugqpNBZSIGrgEJnIXnkUZJbYpYj21K+uwrJc+Qdtegu27+MhzY/hC8c/zfPdeTyjVHf6PZ31tv0Vr/CGsbtcHfpdfB4PBQVpR4KMGfOnDWqOuNIzn0ocVkLXKCqB0XkXGAx8DVgCjBWVa/sYt+rgIvaicssVf1aQpu/AgHgaqACeAuYoKoNIjJYVfeKyMnAa8D5qrpVRAYBNUAO8DiwVVXv7+oiZ8yYoatXrz7kl5GKLQe3kO/KT/oHyVbTuzv9CoQCvLL1FZ5+92n+WfVPcp25/MeY/+CmyTcxZeCUjD19H8vfWXtUlQZvQ8zaiSYdVDXH3W8evydpn6KconicJ+jnnap3klJd3Q43l4+5nJP7nhyzEPbU7MFR6Eh2MfmbOxw7FW6Hu0troDinOKXVEN1WkltCniu1MPTG37K72WLH0+8sHXj8HoYUD+ky3lRZWcns2bNTbhORIxaXQ7nFnAnWyWeAx1X1T8CfRORQQfQqYGjCcgWwN0WbFZEimNtFZBMwClilqnsBVHWbiFQCUzFCUh3Z1yciTwF3HKIfFmCfZx+/e/93/Pbfv6WmpYaKkgq+dfa3uHbitSkD9JbOERH65PehT34fJpZP7LBdVWn0NSbFemJC1LibD2s/7GA1BMIBnttowog5zhyKc4rJ1Vz6O/pTnFPMgD4DjEjkdC4IiZ87E4ZjlXQmjFh6hkOKi4i4VDUInE/EzdTNfVcBo0RkJLAHuAa4rl2bF4BrgadFpP//b+/eo+SqyryPf39V1bd0OgkkJOaCCiYoubwgNJcAakMEgrIUB5cCCbdxYEYHXpEl7xteZrEQZYHIzKgzrNHM6CQBJHITI8NtQIKgEpJgmqSDkSbh0gkjd6FD0peq5/3j7KpUV6q705VTXZXk+azUyql9Tp3z1O6q89TZ+5x9iEZd3ihpP+B9M+sK5ccDNwJImmhmryr65pwBrBv0Xe6jzIwVm1ewaM0iHmh/gN5MLy0fauH6T1/PnIPmkNzX72FSJpIYUz+GMfVjmDl+5k7zp/xT8eFwhGj/3+256ymq9Ve4c7tisARxO/C4pDeIzg57AkDSVHacClyUmfVKugR4iKg/5adm1ibpWmCVmS0L806RtB5IE50F9qak44AfS8oACaI+l+yJALdJOoBopIA1wN8N/W3v3bZ2b+Xu5+5mSesSnnvjOUbXjebCwy/kvMPO4+D9Dq50ePu8SU2TinZQT2qatGddqOfcAAZMLmZ2naRHgYnAw7ajgyZB1PcyIDO7H7i/oOzqvGkDLg+P/GV+B+zc3hDNO2mw7e6rXn7/ZZb+eil3rr+T97rfY+b4mdx08k2c8bEz/AKxKrLghAV9LgqEaESDBScsqGBUzsVr0FORzeypImV/Kk84bqh6M708/MLDLG5dzJMvP0ltspbTp53O+Yefz5ETj9yr2t33Ftm+gz36/jfODaIyg8643fb61te5be1t3Prsrbza+SqTmyZz4Ycv5LK5lzFuxLhKh+cG4R3Ubm/nyWUPYmas2rKKRWsW8V/P/xc9mR4++aFPct1J1zHn4DlsWL3BE4tzrip4ctkDvN/zPvc8dw+LWxez/vX1jKobxXmHncd5h53H1P2nVjo855zbiSeXKvbC2y+weM1i7lx/J+92vcv0A6Zz46dv5AuHfoERNYOOG+qccxXjyaXKpDNpHtn4CItaF/Gbl35DTaKGz077LBccfgHNk5q9g945t0fw5FIl3nj/DW5fdzu3tN7C5vc2M3HkRK447grOmXUO4xuH//7Xzjm3Ozy5VJCZsfrV1Sxes5j7nr+P7nQ3xx94PNe0XMMpHzmFVML/PM65PZPvvSpgW8827v3jvSxqXcS619bRVNvE/FnzOe+w85g2dlqlw3POud3myWUYbXp7E0ueXcId6+7gna53+NjYj3H9nOs589Az/S55zrm9iieXMktn0jy66VGWtC7hsRcfI5VIcdrU07jg8As4ZvIx3kHvnNsreXIpk7e2vcXta2/nlmdv4ZV3X+EDjR/gm7O/yTmzzmHCyAmVDs8558rKk0vM/vDqH1jUuohfbfgVXekuZk+ZzT988h849SOnUpOsqXR4zjk3LDy5xGBbzzaWbVjG4tbFtP65lcaaRs6aeRbnH3Y+Hx330UqH55xzw86Ty2546Z2XWNK6hKVtS3ln+zscMvYQrjvpOs489Eya6poqHZ5zzlWMJ5chyliGp996mht+cQOPbXqMhBLMnTqXCw6/gNlTZnsHvXPO4cmlX7etvY2rHr2Kl//yMpOaJnHp0ZeytWcrS1qX8NJfXmJ843guO/Yy5s2ax8SmiZUO1znnqoonlyJuW3sbF//qYt7veR+Aze9tZsGj0V0Cj518LPMnzueiUy/yDnrnnOtHopwrlzRX0gZJ7ZKK3sNV0pckrZfUJulneeVpSWvCY1le+UGSVkh6XtLPJdXGHfdVj16VSyz5xjeO5+4v382nDviUJxbnnBtA2ZKLpCRwM3AaMB04W9L0gmWmAVcCx5vZDOCyvNnbzOzw8PhcXvl3gX82s2nA28BX4o795b+8XLT89a2vx70p55zbK5XzyOVooN3MNppZN7AU+HzBMhcBN5vZ2wBm9tpAK1TUW34ScFcoWgycEWvUwAdHf7Bo+aSmSXFvyjnn9krl7HOZDLyS97wDOKZgmUMAJP0WSALXmNmDYV69pFVAL3CDmd0LjAXeMbPevHVOLrZxSRcDFwNMmDCB5cuX73Lg8yfO56b3bqIr05Urq0vUMX/SfNpWtrF963baVrbt8vqGS7XGBdUbW7XGBdUbW7XGBdUbW6XiyliGjkQHCfV/HNHZ2Tmk/eOuKmdyKXZOrhXZ/jSgBZgCPCFpppm9A3zQzLZIOhj4taS1wLu7sM6o0GwhsBCgubnZWlpadjnwFlo4dO2hfc4WW3DCAv7q0L8CoG1lGzOOmrHL6xsu1RoXVG9s1RoXVG9s1RoXVG9slYqrs7uTyU2TBxwYd/ny5Qxl/7iryplcOoAD855PAbYUWeYpM+sBNknaQJRsVprZFgAz2yhpOfBx4G5gjKRUOHopts5YzJs1j3mz5tH+VjsNqQa/fsU554agnH0uK4Fp4eyuWuAsYFnBMvcCJwJIGkfUTLZR0n6S6vLKjwfWm5kBjwFfDK8/H/hlGd+Dc865EpQtuYQji0uAh4DngDvMrE3StZKyZ389BLwpaT1R0rjCzN4EDgVWSWoN5TeY2frwmv8LXC6pnagP5ifleg/OOedKU9aLKM3sfuD+grKr86YNuDw88pf5HTCrn3VuJDoTzTnnXJUq60WUzjnn9k2eXJxzzsXOk4tzzrnYeXJxzjkXO08uzjnnYufJxTnnXOw8uTjnnIudJxfnnHOx8+TinHMudp5cnHPOxc6Ti3POudh5cnHOORc7Ty7OOedi58nFOedc7Dy5OOeci50nF+ecc7Hz5OKccy52nlycc87FrqzJRdJcSRsktUta0M8yX5K0XlKbpJ8VzBslabOkf80rWx7WuSY8xpfzPTjnnBu6VLlWLCkJ3AycDHQAKyUtM7P1ectMA64Ejjezt4skim8DjxdZ/TwzW1Wm0J1zzu2mch65HA20m9lGM+sGlgKfL1jmIuBmM3sbwMxey86QdCQwAXi4jDE655wrA5lZeVYsfRGYa2Z/E56fCxxjZpfkLXMv8CfgeCAJXGNmD0pKAL8GzgXmAM3Z10laDowF0sDdwHesyJuQdDFwMcCECROOXLp0aUnvoyvdRUJ9c/D2rdupb6wvaX3lVK1xQfXGVq1xQfXGVq1xQfXGVqm4MpahJlGz0z4sX2dnJyNHjiw678QTT1xtZs2lbLtszWKAipQVJoEUMA1oAaYAT0iaCcwH7jezV6SdVjPPzDZLaiJKLucCS3bakNlCYCFAc3OztbS0lPQm2t9qpyHVQH4cbSvbmHHUjJLWV07VGhdUb2zVGhdUb2zVGhdUb2yViquzu5PJTZNprG3sd5nly5dT6v5xIOVMLh3AgXnPpwBbiizzlJn1AJskbSBKNrOBT0j6GjASqJXUaWYLzGwzgJm9F04AOJoiycU551zllLPPZSUwTdJBkmqBs4BlBcvcC5wIIGkccAiw0czmmdkHzezDwDeBJWa2QFIqLIekGuB0YF0Z34NzzrkSlO3Ixcx6JV0CPETUn/JTM2uTdC2wysyWhXmnSFpP1IdyhZm9OcBq64CHQmJJAo8A/16u9+Ccc6405WwWw8zuB+4vKLs6b9qAy8Ojv3UsAhaF6a3AkWUI1TnnXIz8Cn3nnHOx8+TinHMudp5cnHPOxc6Ti3POudh5cnHOORc7Ty7OOedi58nFOedc7Dy5OOeci50nF+ecc7Hz5OKccy52nlycc87FzpOLc8652Hlycc45F7uyjors9h0Zy9CT7iFtaYRIJpIklSSZSFY6NOdcBXhycUPWm+mlN9NLOpPGzECQVJKGmgYaUg30ZnrpTnfTne5me+92MpYhk8nQ2dWZWzahBKlEioQSFLmVtXNuD+fJxfXLzHYkEguJBKhN1tJY00hDqoHaVC01iZp+j1DMjLSl2ZLawpTRU0hn0nT1dtGV7qI73U1vphfDiP4ZknLJJ3v048nHuT2PJxcH7GjW6s305o5GhKhL1TG6bjT1NfXUJGqoSdaQ0K531UkipRRCjKgZAUBTXVNuvpmRsUwugaUz6dxRT1dvF12ZrlzyQYBBMuHJx7lq58llH5TOpOnJRIkke8SQTCRpSDUwpn4Mdak6ahI1pBKpsu+4s0cqA/XNpDPpXOLJb3LrTnezLb2NdCbq58kmn0QikVunJx/nKqOsyUXSXOAHRPe7/w8zu6HIMl8CriH6bdpqZufkzRsFPAf8wswuCWVHEt32uIHoFspft2x7jesjv1mrN9Mb7WQNapI1NNY0Up+qpzZZS00ySiTVKplIkiQZfYqKKEw+PZkeunq7csknk8nkmtyyySfb5+OcK4+yfbskJYGbgZOBDmClpGVmtj5vmWnAlcDxZva2pPEFq/k28HhB2b8BFwNPESWXucAD5XkXe46MZTAz3u95n0wmk2vWqk3WMqpuFA01DSU1a+0JhpJ80pamJ92T6/fJ2I4TDcwMoVzyyR797G315dxwKOdPt6OBdjPbCCBpKfB5YH3eMhcBN5vZ2wBm9lp2RjhCmQA8CDSHsonAKDP7fXi+BDiDfSy55DdrZQ/akomo+Wdsw9jc0UhNosabhBg4+WxKbmLq2Km5s9+yyac73U1Xuis62y0k6/wmRE8+zg2snMllMvBK3vMO4JiCZQ4BkPRboq/+NWb2oKQE8I/AucCcgnV2FKxzcrGNS7qY6AiHCRMmsHz58pLeRFe6a6edx/at22lb2VbS+obKzHbq0JaEJBJKkGDHqbxd73fRuqJ1WOIaqs7OzpL/BuXU2dnJbx7/zaDLGVESz/49zKJHhkyfvw0hlytM7E5yH87P2VBUa1xQvbFVKq6MZehIdAz4A6hc381yJpdi36rCvpEUMA1oAaYAT0iaCcwH7jezVwq+nLuyzqjQbCGwEKC5udlaWlqGEntO+1vtNKQa+uwk2la2MeOoGSWtrz9mljsaSWfSQLRjqkvWMaJmRK5ZK5VI9dv5vXz5ckp9n+VWrbHFEVfGMn2a3nrSPblTrXvSPfRab67JLdtcmT3bLXutTzHl+JzFoVrjguqNrVJxdXZ3MrlpMo21jf0uU67vZjmTSwdwYN7zKcCWIss8ZWY9wCZJG4iSzWzgE5K+BowEaiV1Ep0cMGWQdVa9bMdz/jUeiUSC+mQ9oxpGUZes82atPUhCCRLJBDXUFJ1fmHx6M710pbuikw56o2t9ir4uk2Fr99bchabZpji/8NTtCcqZXFYC0yQdBGwGzgLOKVjmXuBsYJGkcUTNZBvNbF52AUkXAM1mtiA8f0/SscAK4DzgX8r4HnZbb6Y3GhYlk841rdQkamioaWBEzYg94mwtt3sGSz4QJaD8RzqTpiPZwQEjDsidAZd/QWvGMgNuL3t0lP/whOSGU9n2aGbWK+kS4CGi/pSfmlmbpGuBVWa2LMw7RdJ6IA1cYWZvDrLqr7LjVOQHqJLO/PxmrYztaIevTdTSVNdEQ6ohdzTi4225QtkEUFg2pmFM0eWzF58We+Sffl5KQsqepCDkCWkP0acvMO//bBN7JZT157KZ3U90unB+2dV50wZcHh79rWMRUTLJPl8FzIw51CHr6u3KNWuZGQklqE/V01TfRH2qPnc04mcSuXLIXXza3/nXRRRLSPmnaGePsrMXqg6luS7/4YrL1n+xJJA96zM7nbFM7rqsPieLFPzf5/R5dv571NdH+6JK8LaYXbC9d/uOXwDhD1qfqu8zttZwXM3u3O4oJSEVOzLqSHYwvnH8TkdHAyWk7Paruf+ocIefvXas2M4//wzO7LyBkkEmk2Fb7zYSJPocHQ70yNVXOIIc7P9q48llEE210ThYI2pG5Jq1tiS3MLFpYoUjc678+muuG10/ut/XFOs/iqO5Lj+Wwl/92XVkLDqqKjx9P/86JaDPcEH5wwbl//pPKkkqmRpw55+/gy9Wlv1/S2oLU/efWvLfYU/kyWUQE0ZOqHQIzu1Rhto8NtT+o/xt5D+Sik7tHt84fsAdfX9HBC5enlyccxVVSnNdf5JKDnhU5YaP974555yLnScX55xzsfPk4pxzLnaeXJxzzsXOk4tzzrnYeXJxzjkXO08uzjnnYufJxTnnXOyUHRdnbybpdeClGFc5DngjxvXFpVrjguqNrVrjguqNrVrjguqNrVrjgoFj+5CZHVDKSveJ5BI3SavMrLnScRSq1rigemOr1rigemOr1rigemOr1rigfLF5s5hzzrnYeXJxzjkXO08upVlY6QD6Ua1xQfXGVq1xQfXGVq1xQfXGVq1xQZli8z4X55xzsfMjF+ecc7Hz5OKccy52nlwASQdKekzSc5LaJH09lO8v6b8lPR/+3y+US9IPJbVLelbSEXnrOj8s/7yk82OKLynpD5LuC88PkrQibOPnkmpDeV143h7mfzhvHVeG8g2STo0prjGS7pL0x1B3s6uhziR9I/wd10m6XVJ9pepM0k8lvSZpXV5ZbHUk6UhJa8Nrfqgh3FKxn9i+F/6ez0r6haQxg9WHpLmhrF3SgrzyonVeSlx5874pySSNq5Y6C+WXhjpok3RjNdSZpMMlPSVpjaRVko4e1jozs33+AUwEjgjTTcCfgOnAjcCCUL4A+G6Y/gzwANHdt48FVoTy/YGN4f/9wvR+McR3OfAz4L7w/A7grDD9I+CrYfprwI/C9FnAz8P0dKAVqAMOAl4AkjHEtRj4mzBdC4ypdJ0Bk4FNQENeXV1QqToDPgkcAazLK4utjoCngdnhNQ8Ap+1mbKcAqTD93bzYitZHeLwAHBw+A63A9IE+p6XEFcoPBB4iuiB6XBXV2YnAI0BdeD6+GuoMeDj73kI9LR/OOhu2Hfie9AB+CZwMbAAmhrKJwIYw/WPg7LzlN4T5ZwM/zivvs1yJsUwBHgVOAu4Lf9w32LEDmA08FKYfAmaH6VRYTsCVwJV568wttxtxjSLaiaugvKJ1RpRcXglfkFSos1MrWWfAhwu+9LHUUZj3x7zyPsuVElvBvC8At4XpovWRX5f5yw30OS01LuAu4DDgRXYkl4rXGVFC+HSR5SpaZ2F7X857nz8bzjrzZrECoVnk48AKYIKZvQoQ/h8fFsvuwLI6Qll/5bvj+8D/ATLh+VjgHTPrLbKN3PbD/L+E5csR18HA68B/Kmqy+w9JjVS4zsxsM3AT8DLwKlEdrKY66iwrrjqaHKbLESPAXxP9Si0ltoE+p0Mm6XPAZjNrLZhVDXV2CPCJ0Jz1uKSjSowt1joDLgO+J+kVou/ElSXGVVKdeXLJI2kkcDdwmZm9O9CiRcpsgPJS4zkdeM3MVu/CtoctriBFdBj+b2b2cWArURNPf4arzvYDPk/UDDEJaAROG2Abw1lngxlqLGWLUdJVQC9wW6VjkzQCuAq4utjsSsWVJ0XUjHQscAVwR+iTqHRsXwW+YWYHAt8AfhLKhyUuTy6BpBqixHKbmd0Tiv8saWKYPxF4LZR3ELX/Zk0BtgxQXqrjgc9JehFYStQ09n1gjKRUkW3kth/mjwbeKkNc2W11mNmK8PwuomRT6Tr7NLDJzF43sx7gHuA4qqPOsuKqo44wHWuMoSP3dGCehXaQEmJ7g/7rfKg+QvRjoTV8F6YAz0j6QAlxlaPOOoB7LPI0USvDuBJii7POAM4n+vwD3AkcnRdv+etsKG2Ne+uDKDMvAb5fUP49+na83himP0vfDrGnQ/n+RP0Q+4XHJmD/mGJsYUeH/p307fT7Wpj+e/p2Tt8RpmfQt2NxI/F06D8BfDRMXxPqq6J1BhwDtAEjwrYWA5dWss7YuS08tjoCVoZlsx2tn9nN2OYC64EDCpYrWh9Ev9o3hrJs5/SMgT6npcRVMO9FdvS5VEOd/R1wbZg+hKhpSZWuM+A5oCVMzwFWD2ed7fZOb294ACcQHeY9C6wJj88QtYE+Cjwf/s9WtICbic74WAs0563rr4H28Lgwxhhb2JFcDiY6e6M9fBizZ6nUh+ftYf7Bea+/KsS7gSGcHTNITIcDq0K93Rs+kBWvM+BbwB+BdcAt4ctdkToDbifq++kh+gX4lTjrCGgO7/MF4F8pOMGihNjaiXaO2e/Bjwarj/Bd+VOYd1VeedE6LyWugvkvsiO5VEOd1QK3hnU+A5xUDXVGtF9bTZS8VgBHDmed+fAvzjnnYud9Ls4552LnycU551zsPLk455yLnScX55xzsfPk4pxzLnaeXNxeQdLYMPrrGkn/I2lz3vNdHVn2PyV9dJBl/l7SvHiirg6SnpR0eKXjcHsXPxXZ7XUkXQN0mtlNBeUi+sxnir5wHyXpSeASM1tT6Vjc3sOPXNxeTdJURfd1+RHRBW4TJS0M97dok3R13rJPhntgpCS9I+kGSa2Sfi9pfFjmO5Iuy1v+BklPh3tzHBfKGyXdHV57e9jWTkcGko4KAx2ulvSApAmSasLzE8Iy35P0rTD9LUkrs+8ne0+NEMc/SXpC0npJzYruxfJ8SLTZemiTdEu4L8cdkhqKxHRaeL/PKLqfSGNeHOsV3f/ju7H+kdxeyZOL2xdMB35iZh+3aNTkBWbWTDR8+8mSphd5zWjgcTM7DPg90ZXLxcjMjiYasDCbqC4F/ie89gaiUbb7vkiqA34AnGlmRxJd4f1ti8ZDuxBYKOkUovHkvhNe9gMzOwqYFeKbm7fKbWb2CaLBCe8lGpJkFnCxdtzwazpws5nNArYDf1sQ03ii4WjmmNkRRCMvfF3SBKIrymeY2f8Cru+nLpzL8eTi9gUvmNnKvOdnS3qG6EjmUKKdbqFtZpYdbn410bhNxdxTZJkTiAYaxaIh4tuKvO5QorGnHpG0hminfmB4zbPh9b8kGoKjJ7xmjqSniYbz+FR4fday8P9aYK2Z/dnMthMNlZIddHCTmT0Vpm8NceY7jqgufhdimhfe01tEgzH+u6QvEI2A7dyAUoMv4tweL7czlDQN+DpwtJm9I+lWovHFCnXnTafp/7vSVWSZXbltroBnw9FGMTOJ7i2TbY4bQTSm0xFmtlnSdwrizsaRyZvOPs/GVdjBWvhcwINmdu5OwUrNRDfQO4toKPdT+n9rzvmRi9v3jALeA94Nw92fOsjypXgS+BKApFkUPzJaD0zWjvua10qaEaa/DIwkGqz0ZkmjgAaiRPGGpCbgzBLiOkg7bmR1dogz3++AT0k6OMTRKGla2N4oM7uP6L4gOzXzOVfIj1zcvuYZoh37OqJhz39bhm38C7BE0rNhe+uIjkJyzKxL0heBH4addwr4R0mvE/WxtIQjlB8D/2xmX5G0OKzrJaJRboeqDbhI0k+IRo1eWBDTnyV9Bfh53unb/w/YBtwT+okSwOUlbNvtY/xUZOdipuhmTykz2x6a4R4GptmO29dWIqapwF1m5tezuGHhRy7OxW8k8GhIMgL+tpKJxblK8CMX55xzsfMOfeecc7Hz5OKccy52nlycc87FzpOLc8652Hlycc45F7v/D0Krm9a0660rAAAAAElFTkSuQmCC\n", 527 | "text/plain": [ 528 | "
" 529 | ] 530 | }, 531 | "metadata": {}, 532 | "output_type": "display_data" 533 | } 534 | ], 535 | "source": [ 536 | "plot_learning_curve(tuned_model.model, 'New Model Learning Curve', experiment_2.get_pipeline().fit_transform(X_train), y_train, scoring=scorer);" 537 | ] 538 | }, 539 | { 540 | "cell_type": "markdown", 541 | "metadata": {}, 542 | "source": [ 543 | "High bias, and slightly higher variance. Can we improve it with feature selection?" 544 | ] 545 | }, 546 | { 547 | "cell_type": "markdown", 548 | "metadata": {}, 549 | "source": [ 550 | "### Feature Selection" 551 | ] 552 | }, 553 | { 554 | "cell_type": "code", 555 | "execution_count": 5, 556 | "metadata": {}, 557 | "outputs": [ 558 | { 559 | "data": { 560 | "text/html": [ 561 | "
\n", 562 | "\n", 575 | "\n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | "
agejobmaritaleducationdefaulthousingloancontactmonthday_of_weekdurationcampaignpdayspreviouspoutcomeemp.var.ratecons.price.idxcons.conf.idxeuribor3mnr.employed
025blue-collarsinglehigh.schoolnoyesnocellularjulmon61929990nonexistent1.493.918-42.74.9625228.1
140admin.marriedhigh.schoolnononotelephonejunthu9719990nonexistent1.494.465-41.84.9585228.1
251technicianmarrieduniversity.degreenoyesnocellularaugwed51259990nonexistent1.493.444-36.14.9645228.1
337blue-collarmarriedhigh.schoolunknownyesnocellularjultue42319990nonexistent1.493.918-42.74.9615228.1
454housemaidmarrieduniversity.degreeunknownyesnocellularaugthu29719990nonexistent1.493.444-36.14.9635228.1
\n", 719 | "
" 720 | ], 721 | "text/plain": [ 722 | " age job marital education default housing loan \\\n", 723 | "0 25 blue-collar single high.school no yes no \n", 724 | "1 40 admin. married high.school no no no \n", 725 | "2 51 technician married university.degree no yes no \n", 726 | "3 37 blue-collar married high.school unknown yes no \n", 727 | "4 54 housemaid married university.degree unknown yes no \n", 728 | "\n", 729 | " contact month day_of_week duration campaign pdays previous \\\n", 730 | "0 cellular jul mon 619 2 999 0 \n", 731 | "1 telephone jun thu 97 1 999 0 \n", 732 | "2 cellular aug wed 512 5 999 0 \n", 733 | "3 cellular jul tue 423 1 999 0 \n", 734 | "4 cellular aug thu 297 1 999 0 \n", 735 | "\n", 736 | " poutcome emp.var.rate cons.price.idx cons.conf.idx euribor3m \\\n", 737 | "0 nonexistent 1.4 93.918 -42.7 4.962 \n", 738 | "1 nonexistent 1.4 94.465 -41.8 4.958 \n", 739 | "2 nonexistent 1.4 93.444 -36.1 4.964 \n", 740 | "3 nonexistent 1.4 93.918 -42.7 4.961 \n", 741 | "4 nonexistent 1.4 93.444 -36.1 4.963 \n", 742 | "\n", 743 | " nr.employed \n", 744 | "0 5228.1 \n", 745 | "1 5228.1 \n", 746 | "2 5228.1 \n", 747 | "3 5228.1 \n", 748 | "4 5228.1 " 749 | ] 750 | }, 751 | "execution_count": 5, 752 | "metadata": {}, 753 | "output_type": "execute_result" 754 | } 755 | ], 756 | "source": [ 757 | "X.head()" 758 | ] 759 | }, 760 | { 761 | "cell_type": "code", 762 | "execution_count": 112, 763 | "metadata": {}, 764 | "outputs": [], 765 | "source": [ 766 | "# Create a copy of the data frame with categorical features only\n", 767 | "cat_ct = experiment_2.get_categorical_ct()\n", 768 | "X_fs = cat_ct.fit_transform(X_train)\n", 769 | "features = experiment_2.CATEGORICAL_FEATURES + experiment_2.NEW_CATEGORICAL_FEATURES\n", 770 | "X_fs_df = pd.DataFrame(X_fs, columns=features)" 771 | ] 772 | }, 773 | { 774 | "cell_type": "code", 775 | "execution_count": 221, 776 | "metadata": {}, 777 | "outputs": [ 778 | { 779 | "name": "stdout", 780 | "output_type": "stream", 781 | "text": [ 782 | "Number of features: 66\n" 783 | ] 784 | } 785 | ], 786 | "source": [ 787 | "ohe_features = experiment_2.get_categorical_pipeline().fit_transform(X_train, y_train)\n", 788 | "print('Number of features:', ohe_features.shape[1])" 789 | ] 790 | }, 791 | { 792 | "cell_type": "code", 793 | "execution_count": 211, 794 | "metadata": {}, 795 | "outputs": [ 796 | { 797 | "name": "stdout", 798 | "output_type": "stream", 799 | "text": [ 800 | "==> Starting 3-fold cross validation for nb_fs model, 26360 examples\n", 801 | "==> Elapsed seconds: 351.680\n", 802 | "Best nb_fs model: BernoulliNB(alpha=0, binarize=0.0, class_prior=None, fit_prior=False)\n", 803 | "Best nb_fs score: 0.658\n" 804 | ] 805 | }, 806 | { 807 | "name": "stderr", 808 | "output_type": "stream", 809 | "text": [ 810 | "/anaconda3/envs/py35-ds/lib/python3.5/site-packages/sklearn/naive_bayes.py:480: UserWarning: alpha too small will result in numeric errors, setting alpha = 1.0e-10\n", 811 | " 'setting alpha = %.1e' % _ALPHA_MIN)\n" 812 | ] 813 | } 814 | ], 815 | "source": [ 816 | "param_grid = [{\n", 817 | " 'nb_fs__alpha': [0, 0.01, 0.1, 1],\n", 818 | " 'nb_fs__fit_prior': [True, False],\n", 819 | " 'kbest__k': np.arange(1, ohe_features.shape[1]+1),\n", 820 | " 'kbest__score_func': [chi2, mutual_info_classif]\n", 821 | "}]\n", 822 | "\n", 823 | "ft_pipeline = Pipeline([\n", 824 | " ('cat', experiment_2.get_categorical_pipeline()),\n", 825 | " ('kbest', SelectKBest())\n", 826 | "])\n", 827 | "\n", 828 | "tuned_model = build_tuned_model('nb_fs', BernoulliNB(), X_train, y_train, param_grid, scorer, pipeline=ft_pipeline, cv_folds=3)" 829 | ] 830 | }, 831 | { 832 | "cell_type": "code", 833 | "execution_count": 212, 834 | "metadata": {}, 835 | "outputs": [ 836 | { 837 | "data": { 838 | "text/plain": [ 839 | "('kbest', SelectKBest(k=18, score_func=))" 840 | ] 841 | }, 842 | "execution_count": 212, 843 | "metadata": {}, 844 | "output_type": "execute_result" 845 | } 846 | ], 847 | "source": [ 848 | "tuned_model.pipeline.steps[-1]" 849 | ] 850 | }, 851 | { 852 | "cell_type": "markdown", 853 | "metadata": {}, 854 | "source": [ 855 | "The model achieves maximum performance with 18/66 features. At this point, we could consider adding numeric features and evaluating the performance of more powerful models if stakeholders are not satisfied with a \\$56,316 lift over the baseline heuristic-based model and are willing to incur additional costs to improve the model." 856 | ] 857 | } 858 | ], 859 | "metadata": { 860 | "kernelspec": { 861 | "display_name": "Python 3", 862 | "language": "python", 863 | "name": "python3" 864 | }, 865 | "language_info": { 866 | "codemirror_mode": { 867 | "name": "ipython", 868 | "version": 3 869 | }, 870 | "file_extension": ".py", 871 | "mimetype": "text/x-python", 872 | "name": "python", 873 | "nbconvert_exporter": "python", 874 | "pygments_lexer": "ipython3", 875 | "version": "3.5.5" 876 | } 877 | }, 878 | "nbformat": 4, 879 | "nbformat_minor": 2 880 | } 881 | --------------------------------------------------------------------------------