├── Model Validation Tutorial.ipynb ├── Model_Validation_Framework.pdf ├── ODSC East 2020 - Validation Monitoring Demo.ipynb ├── ODSC_East_2020_Validation_Monitoring_Training.pdf ├── README.md ├── best-weights-complex_model.hdf5 ├── best-weights-simple_model.hdf5 ├── model_0 ├── X_test.npz ├── feature_names.npy ├── history.npy ├── model.json ├── model_weights.h5 └── y_test.npz ├── model_1 ├── X_test.npz ├── feature_names.npy ├── history.npy ├── model.json ├── model_weights.h5 └── y_test.npz ├── model_2 ├── X_test.npz ├── feature_names.npy ├── history.npy ├── model.json ├── model_weights.h5 └── y_test.npz ├── psi.py └── requirements.txt /Model_Validation_Framework.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/Model_Validation_Framework.pdf -------------------------------------------------------------------------------- /ODSC East 2020 - Validation Monitoring Demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 11, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# basics\n", 10 | "import pandas as pd\n", 11 | "import numpy as np\n", 12 | "import os\n", 13 | "\n", 14 | "# keras deep learning\n", 15 | "import tensorflow as tf\n", 16 | "from keras.models import Sequential\n", 17 | "from keras.layers import Dense\n", 18 | "from keras.callbacks import ModelCheckpoint\n", 19 | "\n", 20 | "# preprocessing\n", 21 | "from sklearn.datasets import load_wine\n", 22 | "from sklearn.preprocessing import MinMaxScaler\n", 23 | "from sklearn.pipeline import Pipeline\n", 24 | "from keras.utils import np_utils\n", 25 | "\n", 26 | "# validation\n", 27 | "from keras.wrappers.scikit_learn import KerasClassifier\n", 28 | "from sklearn.model_selection import cross_val_score\n", 29 | "from sklearn.model_selection import KFold\n", 30 | "from sklearn.model_selection import train_test_split\n", 31 | "from sklearn import metrics\n", 32 | "from sklearn.neighbors import LocalOutlierFactor\n", 33 | "\n", 34 | "# interpretability packages\n", 35 | "import shap\n", 36 | "\n", 37 | "# basic adversarial example generator\n", 38 | "from art.classifiers import KerasClassifier\n", 39 | "from art.attacks import FastGradientMethod\n", 40 | "\n", 41 | "# fixes\n", 42 | "import warnings\n", 43 | "%matplotlib inline\n", 44 | "warnings.filterwarnings('ignore',category=FutureWarning)" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "# load data\n", 54 | "data = load_wine()\n", 55 | "X = data['data']\n", 56 | "dummy_y = np_utils.to_categorical(data['target'])\n", 57 | "\n", 58 | "# scale X\n", 59 | "scaler = MinMaxScaler()\n", 60 | "X = scaler.fit_transform(X)\n", 61 | "\n", 62 | "# create test dataset\n", 63 | "X_train, X_test, y_train, y_test = train_test_split(X, dummy_y, test_size=0.2, stratify=data['target'])" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 4, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# load simple model's weights\n", 73 | "simple_model = Sequential()\n", 74 | "simple_model.add(Dense(15, input_dim=X_train.shape[1], activation='relu'))\n", 75 | "simple_model.add(Dense(3, activation='softmax'))\n", 76 | "simple_model.load_weights(\"best-weights-simple_model.hdf5\")\n", 77 | "simple_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[\"accuracy\"])\n", 78 | "\n", 79 | "\n", 80 | "# load complex model's weights\n", 81 | "complex_model = Sequential()\n", 82 | "complex_model.add(Dense(50, input_dim=X_train.shape[1], activation='relu'))\n", 83 | "complex_model.add(Dense(50, activation='relu'))\n", 84 | "complex_model.add(Dense(3, activation='softmax'))\n", 85 | "complex_model.load_weights(\"best-weights-complex_model.hdf5\")\n", 86 | "complex_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[\"accuracy\"])" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 5, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "name": "stdout", 96 | "output_type": "stream", 97 | "text": [ 98 | "classification report, on a testing sample of 36\n", 99 | " precision recall f1-score support\n", 100 | "\n", 101 | " 0 1.00 1.00 1.00 12\n", 102 | " 1 1.00 0.93 0.96 14\n", 103 | " 2 0.91 1.00 0.95 10\n", 104 | "\n", 105 | " accuracy 0.97 36\n", 106 | " macro avg 0.97 0.98 0.97 36\n", 107 | "weighted avg 0.97 0.97 0.97 36\n", 108 | "\n", 109 | "\n", 110 | "ROC-AUC Score on Testing: 1.0\n", 111 | "Cross Entropy Loss: 0.15199867591986227\n", 112 | "\n", 113 | "classification report, on a testing sample of 36\n", 114 | " precision recall f1-score support\n", 115 | "\n", 116 | " 0 1.00 1.00 1.00 12\n", 117 | " 1 1.00 1.00 1.00 14\n", 118 | " 2 1.00 1.00 1.00 10\n", 119 | "\n", 120 | " accuracy 1.00 36\n", 121 | " macro avg 1.00 1.00 1.00 36\n", 122 | "weighted avg 1.00 1.00 1.00 36\n", 123 | "\n", 124 | "\n", 125 | "ROC-AUC Score on Testing: 1.0\n", 126 | "Cross Entropy Loss: 0.044354858682102836\n", 127 | "\n" 128 | ] 129 | } 130 | ], 131 | "source": [ 132 | "def multiple_metrics(model, y, x):\n", 133 | " print('classification report, on a testing sample of', x.shape[0])\n", 134 | " print(metrics.classification_report(np.argmax(y,axis = 1), model.predict_classes(x)))\n", 135 | " print('')\n", 136 | " print('ROC-AUC Score on Testing:', metrics.roc_auc_score(y, model.predict_proba(x)))\n", 137 | " print('Cross Entropy Loss:', metrics.log_loss(y,model.predict_proba(x)))\n", 138 | " print('')\n", 139 | " \n", 140 | "multiple_metrics(simple_model, y_test, X_test)\n", 141 | "multiple_metrics(complex_model, y_test, X_test)" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 25, 147 | "metadata": {}, 148 | "outputs": [ 149 | { 150 | "data": { 151 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGcCAYAAAB5kcI6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdfZRV5X3//fdkZkBvZ4BBZKFifML6+2mtVj7eFgsoRhP9heltDEZbIqJA6s94C1ETcjuAE8D4wJP6AzVVKyZg7DIa7SSYqC1UVoHYr1pcSlEUhdjiw1AchhAUhrn/2Bs9jmdgHs7ZjM7ntdZZc/a193Vd33NgrfNde3/3vkqam5sxMzMzy8qX9ncAZmZm1r04+TAzM7NMOfkwMzOzTDn5MDMzs0w5+TAzM7NMOfkwMzOzTJXt7wC+iOrq6pqrq6v3dxhmZmZZKmnrgT7zYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmSppbm7e3zF84ZTM3uUv1Sz17v11+zsEs/3inrEX7u8QWjVtclF+pkraeqDPfJiZmVmmnHyYmZlZppx8mJmZWaacfJiZmVmmnHyYmZlZpsr2dYCkZcAQYGdO88MRMb6zk0saC0yJiEGdHasDcx8I/BQ4BTgWmBYRM3P29wTuBM4GBgBbgH8ApkbEjqzjNTMz+6LYZ/KRmpH7w9yVSCoBSiNiVzu7NgMrgLuAm/PsLwPqgWpgHTAQeAzoCVzT4YDNzMy6ubYmH62SdAEwleTswSZgZkQsTvcNBO4DBgM9gJeASRHxvKQhwD1AD0nb0uFGpn+fiYiynDlqgaERcU663QxMAi4FTgRGAKskTQAmAkcA64HJEfFUvrjTsxfz0vE+cyYjIv4A1OQ0bZB0L3BV278dMzMza6lTNR+SzgXuJ0kE+gKXAfMlDc8Z/y7gSJJLFy8Aj0kqj4iVwJXA+oioSF/L2jH9OOBioAJ4MU08JgOjgSqSxOExSYW8pPMVYHUBxzMzM+t22nrmo0bS9Tnb50XEKpKzDHdExPK0/TlJi4AxwLMRsRHYuKeTpCkklyyOA9Z0MvbZEfFG+r5J0kRgekTsSQ6WSFoKXAJ0+pKRpEnAmYA6O5aZmVl31tbk46ZWaj6OBkZIujanrRRYDiCpHzAXOAvoA+xOjzmkQ9F+2lt5Ylkg6c6ctjLg7c5OJOl7JGdVzk4TKjMzM+ugztZ8bAAWRsSsVvbfDBwKnB4RmyRVAlv55Pnvu/P0aQRKJfWMiA/TtsPyHNey7wbgxoh4pF2fYB8kTQX+FjgzIl4t5NhmZmbdUWeTj9uBhZJWkdw5UgqcBJRERAC9gO3AFkkVwK0t+r8D9JfUKyK2pm2vAduA8ZLuBs4ARpHUi+zNPKBW0jqSuowDSApd6yNibb4O6e20JSS1KWWSDgCaImJnun8W8C2SxOONfGOYmZlZ+3Sq4DS9k2QCMIvkttRNJElARXrINKA/sJnkTpcVQFPOEEuBp4E3JX0g6cyIaAQuB64DGkjqSh5sQyz3ArcBD5A8k2MjyV045Xvp9irwR2AYcGP6/l4ASUcC15MUyq6WtC19vbKvWMzMzKx1Jc3NXv290Epm7/KXapZ69/66/R2C2X5xz9gL93cIrZo2uSg/UyX7PiThx6ubmZlZppx8mJmZWaacfJiZmVmmXPNRBHV1dc3V1dX7OwwzM7MsuebDzMzMuiYnH2ZmZpYpJx9mZmaWKScfZmZmliknH2ZmZpYpJx9mZmaWKScfZmZmlik/56MIvLaLtZfXP/n86MrrdbSmSOt4mLXk53yYmZlZ1+Tkw8zMzDLl5MPMzMwy5eTDzMzMMlVWiEEkLQOGADtzmh+OiPEFGHssMCUiBnV2rA7OL+Au4E+BTcCNEbFof8RiZmb2RVCQ5CM1IyJmFnC8gpFUApRGxK529usNPAnMBoYBw4FfSnojIlYWPlIzM7MvvkImH62SdAEwFTiW5OzBzIhYnO4bCNwHDAZ6AC8BkyLieUlDgHuAHpK2pcONTP8+ExFlOXPUAkMj4px0uxmYBFwKnAiMAFZJmgBMBI4A1gOTI+KpVkK/ENgO3BYRzcDTkn4JfAdw8mFmZtYBRa/5kHQucD9JItAXuAyYL2l4Tgx3AUcCA4AXgMckladnF64E1kdERfpa1o7pxwEXAxXAi2niMRkYDVQBNelcrV3SORl4MU089nghbTczM7MOKOSZjxpJ1+dsnxcRq0jOMtwREcvT9uckLQLGAM9GxEZg455OkqYA1wDHAWs6GdPsiHgjfd8kaSIwPSJWp21LJC0FLgHyXTKqBBpatH0A9OpkXGZmZt1WIZOPm1qp+TgaGCHp2py2UmA5gKR+wFzgLKAPsDs95pACxPRWnlgWSLozp60MeLuV/o3AUS3a+gBbCxCbmZlZt5RFzccGYGFEzGpl/83AocDpEbFJUiXJj/uex7TuztOnESiV1DMiPkzbDstzXMu+G0juVnmkjbGvBi5o0fbnabuZmZl1QBbJx+3AQkmrgBUkZz1OAkoiIkguYWwHtkiqAG5t0f8doL+kXhGx54zDa8A2YLyku4EzgFEk9Rh7Mw+olbSOJIE4gKTQtT4i1uY5/pfAbZK+D9xBcsfLhcC5bf70ZmZm9ilFLzhN7ySZAMwC6knudplHUgQKMA3oD2wmudNlBdCUM8RS4GngTUkfSDozIhqBy4HrSGoyJgIPtiGWe4HbgAeALSS1JlOB8laO/wD4X8BF6Tz3Alf6NlszM7OO86q2ReBVba29vKrt54dXtTVrlVe1NTMzs67JyYeZmZllysmHmZmZZco1H0VQV1fXXF1dvb/DMDMzy5JrPszMzKxrcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZ8nM+isBru3xxdYc1WD6Pa5d0ZV5XxboRP+fDzMzMuiYnH2ZmZpYpJx9mZmaWKScfZmZmlqmyQgwiaRkwBNiZ0/xwRIwvwNhjgSkRMaizY3Vw/guBWuAY4D/TWB7ZH7GYmZl9ERQk+UjNiIiZBRyvYCSVAKURsaud/f4CWASMBJYBXwcelbQxIn5X8EDNzMy6gUImH62SdAEwFTgW2ATMjIjF6b6BwH3AYKAH8BIwKSKelzQEuAfoIWlbOtzI9O8zEVGWM0ctMDQizkm3m4FJwKXAicAIYJWkCcBE4AhgPTA5Ip5qJfQLgd9GxD+n23WS/hX4W8DJh5mZWQcUveZD0rnA/SSJQF/gMmC+pOE5MdwFHAkMAF4AHpNUHhErgSuB9RFRkb6WtWP6ccDFQAXwYpp4TAZGA1VATTpXa5d0SvjsfctfAk5pRwxmZmaWo5BnPmokXZ+zfV5ErCI5y3BHRCxP25+TtAgYAzwbERuBjXs6SZoCXAMcB6zpZEyzI+KN9H2TpInA9IhYnbYtkbQUuATId8no18Bv0gRqKVAN/CXwVifjMjMz67YKmXzc1ErNx9HACEnX5rSVAssBJPUD5gJnAX2A3ekxhxQgprfyxLJA0p05bWXA2/k6R8QySVem8R2exvwwsF+KX83MzL4Isqj52AAsjIhZrey/GTgUOD0iNkmqBLbyyeWO3Xn6NAKlknpGxIdp22F5jmvZdwNwY3vuVomIhcDCPduS/g34p7b2NzMzs0/LIvm4HVgoaRWwguSsx0lASUQE0AvYDmyRVAHc2qL/O0B/Sb0iYmva9hqwDRgv6W7gDGAUSb3I3swDaiWtA1YDB5AUutZHxNqWB0sqA/4sPbYCuJ6kUHVeOz6/mZmZ5Sh6wWl6J8kEYBZQT3K3yzySH3OAaUB/YDPJnS4rgKacIZYCTwNvSvpA0pkR0QhcDlwHNJDUlTzYhljuBW4DHgC2kNSaTAXKW+lSCvwd8AHwe5JEZGhEvNuWz25mZmaf5VVti8Cr2n5xeVVbay+vamvdiFe1NTMzs67JyYeZmZllysmHmZmZZco1H0VQV1fXXF1dvb/DMDMzy5JrPszMzKxrcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZ8nM+iuDzurZLd1i35PPAa6t0fV6vxSwvP+fDzMzMuiYnH2ZmZpYpJx9mZmaWKScfZmZmlqmyQgwiaRkwBNiZ0/xwRIwvwNhjgSkRMaizY3Vw/oOAm4GLgErg98DfRMSL+yMeMzOzz7uCJB+pGRExs4DjFYykEqA0InZ1oN/jwB+B/zsifi/pGOAPRQjTzMysWyhk8tEqSRcAU4FjgU3AzIhYnO4bCNwHDAZ6AC8BkyLieUlDgHuAHpK2pcONTP8+ExFlOXPUAkMj4px0uxmYBFwKnAiMAFZJmgBMBI4A1gOTI+KpVkL/KvCXwMCI+G+AiFjfya/DzMysWyt6zYekc4H7SRKBvsBlwHxJw3NiuAs4EhgAvAA8Jqk8IlYCVwLrI6IifS1rx/TjgIuBCuDFNPGYDIwGqoCadK7WLumMAN4EfiTpPUlvSPqxpPJ2xGBmZmY5Cnnmo0bS9Tnb50XEKpKzDHdExPK0/TlJi4AxwLMRsRHYuKeTpCnANcBxwJpOxjQ7It5I3zdJmghMj4jVadsSSUuBS4B8l4z6AScATwJfJjlbsoTksstNnYzNzMysWypk8nFTKzUfRwMjJF2b01YKLAeQ1A+YC5wF9AF2p8ccUoCY3soTywJJd+a0lQFvt9K/EWgCaiLiQ2CdpAXA3+Dkw8zMrEOyqPnYACyMiFmt7L8ZOBQ4PSI2SaoEtvLJY1p35+nTCJRK6pkmBQCH5TmuZd8NwI0R8UgbY//3Vtr9bGUzM7MOyiL5uB1YKGkVsILkrMdJQElEBNAL2A5skVQB3Nqi/ztAf0m9ImJr2vYasA0YL+lu4AxgFEm9yN7MA2olrQNWAweQFLrWR8TaPMc/BtxCUvMxDTgc+N/A37f505uZmdmnFL3gNL2TZAIwC6gnudtlHkkRKMA0oD+wmeROlxUklzr2WAo8Dbwp6QNJZ0ZEI3A5cB3QQFJX8mAbYrkXuA14ANhCUmsyFchbQJrO8zWSO162AMuAnwOz2/ThzczM7DO8qm0ReFVb6wyvatv1eVVbs7y8qq2ZmZl1TU4+zMzMLFNOPszMzCxTrvkogrq6uubq6ur9HYaZmVmWXPNhZmZmXZOTDzMzM8uUkw8zMzPLlJMPMzMzy5STDzMzM8uUkw8zMzPLlJMPMzMzy5Sf81EEn9e1Xb5IPu/r1Hh9l/3La7eYdYif82FmZmZdk5MPMzMzy5STDzMzM8uUkw8zMzPLVNm+DpC0DBgC7Mxpfjgixnd2ckljgSkRMaizY3Vg7gOBnwKnAMcC0yJiZotj/i9gPrCn+u9R4OqI+GOWsZqZmX2R7DP5SM1o+cPcVUgqAUojYlc7uzYDK4C7gJtbOeYO4H8Ax6fHPw7MBf53x6I1MzOztiYfrZJ0ATCV5OzBJmBmRCxO9w0E7gMGAz2Al4BJEfG8pCHAPUAPSdvS4Uamf5+JiLKcOWqBoRFxTrrdDEwCLgVOBEYAqyRNACYCRwDrgckR8VS+uCNiBzAvHW9Hns91IPBtYGREvJu2TQXqJH0v7W9mZmbt1KmaD0nnAveTJAJ9gcuA+ZKG54x/F3AkMAB4AXhMUnlErASuBNZHREX6WtaO6ccBFwMVwItp4jEZGA1UATXpXB29pHM8cADwfE7bC8CBwJ90cEwzM7Nur61nPmokXZ+zfV5ErCI5y3BHRCxP25+TtAgYAzwbERuBjXs6SZoCXAMcB6zpZOyzI+KN9H2TpInA9IhYnbYtkbQUuAToyCWjyvRvQ07bnve9OjCemZmZ0fbk46ZWaj6OBkZIujanrRRYDiCpH0mNxFlAH2B3eswhHYr2097KE8sCSXfmtJUBb3dw/Mb0b2/gg5z3AFs7OKaZmVm319majw3AwoiY1cr+m4FDgdMjYpOkSpIf7j2PYN2dp08jUCqpZ0R8mLYdlue4ln03ADdGxCPt+gStexXYAZwK/HPa9ufAH4HXCjSHmZlZt9PZ5ON2YKGkVSR3jpQCJwElEREklye2A1skVQC3tuj/DtBfUq+I2HM24TVgGzBe0t3AGcAoknqLvZkH1EpaB6wmqdcYDNRHxNp8HST1JEmEvgSUSToAaIqInRHxx/QS0nRJL6ddpgM/dbGpmZlZx3Wq4DS9k2QCMAuoJ7nbZR5JESjANKA/sJnkTpcVQFPOEEuBp4E3JX0g6cyIaAQuB64jqbGYCDzYhljuBW4DHgC2kNSaTAXK99LtVZIzGcOAG9P39+bsn0SSDO15vQp8b1+xmJmZWeu8qm0ReFXb/c+r2lpneFVbsw7xqrZmZmbWNTn5MDMzs0w5+TAzM7NMueajCOrq6pqrq6v3dxhmZmZZcs2HmZmZdU1OPszMzCxTTj7MzMwsU04+zMzMLFNOPszMzCxTTj7MzMwsU04+zMzMLFN+zkcRfN7Wdvm8r4PS1Xhdlq7Pa7eYFYWf82FmZmZdk5MPMzMzy5STDzMzM8uUkw8zMzPLVNm+DpC0DBgC7Mxpfjgixnd2ckljgSkRMaizY3Vg7gOBnwKnAMcC0yJiZotjrgFGAycB/7U/4jQzM/ui2WfykZrR8oe5q5BUApRGxK52dm0GVgB3ATe3csx/AbcB/wO4vMNBmpmZ2cfamny0StIFwFSSswebgJkRsTjdNxC4DxgM9ABeAiZFxPOShgD3AD0kbUuHG5n+fSYiynLmqAWGRsQ56XYzMAm4FDgRGAGskjQBmAgcAawHJkfEU/nijogdwLx0vB2tHPOLdP/Y9n0rZmZm1ppO1XxIOhe4nyQR6AtcBsyXNDxn/LuAI4EBwAvAY5LKI2IlcCWwPiIq0teydkw/DrgYqABeTBOPySSXSaqAmnQuXyoxMzPrQtp65qNG0vU52+dFxCqSswx3RMTytP05SYuAMcCzEbER2Link6QpwDXAccCaTsY+OyLeSN83SZoITI+I1WnbEklLgUuALnnJyMzMrDtqa/JxUys1H0cDIyRdm9NWCiwHkNQPmAucBfQBdqfHHNKhaD/trTyxLJB0Z05bGfB2AeYyMzOzAulszccGYGFEzGpl/83AocDpEbFJUiWwlU8ewbo7T59GoFRSz4j4MG07LM9xLftuAG6MiEfa9QnMzMwsU51NPm4HFkpaRXLnSCnJbaklERFAL2A7sEVSBXBri/7vAP0l9YqIrWnba8A2YLyku4EzgFEk9SJ7Mw+olbQOWA0cQFLoWh8Ra/N1kNSTJBH6ElAm6QCgKSJ2pvvLSL6jcqAk3b+nWNXMzMw6oFMFp+mdJBOAWUA9yd0u80iKQAGmAf2BzSR3uqwAmnKGWAo8Dbwp6QNJZ0ZEI8ltrdcBDSR1JQ+2IZZ7SW6LfQDYQlJrMpUkcWjNq8AfgWHAjen7e3P2T0nb/g44Jn3/x33FYmZmZq3zqrZF4FVtuzevatv1eVVbs6LwqrZmZmbWNTn5MDMzs0w5+TAzM7NMueajCOrq6pqrq6v3dxhmZmZZcs2HmZmZdU1OPszMzCxTTj7MzMwsU04+zMzMLFNOPszMzCxTTj7MzMwsU04+zMzMLFN+zkcRfN7WdsnVFdZ56Y5ro3itETP7AvBzPszMzKxrcvJhZmZmmXLyYWZmZply8mFmZmaZKivEIJKWAUOAnTnND0fE+AKMPRaYEhGDOjtWB+Y+EPgpcApwLDAtImZmHYeZmdkXSUGSj9SMrvrDLKkEKI2IXe3s2gysAO4Cbi54YGZmZt1QIZOPVkm6AJhKcvZgEzAzIhan+wYC9wGDgR7AS8CkiHhe0hDgHqCHpG3pcCPTv89ERFnOHLXA0Ig4J91uBiYBlwInAiOAVZImABOBI4D1wOSIeCpf3BGxA5iXjrejAF+FmZlZt1f0mg9J5wL3kyQCfYHLgPmShufEcBdwJDAAeAF4TFJ5RKwErgTWR0RF+lrWjunHARcDFcCLaeIxGRgNVAE16VyZX9IxMzPrrgp55qNG0vU52+dFxCqSswx3RMTytP05SYuAMcCzEbER2Link6QpwDXAccCaTsY0OyLeSN83SZoITI+I1WnbEklLgUuALnnJyMzM7IumkMnHTa3UfBwNjJB0bU5bKbAcQFI/YC5wFtAH2J0ec0gBYnorTywLJN2Z01YGvF2AuczMzKwNsqj52AAsjIhZrey/GTgUOD0iNkmqBLbyyWNad+fp0wiUSuoZER+mbYflOa5l3w3AjRHxSLs+gZmZmRVMFsnH7cBCSatI7hwpBU4CSiIigF7AdmCLpArg1hb93wH6S+oVEVvTtteAbcB4SXcDZwCjSOpF9mYeUCtpHbAaOICk0LU+Itbm6yCpJ0ki9CWgTNIBQFNE7Mx3vJmZme1d0QtO0ztJJgCzgHqSu13mkRSBAkwD+gObSe50WQE05QyxFHgaeFPSB5LOjIhG4HLgOqCBpK7kwTbEci9wG/AAsIWk1mQqUL6Xbq8CfwSGATem7+/d11xmZmaWn1e1LQKvats5XtXWzOxzyavampmZWdfk5MPMzMwy5eTDzMzMMuWajyKoq6trrq6u3t9hmJmZZck1H2ZmZtY1OfkwMzOzTDn5MDMzs0w5+TAzM7NMOfkwMzOzTDn5MDMzs0w5+TAzM7NM+TkfRfB5XdulK6zrUiidWR/G66yYmXWIn/NhZmZmXZOTDzMzM8uUkw8zMzPLlJMPMzMzy1RZIQaRtAwYAuzMaX44IsYXYOyxwJSIGNTZsTow9/HAg8BxQDnwNnB7RPxd1rGYmZl9URQk+UjNiIiZBRyvYCSVAKURsaudXd8BLgXWR0STpD8DnpH0VkQ8VfBAzczMuoFCJh+tknQBMBU4FtgEzIyIxem+gcB9wGCgB/ASMCkinpc0BLgH6CFpWzrcyPTvMxFRljNHLTA0Is5Jt5uBSSTJw4nACGCVpAnAROAIYD0wubVEIiIagIacpub0dTzg5MPMzKwDil7zIelc4H6SRKAvcBkwX9LwnBjuAo4EBgAvAI9JKo+IlcCVJGceKtLXsnZMPw64GKgAXkwTj8nAaKAKqEnn2uslHUkvSfqQJDF6D/h5O2IwMzOzHIU881Ej6fqc7fMiYhXJWYY7ImJ52v6cpEXAGODZiNgIbNzTSdIU4BqSOos1nYxpdkS8kb5vkjQRmB4Rq9O2JZKWApcArV4yiog/k1QODE9ff+hkXGZmZt1WIZOPm1qp+TgaGCHp2py2UmA5gKR+wFzgLKAPsDs95pACxPRWnlgWSLozp62MpJB0ryJiJ/BPkr4JTAP+vwLEZ2Zm1u1kUfOxAVgYEbNa2X8zcChwekRsklQJbOWTx7TuztOnESiV1DMiPkzbDstzXMu+G4AbI+KRdn2CTysjOStjZmZmHZBF8nE7sFDSKmAFyVmPk4CSiAigF7Ad2CKpAri1Rf93gP6SekXE1rTtNWAbMF7S3cAZwCiSepG9mQfUSloHrAYOICl0rY+ItS0PlvQ14APgRZJC0/8FfBv4f9vx+c3MzCxH0QtO0ztJJgCzgHqSu13mkRSBQnIJoz+wmaSgcwXQlDPEUuBp4E1JH0g6MyIagcuB60juRplI8jyOfcVyL3Ab8ACwhaTWZCrJMzzy6QX8PfDfwPtALXBdRNzfho9uZmZmeXhV2yLwqrb7n1e1NTPLnFe1NTMzs67JyYeZmZllysmHmZmZZco1H0VQV1fXXF1dvb/DMDMzy5JrPszMzKxrcvJhZmZmmXLyYWZmZply8mFmZmaZcvJhZmZmmXLyYWZmZply8mFmZmaZ8nM+iqCYa7t0hfVXOrNuSlfjdVzMzArGz/kwMzOzrsnJh5mZmWXKyYeZmZllysmHmZmZZaqsEINIWgYMAXbmND8cEeMLMPZYYEpEDOrsWB2Y+yjgTWA7sKcy8YOIGJh1LGZmZl8UBUk+UjMiYmYBxysYSSVAaUTs6uAQx0fE24WMyczMrLsqZPLRKkkXAFOBY4FNwMyIWJzuGwjcBwwGegAvAZMi4nlJQ4B7gB6StqXDjUz/PhMRZTlz1AJDI+KcdLsZmARcCpwIjABWSZoATASOANYDkyPiqWJ9djMzM/u0otd8SDoXuJ8kEegLXAbMlzQ8J4a7gCOBAcALwGOSyiNiJXAlsD4iKtLXsnZMPw64GKgAXkwTj8nAaKAKqEnn2tclnd9Jel/SMklntWN+MzMza6GQZz5qJF2fs31eRKwiOctwR0QsT9ufk7QIGAM8GxEbgY17OkmaAlwDHAes6WRMsyPijfR9k6SJwPSIWJ22LZG0FLgEyHfJqJ6kluUFoBy4AnhS0ukR8VInYzMzM+uWCpl83NRKzcfRwAhJ1+a0lQLLAST1A+YCZwF9gN3pMYcUIKa38sSyQNKdOW1lQN56jojYBqxKNz8C/o+kvwIuIrk8ZGZmZu2URc3HBmBhRMxqZf/NwKHA6RGxSVIlsJVPHtO6O0+fRqBUUs+I+DBtOyzPcS37bgBujIhH2vUJPjtmmx8ha2ZmZp+WRfJxO7BQ0ipgBclZj5OAkogIoBfJraxbJFUAt7bo/w7QX1KviNiatr0GbAPGS7obOAMYRXJ5ZG/mAbWS1gGrgQNICl3rI2Jty4Ml/UU6z1qS72oMcCZwQzs+v5mZmeUoesFpeifJBGAWSQ3FJpIkoCI9ZBrQH9hMciljBdCUM8RS4GngTUkfSDozIhqBy4HrgAaSupIH2xDLvcBtwAPAFpJak6kk9Rz5HA08ns7xnyR3zlRHxPNt+exmZmb2WV7Vtgi8qu3nh1e1NTMrGK9qa2ZmZl2Tkw8zMzPLlJMPMzMzy5RrPoqgrq6uubq6en+HYWZmliXXfJiZmVnX5OTDzMzMMuXkw8zMzDLl5MPMzMwy5eTDzMzMMuXkw8zMzDLl5MPMzMwy5ed8FEEx13ZpjyzWgfk8rvPi9VzMzIrCz/kwMzOzrsnJh5mZmWXKyYeZmZllysmHmZmZZaqsEINIWgYMAXbmND8cEeMLMPZYYEpEDOrsWB2cfwwwGfgy8N/AA8CPIsJVi1/rMUgAACAASURBVGZmZh1QkOQjNSMiZhZwvIKRVAKURsSudvY7Gfh74EKgDjgeWAr8J3BvoeM0MzPrDgqZfLRK0gXAVOBYYBMwMyIWp/sGAvcBg4EewEvApIh4XtIQ4B6gh6Rt6XAj07/PRERZzhy1wNCIOCfdbgYmAZcCJwIjgFWSJgATgSOA9cDkiHiqldCPBd6LiH9Mt9emZ3lO7sTXYWZm1q0VveZD0rnA/SSJQF/gMmC+pOE5MdwFHAkMAF4AHpNUHhErgSuB9RFRkb6WtWP6ccDFQAXwYpp4TAZGA1VATTpXa5d0fgv8l6RvSPqSpD8FhgP/2MrxZmZmtg+FPPNRI+n6nO3zImIVyVmGOyJiedr+nKRFwBjg2YjYCGzc00nSFOAa4DhgTSdjmh0Rb6TvmyRNBKZHxOq0bYmkpcAlwGcuGUXEHyQ9APwUOBAoBW7dy5kSMzMz24dCJh83tVLzcTQwQtK1OW2lwHIASf2AucBZQB9gd3rMIQWI6a08sSyQdGdOWxnwdr7Oki4HaoFzgH8jOTvzc0kzImJqAeIzMzPrdrKo+dgALIyIWa3svxk4FDg9IjZJqgS28sljWnfn6dMIlErqGREfpm2H5TmuZd8NwI0R8UgbYx8M/HNE/C7dflPSYpLLOU4+zMzMOiCL5ON2YKGkVcAKkrMeJwElERFAL2A7sEVSBXBri/7vAP0l9YqIrWnba8A2YLyku4EzgFEk9SJ7Mw+olbQOWA0cQJJg1EfE2jzH/ytwh6TBaQHsEcC3gefb8fnNzMwsR9ELTtP6iAnALKCe5G6XeSRFoADTgP7AZpI7XVYATTlDLAWeJjnr8IGkMyOiEbgcuA5oIKkrebANsdwL3EbyrI4tJLUmU4HyVo7/eRr3w5IageeAl4Fr8x1vZmZm++ZVbYvAq9p2bV7V1sysKLyqrZmZmXVNTj7MzMwsU04+zMzMLFOu+SiCurq65urq6v0dhpmZWZZc82FmZmZdk5MPMzMzy5STDzMzM8uUkw8zMzPLlJMPMzMzy5STDzMzM8uUkw8zMzPLlJ/zUQRdZW2X1mSx5ksxFGodGa/tYmZWFH7Oh5mZmXVNTj7MzMwsU04+zMzMLFNOPszMzCxTZYUYRNIyYAiwM6f54YgYX4CxxwJTImJQZ8fqwNyjgZ+0aD4Q+HVE/FXW8ZiZmX0RFCT5SM2IiJkFHK9gJJUApRGxqz39ImIxsDhnnN7AfwGLChuhmZlZ91HI5KNVki4ApgLHApuAmekPO5IGAvcBg4EewEvApIh4XtIQ4B6gh6Rt6XAj07/PRERZzhy1wNCIOCfdbgYmAZcCJwIjgFWSJgATgSOA9cDkiHiqjR/lUqAR+GW7vwQzMzMDMqj5kHQucD9JItAXuAyYL2l4Tgx3AUcCA4AXgMcklUfESuBKYH1EVKSvZe2YfhxwMVABvJgmHpOB0UAVUJPO1dZLOn8L/H1E7NznkWZmZpZXIc981Ei6Pmf7vIhYRXKW4Y6IWJ62PydpETAGeDYiNgIb93SSNAW4BjgOWNPJmGZHxBvp+yZJE4HpEbE6bVsiaSlwCbDXS0aS/hI4AXCth5mZWScUMvm4qZWaj6OBEZKuzWkrBZYDSOoHzAXOAvoAu9NjDilATG/liWWBpDtz2sqAt9sw1t8CT0XEmwWIy8zMrNvKouZjA7AwIma1sv9m4FDg9IjYJKkS2Monj2ndnadPI1AqqWdEfJi2HZbnuJZ9NwA3RsQj7fkAkvoCF5GcITEzM7NOyCL5uB1YKGkVsILkrMdJQElEBNAL2A5skVQB3Nqi/ztAf0m9ImJr2vYasA0YL+lu4AxgFEm9yN7MA2olrQNWAweQFLrWR8TavfS7DKgHftWWD2xmZmatK3rBaXonyQRgFskP+CaSJKAiPWQa0B/YTHKnywqgKWeIpcDTwJuSPpB0ZkQ0ApcD1wENJHUlD7YhlnuB24AHgC0ktSZTgfJ9dP0OcF9ENO3jODMzM9sHr2pbBF7Vtji8qq2ZWZfmVW3NzMysa3LyYWZmZply8mFmZmaZcs1HEdTV1TVXV1fv7zDMzMyy5JoPMzMz65qcfJiZmVmmnHyYmZlZppx8mJmZWaacfJiZmVmmnHyYmZlZppx8mJmZWab8nI8i6O5ruxRqDZY9vBaLmXU3tbW1vP766yxatGh/h9IebX7OR1kxozAzM+sqSmbvKur4zde37yf1oYceYu7cuaxdu5bKykpOOeUUampqGDp0aJEibN3UqVN5/PHH+Y//+A+mTJlCbW1tUefzZRczM7OMzZ07l0mTJnHDDTfw7rvvsnHjRq666iqeeOKJ/RLPoEGDuO222/j617+eyXxOPszMzDLU0NDAtGnTWLBgARdeeCEHHXQQ5eXlVFdXM2vWrLx9LrroIgYMGEDv3r0ZPnw4r7zyysf7lixZwgknnEBlZSWHH344s2fPBqC+vp6RI0fSp08f+vbty7Bhw9i9e3fe8S+77DLOP/98KisrC/+B83DyYWZmlqGVK1eyY8cOvvGNb7S5z/nnn8+6det47733OPXUUxk9evTH+8aNG8dPfvITGhsbefnllzn77LMBmDNnDgMHDuT999/n3Xff5cc//jElJW0uyyiqgtR8SFoGDAF25jQ/HBHjCzD2WGBKRAzq7FgdmPsvgKmAgAOA14EZEfF41rGYmdkXw+bNm+nXrx9lZW3/Cb7iiis+fl9bW0tVVRUNDQ307t2b8vJy1qxZw8knn0xVVRVVVVUAlJeXs2nTJjZs2MCgQYMYNmxYwT9LRxXyzMeMiKjIeXU68SgUSSWSOpJo9QX+ATgRqAJmAD+XdFoh4zMzs+7j4IMPpr6+nl272lYA29TUxA9/+EOOPfZYevXqxVFHHQUkl1UAHn30UZYsWcKRRx7JmWeeycqVKwH4/ve/z6BBg/jqV7/KMcccwy233FKUz9MRmdztIukCkjMIxwKbgJkRsTjdNxC4DxgM9ABeAiZFxPOShgD3AD0kbUuHG5n+fSYiynLmqAWGRsQ56XYzMAm4lCR5GAGskjQBmAgcAawHJkfEU/nijoglLZoel7QaGAb8Wwe/DjMz68aGDBlCz549efzxxxk1atQ+j3/ooYd44okneOaZZzjqqKNoaGigqqqKPY/KOO2003jiiSfYuXMn8+fP51vf+ha///3vqaysZM6cOcyZM+fjyzGnnXYaX/nKV4r9Efep6DUfks4F7idJBPoClwHzJQ3PieEu4EhgAPAC8Jik8ohYCVwJrM85o7KsHdOPAy4GKoAX08RjMjCa5ExGTTpXmy7pSBpAksisbkcMZmZmH+vduzfTp0/nu9/9Lo8//jjbt29n586dPPnkk/zgBz/4zPGNjY307NmTgw8+mO3bt3PDDTd8vO+jjz5i8eLFNDQ0UF5eTq9evfjSl5Kf9l/96le8/vrrNDc307t3b0pLSz/e19LOnTvZsWMHu3fvZteuXezYsYOmpqbifAEU9sxHjaTrc7bPi4hVJGcZ7oiI5Wn7c5IWAWOAZyNiI7BxTydJU4BrgOOANZ2MaXZEvJG+b5I0EZgeEXuShyWSlgKXADP3NpCkg4BHgV9HxD91Mi4zM8tYe5/DUUzXXXcdAwYMYObMmYwePZrKykoGDx5MTU3NZ44dM2YMv/3tbzn88MPp27cvM2bM4O677/54/89+9jOuvvpqmpqaOP7441m8eDEA69at4+qrr+b999+nqqqKq666ihEjRuSNZ8KECTz44IMfb99000088MADjB07trAfPFXIf4mbIiLfD/jRwAhJ1+a0lQLLAST1A+YCZwF9gD33AR1SgJjeyhPLAkl35rSVAW/vbRBJlcCvgfdIkiYzM7NOGT169KfuWsmV+5CvioqKzzz/Y8yYT36KfvOb3+Qd43vf+x7f+9732hTLwoULWbhwYZuOLYQs0sANwMKIyH/zMtwMHAqcHhGb0h/6rXzymNZ8NyU3AqWSekbEh2nbYXmOa9l3A3BjRDzS1uAlHQw8SVIf8u2IKO4j8szMzL7gskg+bgcWSloFrCA563ESUBIRAfQCtgNbJFUAt7bo/w7QX1KviNiatr0GbAPGS7obOAMYRVIvsjfzgFpJ60jqNg4gKXStj4i1LQ9OazyeAZ4HroiI4l0AMzMz6yaKXnCa3kkyAZgF1JPc7TKPpAgUYBrQH9hMcqfLCiD3R34p8DTwpqQPJJ0ZEY3A5cB1QANJXcmD7ENE3AvcBjwAbCGpNZkKlLfS5W9JCkxHAQ2StqWvG1o53szMzPbBq9oWgVe19aq2ZmbdUJsfn+rHq5uZmVmmnHyYmZlZppx8mJmZWaZc81EEdXV1zdXV1fs7DDMzsyy55sPMzOzzqra2lm9/+9v7O4yi6TrPmjUzMyui9/7nL4s6fv//+Ea7jn/ooYeYO3cua9eupbKyklNOOYWamhqGDh1apAhb99Zbb3H55Zfzu9/9ji9/+cvMnz+fc845p2jz+cyHmZlZxubOncukSZO44YYbePfdd9m4cSNXXXXVZx6jnpW//uu/5s///M/ZvHkzN910E6NGjeL9998v2nxOPszMzDLU0NDAtGnTWLBgARdeeCEHHXQQ5eXlVFdXM2tW/pVILrroIgYMGEDv3r0ZPnw4r7zyysf7lixZwgknnEBlZSWHH344s2fPBqC+vp6RI0fSp08f+vbty7Bhw9i9+7Mrlrz22mu88MIL/OhHP+LAAw/km9/8JieddBKPPvpocb4AnHyYmZllauXKlezYsYNvfKPtl2nOP/981q1bx3vvvcepp576qQXpxo0bx09+8hMaGxt5+eWXOfvsswGYM2cOAwcO5P333+fdd9/lxz/+MSUln60JfeWVVzjmmGOorKz8uO3kk0/+VIJTaE4+zMzMMrR582b69etHWVnbyy6vuOIKKisr6dmzJ7W1taxevZqGhgYAysvLWbNmDVu3bqWqqopTTz314/ZNmzaxYcMGysvLGTZsWN7kY9u2bfTu3ftTbb1796axsbETn3LvnHyYmZll6OCDD6a+vp5du9q2SHpTUxM//OEPOfbYY+nVqxdHHXUUkFxWAXj00UdZsmQJRx55JGeeeSYrV64E4Pvf/z6DBg3iq1/9Kscccwy33HJL3vErKirYunXrp9q2bt36qTMhhea7XYrgr149H15t23+qYuvK67h4zRYz646GDBlCz549efzxxxk1atQ+j3/ooYd44okneOaZZzjqqKNoaGigqqqKPc/pOu2003jiiSfYuXMn8+fP51vf+ha///3vqaysZM6cOcyZM+fjyzGnnXYaX/nKVz41/oknnsj69etpbGz8OOFYvXo1f/M3f1P4D5/ymQ8zM7MM9e7dm+nTp/Pd736Xxx9/nO3bt7Nz506efPJJfvCDH3zm+MbGRnr27MnBBx/M9u3bueGGTxZW/+ijj1i8eDENDQ2Ul5fTq1cvvvSl5Kf9V7/6Fa+//jrNzc307t2b0tLSj/fl+pM/+RNOOeUUfvSjH7Fjxw5++ctf8tJLL/HNb36zaN+Bz3yYmVm30N7ncBTTddddx4ABA5g5cyajR4+msrKSwYMHU1NT85ljx4wZw29/+1sOP/xw+vbty4wZM7j77rs/3v+zn/2Mq6++mqamJo4//ngWL14MwLp167j66qt5//33qaqq4qqrrmLEiBF543n44YcZO3YsVVVVfPnLX+YXv/gFhxxySHE+PH68elGUzN7VZb5UX3YxM7OM+PHqZmZm1jUV5LKLpGXAEGBnTvPDETG+AGOPBaZExKDOjtXJOM4HlgD3F+JzmZmZdVeFrPmYEREzCzhewUgqAUojokO3oEjqDdwB/GtBAzMzM+uGMik4lXQBMBU4FtgEzIyIxem+gcB9wGCgB/ASMCkinpc0BLgH6CFpWzrcyPTvMxFRljNHLTA0Is5Jt5uBScClwInACGCVpAnAROAIYD0wOSKe2sdHmAvcD/zPDn8JZmZmBmRQ8yHpXJIf7klAX+AyYL6k4Tkx3AUcCQwAXgAek1QeESuBK4H1EVGRvpa1Y/pxwMVABfBimnhMBkYDVUBNOlerl3QkfQ04BZjdjnnNzMysFYU881Ej6fqc7fMiYhXJWYY7ImJ52v6cpEXAGODZiNgIbNzTSdIU4BrgOGBNJ2OaHRFvpO+bJE0EpkfE6rRtiaSlwCXAZy4ZSeoF3A18MyKaJHUyHDMzMytk8nFTKzUfRwMjJF2b01YKLAeQ1I/kssZZQB9gz5J7hbjB+K08sSyQdGdOWxnwdiv9ZwP/EBEvFiAWMzMzI5uajw3AwojIv04w3AwcCpweEZskVQJb+eR+4c+u/wuNQKmknhHxYdp2WJ7jWvbdANwYEY+0MfavAr3TyzWQXL5B0jkRcVQbxzAzM7McWSQftwMLJa0CVpCc9TgJKImIAHoB24EtkiqAW1v0fwfoL6lXROxZ+eY1YBswXtLdwBnAKJJ6kb2ZB9RKWgesBg4gKXStj4i1eY7/Cz79Hc0FdgHX5znWzMysIGpra3n99ddZtGjR/g6lKIqefETEU+mZg1nA8SRnI14BpqWHTAMWApuBd9Pt7+QMsRR4GnhTUinw/0TEv0i6HLgNuAX4DfAgSVKzt1julfQR8ADJJZidJAlL3mQiIt7J3Za0HdgVEf/Vpg9vZmZdxvRb2/wAzg5p71ObH3roIebOncvatWuprKzklFNOoaamhqFDhxYpwvzee+89Jk6cyL/8y7/whz/8gT/90z9l7ty5nH766UWbsyDJR0SctY/9vwZ+3cq+V0keUJZrUc7+ncBnVreJiF8Av9jLnHn/l0XEgySJSrtFxNiO9DMzM8s1d+5cbrnlFu655x6+9rWv0aNHD37zm9/wxBNPZJ58bNu2jdNOO425c+fSv39/7r//fr7+9a/z1ltvUVFRUZQ5/Xh1MzOzDDU0NDBt2jQWLFjAhRdeyEEHHUR5eTnV1dXMmpW/PPKiiy5iwIAB9O7dm+HDh/PKK698vG/JkiWccMIJVFZWcvjhhzN7dvJkiPr6ekaOHEmfPn3o27cvw4YNY/fuz5ZRHnPMMVx77bUceuihlJaW8p3vfIePPvqIV199tThfAE4+zMzMMrVy5Up27NjBN77R9lV2zz//fNatW8d7773HqaeeyujRoz/eN27cOH7yk5/Q2NjIyy+/zNlnnw3AnDlzGDhwIO+//z7vvvsuP/7xjykp2felp3//93/no48+YtCg4q1qkskTTs3MzCyxefNm+vXrR1lZ23+Cr7jiio/f19bWUlVVRUNDA71796a8vJw1a9Zw8sknU1VVRVVVFQDl5eVs2rSJDRs2MGjQIIYNG7bPebZu3cqll17KjTfeSO/evdv/4drIyUcR/OPxT1JdXb2/w0hc3/bMuiOm0b4CKzOz7u7ggw+mvr6eXbt2tSkBaWpqoqamhv+/vTMP16qq9/jnK1MgowpZIILXIUvJZNOVMsMGtVKjckwlsxxSnro5pTjEVVSc7/XmgBMgoSZ5A8EQh/Sp7Ll2V9nVMgfAo4kSBwWEMBlc94+1Xlhnc97hHN6zDxx/n+d5n3fvvebf+q21f2vYe8+YMYPGxka22SYsWixdupQ+ffpw//33M2HCBM477zyGDRvGxIkTGTlyJOeccw7jx4/noIMOAuCUU07hvPPOK5vOO++8w2GHHcZ+++3H+eefX5/ClsGWXQzDMAyjQEaOHEm3bt2YOXNmTf7vvvtuZs2axaOPPsqKFStoaGgAwPsw+BsxYgSzZs1iyZIljB49mqOOOgqAXr16ce2117Jw4UIeeOABrrvuOh577LFm03j33XcZPXo0gwYNYtKkSZtfyCqY8WEYhmEYBdKnTx8uueQSzjjjDGbOnMnq1atZu3Ytc+fO5dxzz93E/8qVK+nWrRvbb789q1evZty4cRvc1qxZw/Tp01mxYgVdunShd+/eG2ZG5syZw/z58/He06dPHzp16rTBLWXt2rUcccQRdO/enalTpzbrp97YsothGIbxvqCl7+FoS8466yx23HFHJkyYwHHHHUevXr0YPnw4F1xwwSZ+x4wZw7x58xg4cCDbbbcdl156KTfffPMG92nTpjF27FjWr1/PHnvswfTp0wF46aWXGDt2LI2NjfTr14/TTz+dAw88cJP4f/e73zFnzhy6d+9O3759N1yfO3duTftEWoNK0zZG/Zg9e7bfYvZ8GIZhGEYx1PwWN1t2MQzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUMz4MAzDMAyjUOzDcm1At27d/rxmzZp/tnc+tnQ6d+68w7p165a2dz62ZExG1TEZVcdkVB2TUXVqkNFS7/0hNcVVpzwZCXvvvfc/nXNZe+djSyfLMmdyqozJqDomo+qYjKpjMqpOPWVkyy6GYRiGYRSKGR+GYRiGYRSKGR9tw63tnYGtBJNTdUxG1TEZVcdkVB2TUXXqJiPbcGoYhmEYRqHYzIdhGIZhGIViT7u0gCzLdgemAtsDbwJjnHMv5fx0Am4ADgE8MNE5d3s1t45CHWQ0HjgdeD16f9I5d0YxuS+GGmV0EHA5sDfwX865sxM30yOqymg8pkdkWXYRcAywHlgLjHPOzYtuPYDJwHBgHXC2c25OcSUohjrIaQrwBaD0iOkM59xlxeS+GGqU0beBHwLvAZ2A25xzN0S3FvdJNvPRMm4BbnTO7Q7cCExqxs9xwK7AbsBIYHyWZUNqcOsobK6MAO5yzu0Tfx3qhhGpRUYLge8CVzfjZnoUqCQjMD0C+D0wwjk3DDgJ+FmWZd2j29nA2865XYHDgNuzLOtZQL6LZnPlBOFmWtKlDmV4RGqR0f3Ax51z+wCfAs7KsmxYdGtxn2TGR41kWTYA2Be4J166B9g3y7L+Oa9HEyzC95xzjcBM4Mga3LZ66iSjDk2tMnLOzXfO/YkwIs3ToeVXJxl1aFogo3nOudXx9BlAhNEtBD2aFP29BDjgS22c9UKpk5w6NC2Q0dvOudIm0R5AF8IsB7SiTzLjo3Z2AhY559YDxP/X4/WUwcAryfmriZ9Kbh2BesgI4Jgsy57JsuzhLMtGtmWG24FaZVQJ06PaMD1qyhhggXPutXje0fUI6iMngDOzLHs2y7KZWZbt2XbZbRdqllGWZYdnWfYXgt5c7Zx7Njq1WJfM+DC2NG4Bhsbpz6uBWVmWvS9GIEZdMT1KyLLss8ClwLHtnZctmTJyugDY1Tm3N/DfwENxj8P7DufcA865jwG7AydkWbZHa+My46N2/gYMLCld/P9wvJ7yKrBzcj448VPJrSOw2TJyzi12zq2Nx4/E63u1cb6LpFYZVcL0qAqmRxuJsz4/BUY7515InDq6HkEd5OScW+Scey8e3wX0BAYVkPeiaHF7c869Stgnc2i81GJdMuOjRpxzS4A/sdEiPhZ4Oq5vpcwATs6ybJu4ZjYa+HkNbls99ZBRlmUDS56yLNsHGAK8QAehBTKqhOlRFUyPAlmWjQB+BhzhnPtjLpoZwKnR327ACOChtsx30dRDTjldOpjwRMyitsx3kbRARnsmxzsABwKlZZcW90n2qG3LOA2YmmXZxcAywtogWZb9ErjYOeeAacC/AqXHlC5xzr0cjyu5dRQ2V0aXZ1k2nNDA1wAnOOcWF1mAAqgqoyzL9gfuBXoDyrLsGOA78fE/06PqMjI9Cm3tJqA7MCnLNnwP7IS4Vn81MCXLsvkEOZ3inFtZcBmKYHPlNDXLsg8SHjF9GzjcOdfRNjnXIqNT4uPtawkbcn/inHs4hm9xn2RvODUMwzAMo1Bs2cUwDMMwjEIx48MwDMMwjEIx48MwDMMwjEIx48MwDMMwjEIx48MwDMMwjEIx48MAQNLBkn6TnI+S1NCOWSoMSVMk1e2rsJKGSPLJeX9Jr0jaoYawp0maVq+8bA1I+oyk5e2dj/cjko5vSTuvd1sxKtNWbaMV9T5R0qX1zIMZHwaSBFwP/LiKv+9J+rOktyUtk+QkHZ24N0g6vplwm1xX4MUYV8+c2yhJXtKq+Htd0mRJ221eSdsH730jcDfV5bstcAkwvoBsbTF473/jve/b3vkoh6Txkh5t73y8H2grWUt6QtKF9Y63rcm3jXbUxSuBMyQNrOqzRsz4MAAOAroCj5fzIOlYws3zO0Afwut3f0h4IU1rOBDYhfDinua+N7Hee9/Te98T2J/wmeb/aGVaWwJ3At+W1LuCn+OBZ733CwrKUxMkdZJkfYJhGE3w3i8D5hLfiFsPrKMpmDgLcKGkx+Oo/llJwyQdK2m+pBWSbpfUOQkzWNLPJS2W9IakWyX1Stwvl7QwxrdA0r8lbkPiLMIJkp6TtFLSw5I+lGRrNPCor/zGuU8Bv/beP+UD70Sr/OEKYSpxKuFVztOootDe+4XAHOATeTdJnaNMRueuT5E0OR5/XtJTcbamUdK9kgaUSy/Ka//kfJSkdcl5Z0nj4szNcklPSsqaj21DGV4ClgJfqOBtNPBILi8/kPR8rLdXJV0hqVN0u1rSzJz/UdHvtvF8L0nzYrlL4btEt5JufEfSc8BqYICkYyT9X5yVekPSpFJ8MdyOkmZHXX0xhveShiR+To6zZCskPS3poHKFbka+UyRNk3RnlO+i2D72kfS/sXyPS/pwEqZB0sWSfhvbgZM0InGvqAOSusQ6fSHGv0DSEQoze+OAUdo4E7dLmXJ8NqaxItbZqYnbKEnrJB0d414h6b60HTcTX2v6imGSfhXLuTCG75S4fzLKZpWk3xIGAGmaPSRdI+llSW9JekjSruXy2Eyet5d0l0JftVjSVCUzlsrNgiY6OKicrCWdGMv7o6iPSyRd24weD0riPVHS/Hj8E+AzwEUxzmZfs68wq/CYpCujjrwp6UxJO0eZrpT0bLyEKAAACnRJREFUB0l7JmE2q60kun5bouub6E08riifXFmaLI/Vqd4fIfRR9cF7b78Cf0AD4RW0ewJdCB8yWgDcCmxL+CDPEuC46P8DwHzCdHx3oB/wS+DOJM7jCTMRAj4HvAMcHN2GAJ5w896B8CrqJ4HbkvBPAd/P5XMU0JCcHwn8E5gAfB7oW6Zsx1e7DvQH3gW+TjAoPDA8l/a65HxXwnc57iwj06uAmcl5T2AV8Jl4vj/huxWdgR2BXwP3JP6nALcn5x7Yv0J+Losy2wXoRJgNWgr0S2XeTD5nAxMq6MbfgcNz174BDI11+4no59To9lHCq8P7J/6nAnfE4wHAmwTjriswEHDAxTndeCzKpWssz5eAjxEGJ7sCzwFXJGk8BtwfdWkA8ESMZ0h0P5mgsx+PcXw51seuZcqdl+8Ugg5/JYY/LYZ/gPBBrx7Ar2iqww2Ez4APj+U4D2gEeteoA1fGcg6Lsh4EDItu4wnGeaV2PTTm+cSYxn7AW8CRSRk9cAdBPz9I6AcuqGNf0Sfqx0VAtxhuIXBO4v5mlE3XKI/FNG3n0wl9xQejn38Hnge6NNdWmsnzQwQ97xd/DwIPVugLhkS5DCon6yjTtcCNhD7wX4AXgXHNxZGEmZ+cPwFcWKUOx8d0vsvGdrAeeDRXB48kYTa3rUwh6M3hMY6vxzzsXKZtlJPP/Ny1DfVUj3qPfoYTZqq7VpJjrb/Cbrr2a9L4zknOvxyVMb2B3AdcH4+PABbk4hhOuHl3KpPGz4Gr4nGpYY5I3M8Ank7OXwROzMUxKlXOeO1Qwiel/x4b5ePAXrmy/QNYnvu9R9MO51xCp1nq0P4ITMql7WPYZcDLhE+kb2LwRP97Em7CA+L5ScCLFergUGBJcr6hocbzssYH4ca0EjggF+ezpTJS3viYDtxUIV9rgFFV9Oca4L7k/Cngh/G4V5T/p+P52cCvcuG/QeyoEt04oEqaY4Hfx+NBMcwuifvnadqh/hkYk4tjNmU6f5o3PtIbVo8Y/5HJtdNpqsMNwKXJuQhf2vxmNR2IflcBXynjdzzVjY9xwJO5a1cA83I6nbbzq4FfVIizgZb1Fd8kfElUifupwAvx+Lgok9T9MmI7JwxOPDA4cd8GWEFsD1QwPggDIA/sllzbI177UFKm1hgf7wI9kmvfJbbxfBxJmNYYH3/JXVvSTB0sq2NbmUKi6/FaI/DVMm2jnHwqGR+bXe/x2m7R34BKcqz1Zx+Wax/eSI5XE/Y3NOaulaZjhwKDtemOZ08YwS2S9H3CaHMQoSPtTtjgWC7NfyTxQ7jBV9qLEBL0fg7BOkbSRwgfY5ojaaiP2kkYlf80DadkV7Ukxbz+1Hu/Nl6+A5go6WzvfenDVut9jZsQvfd/lfRHwgzQdcC3gclJmsOBywkj8R4EGfVsJqpa2CGGna3kiRbCqKjaZ7Z7EwypcmxSDwp7bc4kzLJ0JoxK/ifxMhn4HmHD8FHAa977J6PbUODTOd0RYVSX0pBL84vAxcBHCCPoToROGMLsCYTOrMQrufiGAjdKuiG51hl4jdrZoK/e+9VBbTZpN/kli4YkjJf0KrFOquhAf8JMwostyF+endi0bhcAX03O8+083w6boyV9xU7AK0lbLOVhp3g8qBn3NM9D4/8zUd4luiRxVKLkJ41zQeL2Bq1nifd+dXLeQNt81j6fx9VU0Ls6tJXm0qxFL1pCveq9NxsHhZuN7fnY8nmFYOH3zf0+4L1fJOnThCnjU4Ed4g17NqFzrZWnCVP4NeO9f55ww9uZML1aK58jTE+eVFoXJkzx9SSM3FrLZODEuE65H3BX4nYvYXZld+99b5rf4JqyinAzKvHh5HgpoXP4Qq4+tvXeT6wS714EWZejST1I2okwzTuBMHLsQ5h6Tuv2XmB3SfsSRkCTE7dXCKOkNJ99fNjEm/JekmZXYGaMd3CU14+SNEufEh+chE+PS+melEu3p/f+exXKXg+GlA6ikTuYjQZPJR1oJNxUdisT73tlrqf8LU0/sku8XhR/A3ZW0ztImodFzbgPSY5LN8bdcnXXw3t/T43p5+PcJee2kvJtC8rLeoCkHrl8l+q2NGBpTbytpk5tpaU0V468TKFp+etV73sRZobWtDLvTTDjY8tnDtBVYTNcLwUGSvpadO9NWAJpBLykrxDWIVvCTMJ0YFkknSTpSMV3VcTNXacBz3nv32pBWqcS1ts/AuwTf3sRbpqntDDfKfcSjJobCGuyixK33oQpxJWSBhPWPivxB+BbkrrGjWFnlhzi6OE/gWsk7QYgqafCe1LyHd4GolHUn7B+XI6ZNN2Q2pPQRhuBtZL2A05IA3jvlwO/IBgo+xH2fJS4C8hi3X1A0jZxg9ohFfLQlTCCW+a9f0fSRwlTyaX0XiNMYU+M+tgfyD/CeD0wXmGDqCR1l7R/nC1rS06StK/CRsRzCDMcD0a3sjoQ6/Qm4CqFDbpS2AA5LHpZTJh97Foh7XuA4ZLGKGxI/iRB1++oawkr8yCh7sZF3d2DcDMs5WEOQafOUdhguy9hvxIA3vslhBnTmxQfqZTUV9LXlHscvjm8968DDwPXxnD9gGuBud770uj+D8Cxsc30J+xPSSkn622AK6Mu7UJYUpwa032TaPAqPLG1N2F2NR9vzRtna6QebaWlNCefPxGMs0NjG/8acEDiXq96/yKhj6oLZnxs4cSpxs8RRsTPEzrQxwg3bYB5hJvM7wmj8iMIN6OWMA9YJ2lUBT/LCNP7f5X0D8Jeg+WEtfOaUHi6YDRwjfd+cfojzN58QlWeGimH934FodxfIjzWmnIKYY14JWHPyowq0Y0ldFRvEdbUp+TcfwzMAmZJepuwKfA0Krenk4ApMZ/lmAZ8PHaueO//mqS1nHDDbG4EOplQ7nlJJ0+U64EEmTcQ6vAX5Ha6p3jvVxHq+SpJqwgzLfklvG8SbuyvETYvl+T5bozjNsIm4MkxzVcJN5kuFcpeD24lGJ/LgKMJezhK8q6mAxcQ6npm9PMEG29WMwgj98UKTyQMzYXFe/8yYT/AWMLmvmnARd77++pVuGrEsh5EMGD/zsa+4brovpywifdogoxuAG7ORXMyYXP3E5JWEvYyHUmYbq+F4wnye4HQXy0HxiTuFxIGS28QZHxvLnw5Wb9C0LeXCX3PQwQdK/EtQl+0IpY3b/RdTzDEl0v6S41lqUg92kor2EQ+Pjya/wOC/r8FHELY5FrK52bXu6S+BP2+pZX53gQ1XQYy3q/E0fA47/0B8XwU4WY5pD3ztTUSZ0te9t4rnvcnPGWS5dbrmwt7GmHD6AmV/G1JSDqYYCB19+3UoSjsK7owv9/I2PqRdCKhbus9c1E4W0JbaQ2SriDsN6rbi9psw6kBgPf+IcJowqgz0eDYuUa/t1DH0UVbIGkfwtrzs4TNahOAn21NnalhFEFHaSve+/PrHactuxjlaGDrfqNoe7KcsIm2o9KPsHSxCvgt8Axh2tcwjKZYWymDLbsYhmEYhlEoNvNhGIZhGEahmPFhGIZhGEahmPFhGIZhGEahmPFhGIZhGEahmPFhGIZhGEahmPFhGIZhGEah/D9xq36uvH2QuAAAAABJRU5ErkJggg==\n", 152 | "text/plain": [ 153 | "
" 154 | ] 155 | }, 156 | "metadata": { 157 | "needs_background": "light" 158 | }, 159 | "output_type": "display_data" 160 | } 161 | ], 162 | "source": [ 163 | "# Interpretability\n", 164 | "background = X_test\n", 165 | "e = shap.DeepExplainer(complex_model, background)\n", 166 | "shap_values = e.shap_values(X_train)\n", 167 | "shap.summary_plot(shap_values, X_train, plot_type=\"bar\")" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 27, 173 | "metadata": {}, 174 | "outputs": [ 175 | { 176 | "data": { 177 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAFACAYAAACLCsRFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXxU1fn48c+ZyWRfgEAI+yKyKALCQXHfBWlt3euGVtsi1trqt9Zaq9al1v5stdVKK6hFxdZa9w3XuovbUUAQ2fc1EEjInsnM+f1xh2SSTJKZySSZmTzv1+u+MvfOvec+A4FnzrlnUdZahBBCCNH5XF0dgBBCCNFdSRIWQgghuogkYSGEEKKLSBIWQgghuogkYSGEEKKLSBIWQgghukhKVweQCK6++mr7t7/9ravDEEKIRKNiW9pZzcfU2udie49OJjXhMJSUlHR1CEIIIZKQ1ISFEEIkiISu9IYkSVgIIUSCkCQshBBCdBFJwkIIIUQXSb4kLB2zhBBCiC4iNWEhhBAJIvlqwpKEhRBCJAhJwnFLa/1n4GxgKHCIMWZZiHPcwP3ANMACfzTGPNyZcQohhIhW8iXhZHom/AJwLLCxlXMuAkYABwJHALdqrYd2fGhCCCHaT4XYElvSJGFjzEfGmM1tnPYD4CFjjN8YswsncZ/b8dElBn9NHRVvbqBm6a6uDkUIIUJIviScNM3RYRpM45ryJmBQF8USV2ydn22nPkPVB1tAQd9508i9dGxXhyWEEEESP+k21d2ScNi01jOBmQCFhYVdHE3Hq121x0nAABb2/XOZJGEhRFyxIZJwoqfl7paENwFDgC8C+01rxvWMMXOBuQAzZsxovnJHkknpn42rRxr+khoAPGPyuzgiIYRIft0tCT8N/ERr/RyQD5wBHNO1IcUHd490Brx1LiX3f0XKgGx63XxEV4ckhBBNJHq9t7mkScJa6/uBs4BC4G2tdbEx5mCt9QLgFmOMAeYDhwOrA5fdboxZ3zURx590XUjh49O7OgwhhAgpGZujlbVJ39LabjNmzLDz58/v6jCEECLRxDRH+tUPmyUsl300ofNw0gxREkIIIRKNJGEhhBAJIrpxwkqpkUqpT5RSqwI/D2zhvPOUUkuVUssCP/vGLPQWSBIWQgiRECyq2RamB4HZ1tqRwGxgTtMTlFIauBU4xVo7FjgaKI1N5C2TJCyEECJBRF4TVkoVABOBJwOHngQmKqX6NDn1WuDP1todANbaUmttdYwCb5EkYSGEEAnBhtjCMAjYaq31AQR+bqP5bIkHAcOVUh8opb5SSt2klOrwTl+ShIUQQiSI5jVhpdRMpZQJ2mZGWbgbGAecAhwHnAbMiEXUrUmaccJCCCGSW6hnwNba+tkNW7AZGKCUcltrfUopN9A/cDzYJuAZa20NUKOUehE4DHg8JsG3QGrCQgghEkTkz4SttUXAYuCCwKELgEXW2qbLxf0bOFU5PMBJwJIYBd4iScJCCCESQjt6R88CrlZKrQKuDuyjlFoQ6BUN8B+gCFiOk7S/AR6J6QcIQZqjhRBCJIjo+klZa1fgTFnc9Pj0oNd+4P8CW6eRJCyEECIhRFDzTRjSHC2EEEJ0EakJCyGESAjJWBOWJCyEECJBSBIWQgghuoTUhIUQQoguI0lYCCGE6BLJWBOW3tFCCCFEF5EkLIQQQnQRaY4WQgiREJKxOVqSsBBCiAQhSVgIIYToElITFkIIIbpIMiZh6ZglhBBCdBGpCQshhEgIyVgTliQshBAiQUgSFkIIIbqE1ITjmNZ6JPAYkA8UA5cYY1Y3OacAmAcMAjzAu8DPjTF1nRyuEEKICCVjEk6mjlkPArONMSOB2cCcEOfcCHxrjBkHjAMmAWd1XohCCCGip0JsiS0pknCghjsReDJw6Elgota6T5NTLZCjtXYBaUAqsLXTAhVCCBE1G2JLdEmRhHGal7caY3wAgZ/bAseD3QGMBLYDO4A3jDEfd2agQgghomNRzbZElzTPhMN0LvA1cBKQA7ymtT7HGPNM0xO11jOBmQCFhYWdGqQQQohQEj/pNpUsNeHNwACttRsg8LN/4Hiwq4F/GWP8xphS4EXghFAFGmPmGmO0MUb37NmzA0MXQggRjmSsCSdFEjbGFAGLgQsChy4AFhljdjU5dT0wDUBrnQqcDCzrrDiFEEJET5JwfJsFXK21XoVT450FoLVeoLXWgXOuAY7RWi/FSdqrgIe6IlghhBAiaZ4JG2NWAIeHOD496PVa4JTOjEsIIYRoSdIkYSGEEMktGZqfm5IkLIQQIiFIEhZCCCG6jCRhIYQQokskwwxZTUkSFkIIkRCkOVoIIYToIpKEhRBCiC4jSVgIIYToElITFkIIIbqIdMwSQgghuojUhIUQQoguI0lYCCGE6BJSExZCCCG6SDI+E06mpQyFEEIksWjXE1ZKjVRKfaKUWhX4eWCIc25VShUppRYHttkx/wAhSBIWQgiREKJNwsCDwGxr7UhgNjCnhfMet9ZOCGxXxSLmtkgSFkIIkbSUUgXARODJwKEngYlKqT5dF1UDScJCCCEShGq2KaVmKqVM0DazyUWDgK3WWh9A4Oe2wPGmzldKfa2UelMpdUQHfpB60jFLCCFEQgjVMctaOxeYG4PiHwTutNZ6lVKnAC8qpcZYa4tjUHaLpCYshBAiIUT5THgzMEAp5QYI/OwfON5QtrU7rLXewOu3Au+PjekHCEGSsBBCiIQQTRK21hYBi4ELAocuABZZa3cFn6eUGhD0egIwFFgZq9hbIs3RQgghEkI7JuuYBTymlLoF2AtcAqCUWgDcYq01wB+UUpMAH1ALzLDW7mh/1K2TJCyEECIhRDtZh7V2BXB4iOPTg15fGm1c7SFJWAghRIKQaSuFEEKILiFzRwshhBBdRJKwEEII0UWScQEHScJCCCESgtSE45jWeiTwGJAPFAOXGGNWhzjvPOBmnCf8FjjZGLOzM2MVQggRuWRMwsk0WceDwGxjTIurZGitNXArcIoxZixwNFDamUEKIYSIjg2xJbqkSMJa65CrZGitm66ScS3wZ2PMDgBjTKkxprrzIhVCCBGtdixlGLeSpTl6ELDVGOMDMMb4tNb7V8kInprsIGC91voDIBt4DrjTGJMMX6iEECKpJUPSbSpZknC43MA44BQgFXgd2AQ83vRErfVMYCZAYWFhJ4YohBAilGSsLSVFczSBVTK01m6AwM9mq2TgJNxnjDE1xpgy4EXgsFAFGmPmGmO0MUb37NmzA0MXQggRjmRsjk6KJGyMCblKhjFmV5NT/w2cqrVWWmsPcBKwpPMiFUIIIRokRRIOmAVcrbVeBVwd2EdrvSDQKxrgP0ARsBwnaX8DPNIFsQohhIhQMtaE23wmrLVOARYBk+O5J7ExJuQqGcaY6UGv/cD/BTYhhBAJpFs+EzbG1AE9SM7PL4QQIkEkY0043Obo+4A7A7ViIYQQoguoEFtiCzepXgEMBa7UWm8H/PvfCMxQJYQQQnSoZKj5NhVuEv59h0YhhBBCtCEZn4mGlYSNMY91dCBCCCFEa7pzTRit9WTgcpypIDcD/zTGfNFRgQkhhBDBkrEmHFbHLK31GcAHQB7OcKVc4H2t9ZkdGJsQQghRz49qtiW6cGvCvwPONsYs2H9Aa30a8Efg+Y4ITAghhAiWjM3R4Q5RGoqz2EGwN4AhMY1GCCGEaEF3Xk94I3Byk2Mn4SyIIIQQQnS4ZJysI9zm6DuAF7XWzwDrcWrGZwOXdlBcQgghRCPJkHSbCqsmbIx5FqfmWwlooAo4xRjzTAfGJoQQQtRLxuboFmvCWutnjTFnB15fZoyZByzstMiEEEKIIPFUE1ZKpQIjcUYL7QNWWWtrIy2ntebok4Je3wfMi6RgrXUa8P+A43HW7P2tMWZL0PtLjTGHRFKmEEKI7isekrBSKg24F+dxbGbQW5VKqceA/7PW1oRbXmtJ+But9ZPAUiBVa31jqJOMMX9o4fo7gKOBf+AkdKO1PsEY823g/aHhBimEEELEiQeAg4DvAV8BpThzaEwEbgP+BswMt7DWkvDFwA3ACYAbOCXEORZoKQmfBxxjjNkMzNFaXwP8T2t9rDFmDcnRnC+EEKKTxEnSOBsYaa3dHXSsBHhHKbUMWEEskrAxZj3O6klorRcbY06IMNBeQH3zszHmr1prBbyjtT4uwrKEEEJ0c/HQHE3r6yfaNt5vJtze0RMiKTRgBzCiSTl/AR4C3gU8UZQphBCim4qT3tHPAy8opY5XSuUCKKVylVLHA88GtrCFO1lHNN4BLml60BhzB/A0kNaB9xZCCJFk4mSyjquAb4EFwF6llA/YC7yK0xT980gK68gk/HPgrlBvGGN+hXTMEkIIEYF4SMLW2ipr7U9wHrlOBI4L/My31s601lZGUl7YSxlGyhhTC7Q4ZsoYI1NeCiGECJu/qwMIYq2txhl+2y4dWRNukdY6RWv9z664txBCiMRkXarZFk+UUilKqYhyW1g1Ya21G/gNzuDkAmNMntZ6KjDMGPNg5KHiDpR1eRTXCiGE6IZsfOXcUCLObZEs4HAy8Gtgf5ZfhbOecMgk3NLkHhHeVwghhACIi5qvUiqmuS3cCy4EjjDGbNdaPxw4toHWO1fdAXwGhJq+q0uawYUQQiQuG2XmUEqNBB4D8oFi4BJr7eoWzh0FLAL+bq29LsQpMc1t4SbhTKCoybFUoLqVa9YCvzHGvN/0Da11OlAR5r2FEEIIrDvqmvCDwGxr7RNKqYuBOcCJTU9SSrkD773QSllrgd9Ya5vlNqVUxLkt3Kz9FXBZk2MXAp+3cs1ioKVJPvyA9I4WQggRNr9LNdvaopQqwBlC9GTg0JPARKVUnxCn3wC8gvO4tSUxzW3h1oSvA97TWp8PZGqtX8ZZV7i1qSwvoYUJTQLDl4ZFEmhbtNbNmhuMMSGbG7TW9c0NxphQzQ1CCCHiTJTN0YOArdZaH4C11qeU2hY4vmv/SUqp8cBUnLx2cyvltZjbAksZRpTbwp22chkwBngNeBj4AJhgjFnRyjXVxpiwl3OKgQeB2caYkcBsnCaFZgI9vdtqbhBCCBFnQg1RUkrNVEqZoC3sxRP2U0p5gLnArP3JusUYrK2OZKnCtoTdk8sYswu4J1Y3jiWt9f7mhv0rPT0JPKC17hOIO9j+5obswCaEECIBhBqiZK2di5NAW7IZGKCUcgdqwW6gf+D4fv2AA4AFSimAHoBSSuVaayNO6pEId5xwi12yW1lPuDMNArYaY3wAxhif1rpZc4PWOtzmBiGEEHEmmiFK1toipdRi4ALgicDPRdbaXUHnbAJ6799XSt0KZLfQOzqmwq0JN11LuD9Ou/dHtLyecFzRWu9vbrgskKTbOn8mgTUhCwsLOz5AIYQQHWUW8JhS6hacxRYuAVBKLQBusdaargosrCQcai1hrfXPgFC9y7rCZmCA1todSLCtNjcEEnAPQGmtc40xzZobjDH1TRwzZsyIk7WkhRCi+/JHOULJWrsCODzE8ektnH9rdHeKXHtmrvoHsA34XVsnaq2HA+cDA4wxVwV6J6cYY75px/3rGWOKtNbNmhuCnwcHFoyob27QWt8KZEvvaCGESAzxMGNWMKVUfW6z1l4VmOgjxVobdm5rz8xV46HtdaS01qfgrDQxBZgRONwb+HM77h3KLOBqrfUq4OrAPlrrBbqttmchhBBxz6rmW1dRSsUkt4XbMestGo+LysLpjXxvGJf/ETjXGPO61npv4NhXgetjJjBcqllzgzEmZHODMebWWN5fCCFEx7IqrmrCfwTOtda+rpSKOreF2xz9UZP9MuDGUFNShnCAMeb1wGsLYIypCnSUEkIIIcIS7TPhDnKAtbZRbrPWVgXGHIct3I5Zt0UYXLDNWuuxgQk/gPqhQhvaUaYQQohuJs6eCW9WSo211tbntsCsWxsiKaTFJKy17h9OAcaYbW2ccj/wnNb6dsCttT4buBW4O9wghcNay3UvVvHC0lqOGJrCw+dnke6Jq19KIYToMHG2nvD9wHNKqdsBt1IqqtzWWk14Cy3MjxmgAu+7W7uBMeahQL+oXwfOvQ34qzFmfiSBCnhxqZd733MWrlpXXMu4/m6uPymji6MSQojOEU/PhK21DwVm12qU26y1EeW21pJwuxdYCIzXnQQ8aox5qL3ldXdlNbbVfSGESGbx8kw4MPXlJOBRa227cluLSdgYs7E9BQfK8Gmt30XmaI6Jc8an8sinNby/to4xfV1cdXR6V4ckhBCdJl5qwoE5qGOS28KerENrPRo4HmeWrPo/CWPM7W1cuhwYgnTEareMVMV7V+eyt9JPjwyFipNfSCGE6Axx9kw4Jrkt3HHCFwCPAl8D4wI/x+MsadiW+cALWus/ARtxFj0GwBizMMJ4BdAzsz1zrAghRGLyx1fFYz7wglKqWW6z1oad28KtCf8WmGGM+a/Weq8xZrLW+nJgdBjX/jUo4GBtduoSQggh9ouzmnBMclu4SXgw8HSTY4/jLJBwfWsXGmOk2iaEEKLd4uWZMIC1Nia5LdxCSoC8wOudWusxQC+c6SuFEEKIDmeVarYlunBrwm8DZwLzgP8G9r3Aa21dGGLe6XrGmFPDvL8QQohuLp6ao5VSLeY2a23YuS3caSsvD9r9HbACyAUeC+PypvNO9wfOwenoJYQQQiSimOS2cHtHDw6sx4sxxgL/DvcGoead1lrPx1luUAghhAhLPM0dba1tltuUUhHntnCbo9cFJt14BHjeGFMTyU1C+Bh4tZ1lCCGE6EYS4BlwxLkt3I5ZBwILgbuA7Vrrv+vAhNCRCixhOAvYHc31QgghuifrUs22eBFYwjDi3BbuM+H1OM+Cf6e1Pgm4FHhXa73OGDO+tWu11l4aP7x2A+XAZZEEKoQQopuLo5qwUiomuS3saSuDvIfTKWsQcGwY55/cZL8MWGWMKY/i3t1STZ0lLSV+fvmEEKIrxFPNlxZym7U2otwWydzR43Ay/IVADc5kHT8J41JrjGk2vaXW+hhjzIfh3r878Nf5caU0PCH4dqePqXPK2FxiueKIVB48V4ZlCyG6rzh7Jmyttc1ym1LqGGtt2Lkt3N7Ri4BRwEvAJcBbxhh/61fVewWn5tzUizgTfnR7NWVeXvmZYceSEgZNyWf6fZNISXNz6xtVbC5xWjvmfFLLjw5PY/Jg569s5SYvW3bVMXl0GrlZMimZECL5WRVX/9fFJLeFWxN+CPi3MaYk3IKDNPvqorXOIWiy6+7um2c2sWOJ80e7+dNiVr6yjYPPHkSGp/Ef3f79d7+q5paHS/BbGFTg5pHf5JOVHle/nEIIEXNx1hzdLBilVMS5LdyOWX+PpFAArfVqnIfWGVrrVU3eLgDeirTMZOX2NE6g7lRn/w/TM9iwx8+a3T5+fkw6Y/s5c4K/barwB7oDbC7ysWKjl0mj0jo1ZiGE6Gzx0BytlKrPbUqpdue2aDpmhev3ON8U/gHcGXTcD+wA3u3AeyeUg88dzLZFe9n21R6GHFXAyNP6AdA/z8V7V+U0O3/kIA/vLXKGamd4oPDRd/H2SiHl58eh0jydGrsQQnSars/BEOPc1mFJ2BjzGIDWeoUx5tOOuk8ySElzc9o9E8M+f8bULDLSFJt2eDnpnifp9dVy6gC7fAep3xsBD7wGwwrgL5dDTkbHBS6EEJ0oHmrC1trHAJRSK6y17c5tHVkTBsAY86nW2o0z4Ucfgr7LhOo1LdrmcinOOzELu3Mf1Rctrz/u/3Al/Osl8NY5B9I8MHtm/fule7xkZLlJTZPnx0KIxBNPz4SttZ8qpULmtlC9plvSZhLWWo8ADgGWGGPWRRqo1noi8BzOmsQ2EKgFfEBqpOUlKn9JFTVzP0elp5B2xeGotBh8/ynIwXXUcPwfO38tKUcNhbXfNLy/2Zm4xVrL/L9tw3y4j4xMF1f8ZhDDR2e2//5CCNGJ4qEmvJ9SKia5rdUqkdb6LOBb4FlgudZ6ehSx/hV4Hmc94n04XbrnAD+MoqyEVTZtHlW/fp3KX7xCxSX/jUmZSilS37wKz+MzSH35ClIe+SFMneC8mZkG15wOwLZNNZgP9wFQVennzecaz6pWsWYfn5/+Pz456Q32LCyKSWxCCBFrcbaecExyW1vVsZuAG4G/Az8LvF4QYaCHAKcYY2q01soYU661vh5YTASrMbVFaz0SZ2nFfKAYuMQYs7rJOTcD5+N8U/ECNxpj3ohVDC2xNXX4Pttcv+/9YAMAH8zbyOJXdpJbmMbZt48mJ7/1Hs7Wb7EWXO6GXzyVmUrKjMMaTnr1t7B8C/TNg4IeAGRkunG5wB/oOJ+Z7W5U7uLLPqL0i2IAvjznXU7ech4qjpp9hBAC4qsmTCC3WWtrlFLKWluulIo4t7X1cHAYcI8xpgK4FxgRRaDeoNelWuuCwLHCKMpqzYPAbGPMSGA2zjeSpj4HJhtjxgGXA09prTu855JKSyHlhOH1+55pI9mybB8fz99CxV4v278tZ94VS6irbRhetuuPX7C854OsmfAvataWsOndHTw26RXmjXuJFU+tb/lmbjccMqQ+AQP06uPhoqv6M2BoGmN1Nmde2rfRJbU7q+tfe/fW4g+Ko2LOl+w58ynK/yp964QQXSvOasKNcptSKqrc1lZN2L1/ZixjjFdrHc0z3C+BU3BmF3kPmA9UAl9HUVZIgcQ+MXAfgCeBB7TWfYwxu/af16TW+zVOG34+sCVWsbQk55VLqXliMSo9hdSLJrB7yb5G71fs8bL5630M0z2oWbWXnb9ZiAWqS2rYef1HLCxJxVvhdLj6+PYljDxrCC5P+B2sJh+bx+Rj80K+d+BN41l65SdYn2XErw/Bne7UlKsXrKZ0lrMqV/ULK3H3yyHjBwdH8emFECLpxCS3tZWEU7XWNwbtpzfZxxjzhzbK+DENNe7/w1kOMZfYPhMeBGw1xvgCMfm01tsCx3e1cM0lwFpjTIcnYHCajdNnNjQbDzk0j95DM9i9ocp5X0FOb+c7zvZv9/Hh+APxuV0csKWIg3y2fgIPwEm+LoXfb3FF2Gxs63zY21+CpVtQFx+BOlsz8JID6DOtP/5aPxkDG+anrlu9p9G1dauLI/7cQggRK3HWHB2T3NZWEv6UhtolwGdN9i3QahI2xmwNel0MzGzl9E6htT4OuIPGn6XpOTMJxFpYGOuWc6dT1Q//MZ7//WMDe7dUMe60vvQe6vRY/uTpbXg9zl/N6kF9OeqWiRzrcvP+r7/EV+NjxC8O4ac372BHUR1Tj8viyhkRTMH9p9fgjpcAsC8vhsW3ocYOJK2gcau89Vsyzh5D+d0L8W8rw9U7k4zzx8bmwwshRBTiKQlba7cGvY46t7WahI0xx0dTaFNa66Nxap79jDGna60nAVkxHCe8GRigtXYHasFuoH/geNNYjgCeAL5vjFnZUoHGmLnAXIAZM2bYls5r054yZ6jQmIGQ2ng2K0+am2nXHNDsElfQsoUuj4v00T3JzUzhvDec7wx/ebiYbTsrAXjtvQpOODKL0QeEN22lXb2zYcfnh/W7YezA+kPeah8vXf81m8weBk7owemf/xi1dg8pB/XB3VuGNQkhuk48jRMGUErV5zZr7elKqUlAViTjhKOatUFrrbTW39FavxTGuRfirL5UTcP6wxa4PZp7h2KMKcLpkXZB4NAFwKLg58GBWCYDTwHnGGO+itX9W7R4PYy4Cib8Eo68ESqq274GOOGGg+g5JJPMXqmcdPPBpGY2/q7kdjf+RfS0sNawtRb/T+bhz5mF/7i7sHsrUD86FrICCfuQgXD8qEbXLF+wnU1f7AELWxaVsPyDvaQeM5idL25h3c1fUbm6NMwPL4QQsRVPHbOUUjHJbRHNGKG17o/TDv4joB8QzoDX3wKnGmOM1npG4NgyINY9fGYBj2mtbwH24nw7QWu9ALjFGGNwhlplAHO01vuvm2GMWRrjWBz/eB32BtZ3/nItvLUEzji8zcsKRucy45mjW3z/ojPy2LLDy7YddUw/IZsDhoTuL2ef+QIeDnwh+2AV9tf/xTX3Mlh5F6zbBZOGoDIb16BdTRL6mkdWsefGz3BtcD7H1jkrmbLyLDw9ZcEIIUTniqfmaAK5zVprlFJR57ZwZsxSwGnAFYGfu4EewKQwk1f/QAIE51sCQB3gbuH8qBhjVgDNMpwxZnrQ68mxvGebBvVueK0UDMyPSbH5Pd3c/Zu+Lb5fU1LLhpc3k7ZgHYOD3/hqo/OzMA9fViYpmc0T6UHT+7HZ7GXjZ8WkVXlRS3dTV1JTP/2Ld1c1VWv24ZncJyafRQghwhVtElZKNZtHwlq7usk5lwHX4izE4AYestbe30qx/a217c5tbc2YdTOwHnghcJOzcaboKgV2tnJpsLVa6yObHDsSaPF5bNK4/gy49nQ48RB4+KegoxlmHRlfrZ83zn2Xz2/6ig8Xuvk6N+ie3xlPzdpSlh8wn2U9H2LdGa9ifY2XvnSnuJh++1iufOM4+lVVoQBv6v7fKUtOTyi6aSHF85bTkrqSGiq+3IW/0tviOUIIEal2NEc/CMy21rY2j8SzwHhr7QScHPVLpdS4Vspcq5Rqd25rqyZ8G863hjOMMfUzZQU15Ybj98CLWuv7AI/W+pc43zZ+EkkhCSnVA/de1qm3rNhaQemqhjHI28YcxLg0F/6DBqGunUrRDZ/h3VgGwL4X11P21mZypw1pVs6K+5ZTsakCgJrMFAZdOIyskgoq/ruK8jcrKH9zM+mjepJ1ZL9G11WvKmHlMS9QV1RF2sg8Ri88i5T89A78xEKI7iKamnBgEo1m80gopfpYa+v7DVlrgydvyAQ8NNRwQ/k98KJS6j7Ao5SKKre11TFrBrAceFlrvUhrfbXWulcbgTVijHkBuBCnqXgjcCJwmTHmtUgCFeHJ6pdJ9qCGsb59Tx/OvlOOZcuDm9lc+ABsbzxJiDs39PPkFX/7tv618igmzDmKzP6NhzHVrN/X9DKK562grsgZ+1yzqpSSF9djFpbx6rPF7NhWG/XnEkKIKGvCg4Ct1lofQODn/nkkGlFKfU8p9Q1OrvqTtbbFR67W2pC5zVobUW5ra4jSv4B/aa3H4IyB+h1wN06btybMeaSNMRSpMxYAACAASURBVG8Bb0USWHfnW7Ob2kcNriE9Sf3xYagwvwG6091MfeYE1jy1noyCdIZ/dwCb8+513qzxoZZvJ++M4VQtLabXD0ez9dM9rPrBh2T0z+Swx48hZ5Qzq5bfG9xM7dy71+Vj2PPYCnx7a0gb3ZPc6c1r0KlDchrtf7tT8a83twPwzmt7+d2fh5Lbo8NX0BRCJCEb4r9BpVT9nA4Bc621c6Mq39qXgJeUUoOBF5RSC6y1LTYvW2vbndvC+t/QGPMtcK3W+gbgPJzq9itaa2OMOSzUNVrrucaYmUH7hxljPm9PsN2Fv6SK8qP/jt3p9Ej2by0l49ZTw74+szCDcb84CADr9aGyUrHlTi3UlZ/BsOedvmoVG8r57FBnlFn5qn1887tFTPnP8c4901z4q31OgcoZ7pRxSG/GrLqYmrWlpI/Nx53VeNwzQO+ZB+HdVkH5wh30OGMYL9TlAs7nqCj3s21zrSRhIURUQtV8Awm3taS7GRiglHJba32BNYBDziMRVOYmpdTnwHdp8oxXKTXXWjszaP8wa23UuS2iccLGmBpjzHxjzLHAWGBhK6ef32T/9UiD6678a4vrEzCAb+HGqMtSHjcFz56JZ3wBaUcPpPe877R8ctBDhrEX9GdS2Rfoss8Ze17f+pp4Su8Msg4vDJmAAZRLsbtfHgs31vHWI+sZlNtQo87t4WbgEBnaJISITjTN0dbakPNIBD8PBlBKjQl63Rs4AQjVHB3T3BZ1lcQYsxy4ppVTmv7pxNUAr3jmHl2A68De+Fc76/56Tj+oXeVlnDqcAacOb3Y8a2g2o387jtV/+Yb0/pkcdNuh9e+Nfv0FqAqs1vTGc3DPcWHdq3pPDV/+eRlYqC31UvfSKq6+7XCKttcyXmeTnRu6937xzlpee3o3Lhec9oM+9MxvnOS33/kl+97eQu5JA+l306QwP7kQIpn4ox8nPAt4TCnVaB4JpdQC4JbAUKOZSqlTcVZCUsAD1to3Q5QV09zWahLWWq+mjU5YgaUDQ2l6XfRTP3YzKiuV7IVX4X1+Ga7BPfBMHdX2RVEadd1YRl0XYk7olVsbXq/aHnZ5rhQXrhSF3+v8dbvT3Bw8PouDx2e1et2cP25mxxanyXz7N6X83wOj62vfe55aw7abnNae8ve2kTYil17nHxh2TEKI5GCjzHfW2pDzSFhrpwe9vjbsMFrfj0hbNeHfB71WOOOrfhpm2bFYganbcvXOIu0nbc+u1WGunAZ/fM55fUWL61w0U5viZtH3NOuLYUhNObfcPKDFc+0HK2FfFXbqWIq2N/ScLtpSjb38n6h5P+L198v59gs/Q/r2YshOZ1Wn2s3lLRUphEhicTJjVqpSqlFua7KPtTbs3NZW7+jHgve11vc2PdaKdq/AJLrQXRfDmYc7izwcEX5N/JW3y1i3R4GCDek5fFOSQqiGbHv7i9jfveDsfHc8R009h/VPrMEqxeiytfDyx7x60bnMebIUyCBl2hRmPf8+/Xq66HVRS40vQohkFidJOKa5rcO6qcZqBSYRgX2V8IdnoawKrvs+DGt5asuwHBZ5k69ScNxaw+iiDXw+6GBcrtBzYNv5QX36XllCT3swvk1OTTevuhJG9mXt5rr6U+rcLtLmTeWgqb1wZbjx3vUmdtEW3OdPxH3WhIjjFEIknnhIwtba42NZnowVSSaX3g8vBHrKv7YI1swGV1QLZUXtjNKlpL0/H4DTVn6MvX0YECKZjxsEa4oAsIPzKV6yt/6t4uzeqNfP56iyDN5ZWIHf78yXPeGU3riz3NTd9x51N74MgO/ZxaSZX+E6tNm4eyGEiHuShJPJ10FDmdbvdGrEea13iIq1tKXr6l+7/H5YugEOb56E1bwfYUf2hdIqXNdNo8dJ/2Ovz/nC0MdXgspMYdLQDP5yS182bfMybnQ6eTlOz2r/yqBpy/0W34oiScJCdAOhJutIdJH2js7VWq8KPqeV3tGis804Dm4LrC55uu70BAw4z5HvexVqvNArG04eH/I0lZuBuutcwJkIZNDmnQxJ3U2GLWdQ9WaYthq+uodhg1IZNqjJ1JrnTsT34ELc1k+FO51tX9Yy7oIQNxFCJJV2DFGKW5H0jhbx7tbznRWbyqpg6qFtn98RpoyCRffAonVwzEGNl3NsgVKKHheNpuDxeaQQWHlp0Xqo80FK83HF1QMK+KDXRLJ8VZSmZJO3siLWn0IIEYfi4ZlwrEXUO1okgGMjWk+6Y4wZ6GwR6DdvKnW7l8GCQIet7x8WMgEDZA/NJn18X4qXlQAw4DvSFC1Ed9DtkrDWOgVQxhhv0LEfAhOAD4wxz3VseKK7UC6F55VfwkvHOk3ZZ01p8VxXqpvjXjyJHf9ZSdrgXApOHdyJkQohukoyNke31XX2KaB+QVyt9U04E2UfjbO60o87MDbR3Sjl1IDPO6rFWvB+nl/NZ9BP7qLgnNvg3W+ave9/9nN8o6+Fib+Ez1aFKEEIkWisar4luraSsAZeCdq/GvixMUYDFwNXdlRgQrRozQ548H/O67JquO35Rm/XPv4l+855mrKVHioXlcFFf+2CIIUQsWZRzbZE11YS7mmM2QYQWFM4Dwh0v+UFYGjHhSZEC3IywBNUU+7VuBd49V3vsn9O9Vp64N9b1YnBCSE6il+pZluiaysJV2itswOvNbDMGFMd2FfIOGPRFfrmwfwrYdxgmDYO/nZpo7ddA3LrXyt8qD9d3NkRCiE6QDRLGca7tpLoh8AdWus5wBU0XjdxFBD+8jqi0+0ot9z9iQ+PC359pJteGYn/C1vvB0c4WwiZj55H1TUvY4srSL/5RNSJMpRdiGSQDEm3qbaS8K+BBcAvgGXAvUHvXQR81EFxiRiY/h8vi3Y4c618sc3POzNS27giObgG9iDrmRldHYYQIsb8yZeD2xwnvB4Yo7XuZYzZ0+Ttu4HaEJeJOGCtZcnOhsnOFu9s33LOW0v97Ci3DK6tonR9Bf3H9yAzP629YQohRNi6Y00YgBAJGGNMSezDEbGilOKCg138a5kfgAvHRr+Qw2sr6zjziWpq6mD03hJmfrWc3Pw0zp8/hazekoiFEJ3DnwS9oZuSjlVJ7PHvp3D+wX48LsXUA1pOwm9v9PPUCkt5LXy4xdI3C14920VhlnPN3xZ6qQmsKriiZw+2ZWfh3l3B5s+LGT29PwDbvyrmq4fWkNEjlSN+dRClu2p56eblVJV6OeaKYUw8a0CHf14hRHLrtjVhkZhcSvHdA1uf9OLbYsv0Z/14/Q3HtlbAiU/5+fACxe8+9LO2ygUuP/gtKT4/ObVefErxm9d9HFm0l6vOyua1n35ObbmTqWsr6thTl0Lpdqcj/Tv3rWHMSQVk5Hk67LMKIZJft3smnEi01iOBx4B8oBi4xBizusk5buB+YBrO6lB/NMY83NmxxpMVe2yjBLzftnK45BUfC9ZawEVWTw8ZFbX03l3O8gG9+XpgH9w2hfKFVQzL8dUnYIDyHVXY/JwW7/n003v45NNyhg1L46dXFuB2J+G/LCFEzCXDuOCmOnfF9471IDA7sLTibGBOiHMuAkbgrDJ/BHCr1npop0UYh44dqBiS2/z42SMVK4obOnNV1CluH17Ntr49eGryaLbn5dCnogaA6rQ0dk5wmqU39MrlxomH8p8RQ8goSCc1082B3xnA+6/tYeeWahYuLOfFl0sp2uXjs88rmTdvd6P7VlT6KSv3ddwHFkKIOJIUNWGtdQEwETglcOhJ4AGtdR9jzK6gU38APGSM8QO7tNYvAOcCf+rUgONIfobiyxluPthi6ZcFb22wjOgJF4xxc/en8Ov3nGryyT28bHhmKzMBb2YK3gN7scrC0P4pHDopgxnfjmPA4GFsz8jEX6dYXZfK8hMm8Oiwcp5/ZDt8Ws2HC/Yw5Mheje6/fn0NLy2rpSDbRc3Wav4xrxi/Hy46twffPy2vC/5EhBDxSp4Jx69BwFZjjA/AGOPTWm8LHA9OwoOBjUH7mwLndGv5GYozD3R+uaf0bzh+/RQ3Jw91UVZr2fr0NpbgNJ2kVdZxxemZDBiTQ3amoqoOeqT72NorF6obarFFlbDu24a1fqvKfYwalspHCyucXo7Wsth6+MPD5aDgzMoK3IHL//VMCd+dmovblXz/6IQQ0ZFnwt2I1nomMBOgsLCwi6PpOhMLFaBYpHNZ8oEzKi07L4VBB2aSne08zchKhQVnu7nrMz9F+1x8vtmPUvCHE1IYU5bDkoX7AMjtmYI+ModBIzL4+OMyeg5I5fSXfZDlgR7plK+tIg+n5l2DYk+V5ZyXfSzdDQMyoaLWctJgxZzT3CxeXkX1q+spLExj2MUHoCRZC5H0kmHBhqaSJQlvBgZord2BWrAb6B84HmwTMAT4IrDftGZczxgzF2fZRmbMmNG+mS6SwKHH9SSnZwpFm2sYMzmX7B6Nezof0V/x0pluwE1RhfPHVZClgJ7k9fKwa3sNB0/OISsnheE5KQw/IJ2KGkufd0vYlZcOSvHZwF5M3F6Cy29ZnpXOnZ/5+WCLU/7eagDFw8vh2ZW17PWlouyBXDPnU364sZyBvzgEv4Xeea33BhdCJK5k7JiVFEnYGFOktV4MXAA8Efi5qMnzYICngZ9orZ/D6UV9BnBMpwabwEaMy2HEuJZ7Pe/nJN8GoyZkM2pCdrPzstIUb1yZg37S4gf2pXt4b1gfVHUdfz/ZxTcVCqcTe5A6H3txEq11u3j8uHEULFvHqzfswm/hyjOyuXhqw7321VjW7LWM7KXITm2Iy+e30tQtRIJJxiScTL2jZwFXa61X4ax7PAtAa71Aa60D58wH1gGrgU+B2wNTc4oucuiAFOZ/x03fTCjMhN8fpSi/PpVZU1K54TAX4/vs/yUNJGML2IbE7Kmt490+hfgDhx5+pRwbeH9TqeWguV4m/bOOQx7ysr3cOf6bR0qYOms7J125jVc+a98yh9Za6vyxbyjxF5Xj3yKT0gkRzK+ab4kuKWrCAMaYFcDhIY5PD3rtA67szLhE2y4c4+LCMc2/Dw7IUSy+NAVrLV/usLyw2nLGgR5uf7eWVzYqsv0+Hj/Cy6PfpFK+3Rmn3CPbhQp8W35imZ+tZU5ZG0rhv8v9nDnU8sVnlShA+eCv/y7lu4dnRBX3+5v8nPWcj9Iayw1TXPz+uMb/nHxeP6WbKsjqk05abvgTldT88wuqZj4LPj9pt5xMxm2nRhWfEMlGpq0USadscwU7PttF/tie9Bodn0OClFLofgrdz9l/6cL0oHczOeAIL/f9twyf33LV2Q2Dngc3+TiD8xQ+LBbq/ym3Z0TyL9/xs6fKgoU7P/azdq+XJ89wkm1djY+Xf/IpOxbvJS3Xw+kPTaHPmPD+fKtvfQt8gQ5qd/yP9BtPRKXJP1UhZIiSSCplm8p56XvvULvPi8ujmPbEsfSd3Lurw4rY8P4e7rumV7PjF491s7EU3t3oZ+pwF2eOcgEuxuhMlnxZid+l+MUPov/ikeam0SPr/yy3nDqkjm2lfqbsLWbH4r0A1Ozzsuw/GzjhtvFhlevqm41vs9MUrXplQKp0NhMCkqP5uSlJwt3YtoW7qN3nBcDvtWx+Z3tMkvD+Z7IqDr61/vYoN789qnESe2BmD6q9eXjczvNm67dRDXH6x1Q3x873U+pMHEaKgsufqwELB5QpfhnUryyrIL3FcprK/NcFVP38RWyll4z/Nz0u/hyFiAfJ2DFLknA3ln9wD5RbYX1Opug9vnltMlJPL63jsmechRvmnZPOuYfE36+Yv9ZH5Ztb2L62gvef2QYWTrr5YEZN7RdROeMKFCuv8HDF63Vs2QcDMy0vLnPeW5uTg7r0YAq/3k6vETlM/PGIsMt1j+xD9us/jigWIboDeSYskkrvQ3py6mNHs+Wd7fSZmM/Qae1fbnDW89VU1Da8PveQ5kOTupK1luXfeYPSt7cC0GtgT4oKc3n3j99GnIQB+mYrXjjHeQ789hofL3/jxW8hwwPHXTyEMQXDYhq/EN2ZL/lysCTh7q7/kQX0P7IgZuV53A1tsKkp8fcvpnZbZX0CBsgvLqeoMJeUVBfWbyn9q6F2RTE5l4wl4+iBEZV98gg37/8onc+2+DllhJsxBck0AlCIrifN0UK0Yf55acx83nlIOvfMtC6OpjlP73Q8/TLxbq8EoC4/g9wBGZzwf6PYc+vHlNyxEIDyJ5YzaPmP8AyNrOPW0UPdHD1UOlIJ0RGkY5YQbTjlwBTWXx+/v1auNDdj3/0O2/6yFE9+OgNvnED5JztZ/Z2XqSgvZf/XBltVh3f13oiTsBCi48gzYSGSQOaoHox4sGG20i2//Rx/uZcaPKRShwI8B/Yk/fDwnxF7y7zU7K4ma0i2LCYhRAfxSXO0EMnHnZcKgBcP+1JSGPHPY8n67gG4csNrTi82u/nogvfx7vNSeHI/jnr8GJRbngcLIdom/1OIbm/oP44h59h+ZBzUk6FPnkrujINx9wx/XO/qOSvxBsZb73h7O8WmuKNCFaJbi3buaKXUSKXUJ0qpVYGfB4Y452al1DdKqa+VUl8qpabGOv5QpCYsur30A/IY8/73o74+rXdQwnYp0vLjr0OaEMnAF/0z4QeB2dbaJ5RSFwNzgBObnPM5cI+1tlIpNR54XynVz1rbvlVe2iBJWIh2GnvjOLxlXsrXlXHAD0eQMyK37YuEEBGLZpywUqoAmAicEjj0JPCAUqqPtbZ+uVtr7RtBl32NM8V8PrAl2njDIUlYiHby5Hg47IEpXR2GEEkvynHCg4Ct1lofgLXWp5TaFjjedM35/S4B1lprOzQBgyRhIYQQCSJU72il1ExgZtChudbaudHeQyl1HHAHDTXnDiVJWAghREKoC3EskHBbS7qbgQFKKXegFuwG+geON6KUOgJ4Avi+tXZlLGJui/SOFiIKtZ9sYveY+9k17F5qXl7R1eEI0S34lGq2tcVaWwQsBi4IHLoAWBT8PBhAKTUZeAo4x1r7VYxDb5EkYSGisO/yF/Ct2I1/QwmlFz6D9fu7OqSorHxmA/+b9QlLbzZ4dzhTeZbvrqG2MlSdQ4iuVaeab2GaBVytlFoFXB3YRym1QCmlA+f8HcgA5iilFge2Q2L8EZqR5mghomCrG5KUrfWB32K9dai00P+k/D7LlkV7SctOoe/o2PWernzkA2o/WEXGD48i7YQxbZ7vr/JSfMOH1K7aQ8Wxw/n44fWM2lxMrdfH0r8sYsfPDuebj/fiSXfx/TsPZthh7V/eUohYqYtyiJK1dgVweIjj04NeT44+suhJEhYiCrn3T3dqwLU+Mm86nqIDHsC3qZTMHx9Kj4dOb3b+qzctZc17TuvXMT8bgb5oSLtjKLn7DXr8eg6ZQN0Tb1D98d2kT2l96cTdN37ExvtX4MNN5Ucl5GVlku71AeCt8rP8wz3gUnir/Xzy6EZ6D83EX2fJ65/R7niFaC9v8s1aKc3RQkQj7fTR9Cm9kYKKm6j7thjfplIAKh9eRO2njUc1VJV66xMwwLKXtsUkhsrnF9e/TvH7KHl2SZvX7Fiwg0rSqcGDu1yRXuMNKsPP4O0Ns33VVtTx8Bkf88+zF/LJw+tiErMQ7eFVqtmW6CQJCxEl5XKhUtyojMYNSirT02g/LctNVu+GWbR6Dc2Myf3rjj+kflWZGlcqnlPHtn1Rv5ygHUVhaVmjt0cf1oO+o7IZfkQvqnZW7l8ams8e3YDfZ2MStxDR8obYEp00RwvRTjl3nkjdur3Urd5D1lWT8Yzr2+h9V4qLs/92KGb+RlKz3Rzxo+Exue+gP5zGWk86vk/XkDljCoNOGdHiuZVf7WLdeW9Su7MGlerC1vrJGp1L+opSalH4cZHSK41Rf5jMuOHOM+snLvmM6tJyALJ6peJyJ36tQyS2yiSo+TalrJVvt22ZMWOGnT9/fleHIUTUlkx5jpVbvdS5XfTbXcahr55G7pF92fuXLyl7bQNph/al751H4gqq1e/dVMn7963CV2c55qcjKBiV08od2uav8bHi0g8ofX8HPacNZNTDR0W02lTdmj3UfrQJz+ED8Izp01BunZ+9z62lZPYS3Bkp9L3rSDIOLWhXrLFmntvGB/M2kdnDw5m3jqbvAVldHVJniWnWTLumuFnCqvlrfkJnZqkJC9ENrKxysyfPaSavyEjliANzcXlc5F8/mfzrQ3cK7Tk4kzPumdBquf7yGlzZ4S1YsX3uSnY9tR6AnY+upsfxhRRe2mwxm5C8y3ex+/BHsOW1kJ5C749+SOqk/ix5ahMf3LMCl9fHIetK6LWvguqluxm99cdhldsZKvbU8tbs9WChpsLH27PXcdG9HT7yJSnVxjanxwV5JixEN2D7NdS8fG4X/lR3VOV499Xy+Xff5s2C//DF4McoyrmT4nGz8ReVt3mtr6rx2GN/tS/s+9a8udZJwADVddQsWIO32scH967AWvCluFk9yHkMULejElsXP+O2m1XdpPExeirEluAkCQvRDYw9IR8VePQ0fKiHzN7hr5ccbMMDK9j99nbq9nnZVeSixJVN3dKdVM7+3DnB13Ly63/FaHIOd5qR844vpO+Mlp9hN+WZ1K/Rf7ieyf1RLoUrpeG/MBWYMKXPDRqV0jX/tZW+vJ4dt31O5aKG3vDZvVI5adZQPOku8grTOOmnrQ8jE91LwjdHa60zgXnAJJypRa8zxrwS4rzvA7cAaTj/nP9pjLmnM2MVopk95XDRbFi2BS47Dm4/p0Nu0/vprzhx1W7q3C5yvqnBt/tY3L0j76Vtm/SQ3r+nsjxw7wK44T+QmwHPXQPHNp48JCUvlYmfns4X9yxj/Ue7qLx3OVN+c0hYHb7SjhlCrwUXUvPWOlKPHUL6NCeBn3r7WD788woo83JI/wyGzDmXnGP6R/y5YmHvU6vZeL6zGl7R3V8xasn5pI3oAcBh5w7gsHMHdElcSSUJO2YlQ034OmCfMWYEcDrwsNY6O8R5O4DTjTFjgSOBK7XWx3RinEI0d8fz8PrXsGWP8/rjjpkzXmV7yKjzklNTA6luVJTN0cN+NpqeU/qg3Ir8YR7y+7tJO28smTMOgev+DV4fFJfDtU+EvH79m9tYPGc1u78p4ZvH17Ii8Iw4HOnTRpB3z6nYXllsn/U2e+5fxIgT+nL5a8dz+UenMPnFU9uVgH07y9k7fT67x9xP5VwT8fUVHzSM//ZX1lFpiqKORXQfCV8TBn4AXApgjFmttTbAacDTwScZYz4Lel2qtf4WGAJ82ImxCtFYZW3r+zHS88Hp7P3hy/iLK8m98wRcueF1pmrK0zONI96bhrUWFVwr2VcJKS4nCQOkeUJeX72nptF+VXFNyPNaUruuhE2nPocNPE+2Xj/5v5wUURktKb/+TWpfWw1A2ayXST1xGCkj8sO+PmfqYHb/YylYcOWlknVEYbtjslW11N3yCnZDMe6rjsN9/Mh2l5nQpCYclwYDG4P2N+Es1twirfVoYArwTgfGJUTbbjgdDgiMK/7BFDjp4A65jWd0bwo+vYzC1VeRed5B7S5PNf3PMDcTHvkJ9OsBBw+Ef1wW8rrh0wfQa3QeADmDshh93tCI7lu7Ym99AgYou+FNykb8Ad+S9s9CZkurg3Ysdl9kXxDyvjeMEe+dyYD7jmHk5+eSOqT9c4TX/foFfH9+G/8zi/B+5+/YHaXtLrMttrYO+9rXWBN+K0WnScKOWXFfE9Zaf4WTaEPp28Lx1srrB7wI/NQY0+K/XK11/ULRhYXt/0YrREjDCmDNvVBVCxmpXR1N+8w4xtlakZabypnPHk/5jiqy+qbjjrBZPOPIfniG5+FdVwpYMurKsWurqb72RbLeubIdwUPWzcdT+/Em7O5K0i+dgGdi5E3b2ccOIPvY2D37tauCmrQra7FbS1GFeTErv9n9/H7s9L/A/5Y7B/52EepnJ3fY/SKXBFm3ibhPwsaYia29r7XehNOsvL874mDg3RbOLQDeBu42xjwd6pyg+9YvFD1jxgwZVCA6VqIn4Ai4PC5yB0U3WYXr988xtGYZFYcOoG5REZ79y7y3MOmQb+NefEt3kHL4IFx9QnUVaeCZ1J8+W6/DltXiyo/N1KLhsF4fe276kJqvd5Fz0UHkXNzQGuK+4mj8/1sBdX7U0QegxnVw567NexoSMGDnfRRfSTj5cnD8J+EwPA1cARit9YHAZBoWb66ntc4H3gIeMMY80rkhCiHay766GO55DTeQu3UXdVMnUvVREapvDul/+X6z8+u+3Mq+4+ZCRS2qXw55X1yFa0DrtUiVmoLK79z/Fkv+/AUldztDvKreWE/qQfmkTXRa39xnTkB9ewtsK0VNGYbyRNehLmx9cqAgF4r2Oftj46xHtyThuPQn4FGt9RrAB8w0xpQBaK1vB7YZYx4EbgBGAldora8IXHufMWZeVwQtRHfir/Gx8RcfU2F2kX/ecPpdf2jkhVQ0fkabMq6Q3NevbfH02v9+DRVORze7vQzv66tI+1GXLBnbqroNQc95LXg37qtPwgCuEQUwonOm4VSZafC/X2H/8iYU5KBu+l6n3Dd8yZeFEz4JG2MqgHNbeO+WoNe/An7VWXF1Nf8zBv91T0NuOu7Hf4Sa0NJjdSE63o6/fs2uOU4zZ+WXu8ic1Ie8kwZGVsgZk2DaOGdI1+j+cM20Vk93HxzUZUQpXGPiaz5pAHaXkbNsCeUuP36/i9TxBWSeMrRLQ1JjB6IeubxLY2hR8uXgxE/Cojlb7cV/8cNQ4zwv8818nJTPb+riqER3ULdyNyonDXf/xos91BVVtbofDpWaAq/9Cltejcpue8avtEsmYitqqftkE6nfPwjPkUMivmeH++1/SV+4hMG48ZJK6iPn4cruPv0DIpd8WTgZhiiJpup8DeM1ocPGnor4YCtqsMv/f3t3HyRXVeZx/HvmPW8QkMQAIQkCYV0XWMPDImwAgwkiSskukRe3gsJCiG4Fo6ZKihJEOgAADRhJREFUi/cSldWwgLhhgWDxIoKIRblWihLUIosgKeoxYBSFRCAJCeSNMglhQszM3P3j3IROpjPpmb7Td/rO71PVNdO3T/d9nszkPnPuPfecNSTv5ru66qbLF7L+725n3fjv0f7gH3Z7bfQXP0TzoXEw1rATRzPy0xP6vJ/SApxs2UbH9/+PjnsXk5SZMrPtCx9h+P3n0XJuBWst52FzOwCNdNLGNhq26/9qj3SLktSDMLyNhv86L56OHtZKwy3n5x2S9JPk9bdIJt8Iq96Co8fA01cSDhrB9qXr2Xyz0zBqCAde98/93rvq3PAO7XctiU86uth64zMM/ex7KwW1HbE/xy2/kB1r22k5bHhmcztvnzaf5Lk4TUDX4hW03HFBJp9bM1d9Ghb9GdZths+eDCdVtqrUoFWAorsnFeGCapgzjfAfU6CxgdCgEx6Fde8zsQADvLwWHn6OrotP5Y2pP6FrQ+xlda5r5/33f7Jfw2gY3kLYr3XXBBeNY7uvPdwwpInWw6ufwGKnZFP7rgIM0PXES5l9ds0cMw5Wfx/efhcOGDRrDFeheFVYR+cCC81NKsBFd8jIbs87N7TvKsAAO/60sapddKzcxMYp97Hu6Pm0/3Bp2TZhSDMHLryAltMn0Db9g4y8++yq9lmR/YcQPvze4K6GKXXai2xqVAGulE5Hi8iAcvFkeGU9PLWMcNaxhH85nqauhCFnTGDbEysAGHHJsVXtYssVv+Bvi2KPc9MlP6d12gdoHNN94ovWU8bT+uuLqtpXb4QQaP31bDp+8CxhvzYaL/lIzfYtOSng3NEqwiJ1LDQ0EL49fY9tgYMXnsu2J1fROGoorR/u9eyuu+kqnUO5o4vknYEzeCgcMJTmuR/LOwyRPtO5SpECCs2NDD3j8KoLMMCIG6YQRsYRycPmnEjTEQdW/ZkiEqknLCI9ap08jjHrv0rSvoOG/fd9f65Ivyne2WgVYRHZt9DcSNi/n+ctFtmn4lVhFWEREakPxavBKsIiIlInVIRFRETyUrwqrCIsIiL1oXg1WLcoiYiI5EU9YRERqQ8F7AmrCIuISH0o4LSVOh0tIiKSE/WERUSkPhSvI6wiLCIi9aJ4VVhFWERE6kPxarCuCYuISLGFECaGEJ4NISxLvx5Vps0ZIQQPIWwPIdxUq9hUhEUKquOJl9h23HfZdvKtdP3xzbzDEaleKPOozB3A/CRJJgLzgTvLtHkVuBSYV3WcvaAiLFJASUcn26ffQ9fSN+h6dgXbP/9g3iGJ5CKEMBqYBDyUbnoImBRCGFXaLkmSvyRJ8gLQUcv4VIRFiqijC7b+bdfTZNO2HIMRyUgI3R/7dhiwJkmSToD06xvp9typCIsUUGhrpvnbn4wHqdYmWr5zdt4hifSLEMLM9FruzsfMvGPqDY2OFimolq9PpXnWydDcSBjWmnc4ItUr0/FNkuQu4K4e3vU6cGgIoTFJks4QQiNwSLo9d3VfhM1sKHAPcDzxXP5cd1/YQ/s24HfANne32kQpko8wcmjeIYjkKkmS9SGEF4ALgQfSr88nSbIh38iiIpyOngtscfcjgbOBu81seA/tvwUsrklkIiKSnb6Pjp4FzA4hLANmp88JITwWQrD0+8khhNXAV4DLQwirQwgfzzaB7uq+JwycD3wOwN2Xm5kDnwAe2bOhmZ0CHAXcDBxXyyBFRKRafZutI0mSl4ATy2w/q+T7p4GxfQ6tj4rQEx4HrCx5vooyo97MbBhwK/CFGsUlIiJZ6ntPeMAa8D1hM1tCLLTlvL8XHzUPmO/ua8ys22wpZfY7E5gJMGbMmF7sRkREpDIDvgi7+6SeXjezVcB4YOdF9nHAk2WaTgbOMrNrgTbgADNb6u7H7mW/u0bczZgxI+lj+CIikpUC9Hz3NOCLcAUeAS4HPO3hnkAc/bab0mJrZh8FbtLoaBERyVMRrgnPA0aa2V+AhcBMd38bwMy+YWazco1ORESyoWvCA4+7vwN8Zi+vXbuX7YsA9YJFRCRXdV+ERURkkKhsrui6oiIsIiL1oXg1mJAkGvi7L2a2gd3vRe6tg4CNGYVTLwZbzsq3+AZbzlnku9Hdz8wimKJSEa4BM/PBNhJ7sOWsfItvsOU82PLNSxFGR4uIiNQlFWEREZGcqAjXRk9rXRbVYMtZ+RbfYMt5sOWbC10TFhERyYl6wiIiIjnRfcIZMbOJwH3A+4C3gIvcffkebRqB24AzgQT4T3e/u9axZqXCnK8BLgA6gR3Ale7+eK1jzUIl+Za0PRp4Hrjd3efWLsrsVJqvmZ0HXEO8izMBprr7ulrGmpUKf6dHA/cQl0xtJi4Yc4W7d9Q43KqY2U3AucAE4Bh3/2OZNoU6Zg1E6gln5w7iUokTgfnAnWXa/BtwJHAUcBJwvZlNqFmE2ask5+eAE9IFNC4BHjazITWMMUuV5LvzwHUn8LMaxtYf9pmvmRlwPTDN3f+BuFrZ5loGmbFKfsZXAn9Of6ePBY4H/rV2IWbmZ8Cp9DwHQtGOWQOOinAG0r+MJwEPpZseAiaZ2ag9mp4PLHD3LnffQPxPUHbe64Gu0pzd/XF3b0+fLiX2lt5Xs0Az0oufMcDXiYuJLKtReJnrRb5fJq5IthbA3Te7+7u1izQ7vcg5AUaYWQPQCrQAa2oWaEbc/Wl3f30fzQpzzBqoVISzcRiwxt07AdKvb6TbS41j9786V5VpUy8qzbnURcAr7r66BvFlraJ8zew44OPALTWPMFuV/nz/HviAmT1lZkvM7Gozq9fJBSvN+QZgIvAmsBZ43N2fqWWgNVSkY9aApCIsNWFmpxEPXt3Wei4KM2sm3tYxa+eBfBBoJJ6SnQacBnwCmJFrRP3vM8SzOgcDhwKnmtn0fEOSeqUinI3XgUPTa4E7rwkekm4vtQoYX/J8XJk29aLSnDGzk4AHgHPc/eWaRpmdSvI9GDgCeMzMVgBzgMvMrB7vt+zN7/RP3X17uo73/wL/VNNIs1NpzrOBH6WnaDcTc55S00hrp0jHrAFJRTgD7r4eeIH3enkXAs+n11BKPUI8KDek15nOAX5au0izU2nOZnYC8DAw3d2X1DbK7FSSr7uvcveD3H2Cu08AbiVeT5tZ84Cr1Ivf6QeBM8wspGcCPgb8vnaRZqcXOb9GHC2MmbUAU4FuI4sLojDHrIFKRTg7s4DZZraM+JfyLAAzeywdQQrwQ+BVYDmwGPiGu7+WR7AZqSTn24EhwJ1m9kL6OCafcKtWSb5FUkm+PwbWA38iFrAXgR/kEGtWKsl5DnCKmf2BmPMyYEEewVbDzG4zs9XAWOBXZvZiur3Ix6wBRzNmiYiI5EQ9YRERkZyoCIuIiORERVhERCQnKsIiIiI5UREWERHJiYqwiIhITrSUoUgfmNki4qoyO0o2/9jdL83gsz8PXO3uR1b7WX3Y9xDgfuAfibN/Xevu3yx5vZW4tN3pwBjgr8TJWK6p14UbRPKkIizSdzeUFqiBJF1EobEPa9wmwG+Jk6zcWOb1JmAjcDZxAoexwKPE1YSu6HPAIoOUirBIPzCzc4gL3R9BXG3nm+7+o/S1scDdxHVoW4iLAcxx99+l82zfAbSY2db04z6Vfv2VuzeV7ON6YLK7T02fJ8TZnGYAHyLOZ7zYzC4DvkRc/eZV4Gvu/kS5uNPe7C3p53Xr2br7O8BVJZtWmtkC4IuV/+uIyE66JiySMTObRpy6cQ5wIPA54L/N7NS0SQOxpzmeeEp3CfComTW7+7PEqRJfdffh6WNRL3b/78Q1YIcDz6cF+GvExdkPIBbQR80sy1PddTtftEje1BMW6burzGxuyfMz3X0xsdf5PXf/Tbr9OTN7gLie8lPuvoq4Og0AZnY18VTuUcQ5mKtxk7u/kn7faWZfIs73u7NIPmZmTwIXAFWfSjezOcQlDIs4d7ZIv1MRFum7b+3lmvDhwBQz+0rJtkbgNwBmdhBwM/BRYCTQlbYZlUFMK8rEMt/MbivZ1gSsrnZHZvZlYi/79PQPCxHpJRVhkeytBO5193l7ef1G4trDJ7r7m2Y2AtgChPT1rjLveRtoNLNWd9+ebjukTLs937sSuM7dH+lVBvtgZtcAlwOn1fEa0SK5UxEWyd6twL1mtpg40rgROAYI7u7AfkA78FczGw58Z4/3rwVGm9l+7r4l3bYM2Apcamb/A5wMTCdeT+7JLcD1ZraceN22jTggbKO7v1TuDeltSIF47brJzNqATnffkb4+DziPWIBfKfcZIlIZDcwSyVg68vgyYB7xdp43icVweNrkWmA08BZxZPRvgc6Sj3gS+CXwmpltMrPT3P1t4GLgq8Bm4nXn+yqIZQHwXeAe4j29q4ijtpt7eNvLwDbgFOC69PsFAGY2HphLHFD2ezPbmj5e3FcsItKd1hMWERHJiXrCIiIiOVERFhERyYmKsIiISE5UhEVERHKiIiwiIpITFWEREZGcqAiLiIjkREVYREQkJyrCIiIiOfl/plsNPi/MEWMAAAAASUVORK5CYII=\n", 178 | "text/plain": [ 179 | "
" 180 | ] 181 | }, 182 | "metadata": { 183 | "needs_background": "light" 184 | }, 185 | "output_type": "display_data" 186 | } 187 | ], 188 | "source": [ 189 | "shap.dependence_plot(12, shap_values[1], X_train)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 7, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "name": "stdout", 199 | "output_type": "stream", 200 | "text": [ 201 | "creation of adversarial examples for simple model\n", 202 | "Training of ART classifier\n", 203 | "Creation of adversarial examples\n", 204 | "Adversarial examples created\n", 205 | "\n", 206 | "creation of adversarial examples for simple model\n", 207 | "Training of ART classifier\n", 208 | "Creation of adversarial examples\n", 209 | "Adversarial examples created\n" 210 | ] 211 | } 212 | ], 213 | "source": [ 214 | "def adversarial_examples(model):\n", 215 | " classifier = KerasClassifier(model=model, use_logits=False)\n", 216 | " probs_adv = classifier.predict(X_test)\n", 217 | "\n", 218 | " print('Training of ART classifier')\n", 219 | " classifier.fit(X_test, y_test, batch_size=5, nb_epochs=5, verbose = 0)\n", 220 | " #Evaluate the ART classifier on benign test examples\n", 221 | " predictions = classifier.predict(X_test)\n", 222 | " accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test)\n", 223 | "\n", 224 | " #Generate adversarial test examples\n", 225 | " print('Creation of adversarial examples')\n", 226 | " attack = FastGradientMethod(classifier=classifier, eps=0.05)\n", 227 | " x_test_adv = attack.generate(x=X_train)\n", 228 | " print('Adversarial examples created')\n", 229 | " return x_test_adv\n", 230 | "\n", 231 | "print('creation of adversarial examples for simple model')\n", 232 | "x_test_adv_simple = adversarial_examples(simple_model)\n", 233 | "print('')\n", 234 | "print('creation of adversarial examples for simple model')\n", 235 | "x_test_adv_complex = adversarial_examples(complex_model)" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": 8, 241 | "metadata": {}, 242 | "outputs": [ 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "classification report, on a testing sample of 142\n", 248 | " precision recall f1-score support\n", 249 | "\n", 250 | " 0 0.91 0.89 0.90 47\n", 251 | " 1 0.78 0.86 0.82 57\n", 252 | " 2 0.88 0.76 0.82 38\n", 253 | "\n", 254 | " accuracy 0.85 142\n", 255 | " macro avg 0.86 0.84 0.85 142\n", 256 | "weighted avg 0.85 0.85 0.85 142\n", 257 | "\n", 258 | "\n", 259 | "ROC-AUC Score on Testing: 0.9759297215311311\n", 260 | "Cross Entropy Loss: 0.3357352063594333\n", 261 | "\n", 262 | "classification report, on a testing sample of 142\n", 263 | " precision recall f1-score support\n", 264 | "\n", 265 | " 0 0.87 0.87 0.87 47\n", 266 | " 1 0.80 0.86 0.83 57\n", 267 | " 2 0.94 0.84 0.89 38\n", 268 | "\n", 269 | " accuracy 0.86 142\n", 270 | " macro avg 0.87 0.86 0.86 142\n", 271 | "weighted avg 0.86 0.86 0.86 142\n", 272 | "\n", 273 | "\n", 274 | "ROC-AUC Score on Testing: 0.9734526809546571\n", 275 | "Cross Entropy Loss: 0.29866210325483955\n", 276 | "\n" 277 | ] 278 | } 279 | ], 280 | "source": [ 281 | "multiple_metrics(simple_model, y_train, x_test_adv_simple)\n", 282 | "multiple_metrics(complex_model, y_train, x_test_adv_complex)" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": 9, 288 | "metadata": {}, 289 | "outputs": [ 290 | { 291 | "data": { 292 | "text/plain": [ 293 | "
" 294 | ] 295 | }, 296 | "metadata": {}, 297 | "output_type": "display_data" 298 | } 299 | ], 300 | "source": [ 301 | "%run 'psi.py'" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 10, 307 | "metadata": {}, 308 | "outputs": [ 309 | { 310 | "data": { 311 | "text/html": [ 312 | "
\n", 313 | "\n", 326 | "\n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | "
column namebaseline mean (std dev)new mean (std dev)stability indexconclusion
000.52 (0.22)0.5199999809265137 (0.1899999976158142)0.20Significant change
110.31 (0.22)0.30000001192092896 (0.20999999344348907)0.11Some minor change
220.54 (0.15)0.5299999713897705 (0.14000000059604645)0.21Significant change
330.46 (0.18)0.44999998807907104 (0.1599999964237213)0.16Some minor change
440.32 (0.16)0.3100000023841858 (0.15000000596046448)0.20Some minor change
550.46 (0.22)0.44999998807907104 (0.20999999344348907)0.06Very slight change
660.36 (0.21)0.36000001430511475 (0.20000000298023224)0.10Very slight change
770.43 (0.24)0.41999998688697815 (0.23000000417232513)0.12Some minor change
880.38 (0.18)0.38999998569488525 (0.1899999976158142)0.13Some minor change
990.33 (0.2)0.3199999928474426 (0.17000000178813934)0.28Significant change
10100.39 (0.19)0.4000000059604645 (0.18000000715255737)0.38Significant change
11110.49 (0.26)0.49000000953674316 (0.23999999463558197)0.11Some minor change
12120.33 (0.23)0.33000001311302185 (0.20999999344348907)0.22Significant change
\n", 444 | "
" 445 | ], 446 | "text/plain": [ 447 | " column name baseline mean (std dev) \\\n", 448 | "0 0 0.52 (0.22) \n", 449 | "1 1 0.31 (0.22) \n", 450 | "2 2 0.54 (0.15) \n", 451 | "3 3 0.46 (0.18) \n", 452 | "4 4 0.32 (0.16) \n", 453 | "5 5 0.46 (0.22) \n", 454 | "6 6 0.36 (0.21) \n", 455 | "7 7 0.43 (0.24) \n", 456 | "8 8 0.38 (0.18) \n", 457 | "9 9 0.33 (0.2) \n", 458 | "10 10 0.39 (0.19) \n", 459 | "11 11 0.49 (0.26) \n", 460 | "12 12 0.33 (0.23) \n", 461 | "\n", 462 | " new mean (std dev) stability index \\\n", 463 | "0 0.5199999809265137 (0.1899999976158142) 0.20 \n", 464 | "1 0.30000001192092896 (0.20999999344348907) 0.11 \n", 465 | "2 0.5299999713897705 (0.14000000059604645) 0.21 \n", 466 | "3 0.44999998807907104 (0.1599999964237213) 0.16 \n", 467 | "4 0.3100000023841858 (0.15000000596046448) 0.20 \n", 468 | "5 0.44999998807907104 (0.20999999344348907) 0.06 \n", 469 | "6 0.36000001430511475 (0.20000000298023224) 0.10 \n", 470 | "7 0.41999998688697815 (0.23000000417232513) 0.12 \n", 471 | "8 0.38999998569488525 (0.1899999976158142) 0.13 \n", 472 | "9 0.3199999928474426 (0.17000000178813934) 0.28 \n", 473 | "10 0.4000000059604645 (0.18000000715255737) 0.38 \n", 474 | "11 0.49000000953674316 (0.23999999463558197) 0.11 \n", 475 | "12 0.33000001311302185 (0.20999999344348907) 0.22 \n", 476 | "\n", 477 | " conclusion \n", 478 | "0 Significant change \n", 479 | "1 Some minor change \n", 480 | "2 Significant change \n", 481 | "3 Some minor change \n", 482 | "4 Some minor change \n", 483 | "5 Very slight change \n", 484 | "6 Very slight change \n", 485 | "7 Some minor change \n", 486 | "8 Some minor change \n", 487 | "9 Significant change \n", 488 | "10 Significant change \n", 489 | "11 Some minor change \n", 490 | "12 Significant change " 491 | ] 492 | }, 493 | "execution_count": 10, 494 | "metadata": {}, 495 | "output_type": "execute_result" 496 | } 497 | ], 498 | "source": [ 499 | "dataset_validation(X_train, x_test_adv_simple)" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": 12, 505 | "metadata": {}, 506 | "outputs": [ 507 | { 508 | "name": "stdout", 509 | "output_type": "stream", 510 | "text": [ 511 | "fitting completed\n", 512 | "prediction of testing dataset completed\n", 513 | "prediction of new dataset completed\n", 514 | "error ratio testing dataset: 0.00%\n", 515 | "error ratio new dataset: 0.04%\n" 516 | ] 517 | } 518 | ], 519 | "source": [ 520 | "clf = LocalOutlierFactor(n_neighbors=10, novelty=True, contamination='auto', n_jobs=-1)\n", 521 | "clf.fit(X_train)\n", 522 | "print('fitting completed')\n", 523 | "\n", 524 | "# DO NOT use predict, decision_function and score_samples on X_train as this\n", 525 | "# would give wrong results but only on new unseen data (not used in X_train),\n", 526 | "# e.g. X_test, X_outliers or the meshgrid\n", 527 | "y_pred_val = clf.predict(X_test)\n", 528 | "print('prediction of testing dataset completed')\n", 529 | "\n", 530 | "y_pred_new = clf.predict(x_test_adv_simple)\n", 531 | "print('prediction of new dataset completed')\n", 532 | "\n", 533 | "n_error_val = y_pred_val[y_pred_val == -1].size\n", 534 | "print('error ratio testing dataset: %.2f%%' % (n_error_val / len(y_pred_val)))\n", 535 | "n_error_new = y_pred_new[y_pred_new == -1].size\n", 536 | "print('error ratio new dataset: %.2f%%' % (n_error_new / len(y_pred_new)))" 537 | ] 538 | } 539 | ], 540 | "metadata": { 541 | "kernelspec": { 542 | "display_name": "Python 3", 543 | "language": "python", 544 | "name": "python3" 545 | }, 546 | "language_info": { 547 | "codemirror_mode": { 548 | "name": "ipython", 549 | "version": 3 550 | }, 551 | "file_extension": ".py", 552 | "mimetype": "text/x-python", 553 | "name": "python", 554 | "nbconvert_exporter": "python", 555 | "pygments_lexer": "ipython3", 556 | "version": "3.6.8" 557 | } 558 | }, 559 | "nbformat": 4, 560 | "nbformat_minor": 2 561 | } 562 | -------------------------------------------------------------------------------- /ODSC_East_2020_Validation_Monitoring_Training.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/ODSC_East_2020_Validation_Monitoring_Training.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | Hello fellow data geeks! Thanks for joining me to our session on model validation and monitoring training. 3 | 4 | Agenda: 5 | * We will have a framework review. Here are the slides [(Link)](https://github.com/moovai/model_validation_tutorial/blob/master/ODSC_East_2020_Validation_Monitoring_Training.pdf "(Link)") 6 | * Then, a short technical demo 7 | 8 | *** 9 | 10 | ## Tutorial 11 | If time, there will be a technical demo. No need to run the code, but you can clone this repo and follow along! 12 | 13 | You can review the code here: 14 | [(Tutorial Code)](https://github.com/moovai/model_validation_tutorial/blob/master/ODSC%20East%202020%20-%20Validation%20Monitoring%20Demo.ipynb "(Tutorial Code)") 15 | 16 | ... and you can also run the code from the repo 17 | [(Code requirements)](https://github.com/moovai/model_validation_tutorial/blob/master/requirements.txt "(Code requirements)") 18 | 19 | 20 | Also, if you have any questions, feel free to contact me at https://https://www.linkedin.com/in/olivierblais/ 21 | 22 | 23 | 24 | Cheers, 25 | Olivier 26 | -------------------------------------------------------------------------------- /best-weights-complex_model.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/best-weights-complex_model.hdf5 -------------------------------------------------------------------------------- /best-weights-simple_model.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/best-weights-simple_model.hdf5 -------------------------------------------------------------------------------- /model_0/X_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_0/X_test.npz -------------------------------------------------------------------------------- /model_0/feature_names.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_0/feature_names.npy -------------------------------------------------------------------------------- /model_0/history.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_0/history.npy -------------------------------------------------------------------------------- /model_0/model.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "batch_input_shape": [null, 22], "dtype": "float32", "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": true, "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": true, "units": 3, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} -------------------------------------------------------------------------------- /model_0/model_weights.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_0/model_weights.h5 -------------------------------------------------------------------------------- /model_0/y_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_0/y_test.npz -------------------------------------------------------------------------------- /model_1/X_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_1/X_test.npz -------------------------------------------------------------------------------- /model_1/feature_names.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_1/feature_names.npy -------------------------------------------------------------------------------- /model_1/history.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_1/history.npy -------------------------------------------------------------------------------- /model_1/model.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "batch_input_shape": [null, 21], "dtype": "float32", "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": true, "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": true, "units": 3, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} -------------------------------------------------------------------------------- /model_1/model_weights.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_1/model_weights.h5 -------------------------------------------------------------------------------- /model_1/y_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_1/y_test.npz -------------------------------------------------------------------------------- /model_2/X_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_2/X_test.npz -------------------------------------------------------------------------------- /model_2/feature_names.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_2/feature_names.npy -------------------------------------------------------------------------------- /model_2/history.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_2/history.npy -------------------------------------------------------------------------------- /model_2/model.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "batch_input_shape": [null, 21], "dtype": "float32", "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": true, "units": 70, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": true, "units": 3, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} -------------------------------------------------------------------------------- /model_2/model_weights.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_2/model_weights.h5 -------------------------------------------------------------------------------- /model_2/y_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moovai/model_validation_tutorial/fa074bc3adf20f25debf35364a8ed9748a195319/model_2/y_test.npz -------------------------------------------------------------------------------- /psi.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import random 4 | 5 | def calculate_psi(expected, actual, buckettype='bins', buckets=10, axis=0): 6 | '''Calculate the PSI (population stability index) across all variables 7 | Args: 8 | expected: numpy matrix of original values 9 | actual: numpy matrix of new values, same size as expected 10 | buckettype: type of strategy for creating buckets, bins splits into even splits, quantiles splits into quantile buckets 11 | buckets: number of quantiles to use in bucketing variables 12 | axis: axis by which variables are defined, 0 for vertical, 1 for horizontal 13 | Returns: 14 | psi_values: ndarray of psi values for each variable 15 | Author: 16 | Matthew Burke 17 | github.com/mwburke 18 | worksofchart.com 19 | ''' 20 | 21 | def psi(expected_array, actual_array, buckets): 22 | '''Calculate the PSI for a single variable 23 | Args: 24 | expected_array: numpy array of original values 25 | actual_array: numpy array of new values, same size as expected 26 | buckets: number of percentile ranges to bucket the values into 27 | Returns: 28 | psi_value: calculated PSI value 29 | ''' 30 | 31 | def scale_range (input, min, max): 32 | input += -(np.min(input)) 33 | input /= np.max(input) / (max - min) 34 | input += min 35 | return input 36 | 37 | 38 | breakpoints = np.arange(0, buckets + 1) / (buckets) * 100 39 | 40 | if buckettype == 'bins': 41 | breakpoints = scale_range(breakpoints, np.min(expected_array), np.max(expected_array)) 42 | elif buckettype == 'quantiles': 43 | breakpoints = np.stack([np.percentile(expected_array, b) for b in breakpoints]) 44 | 45 | 46 | 47 | expected_percents = np.histogram(expected_array, breakpoints)[0] / len(expected_array) 48 | actual_percents = np.histogram(actual_array, breakpoints)[0] / len(actual_array) 49 | 50 | def sub_psi(e_perc, a_perc): 51 | '''Calculate the actual PSI value from comparing the values. 52 | Update the actual value to a very small number if equal to zero 53 | ''' 54 | if a_perc == 0: 55 | a_perc = 0.0001 56 | if e_perc == 0: 57 | e_perc = 0.0001 58 | 59 | value = (e_perc - a_perc) * np.log(e_perc / a_perc) 60 | return(value) 61 | 62 | psi_value = sum(sub_psi(expected_percents[i], actual_percents[i]) for i in range(0, len(expected_percents))) 63 | 64 | return(psi_value) 65 | 66 | if len(expected.shape) == 1: 67 | psi_values = np.empty(len(expected.shape)) 68 | else: 69 | psi_values = np.empty(expected.shape[axis]) 70 | 71 | for i in range(0, len(psi_values)): 72 | if len(psi_values) == 1: 73 | psi_values = psi(expected, actual, buckets) 74 | elif axis == 0: 75 | psi_values[i] = psi(expected[:,i], actual[:,i], buckets) 76 | elif axis == 1: 77 | psi_values[i] = psi(expected[i,:], actual[i,:], buckets) 78 | 79 | return(psi_values) 80 | 81 | def dataset_validation(initial_df, new_df): 82 | 83 | initial = pd.DataFrame(initial_df) 84 | new = pd.DataFrame(new_df) 85 | 86 | if initial.shape[1] == new.shape[1]: 87 | df = pd.DataFrame() 88 | for col in initial.columns: 89 | try: 90 | col_initial = pd.to_numeric(initial[col], errors='raise') 91 | col_new = pd.to_numeric(new[col], errors='raise') 92 | 93 | if min(col_initial) == max(col_initial): 94 | col_initial = random.choices([-0.000001, 0.000001], k=len(col_initial)) + col_initial 95 | 96 | if min(col_new) == max(col_new): 97 | col_new = random.choices([-0.000001, 0.000001], k=len(col_new)) + col_new 98 | 99 | psi = calculate_psi(col_initial, col_new, buckettype='bins', buckets=10, axis=0) 100 | initial_desc_stat = '{} ({})'.format(round(np.mean(col_initial), 2),round(np.std(col_initial), 2)) 101 | new_desc_stat = '{} ({})'.format(round(np.mean(col_new), 2),round(np.std(col_new), 2)) 102 | 103 | if psi < 0.1: 104 | conclusion = 'Very slight change' 105 | elif psi < 0.2: 106 | conclusion = 'Some minor change' 107 | else: 108 | conclusion = 'Significant change' 109 | 110 | df = df.append(pd.DataFrame(data={ 111 | 'column name': col 112 | ,'baseline mean (std dev)': initial_desc_stat 113 | ,'new mean (std dev)': new_desc_stat 114 | ,'stability index': round(psi, 2) 115 | ,'conclusion': conclusion 116 | }, index = [0])) 117 | except ValueError: continue 118 | df = df.reset_index(drop = True) 119 | else: 120 | df = pd.DataFrame() 121 | print('The 2 csv do not have the same number of columns') 122 | return df 123 | 124 | def dataset_validation_csv(initial_path, new_path): 125 | 126 | initial = pd.read_csv(initial_path) 127 | new = pd.read_csv(new_path) 128 | 129 | if initial.shape[1] == new.shape[1]: 130 | df = pd.DataFrame() 131 | for col in initial.columns: 132 | try: 133 | col_initial = pd.to_numeric(initial[col], errors='raise') 134 | col_new = pd.to_numeric(new[col], errors='raise') 135 | 136 | if min(col_initial) == max(col_initial): 137 | col_initial = random.choices([-0.000001, 0.000001], k=len(col_initial)) + col_initial 138 | 139 | if min(col_new) == max(col_new): 140 | col_new = random.choices([-0.000001, 0.000001], k=len(col_new)) + col_new 141 | 142 | psi = calculate_psi(col_initial, col_new, buckettype='bins', buckets=10, axis=0) 143 | initial_desc_stat = '{} ({})'.format(round(np.mean(col_initial), 2),round(np.std(col_initial), 2)) 144 | new_desc_stat = '{} ({})'.format(round(np.mean(col_new), 2),round(np.std(col_new), 2)) 145 | 146 | if psi < 0.1: 147 | conclusion = 'Very slight change' 148 | elif psi < 0.2: 149 | conclusion = 'Some minor change' 150 | else: 151 | conclusion = 'Significant change' 152 | 153 | df = df.append(pd.DataFrame(data={ 154 | 'column name': col 155 | ,'baseline mean (std dev)': initial_desc_stat 156 | ,'new mean (std dev)': new_desc_stat 157 | ,'stability index': round(psi, 2) 158 | ,'conclusion': conclusion 159 | }, index = [0])) 160 | except ValueError: continue 161 | df = df.reset_index(drop = True) 162 | else: 163 | df = pd.DataFrame() 164 | print('The 2 csv do not have the same number of columns') 165 | return df 166 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==0.25.2 2 | matplotlib==3.1.1 3 | shap==0.29.3 4 | tensorflow==2.5.0 5 | imbalanced_learn==0.5.0 6 | numpy==1.17.2 7 | Adversarial_Robustness_Toolbox==1.0.1 8 | Keras==2.2.4 9 | art==4.1 10 | imblearn==0.0 11 | scikit_learn==0.21.3 12 | plotly==4.2.1 --------------------------------------------------------------------------------