├── config.yml ├── codalab ├── CheXpert-v1.0 │ └── valid │ │ └── patient00000 │ │ ├── study1 │ │ ├── view1_frontal.jpg │ │ └── view2_lateral.jpg │ │ └── study2 │ │ └── view1_frontal.jpg ├── valid_image_paths.csv └── CheXpert_predict.py ├── README.md ├── .gitignore ├── data_processing.ipynb └── replicating_chexpert.ipynb /config.yml: -------------------------------------------------------------------------------- 1 | path: /home/jupyter/chestxrays/chexpert-entries 2 | chexpert_url: http://download.cs.stanford.edu/deep/CheXpert-v1.0-small.zip -------------------------------------------------------------------------------- /codalab/CheXpert-v1.0/valid/patient00000/study1/view1_frontal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simongrest/chexpert-entries/HEAD/codalab/CheXpert-v1.0/valid/patient00000/study1/view1_frontal.jpg -------------------------------------------------------------------------------- /codalab/CheXpert-v1.0/valid/patient00000/study1/view2_lateral.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simongrest/chexpert-entries/HEAD/codalab/CheXpert-v1.0/valid/patient00000/study1/view2_lateral.jpg -------------------------------------------------------------------------------- /codalab/CheXpert-v1.0/valid/patient00000/study2/view1_frontal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simongrest/chexpert-entries/HEAD/codalab/CheXpert-v1.0/valid/patient00000/study2/view1_frontal.jpg -------------------------------------------------------------------------------- /codalab/valid_image_paths.csv: -------------------------------------------------------------------------------- 1 | Path 2 | CheXpert-v1.0/valid/patient00000/study1/view1_frontal.jpg 3 | CheXpert-v1.0/valid/patient00000/study1/view2_lateral.jpg 4 | CheXpert-v1.0/valid/patient00000/study2/view1_frontal.jpg 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Repo for replication of CheXpert paper and submission to CheXpert competition 2 | 3 | This repo contains my notebooks and code that attempt to replicate and improve on the results from 4 | 5 | **CheXpert: A large chest radiograph dataset with uncertainty labels and expert comparison** 6 | by Jeremy Irvin, Pranav Rajpurkar et al from Stanford's ML Group 7 | https://arxiv.org/abs/1901.07031 8 | 9 | It also contains the script required to make submissions to the associated competition https://stanfordmlgroup.github.io/competitions/chexpert/ 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # data directory 107 | data/ 108 | -------------------------------------------------------------------------------- /data_processing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 12, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from pathlib import Path\n", 10 | "import shutil\n", 11 | "import zipfile\n", 12 | "import yaml\n", 13 | "import wget" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 13, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "with open(\"config.yml\", 'r') as ymlfile:\n", 23 | " cfg = yaml.load(ymlfile)\n", 24 | " path = cfg['path']\n", 25 | " chexpert_url = cfg['chexpert_url']" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 14, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "root = Path(path)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "## Download CheXpert Dataset" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "zipname = wget.download(chexpert_url,root)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 15, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "zipname = 'small_images.zip'" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 17, 65 | "metadata": {}, 66 | "outputs": [], 67 | "source": [ 68 | "zip = zipfile.ZipFile(root/zipname)\n", 69 | "zip.extractall(root/'data')\n", 70 | "zip.close()" 71 | ] 72 | } 73 | ], 74 | "metadata": { 75 | "kernelspec": { 76 | "display_name": "Python 3", 77 | "language": "python", 78 | "name": "python3" 79 | }, 80 | "language_info": { 81 | "codemirror_mode": { 82 | "name": "ipython", 83 | "version": 3 84 | }, 85 | "file_extension": ".py", 86 | "mimetype": "text/x-python", 87 | "name": "python", 88 | "nbconvert_exporter": "python", 89 | "pygments_lexer": "ipython3", 90 | "version": "3.7.1" 91 | } 92 | }, 93 | "nbformat": 4, 94 | "nbformat_minor": 2 95 | } 96 | -------------------------------------------------------------------------------- /codalab/CheXpert_predict.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import sys 3 | import pandas as pd 4 | from joblib import Parallel, delayed 5 | from fastai.vision import * 6 | from torchvision.models import * 7 | 8 | def load_and_resize_img(path): 9 | """ 10 | Load and convert the full resolution images on CodaLab to 11 | low resolution used in the small dataset. 12 | """ 13 | img = cv2.imread(path, 0) 14 | 15 | size = img.shape 16 | max_dim = max(size) 17 | max_ind = size.index(max_dim) 18 | 19 | if max_ind == 0: 20 | # width fixed at 320 21 | wpercent = (320 / float(size[0])) 22 | hsize = int((size[1] * wpercent)) 23 | new_size = (hsize, 320) 24 | 25 | else: 26 | # height fixed at 320 27 | hpercent = (320 / float(size[1])) 28 | wsize = int((size[0] * hpercent)) 29 | new_size = (320, wsize) 30 | 31 | resized_img = cv2.resize(img, new_size) 32 | 33 | cv2.imwrite(path, resized_img) 34 | 35 | 36 | def main(): 37 | 38 | #python src/ 39 | 40 | infile = sys.argv[1] 41 | outfile = sys.argv[2] 42 | 43 | print(infile) 44 | print(outfile) 45 | 46 | test_df = pd.read_csv(infile) 47 | 48 | Parallel(n_jobs=-1)(delayed(load_and_resize_img)(path) for path in test_df.Path.values) 49 | 50 | chexpert_learn = load_learner('src/','chexpert_densenet.pkl') 51 | 52 | test_data_src = (ImageList.from_df(test_df, '.','Path')) 53 | 54 | chexpert_learn.data.add_test(test_data_src) 55 | 56 | chexpert_learn.data.batch_size = 8 57 | 58 | test_preds=chexpert_learn.get_preds(ds_type=DatasetType.Test)[0] 59 | 60 | i = 0 61 | for c in chexpert_learn.data.classes: 62 | 63 | test_df[c] = test_preds[:,i] 64 | i = i+1 65 | 66 | #CheXpert-v1.0/{valid,test}// 67 | 68 | test_df.Path.str.split('/') 69 | 70 | def get_study(path): 71 | return path[0:path.rfind('/')] 72 | 73 | test_df['Study'] = test_df.Path.apply(get_study) 74 | 75 | study_df = test_df.drop('Path',axis=1).groupby('Study').max().reset_index() 76 | 77 | study_df.to_csv(outfile,index=False) 78 | 79 | if __name__ == '__main__': 80 | main() -------------------------------------------------------------------------------- /replicating_chexpert.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Attempting to replicate the results from:\n", 8 | "# **CheXpert: A large chest radiograph dataset with uncertainty labels and expert comparison**\n", 9 | "## https://arxiv.org/abs/1901.07031\n", 10 | "##### Irvin, Jeremy and Rajpurkar, Pranav and Ko, Michael and Yu, Yifan and Ciurea-Ilcus, Silviana and Chute, Chris and Marklund, Henrik and Haghgoo, Behzad and Ball, Robyn and Shpanskaya, Katie and others" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import warnings\n", 20 | "warnings.filterwarnings('ignore')" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "%reload_ext autoreload\n", 30 | "%autoreload 2\n", 31 | "%matplotlib inline" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 3, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "from fastai.vision import *\n", 41 | "from torchvision.models import *\n", 42 | "import yaml\n", 43 | "import pandas as pd\n", 44 | "import datetime\n", 45 | "\n", 46 | "from sklearn.metrics import roc_auc_score" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "### Load configuration with local path and url for dataset" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "with open(\"config.yml\", 'r') as ymlfile:\n", 63 | " cfg = yaml.load(ymlfile)\n", 64 | " path = cfg['path']" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 5, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "chestxrays_root = Path(path)\n", 74 | "data_path = chestxrays_root/'data'" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "### Load Data" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 6, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "full_train_df = pd.read_csv(data_path/'CheXpert-v1.0-small/train.csv')\n", 91 | "full_valid_df = pd.read_csv(data_path/'CheXpert-v1.0-small/valid.csv')" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 7, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "chexnet_targets = ['No Finding',\n", 101 | " 'Enlarged Cardiomediastinum', 'Cardiomegaly', 'Lung Opacity',\n", 102 | " 'Lung Lesion', 'Edema', 'Consolidation', 'Pneumonia', 'Atelectasis',\n", 103 | " 'Pneumothorax', 'Pleural Effusion', 'Pleural Other', 'Fracture',\n", 104 | " 'Support Devices']\n", 105 | "\n", 106 | "chexpert_targets = ['Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Pleural Effusion']" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "### Uncertainty Approaches\n", 114 | "\n", 115 | "The CheXpert paper outlines several different approaches to mapping using the uncertainty labels in the data:\n", 116 | "\n", 117 | "- Ignoring - essentially removing from the calculation in the loss function\n", 118 | "- Binary mapping - sending uncertain values to either 0 or 1\n", 119 | "- Prevalence mapping - use the rate of prevelance of the feature as it's target value\n", 120 | "- Self-training - consider the uncertain values as unlabeled\n", 121 | "- 3-Class Classification - retain a separate value for uncertain and try to predict it as a class in its own right\n", 122 | "\n", 123 | "\n", 124 | "The paper gives the results of different experiments with the above approaches and indicates the most accurate approach for each feature.\n", 125 | " \n", 126 | "|Approach/Feature|Atelectasis|Cardiomegaly|Consolidation|Edema|PleuralEffusion|\n", 127 | "|-----------|-----------|-----------|-----------|-----------|-----------|\n", 128 | "|`U-Ignore`|0.818(0.759,0.877)|0.828(0.769,0.888)|0.938(0.905,0.970)|0.934(0.893,0.975)|0.928(0.894,0.962)|\n", 129 | "|`U-Zeros`|0.811(0.751,0.872)|0.840(0.783,0.897)|0.932(0.898,0.966)|0.929(0.888,0.970)|0.931(0.897,0.965)|\n", 130 | "|`U-Ones`|**0.858(0.806,0.910)**|0.832(0.773,0.890)|0.899(0.854,0.944)|0.941(0.903,0.980)|0.934(0.901,0.967)|\n", 131 | "|`U-Mean`|0.821(0.762,0.879)|0.832(0.771,0.892)|0.937(0.905,0.969)|0.939(0.902,0.975)|0.930(0.896,0.965)|\n", 132 | "|`U-SelfTrained`|0.833(0.776,0.890)|0.831(0.770,0.891)|0.939(0.908,0.971)|0.935(0.896,0.974)|0.932(0.899,0.966)|\n", 133 | "|`U-MultiClass`|0.821(0.763,0.879)|**0.854(0.800,0.909)**|0.937(0.905,0.969)|0.928(0.887,0.968)|0.936(0.904,0.967)|\n", 134 | "\n", 135 | "The binary mapping approaches (U-Ones and U-Zeros) are easiest to implement and so to begin with we take the best option between U-Ones and U-Zeros for each feature\n", 136 | "\n", 137 | "- Atelectasis `U-Ones`\n", 138 | "- Cardiomegaly `U-Zeros`\n", 139 | "- Consolidation `U-Zeros`\n", 140 | "- Edema `U-Ones`\n", 141 | "- Pleural Effusion `U-Zeros`" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 8, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "u_one_features = ['Atelectasis', 'Edema']\n", 151 | "u_zero_features = ['Cardiomegaly', 'Consolidation', 'Pleural Effusion']" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "### Add target features string" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 9, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "def feature_string(row):\n", 168 | " feature_list = []\n", 169 | " for feature in u_one_features:\n", 170 | " if row[feature] in [-1,1]:\n", 171 | " feature_list.append(feature)\n", 172 | " \n", 173 | " for feature in u_zero_features:\n", 174 | " if row[feature] == 1:\n", 175 | " feature_list.append(feature)\n", 176 | " \n", 177 | " return ';'.join(feature_list)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 10, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "full_train_df['train_valid'] = False\n", 187 | "full_valid_df['train_valid'] = True" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": {}, 193 | "source": [ 194 | "##### Create patient and study columns" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 11, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "full_train_df['patient'] = full_train_df.Path.str.split('/',3,True)[2]\n", 204 | "full_train_df ['study'] = full_train_df.Path.str.split('/',4,True)[3]\n", 205 | "\n", 206 | "full_valid_df['patient'] = full_valid_df.Path.str.split('/',3,True)[2]\n", 207 | "full_valid_df ['study'] = full_valid_df.Path.str.split('/',4,True)[3]" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 12, 213 | "metadata": {}, 214 | "outputs": [], 215 | "source": [ 216 | "full_df = pd.concat([full_train_df, full_valid_df])" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 13, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "full_df['feature_string'] = full_df.apply(feature_string,axis = 1).fillna('')" 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": {}, 231 | "source": [ 232 | "### Set up a small sample for fast iteration" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": 14, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "def seed_everything(seed):\n", 242 | " random.seed(seed)\n", 243 | " os.environ['PYTHONHASHSEED'] = str(seed)\n", 244 | " np.random.seed(seed)\n", 245 | " torch.manual_seed(seed)\n", 246 | " torch.cuda.manual_seed(seed)\n", 247 | " torch.backends.cudnn.deterministic = True" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 15, 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [ 256 | "def get_sample_df(sample_perc = 0.05):\n", 257 | " \n", 258 | " train_only_df = full_df[~full_df.train_valid]\n", 259 | " valid_only_df = full_df[full_df.train_valid]\n", 260 | " unique_patients = train_only_df.patient.unique()\n", 261 | " mask = np.random.rand(len(unique_patients)) <= sample_perc\n", 262 | " sample_patients = unique_patients[mask]\n", 263 | "\n", 264 | " sample_df = train_only_df[full_train_df.patient.isin(sample_patients)]\n", 265 | " return pd.concat([sample_df,valid_only_df])" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "### Set up data set using Fastai datablock" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 16, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "def get_src(df = full_df):\n", 282 | " return (ImageList\n", 283 | " .from_df(df, data_path, 'Path')\n", 284 | " .split_from_df('train_valid')\n", 285 | " .label_from_df('feature_string',label_delim=';')\n", 286 | " )\n", 287 | "\n", 288 | "def get_data(size, src, bs=32):\n", 289 | " return (src.transform(get_transforms(do_flip=False), size=size, padding_mode='zeros')\n", 290 | " .databunch(bs=bs).normalize(imagenet_stats))" 291 | ] 292 | }, 293 | { 294 | "cell_type": "markdown", 295 | "metadata": {}, 296 | "source": [ 297 | "## Create a function to evaluate performance of all features" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 17, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "def validation_eval(learn):\n", 307 | " acts = full_valid_df.groupby(['patient','study'])[learn.data.classes].max().values\n", 308 | "\n", 309 | " valid_preds=learn.get_preds(ds_type=DatasetType.Valid)\n", 310 | " preds = valid_preds[0]\n", 311 | " preds_df = full_valid_df.copy()\n", 312 | "\n", 313 | " for i, c in enumerate(learn.data.classes):\n", 314 | " preds_df[c] = preds[:,i]\n", 315 | "\n", 316 | " preds = preds_df.groupby(['patient','study'])[learn.data.classes].mean().values\n", 317 | "\n", 318 | " auc_scores = {learn.data.classes[i]: roc_auc_score(acts[:,i],preds[:,i]) for i in range(len(chexpert_targets))}\n", 319 | "\n", 320 | " #average results reported in the associated paper\n", 321 | " chexpert_auc_scores = {'Atelectasis': 0.858,\n", 322 | " 'Cardiomegaly': 0.854,\n", 323 | " 'Consolidation': 0.939,\n", 324 | " 'Edema': 0.941,\n", 325 | " 'Pleural Effusion': 0.936}\n", 326 | "\n", 327 | " max_feat_len = max(map(len, chexpert_targets))\n", 328 | "\n", 329 | " avg_chexpert_auc = sum(list(chexpert_auc_scores.values()))/len(chexpert_auc_scores.values())\n", 330 | " avg_auc = sum(list(auc_scores.values()))/len(auc_scores.values())\n", 331 | "\n", 332 | " [print(f'{k: <{max_feat_len}}\\t auc: {auc_scores[k]:.3}\\t chexpert auc: {chexpert_auc_scores[k]:.3}\\t difference:\\\n", 333 | " {(chexpert_auc_scores[k]-auc_scores[k]):.3}') for k in chexpert_targets]\n", 334 | "\n", 335 | " print(f'\\nAverage auc: {avg_auc:.3} \\t CheXpert average auc {avg_chexpert_auc:.3}\\t Difference {(avg_chexpert_auc-avg_auc):.3}')\n", 336 | "\n", 337 | "def avg_auc_metric(input, targs):\n", 338 | " input=input.detach().cpu()\n", 339 | " targs=targs.detach().cpu().byte()\n", 340 | " auc_scores = [roc_auc_score(targs[:,i],input[:,i]) for i in range(targs.shape[1])]\n", 341 | " auc_scores = torch.tensor(auc_scores)\n", 342 | " return auc_scores.mean()" 343 | ] 344 | }, 345 | { 346 | "cell_type": "markdown", 347 | "metadata": {}, 348 | "source": [ 349 | "### Create callbacks to evaluate and save learner" 350 | ] 351 | }, 352 | { 353 | "cell_type": "code", 354 | "execution_count": 18, 355 | "metadata": {}, 356 | "outputs": [], 357 | "source": [ 358 | "class SaveCallback(LearnerCallback):\n", 359 | " _order = 99\n", 360 | " def __init__(self, learn):\n", 361 | " super().__init__(learn)\n", 362 | " self.epoch = 0\n", 363 | " self.skip = False\n", 364 | " def on_epoch_end(self, **kwargs):\n", 365 | " self.epoch += 1\n", 366 | " if self.skip: return\n", 367 | " learn.save(f'{datetime.datetime.now():%Y-%m-%d %H:%M} avg AUC:{learn.recorder.metrics[-1][-1].item():.3}')" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": 19, 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [ 376 | "def get_batch_size(img_size, data_image_size=320, base_batch_size=32):\n", 377 | " pixel_ratio = (img_size/data_image_size)**2\n", 378 | " return 2**math.floor(math.log2(base_batch_size/pixel_ratio))\n", 379 | "\n", 380 | "def get_chexpert_learner(learn=None, img_size=320, size=1, mixup=True, pretrained=True, callback_fns=[]):\n", 381 | " bs = get_batch_size(img_size)\n", 382 | "\n", 383 | " data = get_data(img_size, get_src(get_sample_df(size)), bs=bs)\n", 384 | " \n", 385 | " if learn:\n", 386 | " learn.data = data\n", 387 | " elif mixup:\n", 388 | " learn = cnn_learner(data, densenet121, callback_fns=callback_fns, pretrained=pretrained, metrics=avg_auc_metric).mixup(stack_y=False)\n", 389 | " else:\n", 390 | " learn = cnn_learner(data, densenet121, callback_fns=callback_fns, pretrained=pretrained, metrics=avg_auc_metric)\n", 391 | " return learn" 392 | ] 393 | }, 394 | { 395 | "cell_type": "markdown", 396 | "metadata": {}, 397 | "source": [ 398 | "### Alter LR_Finder to remove my callbacks before running" 399 | ] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": 20, 404 | "metadata": {}, 405 | "outputs": [], 406 | "source": [ 407 | "cbfs = [SaveCallback]\n", 408 | "def lr_find_no_cbs(learn):\n", 409 | " learn.callback_fns = [cbf for cbf in learn.callback_fns if cbf not in cbfs]\n", 410 | " lr_find(learn)\n", 411 | " learn.recorder.plot(suggestion=True)\n", 412 | " learn.callback_fns += cbfs" 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "metadata": {}, 418 | "source": [ 419 | "### Train on sample set on small images" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": 25, 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "seed_everything(2019)" 429 | ] 430 | }, 431 | { 432 | "cell_type": "code", 433 | "execution_count": 26, 434 | "metadata": {}, 435 | "outputs": [], 436 | "source": [ 437 | "learn = get_chexpert_learner(img_size=64, size=1, callback_fns=cbfs)" 438 | ] 439 | }, 440 | { 441 | "cell_type": "code", 442 | "execution_count": 23, 443 | "metadata": {}, 444 | "outputs": [ 445 | { 446 | "data": { 447 | "text/html": [], 448 | "text/plain": [ 449 | "" 450 | ] 451 | }, 452 | "metadata": {}, 453 | "output_type": "display_data" 454 | }, 455 | { 456 | "name": "stdout", 457 | "output_type": "stream", 458 | "text": [ 459 | "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n", 460 | "Min numerical gradient: 1.58E-02\n" 461 | ] 462 | }, 463 | { 464 | "data": { 465 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEKCAYAAAArYJMgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8FdXZwPHfc7PvCSQgEJawg4ogETdc6oq7VmvBfWnVVq1LtdW31vpqbavW+tatrfsupa6oKK5oVVCC7HtYhLAGyL4vz/vHneAlJCSQO7lzb57v53M/uXPmzNznkIQnc+bMOaKqGGOMMW7xhToAY4wxkc0SjTHGGFdZojHGGOMqSzTGGGNcZYnGGGOMqyzRGGOMcZUlGmOMMa5yNdGIyAQRWS4i+SJyWwv7+4vIJyKyQERmiEh2wL5LRWSl87o0oHysiCx0zvmwiIibbTDGGNMx4tYDmyISBawATgQKgNnAJFVdElDnP8C7qvq8iBwHXK6qF4tINyAPyAUUmAOMVdUiEfkWuAGYBUwDHlbV911phDHGmA6LdvHc44B8VV0NICKTgbOAJQF1RgI3Oe8/A95y3p8MfKSqO5xjPwImiMgMIFVVZzrlLwBnA3tMNJmZmTpgwIAgNMkYY7qOOXPmbFPVrI6ex81E0wdYH7BdABzarM584Fzg78A5QIqIdG/l2D7Oq6CF8j0aMGAAeXl5exu/McZ0aSLyfTDO4+Y9mpbunTTvp7sFOEZE5gLHABuA+j0c255z+j9c5CoRyRORvMLCwvZHbYwxJqjcTDQFQN+A7WxgY2AFVd2oqj9W1THA75yykj0cW+C8b/WcAed+QlVzVTU3K6vDV37GGGP2kZuJZjYwRERyRCQWmAhMDawgIpki0hTD7cAzzvvpwEkikiEiGcBJwHRV3QSUichhzmizS4C3XWyDMcaYDnIt0ahqPXAd/qSxFJiiqotF5G4ROdOpdiywXERWAD2Be51jdwD34E9Ws4G7mwYGAL8AngLygVW0MRDAGGNMaLk2vNlLcnNz1QYDGGPM3hGROaqa29Hz2MwAxhhjXGWJxhhjjKss0RhjTARatrmUv324nMKymlCHYonGGGMi0ZKNpTz8aT7lNfWhDsUSjTHGRKIKJ8EkxUWFOBJLNMYYE5HKaxoASI5zc6ax9rFEY4wxEaiiph6fQEKMXdEYY4xxQXlNPUmx0XhhyS5LNMYYE4EqaupJjg99txlYojHGmIhUXlNPkgfuz4AlGmOMiUiWaIwxxriqoqaeZA8MbQZLNMYYE5EqahpIirUrGmOMMS4pr6n3xDM0YInGGGMiUkWtjTozxhjjElWlwgYDGGOMcUtNfSN1DWpdZ8YYY9yxc0LN2C4w6kxEJojIchHJF5HbWtjfT0Q+E5G5IrJARE51yi8UkXkBr0YRGe3sm+Gcs2lfDzfbYIwx4abCmVDTK11nrkUhIlHAY8CJQAEwW0SmquqSgGp3AFNU9R8iMhKYBgxQ1ZeBl53zHAi8rarzAo67UFXz3IrdGGPCWdMaNF2h62wckK+qq1W1FpgMnNWsjgKpzvs0YGML55kEvOpalMYYE2Eqap1E0wVGnfUB1gdsFzhlge4CLhKRAvxXM9e3cJ6fsnuiedbpNvu9tDI1qYhcJSJ5IpJXWFi4Tw0wxphwVL5z0bPITzQtJQBttj0JeE5Vs4FTgRdFZGdMInIoUKmqiwKOuVBVDwSOcl4Xt/ThqvqEquaqam5WVlZH2mGMMWGlvLrrdJ0VAH0DtrPZvWvsSmAKgKrOBOKBzID9E2l2NaOqG5yvZcAr+LvojDHGOCq60BXNbGCIiOSISCz+pDG1WZ11wPEAIjICf6IpdLZ9wE/w39vBKYsWkUznfQxwOrAIY4wxO+0cDOCRuc5ci0JV60XkOmA6EAU8o6qLReRuIE9VpwK/Bp4UkZvwd6tdpqpN3WtHAwWqujrgtHHAdCfJRAEfA0+61QZjjAlHPwxv9sZzNK6mO1Wdhv8mf2DZnQHvlwBHtnLsDOCwZmUVwNigB2qMMRGkorae+Bgf0VHeeCbfG1EYY4wJGi/N3AyWaIwxJuJ4aUJNsERjjDERp7y63jOLnoElGmOMiTjWdWaMMcZVFbX1nhlxBpZojDEm4lTUNJAcHxPqMHayRGOMMRHG33VmVzTGGGNcUlFjgwGMMca4pKFRqaxtsOHNxhhj3LFzLRpLNMYYY9zgtZmbwRKNMcZElKZE45XVNcESjTHGRJRyZ+ZmG3VmjDHGFTu7zmzUmTHGGDeU2z0aY4wxbiqvtlFnxhhjXNQ0vLnLXNGIyAQRWS4i+SJyWwv7+4nIZyIyV0QWiMipTvkAEakSkXnO658Bx4wVkYXOOR8WEXGzDcYYE06aus5SusKoMxGJAh4DTgFGApNEZGSzancAU1R1DDAReDxg3ypVHe28rgko/wdwFTDEeU1wqw3GGBNuKmrqifIJcdHe6bByM5JxQL6qrlbVWmAycFazOgqkOu/TgI17OqGI9AJSVXWmqirwAnB2cMM2xpjwVVHTQFJsFF7q7HEz0fQB1gdsFzhlge4CLhKRAmAacH3AvhynS+1zETkq4JwFbZzTGGO6LK8tegbuJpqW0qk2254EPKeq2cCpwIsi4gM2Af2cLrWbgVdEJLWd5/R/uMhVIpInInmFhYX73IhIV1FTT11DY6jDMMYESXl1vacGAgC4GU0B0DdgO5vdu8auxLnHoqozRSQeyFTVrUCNUz5HRFYBQ51zZrdxTpzjngCeAMjNzW0xGXmRqrK9opaNxVVsKKqisLyGxNhoMhJjSE+MIT0xlr4ZicTuY/9rbX0jc9cV8WX+Nv67chsLCopJiY/hpJE9OfXAXhwxuDtx0bs/Uayq7KioZXNpNQ2NSo+UeDKTY4mO8k4/sDGmaXXNrpNoZgNDRCQH2ID/Zv8FzeqsA44HnhOREUA8UCgiWcAOVW0QkYH4b/qvVtUdIlImIocB3wCXAI+42IZOUdfQyBcrCnlj7gY+XbqVqrqGPdaP9gmDspIZul8Kw3omMzArmZzMJAZ0TyIhNmpnsiooqmL9jkpWFZazcms5+VvKWbOtgtqGRnwCB/VN55fHDmZjcRUfLNrMf+YUkBIfzcDMJBRoVEUVyqrr2VxaTW39rlc+ItA9KZZ+3RIZlZ3OqOw0RmWnMzAzCZ/PO/3DxnQlXuw6cy0aVa0XkeuA6UAU8IyqLhaRu4E8VZ0K/Bp4UkRuwt8FdpmqqogcDdwtIvVAA3CNqu5wTv0L4DkgAXjfeYVcVW0DOypr2VBUxeKNJSzZWMrijaVsKqkiMTaa5LhokuP9X1MTYkiJjyY1PoaKmnqmLdzE9opauiXF8uOD+zC0Zwq90xPonR5Pj5R4qmobKKqspbiqju3lNeRvLWf55jLmrivinfm7XtBlpcRRVl1Hdd0PSUEE+mYkMqRHMscOy+Lg/hkcNrA7aQk/LPVaU9/A1/nb+WDRZraUVSOAiCDAoKxoeqXFs19aPL3S4hERCstq2FpWQ2FZNau2VvDv2et57uu1ACTGRjGkZwpDeyQztGcKfTISUPUnrkZV6hqUqroGqmsb/F/rGmho9O9raASf4HyW/98gOyORrJS4TvguGhP+Kmrq6ZkSH+owdiH+wVuRLTc3V/Py8vb6uGWbS6moaSAhJoqE2CgSYqIoqapjyaYfEsn32yvZUVG721VIZnIsI3un0a9bAlW1jZTX1FFeU09pVT1l1XWUVddTWl2HIJwwsgc/HpPNMcOyiNnLrqiKmnrWbKtg7fYK1hRWsL6okrSEGPqkJ5CdkUh2twQGdE8iPsbdCfYaGpX8reXMX1/Mkk2lrNhSxootZWwrr23z2Gif4PMJUSJE+YT6xsZdEiVAn/QEcgdkkNs/gzH9MtgvLZ70hBjrujOmmSP/8imHDezOg+cf1OFzicgcVc3t6Hm8dX3lMfe9v4zPlrc8kCAu2sfw/VI4NKcb3ZNjyUiKpVtiLD3T4tm/Vyo9Utv+i0KdrqmOdDMlxUVzQJ80DuiTts/nCIYonzBsvxSG7ZeyS/mOilq2lFbjEyHK579KivbJzsSdEBO1W7JQVUqr6tlYUsXG4irWbKtg7rpiZq7aztvzdr2CS0uIoVtSLD1S4uiVFk/PtHh6pyVw6MBuDOuZ4qkhnsZ0Bn/XmXdmbgZLNHt068nDufSIAVTXNVJd5+/mSYiJYmTvVAZmJnX4r2kRIdL/H+yWFEu3pNi9OkZESEuMIS0xhhG9UneWqyoFRVUsKChhW3kNOypqKaqsZXu5P5nlfV/EltJq6hr8V+l90hM4bngPjhveg77dEkmN93dfJsR46xkDY4JFVamo6VqDAcLeyN6pjCS17YqmU4gIfbsl0rdbYqt1GhuVzaXVfLGikE+WbeW1OQW8OOv7XepE+YQeKXH0Tk+gV1o8fdITGNQjmf17pzKkR8o+j+gzJtRq6hupb1RLNMa4yecTeqcnMHFcPyaO60d1XQNzvi9iW3kN5TX1lDv3xraW1rCxpIrFG0v5cMmWnSPqYqKEoT1TGN03nUMHduewnG7t6gY1xgua5jnrMqPOjPGC+Jgojhycucc6jY3K2u0VLHYGeCzeWMLb8zby8jfrABjQPZHxQzI5fkRPDh/Y3fWBFcbsqwpLNMZ4k88nDMzyP490xkG9AahvaGTJplK+XbODWau388Z3G3hp1joSY6MYPziTHx+czcn797R7PcZTvLjoGViiMaZF0VE+5yHUdH521ECq6xqYtXo7nyzdysdLt/Dhki2Myk7jtxOGt3nFZExnqajxP2bhtSsau+tpTDvEx0Rx7LAe3HP2AXz52+N44LxRbCur4cKnvuHCp2bxVf42mzPOhFzFzisab3XveivtGRMGonzCT3L7csZBvXn5m3U89lk+Fz71DSnx0Rw9JIsfDe/Bj4Zl0T3ZZjMwncsGAxgTYeJjorhyfA6TxvXlixXb+GzZVj5bvpX3Fm5CBMb0Tef4ET05YURPhvZMtvs5xnV2j8aYCJUYG82EA/ZjwgH70dioLN5YyifLtvDJ0q08MH05D0xfzoDuiZx7cDbnjs2md3pCqEM2EWrnqDMPLeMMlmiMCSqfTzgwO40Ds9O48YShbCmt5pOlW5k6fwMPfrSCv328gvGDM5l4SD9O3r+nzdVmgmrnFU2st/5r91Y0xkSYnqnxXHBoPy44tB/rtlfy2ncFvD6ngGtf+Y6+3RL4+VED+cnYviTEeuvmrQlPFTX1JMREEeWxZTrszyljOkm/7oncfOJQ/vubH/Gvi8eSmRzHnW8v5oi/fMLfP15JWXVdqEM0Ya68psFz92fArmiM6XQ+n3Dy/vtx0sie5H1fxD9nrOKhj1fw/My1XH/cYC44tF+Lq5wa05YKD87cDHZFY0zIiAiHDOjG05cdwlvXHsmwnin87ztLOP7Bz3lr7gYaGyN/rSgTXOUenLkZLNEY4wmj+6bzys8P5fkrxpESH8ON/57Huf/8mnnri0MdmgkjXlzGGSzRGOMZIsIxQ7N47/rxPHDeKNbvqOLsx77ilv/MZ2tpdajDM2GgoismGhGZICLLRSRfRG5rYX8/EflMROaKyAIROdUpP1FE5ojIQufrcQHHzHDOOc959XCzDcZ0Np8z88BntxzD1ccM5O15G/jRX2fwjxmrqKlvaPsEpsvy4qJn4GKiEZEo4DHgFGAkMElERjardgcwRVXHABOBx53ybcAZqnogcCnwYrPjLlTV0c5rq1ttMCaUUuJjuP2UEXx40zEcPqg7932wjJMf+oKPl2xB1e7fmN15ddSZm1c044B8VV2tqrXAZOCsZnUUdi5hmQZsBFDVuaratDj8YiBeRGziKNMl5WQm8dSlh/D8FeOI8gk/eyGPS5+dzZzvd4Q6NOMxXXHUWR9gfcB2gVMW6C7gIhEpAKYB17dwnnOBuapaE1D2rNNt9nuxCaRMF3HM0Cw+uPFo7jhtBPPXF3PuP2by48e/4oNFm2mwEWpdXkOjUlXX9a5oWkoAzX8bJgHPqWo2cCrwoojsjElE9gfuA64OOOZCp0vtKOd1cYsfLnKViOSJSF5hYWEHmmGMd8RE+fjZUQP5+rbjuOuMkWwtq+Gal+Zwwt8+Z8rs9bZUQRfm1Zmbwd1EUwD0DdjOxukaC3AlMAVAVWcC8UAmgIhkA28Cl6jqqqYDVHWD87UMeAV/F91uVPUJVc1V1dysrKygNMgYr0iKi+ayI3OYccuxPHrBGBJjo/jN6ws49oEZvDTrexs00AV5dRlncDfRzAaGiEiOiMTiv9k/tVmddcDxACIyAn+iKRSRdOA94HZV/aqpsohEi0hTIooBTgcWudgGYzwtOsrH6aN68+7143nmslyyUuK4461FHHP/DN6et8EGDXQhFR5dIgBcTDSqWg9cB0wHluIfXbZYRO4WkTOdar8Gfi4i84FXgcvU/5txHTAY+H2zYcxxwHQRWQDMAzYAT7rVBmPChYhw3PCevPnLI3jxynH0SI3jhsnzuOSZb/l+e0WowzOdwMtdZ9IV/uLJzc3VvLy8UIdhTKdpaFRemvU9D0xfTl1DI9f9aDBXHzOI2Gh7RjtSfblyGxc9/Q1Trj6ccTndgnJOEZmjqrkdPY/91BkTgaJ8wqVHDOCTXx/DCSN68uBHKzjn8a9YsaUs1KEZl/ywumbXGt5sjAmxnqnxPHbhwfzr4rFsLqnm9Ee+5Kn/rrYJOyNQU6JJiYsJcSS7s0RjTBdw8v77Mf2mozl6SBZ/fG8pFzw1i80lNn9aJCl31jPy2jLOYInGmC4jMzmOJy8Zy/3njmJhQQlnPPolc9cVhTosEyRFlf5Ek5ZgVzTGmBASEc4/pC9vXnskCTFR/PSJWbzxXUGowzJBUFJVR2p8tOeWcQZLNMZ0SUN7pvD2tUcytl8GN0+Zz5+nLbVpbMJccWUt6YmxoQ6jRZZojOmiMpJieeHKcVx8WH/+9cVqLnv2W3ZU1IY6LLOPiirryEj0XrcZWKIxpkuLifJxz9kH8JcfH8g3a3Zw+sP/tfs2Yaq4qo40u6IxxnjVxHH9eP2aI/D5hPP/NZMXZ6616WvCTEllLekeHAgAlmiMMY4Ds9N49/rxjB+cye/fXswdby2y523CiHWdGWPCQnpiLE9feghXHzOQl79Zx29eX2CDBMJAQ6NSWu3drjPvPdljjAkpn0+4bcJwEmKi+L+PV1Jb38jfzj+I6Cj7u9SryqrrUMWzXWeWaIwxuxERbjxhKHHRUdz3wTLqGhr5+8QxNimnRzU9rJmR5M1E066fGhEZJCJxzvtjReRXzpoxxpgI9otjB3Hn6SN5f9Fmbpg817rRPKq40j8sPT3Bm11n7f3z5HWgQUQGA08DOfhXtzTGRLgrxufweyfZ/P7tRTYazYOKq5zpZzw6GKC9XWeNqlovIucA/6eqj4jIXDcDM8Z4x5Xjc9hWXsM/ZqwiMzmOm08cGuqQTICmK5qMMB8MUCcik4BLgTOcMm+mTmOMK35z8jC2l9fw8CcryUqO5eLDB4Q6JOModu7RhPtggMuBa4B7VXWNiOQAL7kXljHGa0SEP51zIDsq6rhz6mLSE2M546DeoQ7L4B8MIAKpHk007bpHo6pLVPVXqvqqiGQAKar6l7aOE5EJIrJcRPJF5LYW9vcTkc9EZK6ILBCRUwP23e4ct1xETm7vOY0x7omO8vHoBWM4pH83bvr3PD5asiXUIRn8swKkxsd4cuZmaP+osxkikioi3YD5wLMi8rc2jokCHgNOAUYCk0RkZLNqdwBTVHUMMBF43Dl2pLO9PzABeFxEotp5TmOMi+Jjonj6slz275PGtS9/x+crCkMdUpdXXFVHukcHAkD7R52lqWop8GPgWVUdC5zQxjHjgHxVXa2qtcBk4KxmdRRIbfoMYKPz/ixgsqrWqOoaIN85X3vOaYxxWUp8DC9cPo7BPZK56oU8Zq7aHuqQurSiyjrPLhEA7U800SLSCzgfeLedx/QB1gdsFzhlge4CLhKRAmAacH0bx7bnnACIyFUikicieYWF9heXMcGWlhjDi1eOo1+3RK58fjZzvrdZn0PFyxNqQvsTzd3AdGCVqs4WkYHAyjaOaamzsPkA/EnAc6qaDZwKvCgivj0c255z+gtVn1DVXFXNzcrKaiNUY8y+6J4cx8s/O5QeKXH87PnZrN9RGeqQuqSI6DpT1f+o6ihV/YWzvVpVz23jsAKgb8B2Nj90jTW5EpjinHMmEA9k7uHY9pzTGNOJeqTG8+zl42hoVH7+Qh7lNfWhDqnLKaqo9ewzNND+wQDZIvKmiGwVkS0i8rqIZLdx2GxgiIjkiEgs/pv7U5vVWQcc73zGCPyJptCpN1FE4pyh1EOAb9t5TmNMJ8vJTOKxCw9m5dZybv73PFteoBP5Z26uJy0Cus6exf8fem/890Teccpapar1wHX4u9yW4h9dtlhE7haRM51qvwZ+LiLzgVeBy9RvMf4rnSXAB8C1qtrQ2jnb31xjjFuOGpLFHaeN4MMlW3jo4xWhDqfLKHWmn/Fy11l7H9jMUtXAxPKciNzY1kGqOg3/Tf7AsjsD3i8Bjmzl2HuBe9tzTmOMN1x2xACWbSrjkU/zGbZfCqePsgc63Vbk8elnoP1XNNtE5KKmZ1lE5CLAxjMaY3YhItxz9gHk9s/gttcXsqG4KtQhRTyvT6gJ7U80V+Af2rwZ2ASch39aGmOM2UVstI+HfjqaRlV++9oCm+3ZZSUen+cM2j/qbJ2qnqmqWaraQ1XPxv/wpjHG7KZvt0T+59QRfJm/jVe+XRfqcCJaJHWdteTmoEVhjIk4Fx7aj/GDM7n3vaX2fI2Lds7cHAFdZy3x5uxtxhhPEBHuO28UPhFufW2+DXl2SXGVf+bmlPjITDT2U2OM2aM+6Qn8/vQRzFq9gxdmrg11OBGpuLKWtATvztwMbSQaESkTkdIWXmX4n6kxxpg9Oj+3L8cMzeKB6cvZUlod6nAiTnFlnacHAkAbiUZVU1Q1tYVXiqq29xkcY0wXJiLcfdb+1DUqf562NNThRJziqjrSPDwQADrWdWaMMe3Sv3sSVx89kLfmbeSb1fYIXjAVV9aS4eGBAGCJxhjTSX557GD6pCfwh6mLqW9oDHU4ESPsu86MMSZYEmKjuOO0ESzbXMbL39izNcFSXFnr6UXPwBKNMaYTTThgP8YPzuTBD5ezrbwm1OGEvfqGRkqr6z39DA1YojHGdCIR4a4zR1JZ28D9HywLdThhr7Tav/aPdZ0ZY0yAwT1SuGJ8DlPyCpi3vjjU4YS14qbpZ5Ks68wYY3Zx/XGD6ZESx51vL6LBZgzYZ0XO9DNeXvQMLNEYY0IgJT6G/zl1BAsKSpiStz7U4YStkir/FY0NBjDGmBacNbo34wZ04/4Plu3sAjJ7p6jCf0Vjz9EYY0wLRIT/PWt/Sqrq+OuHy0MdTlhqWvQsPaELX9GIyAQRWS4i+SJyWwv7HxKRec5rhYgUO+U/CiifJyLVInK2s+85EVkTsG+0m20wxrhnRK9ULjl8AC9/s45FG0pCHU7YKamsxSeQEu/tGcFcSzQiEgU8BpwCjAQmicjIwDqqepOqjlbV0cAjwBtO+WcB5ccBlcCHAYfe2rRfVee51QZjjPtuOnEo3RJj+cPUxbYa514qqqwjLSEGn4dnbgZ3r2jGAfmqulpVa4HJwFl7qD8JeLWF8vOA91XVVk4yJgKlJcRw68nDmPN9EdMXbw51OGGluKrO8wMBwN1E0wcIHE5S4JTtRkT6AznApy3snsjuCeheEVngdL3FtXLOq0QkT0TyCgsL9z56Y0ynOW9sNoN7JHP/B8ups3nQ2q1pLRqvczPRtHQt19p18UTgNVVt2OUEIr2AA4HpAcW3A8OBQ4BuwG9bOqGqPqGquaqam5WVtbexG2M6UXSUj99OGE79ynzWnH8ppKaCz+f/+stfwqpVoQ7Rk4or6zw/4gzcTTQFQN+A7WxgYyt1W7pqATgfeFNV65oKVHWT+tUAz+LvojPGhLkT1s7hw+euJ+etV6GsDFT9X596CkaNgvffD3WInlNc5f0JNcHdRDMbGCIiOSISiz+ZTG1eSUSGARnAzBbOsdt9G+cqBxER4GxgUZDjNsZ0tlWrkJ/8hPjaamIaG3bdV1cHlZVw3nl2ZdNMsTMYwOtcSzSqWg9ch7/baykwRVUXi8jdInJmQNVJwGRtNtxERAbgvyL6vNmpXxaRhcBCIBP4ozstMMZ0mgcf9CeUPamrg4ce6px4wkB9QyNl1fVkhMEVjauDr1V1GjCtWdmdzbbvauXYtbQweEBVjwtehMYYT3jppfYlmhdfhEcf7ZyYPK6k6WHNLn6Pxhhj2qe8PLj1uoBiSzTGGLMXkpODW68LaJofrqsPBjDGmPa56CKIaeMv85gYuPjizoknDBRXNs1zZlc0xhjTtl//un2J5qabOieeMLAz0VjXmTHGtMOgQfDaa5CYuFvCqfVF0ZCQ4N8/aFCIAvSeoqauM4/P3AyWaIwxXnHKKbBgAVx11c6ZATQ1lXcPPZ2f3fw0DSdPCHWEnrJmWwVpCTGkJnh75mawRGOM8ZJBg/zDl0tKoKEBKSkh5h+P81l9Km/O3RDq6DxlxZYyhvZMxv/surdZojHGeNppB/ZiVHYaf/twOVW1DW0f0AWoKss3lzG0Z0qoQ2kXSzTGGE/z+YQ7ThvJxpJqHvl0ZajD8YQtpTWUVtczbD9LNMYYExTjcrpx3thsnvhiNSu3lIU6nJBb7vwb2BWNMcYE0e2nDCcpLprfvbWoy6/EuWKzJRpjjAm67slx3H7KcL5ds4PXv+vaAwOWbykjKyWObkneH9oMlmiMMWHk/Ny+jO2fwZ+mLaWoojbU4YTMii1lDAuTqxmwRGOMCSM+n3DvOQdQUlXHfR8sC3U4IdHYqKzcUh423WZgicYYE2aG75fKz8bnMHn2er5dsyPU4XS6gqIqquoaGLZf+EwwaonGGBN2bjhhCNkZCdz2xgKq67rWszXhNuIMLNEYY8JQYmw0fzrnQFYXVvDop/mhDqdTrXASzRBLNH4iMkFElotIvojc1sL+h0RknvNaISLFAfsaAvZNDSjPEZFvRGSliPxbRMJj2IUxJqiOHprFuQcuW9Y7AAAT30lEQVRn88/PV7F0U2mow+k0yzeXkZ2RQHKc9+c4a+JaohGRKOAx4BRgJDBJREYG1lHVm1R1tKqOBh4B3gjYXdW0T1XPDCi/D3hIVYcARcCVbrXBGONtd5w2grSEGG57fQENjV3j2ZpwG3EG7l7RjAPyVXW1qtYCk4Gz9lB/EvDqnk4o/tnjjgNec4qeB84OQqzGmDCUkRTLXWfuz/yCEp79ak2ow3FdXUMjqwrLGRomU880cTPR9AHWB2wXOGW7EZH+QA7waUBxvIjkicgsEWlKJt2BYlWtb+ucxpiu4fRRvTh+eA8e/HAFqwrLQx2Oq9Zuq6CuQe2KJkBLc1e3dm07EXhNVQOHj/RT1VzgAuD/RGTQ3pxTRK5yElVeYWHh3sRtjAkjIsIfzzmAhNgofvnSdxE9w3M4jjgDdxNNAdA3YDsb2NhK3Yk06zZT1Y3O19XADGAMsA1IF5Gmu2CtnlNVn1DVXFXNzcrK2tc2GGPCQK+0BP7vp6NZsbWM3721MGLnQluxuYwonzAwKynUoewVNxPNbGCIM0osFn8ymdq8kogMAzKAmQFlGSIS57zPBI4Elqj/p+cz4Dyn6qXA2y62wRgTJo4emsUNxw/hje82MHn2+rYPCEPLt5QxoHsi8TFRoQ5lr7iWaJz7KNcB04GlwBRVXSwid4tI4CiyScBk3fVPkBFAnojMx59Y/qKqS5x9vwVuFpF8/PdsnnarDcaY8PKr44Zw9NAs/jB1MYs2lIQ6nKBbsaU8bNagCSSReokZKDc3V/Py8kIdhjGmE+yoqOX0h/+LzydMu+EoUuNjQh1SUFTXNTDizg+44fgh3HjC0E75TBGZ49wr7xCbGcAYE1G6JcXy6IUHs6G4ioc+WhHqcIImf2s5qoTdiDOwRGOMiUAH98vgwkP78cLM71m2OTJmDVjetNhZGHadWaIxxkSkW04aRmp8NHe+vTgiRqEt3lhKXLSP/t0SQx3KXrNEY4yJSOmJsdx6sn9FzqnzW3uyInx8vWobuQMyiI4Kv/+2wy9iY4xpp58e0pdR2Wn8adpSymvq2z7Ao7aX17BscxlHDMoMdSj7xBKNMSZiRfmE/z1zf7aU1vDIJytDHc4+m7Xav8DbEYO6hziSfWOJxhgT0cb0y+D83Gye/nLNzrVcws1Xq7aRHBfNgX3SQh3KPrFEY4yJeL+dMJy0hBhumDyPmvrwmwtt5qrtHJrTLSzvz4AlGmNMF9A9OY77zxvF0k2l/HX68lCHs1c2FlexZlsFRwwOz/szYInGGNNFHD+iJxcf1p8n/7uGL1duC3U47fb1qu1A+N6fAUs0xpgu5H9OHcHgHsncPGUeRRW1oQ6nXb5etY1uSbFhOSNAE0s0xpguIyE2ir9PHE1RZS23vbHA8w9yqipf52/n8EHd8flaWo4rPFiiMcZ0Kfv3TuM3Jw9n+uItPP/12lCHs0drtlWwubQ6rLvNwBKNMaYLunJ8DieM6Mk97y3lq3zv3q/5auf9mfAdCACWaIwxXZDPJzz004MYlJXEL1/+ju+3V4Q6pBbNXLWN3mnxDOgefvObBbJEY4zpklLiY3jyklxE4Mrn8yirrgt1SLtobFRmrtrO4YMyEQnf+zNgicYY04X1757E4xcczJptFdwweR4Njd4ZHLB0cylFlXVhf38GLNEYY7q4IwZn8oczRvLpsq3c8+4Sz4xEm9l0f2awJZo9EpEJIrJcRPJF5LYW9j8kIvOc1woRKXbKR4vITBFZLCILROSnAcc8JyJrAo4b7WYbjDGR7+LD+nPFkTk89/VaHv00P9ThAPDhki0M6ZFMr7SEUIfSYdFunVhEooDHgBOBAmC2iExV1SVNdVT1poD61wNjnM1K4BJVXSkivYE5IjJdVYud/beq6mtuxW6M6VpEhDtOG0FRZS0PfrSCjKRYLjqsf8ji2VJazey1O/jVcUNCFkMwuZZogHFAvqquBhCRycBZwJJW6k8C/gCgqjsX+lbVjSKyFcgCils51hhjOsTnE+4/bxQlVXX8/u1FZCTGctqoXiGJ5f2Fm1CF00P0+cHmZtdZH2B9wHaBU7YbEekP5ACftrBvHBALrAoovtfpUntIROKCF7IxpiuLifLx2AUHM7ZfBjf+e27InrF5b+EmhvVMYUgYTzsTyM1E09J4vNbusk0EXlPVXebvFpFewIvA5ara6BTfDgwHDgG6Ab9t8cNFrhKRPBHJKyws3Jf4jTFdUEJsFE9feggDM5O5+sU5LNlY2qmfv6mkitlri0J2NeUGNxNNAdA3YDsbaG3h7onAq4EFIpIKvAfcoaqzmspVdZP61QDP4u+i242qPqGquaqam5WV1YFmGGO6mrTEGJ69/BCS46K5/Llv2VBc1WmfPW3hZgBLNO00GxgiIjkiEos/mUxtXklEhgEZwMyAsljgTeAFVf1Ps/q9nK8CnA0scq0Fxpguq3d6As9dcQiVNQ1c9sy3lFR2zgOd7y3YyIheqQzKSu6Uz+sMriUaVa0HrgOmA0uBKaq6WETuFpEzA6pOAibrroPXzweOBi5rYRjzyyKyEFgIZAJ/dKsNxpiubfh+qfzrkrF8v72Sn7+YR3Wdu6tzbiiu4rt1xREzCKCJeOXhJDfl5uZqXl5eqMMwxoSpqfM38qtX53L6qF48PHGMa1P2P/nFau6dtpQZtxzLgMwkVz5jb4jIHFXN7eh53BzebIwxEeHMg3qzoaiK+z5YRv/uidx68nBXPufdhZs4oE+qJ5JMMNkUNMYY0w7XHDOQSeP68thnq5gye33bB+yl9Tsqmb++mNMO7B30c4eaXdEYY0w7iAh3n3UABUVV/M+bC+mTkcCRg4O3Tsx7CzcBcNqBkXV/BuyKxhhj2i0mysdjFx7MoKxkrnlpDos2lATlvPUNjbzyzTrG9s+gX5ivPdMSSzTGGLMXUuNjeObyQ0iIieLsx77iz9OWUlFT36Fzvr9oM+t2VPLzowYGKUpvsURjjDF7qU96Au/fcBTnHpzNv75YzfEPfs67Czbu0xIDqso/P1/FwKwkThrZ04VoQ88SjTHG7IPuyXHcd94oXv/FEXRPjuW6V+Zy5fN5bC+v2avzfJm/jcUbS7n66IGuDZsONUs0xhjTAWP7ZzD1uvHcefpIvly5jVP+/l++XtX+yTj/+fkqeqbGcfaYFuccjgiWaIwxpoOifMIV43N489ojSI6P5sKnvuFvH62gvqFxj8ctLCjhq/ztXHFkDnHRUZ0UbeezRGOMMUGyf+803rluPOcenM3Dn6zkZy/kUVPf+rQ1//x8FSnx0VxwaL9OjLLzWaIxxpggSoqL5q8/OYg/nn0AM5YXcu3L31Fbv/uVzdptFby/aBMXHdaflPiYEETaeSzRGGOMCy46rD/3nH0AHy/dyq9enUtdQDfako2l/Ob1BURH+bj8yAGhC7KT2MwAxhjjkosP609dfSN3v7uEm6fM5+qjB/Lop/l8sHgzKXHR3HXG/vRIiQ91mK6zRGOMMS66YnwOtQ2N/OX9ZbwzfyMpcdHccPwQrjgyh7TEyO4ya2KJxhhjXHbNMYNIjoumqKKWSw4f0GUSTBNLNMYY0wkuOqx/qEMIGRsMYIwxxlWWaIwxxrjK1UQjIhNEZLmI5IvIbS3sf0hE5jmvFSJSHLDvUhFZ6bwuDSgfKyILnXM+LCKROTmQMcZECNfu0YhIFPAYcCJQAMwWkamquqSpjqreFFD/emCM874b8AcgF1BgjnNsEfAP4CpgFjANmAC871Y7jDHGdIybVzTjgHxVXa2qtcBk4Kw91J8EvOq8Pxn4SFV3OMnlI2CCiPQCUlV1pvrn434BONu9JhhjjOkoNxNNHyBwYe0Cp2w3ItIfyAE+bePYPs779pzzKhHJE5G8wsLCfWqAMcaYjnMz0bR076S1VYEmAq+patPsc60d2+5zquoTqpqrqrlZWVltBmuMMcYdbiaaAqBvwHY2sLGVuhP5odtsT8cWOO/bc05jjDEeIPuy9Gi7TiwSDawAjgc2ALOBC1R1cbN6w4DpQI5z36VpMMAc4GCn2nfAWFXdISKzgeuBb/APBnhEVae1EUsh8H2z4jSgpI2ywO223mcC7V/tqO1Y9qZOpLRlb9vRfLvpfWBZOLYl2N+TPcXZnjqR8vPV2r5wbEtn/a70V9WOdwmpqmsv4FT8yWYV8Dun7G7gzIA6dwF/aeHYK4B853V5QHkusMg556M4yXIfYnuirbLA7bbeA3kd+HfaLZa9qRMpbdnbduwh/sCysGtLsL8nnd0Wr/58RVJbvPa70tbL1Slo1H+lMa1Z2Z3Ntu9q5dhngGdaKM8DDghCeO+0o+ydvXwfzFj2pk6ktGVv29F8+51W6uyrULUl2N+T9p4nWG3x6s9Xa/vCsS1e+13ZI9e6zroaEclT1dxQxxEM1hbviZR2gLXFq9xsi01BEzxPhDqAILK2eE+ktAOsLV7lWlvsisYYY4yr7IrGGGOMqyzRtEBEnhGRrSKyaB+ObXXSTxG53plkdLGI3B/cqFuNJ+htEZG7RGRDwISopwY/8t1iceV74uy/RURURDKDF/Ee43Hje3KPiCxwvh8fikjv4EfeYjxutOUBEVnmtOdNEUkPfuQtxuNGW37i/L43ioir93I6En8r5wvexMZuDWcL5xdwNP5neBbtw7HfAofjn8XgfeAUp/xHwMdAnLPdI4zbchdwS7h/T5x9ffE/x/U9kBmubcE/B2BTnV8B/wzjtpwERDvv7wPuC+O2jACGATOAXC/G78Q2oFlZN2C18zXDeZ+xp7bu6WVXNC1Q1S+AHYFlIjJIRD4QkTki8l8RGd78uDYm/fwF/ueFapzP2OpuK/xcakunc7EdDwG/ofXpkYLOjbaoamlA1SQ6qT0uteVDVa13qs5i19lAXONSW5aq6nIvx9+KoE5sbImm/Z4ArlfVscAtwOMt1NnTpJ9DgaNE5BsR+VxEDnE12j3raFsArnO6Np4RkQz3Qt2jDrVDRM4ENqjqfLcDbYcOf09E5F4RWQ9cCNxJ6ATj56vJFYR2GZBgtiUU2hN/Szo8sXEgVx/YjBQikgwcAfwnoDsyrqWqLZQ1/WUZjf8S9DDgEGCKiAx0/iroNEFqyz+Ae5zte4AH8f+H0Gk62g4RSQR+h7+bJqSC9D1BVX8H/E5Ebgeuw7+mU6cKVlucc/0OqAdeDmaM7RXMtoTCnuIXkcuBG5yywcA0EakF1qjqOQRhYuNAlmjaxwcUq+rowELxL+42x9mciv8/4NYm/SwA3nASy7ci0oh/bqHOXsOgw21R1S0Bxz0JvOtmwK3oaDsG4V+aYr7zS5gNfCci41R1s8uxNxeMn69ArwDvEYJEQ5Da4tx8Ph04vrP/GAsQ7O9LZ2sxfgBVfRZ4FkBEZgCXqeragCoFwLEB29n47+Xs28TGbt6cCucXMICAm2rA18BPnPcCHNTKcbPxX7U03Sg71Sm/BrjbeT8U/2XpPs3T5oG29AqocxMwORzb0azOWjppMIBL35MhAXWux7/sRri2ZQKwBMjqrDa4/TNGJwwG2Nf4aX0wwBr8vTAZzvtu7Wlri3F19jcyHF74lyzYBNThz+BX4v/r9wNgvvNLcGcrx7Y46ScQC7zk7PsOOC6M2/IisBBYgP8vul7h2I5mddbSeaPO3PievO6UL8A/f1WfMG5LPv4/xOY5r84aQedGW85xzlUDbAGmey1+Wkg0TnnQJja2mQGMMca4ykadGWOMcZUlGmOMMa6yRGOMMcZVlmiMMca4yhKNMcYYV1miMV2SiJR38uc9JSIjg3SuBvHP0rxIRN5pa3ZjEUkXkV8G47ON2Rc2vNl0SSJSrqrJQTxftP4wEaSrAmMXkeeBFap67x7qDwDeVdUDOiM+Y5qzKxpjHCKSJSKvi8hs53WkUz5ORL4WkbnO12FO+WUi8h8ReQf4UESOFZEZIvKa+NdTeblprQ6nPNd5X+5MgDlfRGaJSE+nfJCzPVtE7m7nVddMfpgkNFlEPhGR78S/XshZTp2/AIOcq6AHnLq3Op+zQET+N4j/jMbsxhKNMT/4O/CQqh4CnAs85ZQvA45W1TH4Z0X+U8AxhwOXqupxzvYY4EZgJDAQOLKFz0kCZqnqQcAXwM8DPv/vzue3OX+UM+fW8fhnZwCoBs5R1YPxr3/0oJPobgNWqepoVb1VRE4ChgDjgNHAWBE5uq3PM2Zf2aSaxvzgBGBkwEy3qSKSAqQBz4vIEPwz1cYEHPORqgauAfKtqhYAiMg8/HNPfdnsc2r5YSLSOcCJzvvD+WFtj1eAv7YSZ0LAuefgXysE/HNP/clJGo34r3R6tnD8Sc5rrrOdjD/xfNHK5xnTIZZojPmBDzhcVasCC0XkEeAzVT3Hud8xI2B3RbNz1AS8b6Dl37E6/eHmaGt19qRKVUeLSBr+hHUt8DD+dWiygLGqWicia4H4Fo4X4M+q+q+9/Fxj9ol1nRnzgw/xr+MCgIg0Ta+eBmxw3l/m4ufPwt9lBzCxrcqqWoJ/2eZbRCQGf5xbnSTzI6C/U7UMSAk4dDpwhbNeCSLSR0R6BKkNxuzGEo3pqhJFpCDgdTP+/7RznRvkS/Av7QBwP/BnEfkKiHIxphuBm0XkW6AXUNLWAao6F//MvBPxLxCWKyJ5+K9uljl1tgNfOcOhH1DVD/F3zc0UkYXAa+yaiIwJKhvebIxHOKt+VqmqishEYJKqntXWccZ4nd2jMcY7xgKPOiPFiunk5bGNcYtd0RhjjHGV3aMxxhjjKks0xhhjXGWJxhhjjKss0RhjjHGVJRpjjDGuskRjjDHGVf8P/56Ks0E6ldoAAAAASUVORK5CYII=\n", 466 | "text/plain": [ 467 | "
" 468 | ] 469 | }, 470 | "metadata": { 471 | "needs_background": "light" 472 | }, 473 | "output_type": "display_data" 474 | } 475 | ], 476 | "source": [ 477 | "lr_find_no_cbs(learn)" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": {}, 484 | "outputs": [ 485 | { 486 | "data": { 487 | "text/html": [ 488 | "\n", 489 | "
\n", 490 | " \n", 502 | " \n", 503 | " 40.00% [2/5 16:04<24:06]\n", 504 | "
\n", 505 | " \n", 506 | "\n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | "
epochtrain_lossvalid_lossavg_auc_metrictime
00.4633980.4868260.81689211:27
10.4438220.4811300.84900904:36

\n", 533 | "\n", 534 | "

\n", 535 | " \n", 547 | " \n", 548 | " 6.42% [28/436 00:21<05:15 0.4433]\n", 549 | "
\n", 550 | " " 551 | ], 552 | "text/plain": [ 553 | "" 554 | ] 555 | }, 556 | "metadata": {}, 557 | "output_type": "display_data" 558 | } 559 | ], 560 | "source": [ 561 | "lr = 1e-2\n", 562 | "learn.fit_one_cycle(5,slice(lr))" 563 | ] 564 | }, 565 | { 566 | "cell_type": "code", 567 | "execution_count": 29, 568 | "metadata": {}, 569 | "outputs": [ 570 | { 571 | "name": "stdout", 572 | "output_type": "stream", 573 | "text": [ 574 | "Atelectasis \t auc: 0.792\t chexpert auc: 0.858\t difference: 0.066\n", 575 | "Cardiomegaly \t auc: 0.804\t chexpert auc: 0.854\t difference: 0.0502\n", 576 | "Consolidation \t auc: 0.893\t chexpert auc: 0.939\t difference: 0.0456\n", 577 | "Edema \t auc: 0.874\t chexpert auc: 0.941\t difference: 0.0665\n", 578 | "Pleural Effusion\t auc: 0.89\t chexpert auc: 0.936\t difference: 0.0464\n", 579 | "\n", 580 | "Average auc: 0.851 \t CheXpert average auc 0.906\t Difference 0.0549\n" 581 | ] 582 | } 583 | ], 584 | "source": [ 585 | "validation_eval(learn)" 586 | ] 587 | }, 588 | { 589 | "cell_type": "code", 590 | "execution_count": 30, 591 | "metadata": {}, 592 | "outputs": [ 593 | { 594 | "data": { 595 | "text/html": [], 596 | "text/plain": [ 597 | "" 598 | ] 599 | }, 600 | "metadata": {}, 601 | "output_type": "display_data" 602 | }, 603 | { 604 | "name": "stdout", 605 | "output_type": "stream", 606 | "text": [ 607 | "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n", 608 | "Min numerical gradient: 7.59E-07\n" 609 | ] 610 | }, 611 | { 612 | "data": { 613 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcVNWd9/HPr6t3emFrFtllUTHiQkuiThKXqGR5MBmTjDEmkoxjJgadSWLy6CsZJ48+mckyGWfyjLOo0TGrRrKhwQVNTCauNIgICAjI0mzdQHfTe22/54+6YNk20NB1u6q6v+/X676oe+rcqt+huuvX59x7zzF3R0REpL8Ksh2AiIgMDkooIiKSEUooIiKSEUooIiKSEUooIiKSEUooIiKSEUooIiKSEUooIiKSEUooIiKSEYXZDiBTRo8e7VOnTs12GCIieWXFihX73L0mE681aBLK1KlTqaury3YYIiJ5xcy2Zeq1NOQlIiIZoYQiIiIZoYQiIiIZEWpCMbP5ZrbBzDaZ2S1HqPNxM1tnZmvN7Kdp5dea2evBdm2YcYqISP+FdlLezCLAXcClQD2w3MyWuPu6tDozgVuBC9y9yczGBOUjgb8HagEHVgTHNoUVr4iI9E+YPZR5wCZ33+LuUeBB4Ioedf4KuOtQonD3hqD8cmCZux8InlsGzA8xVhER6acwE8oEYEfafn1Qlm4WMMvMnjWzF8xs/nEci5ldb2Z1ZlbX2NiYwdBFROR4hZlQrJeynusNFwIzgQuBTwD3mtnwPh6Lu9/t7rXuXltTk5H7ckRE8sriFfX89MXt2Q4DCDeh1AOT0vYnArt6qfMbd4+5+xvABlIJpi/HiogMeb9cWc8vV9ZnOwwg3ISyHJhpZtPMrBi4CljSo86vgYsAzGw0qSGwLcATwGVmNsLMRgCXBWUiIpKmuSPG8PKibIcBhHiVl7vHzWwRqUQQAe5z97VmdjtQ5+5LeDNxrAMSwFfcfT+Amd1BKikB3O7uB8KKVUQkX7V0xjhtfFW2wwBCnsvL3ZcCS3uU3Zb22IEvBVvPY+8D7gszPhGRfNfcEc2ZHorulBcRyVPReJL2aILhZUooIiLSDy2dMQD1UEREpH9aOqMAVJcXZzmSFCUUEZE81dwR9FA05CUiIv1xaMirWglFRET643APRedQRESkP5oPnZQv0zkUERHph5aOKGZQWRrqLYV9poQiIpKnmjtjVJcVUVDQ23y6A08JRUQkTzV3xHLmCi9QQhERyVvNnbGcuQcFlFBERPJWSzDklSuUUERE8lRLR1RDXiIi0n/NnbmzFgoooYiI5KVk0mnp1El5ERHpp9auOO65MzEkKKGIiOSl5mCmYfVQRESkX3JtHi8IOaGY2Xwz22Bmm8zsll6eX2hmjWa2KtiuS3vuO2a21sxeM7Pvm1lu3AoqIpIDcm1xLQhxTXkziwB3AZcC9cByM1vi7ut6VH3I3Rf1OPZ84AJgTlD0J+C9wDNhxSsikk+ac2zqegi3hzIP2OTuW9w9CjwIXNHHYx0oBYqBEqAI2BtKlCIieailI1itMUdmGoZwE8oEYEfafn1Q1tOVZrbazBab2SQAd38e+D2wO9iecPfXQoxVRCSvHDqHMlR6KL2d8/Ae+48AU919DvAU8ACAmc0ATgMmkkpCF5vZe972BmbXm1mdmdU1NjZmNHgRkVzW3BljWHGE4sLcubYqzEjqgUlp+xOBXekV3H2/u3cHu/cAc4PHHwFecPc2d28DHgPe1fMN3P1ud69199qampqMN0BEJFc1d8QYnkP3oEC4CWU5MNPMpplZMXAVsCS9gpmNT9tdABwa1toOvNfMCs2siNQJeQ15iYgEWjqjOTXcBSFe5eXucTNbBDwBRID73H2tmd0O1Ln7EuAmM1sAxIEDwMLg8MXAxcCrpIbJHnf3R8KKVUQk36R6KEMkoQC4+1JgaY+y29Ie3wrc2stxCeBzYcYmIpLPWjpjzBhTke0w3iJ3zuaIiEif5dpMw6CEIiKSd9ydlo5YTt2DAkooIiJ5pzOWIJpIqociIiL9c3hiyBy7yksJRUQkz+TiTMOghCIikncOrYWicygiItIvB3Nw6npQQhERyTu5ODEkKKGIiOSdZvVQREQkE5o7YhRHCigrimQ7lLdQQhERyTMtnVGqy4vItZXRlVBERPJMc0cs5+5BASUUEZG8k4szDYMSiohI3mnpzL15vEAJRUQk76QSinooIiLST80dUQ15iYhI/0TjSdqjCZ2UFxGR/mnJ0ZsaQQlFRCSvtByaGLJ8iJ2UN7P5ZrbBzDaZ2S29PL/QzBrNbFWwXZf23GQze9LMXjOzdWY2NcxYRUTyQa6uhQJQGNYLm1kEuAu4FKgHlpvZEndf16PqQ+6+qJeX+CHwTXdfZmYVQDKsWEVE8kWuroUC4fZQ5gGb3H2Lu0eBB4Er+nKgmc0GCt19GYC7t7l7R3ihiojkh8PnUIbYfSgTgB1p+/VBWU9XmtlqM1tsZpOCsllAs5n90sxeNrPvBj2etzCz682szszqGhsbM98CEZEcc2im4aF2H0pvs5Z5j/1HgKnuPgd4CnggKC8E3g3cDJwLnAwsfNuLud/t7rXuXltTU5OpuEVEclZLRxQzqCwN7YzFCQszodQDk9L2JwK70iu4+3537w527wHmph37cjBcFgd+DZwTYqwiInmhObhLvqAgt2YahnATynJgpplNM7Ni4CpgSXoFMxuftrsAeC3t2BFmdqjbcTHQ82S+iMiQk6szDUOIV3m5e9zMFgFPABHgPndfa2a3A3XuvgS4ycwWAHHgAMGwlrsnzOxm4GlLTfi/glQPRkRkSGvujOXkPSgQYkIBcPelwNIeZbelPb4VuPUIxy4D5oQZn4hIvmnpjFGVg+dPQHfKi4jklfbueE6ekAclFBGRvNLWFaeiRAlFRET6qb07zjAlFBER6Q93py2qHoqIiPRTRzSBO0ooIiLSP+3dcQANeYmISP+0BQlFPRQREekXJRQREcmINg15iYhIJrR3JwD1UEREpJ/aulNroVToTnkREemPtqCHMqzkbesN5gQlFBGRPNGuk/IiIpIJbV1xCgzKitRDERGRfmgL5vFKLROVe5RQRETyRHt37s7jBUooIiJ5oz2auzMNgxKKiEjeaM3htVAg5IRiZvPNbIOZbTKzW3p5fqGZNZrZqmC7rsfzVWa208z+Lcw4RUTyQa4PeYUWmZlFgLuAS4F6YLmZLXH3dT2qPuTui47wMncAfwgrRhGRfNLenaCmsiTbYRxRmD2UecAmd9/i7lHgQeCKvh5sZnOBscCTIcUnIpJX2rrjVJQUZTuMIwozoUwAdqTt1wdlPV1pZqvNbLGZTQIwswLge8BXQoxPRCSvpBJKbt6DAuEmlN4ulPYe+48AU919DvAU8EBQfgOw1N13cBRmdr2Z1ZlZXWNjY78DFhHJVe6e0+vJQ4jnUEj1SCal7U8EdqVXcPf9abv3AN8OHp8HvNvMbgAqgGIza3P3W3ocfzdwN0BtbW3PZCUiMmh0x5PEk56zE0NCuAllOTDTzKYBO4GrgKvTK5jZeHffHewuAF4DcPdPptVZCNT2TCYiIkNJri+uBSEmFHePm9ki4AkgAtzn7mvN7Hagzt2XADeZ2QIgDhwAFoYVj4hIPju8nnzxEEwoAO6+FFjao+y2tMe3Arce4zX+G/jvEMITEckbrV1BDyWHh7x0p7yISB7I9anrQQlFRCQvtEdzez15UEIREckLh4e8huh9KCIikiHtwfK/eX+nvJlNN7OS4PGFZnaTmQ0PNzQRETnk8FVeg6CH8gsgYWYzgB8A04CfhhaViIi8RWseXDbc14SSdPc48BHgX9z9i8D48MISEZF07d1xhhVHKCjIzeV/oe8JJWZmnwCuBR4NynJ3IE9EZJDJ9Xm8oO8J5TOk5tf6pru/EUyn8uPwwhIRkXStOb64FvTxTvlgUaybAMxsBFDp7t8KMzAREXlTe3c8p++Sh75f5fVMsBzvSOAV4H4z++dwQxMRkUNS51AGQUIBqt39IPDnwP3uPhd4X3hhiYhIurbuxKA5h1JoZuOBj/PmSXkRERkgbd0xKgfDkBdwO6lp6De7+3IzOxl4PbywREQkXXt3IqdvaoS+n5R/GHg4bX8LcGVYQYmIyFu1DZbLhs1sopn9yswazGyvmf3CzCaGHZyIiEA0niQaT1IxSE7K3w8sAU4CJgCPBGUiIhKyw2uhDJJzKDXufr+7x4Ptv4GaEOMSEZFAW3fur4UCfU8o+8zsGjOLBNs1wP4wAxMRkZS2PFitEfqeUD5L6pLhPcBu4KOkpmM5KjObb2YbzGyTmd3Sy/MLzazRzFYF23VB+Vlm9ryZrTWz1Wb2F31vkojI4JIPy/9C36/y2g4sSC8zs78F/uVIx5hZBLgLuBSoB5ab2ZJgGpd0D7n7oh5lHcCn3f11MzsJWGFmT7h7c1/iFREZTAbbkFdvvnSM5+cBm9x9i7tHgQeBK/rywu6+0d1fDx7vAhrQORsRGaIG25BXb441Kf8EYEfafn1Q1tOVwbDWYjOb9LY3MZsHFAObe3nuejOrM7O6xsbG4whdRCR/DLarvHrjx3i+t4TT85hHgKnuPgd4CnjgLS+Qmu7lR8Bn3D35thdzv9vda929tqZGHRgRGZzaDq0nn+P3oRw1OjNrpffEYUDZMV67HkjvcUwEdqVXcPf0K8XuAb6d9t5VwG+Br7v7C8d4LxGRQautK/fXk4djJBR3r+zHay8HZgaLce0ErgKuTq9gZuPdfXewuwB4LSgvBn4F/DCY9kVEZMhqj8YpLSqgMNKfQaXwhdZ/cve4mS0iNalkBLjP3dea2e1AnbsvAW4yswVAHDgALAwO/zjwHmCUmR0qW+juq8KKV0QkV7XlwWqNEGJCAXD3pcDSHmW3pT2+Fbi1l+N+jJYYFhEBUkNeuX7JMPTvpLyIiAyAfFitEZRQRERyXlserCcPSigiIjkvX86hKKGIiOS49jxYXAuUUEREcl5bd0I9FBER6b/27jgVOX5TIyihiIjktHgiSWcsoSEvERHpn/ZoMI+XEoqIiPRHviyuBUooIiI5LV8W1wIlFBGRnNaWJ2uhgBKKiEhO05CXiIhkxOG1UDSXl4iI9MehIa9KDXmJiEh/tOukvIiIZMKbV3npTnkREemHtu4ERRGjpFAJRURE+qE9T6auh5ATipnNN7MNZrbJzG7p5fmFZtZoZquC7bq05641s9eD7dow4xQRyVVteTJ1PYS4pryZRYC7gEuBemC5mS1x93U9qj7k7ot6HDsS+HugFnBgRXBsU1jxiojkonxZXAvC7aHMAza5+xZ3jwIPAlf08djLgWXufiBIIsuA+SHFKSKSszTklTIB2JG2Xx+U9XSlma02s8VmNuk4jxURGdTyacgrzIRivZR5j/1HgKnuPgd4CnjgOI7FzK43szozq2tsbOxXsCIiuWh/W5TqsqJsh9EnYSaUemBS2v5EYFd6BXff7+7dwe49wNy+Hhscf7e717p7bU1NTcYCFxHJBQ0Hu9jZ3MmcidXZDqVPwkwoy4GZZjbNzIqBq4Al6RXMbHza7gLgteDxE8BlZjbCzEYAlwVlIiJDxsrtqeuQzp48IsuR9E1oA3PuHjezRaQSQQS4z93XmtntQJ27LwFuMrMFQBw4ACwMjj1gZneQSkoAt7v7gbBiFRHJRSu3N1McKeAdE6qyHUqfmPvbTk3kpdraWq+rq8t2GCIiGXPlfzwHwC8+f35o72FmK9y9NhOvpTvlRURyUHc8wav1Lcydkh/DXaCEIiKSk9bsPEg0keScycOzHUqfKaGIiOSgl4MT8ufkyQl5UEIREclJK7Y1MWlkGWOqSrMdSp8poYiI5Bh3Z8W2przqnYASiohIztnZ3ElDa3denZAHJRQRkZyzYlv+nT8BJRQRkZyzclsT5cURTh1Xme1QjosSiohIjlm5vZkzJw6nMJJfX9H5Fa2IyCDXEY2zbvfBvDt/AkooIiI55ZUdLSSSroQiIiL98+YMw/lzh/whSigiIjlk5bYmptcMY3h5cbZDOW5KKCIiOaK+qYPnNu9n3rRR2Q7lhCihiIjkAHfna79agxl84aLp2Q7nhCihiIjkgF+v2skfNjby1ctPYeKI8myHc0KUUEREsmx/Wze3P7KOsycP51PnTc12OCdMCUVEJMtuf3Qdbd1xvn3lHCIFlu1wTpgSiohIFv1u/V5+s2oXX7hoBrPG5tdUKz2FmlDMbL6ZbTCzTWZ2y1HqfdTM3Mxqg/0iM3vAzF41s9fM7NYw4xQRyYZE0rntN2uZOaaCz1+Ynyfi04WWUMwsAtwFvB+YDXzCzGb3Uq8SuAl4Ma34Y0CJu58BzAU+Z2ZTw4pVRCQbVmxror6pkxsvmUlJYSTb4fRbmD2UecAmd9/i7lHgQeCKXurdAXwH6Eorc2CYmRUCZUAUOBhirCIiA27pq7spKSzg4lPHZDuUjAgzoUwAdqTt1wdlh5nZ2cAkd3+0x7GLgXZgN7Ad+Cd3P9DzDczsejOrM7O6xsbGjAYvIhKmZNJ5fM0e3jurhoqSwmyHkxFhJpTeLlXww0+aFQB3Al/upd48IAGcBEwDvmxmJ7/txdzvdvdad6+tqanJTNQiIgPg5R3N7DnYxQfOGJ/tUDImzLRYD0xK258I7ErbrwTeATxjZgDjgCVmtgC4Gnjc3WNAg5k9C9QCW0KMV0RkwDz26m6KIwVcfNrgGO6CcHsoy4GZZjbNzIqBq4Alh5509xZ3H+3uU919KvACsMDd60gNc11sKcOAdwHrQ4xVRGTAuDuPrdnDu2eOpqq0KNvhZExoCcXd48Ai4AngNeDn7r7WzG4PeiFHcxdQAawhlZjud/fVYcUqIjKQVte3sLO5k/cPouEuCHfIC3dfCiztUXbbEepemPa4jdSlwyIig87SNbspLDAuPW1stkPJKN0pLyIygNydx17dwwUzRlNdPniGu0AJRURkQK3ddZDtBzr4wBnjsh1KximhiIgMoMfW7CZSYFw6WwlFRERO0KHhrvNOHsXIYfm3xO+xKKGIiAyQZev2smVfOx8+e8KxK+chJRQRkQGQSDrfe3Ij00YP48NnnZTtcEKhhCIiMgAeeWUXG/a28qVLZ1EYGZxfvYOzVSIiOSQaT/LPyzYye3wVHxxkNzOmU0IREQnZz+t2sP1AB1+5/BQK8niJ32NRQhERCVFXLMH3n36d2ikjuPCUwT0ruhKKiEiIfvj8Vhpau/nq/FMJZlYftJRQ0m3eDDfcAFVVUFCQ+veGG1LlIiLHafv+Dv79mc28d1YN86aNzHY4oRvyCSWeSHLrL19l509/AXPmwL33QmsruKf+vffeVPljj2U7VBHJI/VNHXzinhcA+LsPzc5yNANjyCeUHU2drPvTSkYs/CR0dEAs9tYKsViq/KMfVU9FRPpkT0sXn7z3RVq7Yvz4L9/JjDEV2Q5pQAz5hDJt9DB+dvA5ipKJo1eMxeDOOwcmKBHJWw2tXVx97wvsb4vywGfn8Y4J1dkOacAM+YQCUP7zn1GUiB+9UiwGP/rRwAQkInmpobWLa+59kT0tXdz/mXM5e/KIbIc0oEJdYCtvtLVltp6IDDnb9rfzqR+8xL62bu69tpZzpw7+k/A9KaEAVFSkTsD3pZ6ISA/rdh3k0/e9RDyZ5CfXvXPI9UwOCXXIy8zmm9kGM9tkZrccpd5HzczNrDatbI6ZPW9ma83sVTMrDS3Qa66BomOsnFZUBJ/6VGghiEh+eumNA/zF3c9TFDEW//V5QzaZQIgJxcwiwF3A+4HZwCfM7G3XzplZJXAT8GJaWSHwY+Cv3f104EIg1vPYjPnyl4+ZUOKRQlb8+bWs33OQhoNdJJMeWjiSG3Yc6KCpPYq7Pmt5u4bWLu54dB3X/OBFaipLWPz585kxpjLbYWVVmENe84BN7r4FwMweBK4A1vWodwfwHeDmtLLLgNXu/gqAu+8PMU6YPh0WL05dGhyLveXS4WRhEd1WwOc/9L955skGeLIBgEkjy/j43El8tHYi46vLQg1PBtbre1v51mPreXp96rOuLClk0shypowq5/LTx/H+M8ZRUhjJcpSSLQ2tXfzXH7bw4xe2EUsk+fDZE/j6B2cPygWzjleYCWUCsCNtvx54Z3oFMzsbmOTuj5pZekKZBbiZPQHUAA+6+3dCjBXe/35YvTp1afCPfpQ6AV9RQcGnPkXyhhu5ZeR4Ptceo6kjSmNrN0+s3cP3lm3kzqc28t5ZNSy6eCZzpwzdru5g0HCwizuf2shDy3cwrLiQL75vFsNKItQ3dbL9QAer61t4bM0e7ni0mKvmTeLqd06hKGJs2tvGxr2tbG5sB6CitJCKkkIqSwsZX13G9JphTB5ZPminLB8q1u5q4YfPbePXq3YSTzofPmsCiy6ewbTRw7IdWs6wsLrzZvYx4HJ3vy7Y/xQwz91vDPYLgN8BC919q5k9A9zs7nVBcvkCcC7QATwNfN3dn+7xHtcD1wNMnjx57rZt20Jpy5Fs29/Ow3X1PFS3g5aOGN/7+Jn8rzP7v3DOlsY2lm89wOt729jY0Mbre1vZ3xalprKEsVUljKsuZeSwYgqCeYEMKC2KMGficGqnjmBsVep0U3c8Qd3WJp7Z0MCrO1s4Y0I1756ZmgKitOjtf2F3RhO8vKOJuq1NvLqzhVljK7jktLGcNXF4Xs+Qun1/B/c9+wbL1u0FoChiFEUKiBQY3fEkXbEEnbEErV1xCgyuedcUbrx45tv+4kwmnT9t2scPn9/G79bvpeeoZ2VJIYURo7UrTrzHk0URY8qoYcwcU8GssZXMGlvJKeMqmDa6gsgR/m/3t3XTHU9y0nD1gLMlnkjy+No9PPDcVpZvbaK0qICPnD2Rz73nZKYOkkRiZivcvfbYNfvwWiEmlPOAb7j75cH+rQDu/o/BfjWwGTh0Le444ACwAJgBzHf3hUHdvwO63P27R3q/2tpar6urC6Utx9LcEeWvfljH8q1NfP2Dp3Hdu08+ruO7Yqkv/qfX7+X36xvYur8DgJLCAmYEX0A1lSXsa+1mz8Eu9h7soqkjhrtz6NPr6E4QTSQBmDiijKmjhrFyexMd0QTFkQJmjq3g9b1tRBNJigsLOGfycEqLIsQSSWJxp607zsa9rcSTjhlMGVnOjqZOEklndEUJF51SQ3VZEe3RBB3ROB3RBO+cNpKr5k2moiS8ju6OAx08u2kff9q0j6372yksKDicEArM6I4n6I4n6Y4lKSgwTh1XyezxVcw+qYoCMx54bitPrttDpMC45NSxVJQWEkskicaTxJNOSWEBZUURyoojVJUW8bHaiUwZdewvivqmDn6zahfDiiPMHFvJzDEV1FSWYGa4O93xJK1dceqbOtjc2M7mxjY2NaS2rfvbOfRrN6K8iItOHcOlp43l3bNqiMVTX2CPrt7F85v3k3Q4ZWwlF582hktOHcPsk6roiiUPfwaJpFNaFEm1oSjCsJJIaD0hd2fPwS4KCwqoKit8y7BfdzxBc0eM1q44U0aVU5TnvbFYIsmvXt7JXb/fxLb9HUweWc6nz5vCx+ZOorr8GBfw5Jl8SSiFwEbgEmAnsBy42t3XHqH+M7zZQxlBqlfyZ0AUeBy4091/e6T3y2ZCgVRS+OJDq3hszR4+c8FUvv7B2b3+5XmwK8bre1tZu+sgr9a38OrOFl5vaCMRfLmdN30UF586hj+bMZopo4Yd8a/XnmKJJGt3HaRu6wFWbm/ijX0dzJ0ynAtnjeG86aMYVlJIZzTBi2/s539e30fdtiZwpzCS+oIuKYww+6Qq5k0dyTlTRlBdVkRLR4xnNjbw1GsN/HFjI/FEkvKSQoYVRygoMLY0tlNdVsS1503h2vOnMqqiJCP/l8mkc9+zb/CjF7axLUiuY6tKmD2+ioRDLJ4klkiScKe0MEJJUQGlhRG64wnW72lld0vX4deqLivik++czLXnTz3cc8u2rliCTQ1trN/TynOb9vG7DQ00d8QojhSQdCeedKaNHsaH5oynuqyI361v4KU3Dryt19Ob4kgBp42vZM7E4cyZWM25U0ce8S9pd2dXSxcRM8qKUv+PJYUFb5kRN5F0VmxrYtm6PSxbt/fwHzsApUUFVJQUHU5uh9RUlvDRuRO56txJfUrOA2FPSxev1Dfzyo5mdjV3MmJYMaMrSqipKGF4eRGFEaPAjEiBsXV/B//1h83UN3Vy+klV3HjxTC6dPbbPv4v5Ji8SCoCZfQD4FyAC3Ofu3zSz24E6d1/So+4zBAkl2L8GuBVwYKm7f/Vo75XthAKpL8L/+9vXuO/ZNxhXVUpNZeqHdUR5MW3dcTbsaWVnc+fh+iOHFfOOCdWcMaGKcyaP4Pzpoykrzp+TvS9vb+I//7CZJ9bupbSogPedNpYLZozmgumjmTyq/IRes6k9ypcffoXfrW/gXSePZP7p4/izmaOZXlPR56m/D7RHeW33QQ60R7nktDGUF+f27VbxRJIV25p4en0DkQLjg2eM5/STqt7S3oNdMf64sZHtBzooL4pQXlxIWXGEwgKjK56gM5qkM5ag4WDqi3PNzoO0dadmf3jfaWP5wkXTD1/Omkg6j6/Zw3/8YRNrdh58SyxmUBQpoDj4QyOWSPVeiyLGu04exYWnjKE4YrR0xjjYFae1K0Z5cSEjyosYXl5MSWEBT6zde3hI8IIZo3j3zBpOGVfJKWMrGV9dGtoU7hv3tvJw3Q5eqW8hnkiSSDqxhLOvrZuG1m4ACguMsVWlNHdEaY8eebqlMycN528umcFFp4wZ9FPO501CGUi5kFAOebhuB89t3k9TR5SmjhjNHVHKiiKcMi41dn7quEpOG18V6i/XQNrU0MoP/vQGT7/WcPgXd+KIMt532lgWnHUSZ08a3qd2rtzexKKfrKSxrZuvf3A2nz5vyqD4/8mGZNLZsq+N367ew/3PvUFzR4wLZoziPTNr+NlL29m6v4OTRw/j6ndOpry48PB5pO5YgmjCU0OhiSTuMG/aSN57Sg1VpX0f6tnT0sXDdTtYvLL+cC8ToLK0kHFVpVSWFlJVVkRVaRHTRg/jnCkjOGvScKrLjm84qak9ytI1u/l5XT1egAOMAAAKPElEQVSv7GimKGKcOXE4ZcURIgV2eHjujAnVzJk4nNNPqjp8/rAzmmBfWzfNHTES7iSSjntqCLFnQh/MlFB6kUsJZahydzY3tvHsptSw2h9fbyQaTzJlVDlXnHkS588YzYThZYytKqW4sIBE0tnS2MaqHc2s3N7Mw3U7GD+8lLuuPoc5E4dnuzmDRlt3nJ++uI17/ucNGlu7OWNCNTdcOJ3LTh83IMM4zR1RNu5tY8PeVjbuaWVfWzcHu2Ic7IzT0hmjvqnj8AUOM8dUMGlkOcWRAooLU9vwsiLGVZcyvrqMcdWlNLVHeW7zfp7fsp/Xdqd6WKeOq+RjtZP48FknZWzodahQQumFEkruOdgV4/E1e1iyahfPbd53+EvDDEZXlNDRHT887FBRUsils8fyjQWnH/dfqdI3XbEE9U0dxzV8OBDauuO8sqOZlduaWLm9ica2bqLx5OGtqSNGZ+ytw1MlhQXUTh3BeSeP4r2zxvCOCUOnR5FpSii9UELJbQ2tXWzY08ru5i52tXSyq7mTksIIZ04azpkTqzm55siXz8rQ5u4c7Iyz+2Anu5u7KC+OcNbk4bq5NEMymVBy+2ylDBpjKksZU5kbV1lJfjEzqsuLqC4v4tRxVdkOR44ivy8WFxGRnKGEIiIiGaGEIiIiGaGEIiIiGaGEIiIiGaGEIiIiGaGEIiIiGaGEIiIiGTFo7pQ3s0agtxW2qoGWY5Sl7/f2OL1sNLDvBMPsLZa+1jnedvTcz+e2HOtxf9pxtDj78nwutaU/n0lvzw2Vn6+e+z3bEvbP19HqDMTP1xR3rzlGfH3j7oN6A+4+Vln6fm+Pe5TVZTKWvtY53nYMprb04fM54Xb0pS1Hez6X2tKfz+R4f54G08/XsdoS9s9XJtsS9u/KsbahMOT1SB/KHjnG495eI1Ox9LXO8baj534+t6Uvj/vjWK9ztOdzqS39+Ux6e26o/Hz13M/ntoT9u3JUg2bIa6CYWZ1naCK1bBssbRks7QC1JRcNlnZA+G0ZCj2UTLs72wFk0GBpy2BpB6gtuWiwtANCbot6KCIikhHqoYiISEYM6YRiZveZWYOZrTmBY+ea2atmtsnMvm9py8WZ2Y1mtsHM1prZdzIbda+xZLwdZvYNM9tpZquC7QOZj7zXeEL5TILnbzYzN7PRmYv4qPGE8bncYWarg8/kSTM7KfORvy2WMNrxXTNbH7TlV2Y2IGs+h9SWjwW/60kzC/1cS3/acITXu9bMXg+2a9PKj/r71KswLyHL9Q14D3AOsOYEjn0JOA8w4DHg/UH5RcBTQEmwPyZP2/EN4ObB8JkEz00CniB1r9LofG0LUJVW5ybgP/O0HZcBhcHjbwPfzuPP5DTgFOAZoDZX2xDEN7VH2UhgS/DviODxiKO192jbkO6huPsfgQPpZWY23cweN7MVZvY/ZnZqz+PMbDypX+znPfU//0Pgw8HTnwe+5e7dwXs0hNuK0NqRFSG25U7gq8CAnTQMoy3ufjCt6jAGoD0hteNJd48HVV8AJobbipSQ2vKau28YiPiD9zuhNhzB5cAydz/g7k3AMmD+iX43DOmEcgR3Aze6+1zgZuDfe6kzAahP268PygBmAe82sxfN7A9mdm6o0R5Zf9sBsCgYkrjPzEaEF+ox9astZrYA2Onur4QdaB/0+3Mxs2+a2Q7gk8BtIcZ6NJn4+Trks6T+As6WTLYlW/rSht5MAHak7R9q1wm1V2vKpzGzCuB84OG04cKS3qr2UnboL8VCUl3HdwHnAj83s5ODLD8gMtSO/wDuCPbvAL5H6hd/QPW3LWZWDnyN1BBLVmXoc8HdvwZ8zcxuBRYBf5/hUI8qU+0IXutrQBz4SSZj7KtMtiVbjtYGM/sM8DdB2QxgqZlFgTfc/SMcuV0n1F4llLcqAJrd/az0QjOLACuC3SWkvmzTu+gTgV3B43rgl0ECecnMkqTmz2kMM/Ae+t0Od9+bdtw9wKNhBnwU/W3LdGAa8ErwyzYRWGlm89x9T8ix95SJn690PwV+ywAnFDLUjuAE8IeASwbyD64eMv2ZZEOvbQBw9/uB+wHM7BlgobtvTatSD1yYtj+R1LmWek6kvWGfQMr1DZhK2skt4DngY8FjA848wnHLSfVCDp2w+kBQ/tfA7cHjWaS6k5aH7RifVueLwIP5+pn0qLOVATopH9LnMjOtzo3A4jxtx3xgHVAzUJ9F2D9fDNBJ+RNtA0c+Kf8GqVGVEcHjkX1pb69xDfSHmUsb8DNgNxAjlZH/ktRfs48DrwQ/8Lcd4dhaYA2wGfg33rxJtBj4cfDcSuDiPG3Hj4BXgdWk/kIbH3Y7wmpLjzpbGbirvML4XH4RlK8mNT/ThDxtxyZSf2ytCrbQr1YLsS0fCV6rG9gLPJGLbaCXhBKUfzb4PDYBnzlWe4+26U55ERHJCF3lJSIiGaGEIiIiGaGEIiIiGaGEIiIiGaGEIiIiGaGEIoOambUN8Pvda2azM/RaCUvNKrzGzB451oy8ZjbczG7IxHuLnAhdNiyDmpm1uXtFBl+v0N+c1DBU6bGb2QPARnf/5lHqTwUedfd3DER8Ij2phyJDjpnVmNkvzGx5sF0QlM8zs+fM7OXg31OC8oVm9rCZPQI8aWYXmtkzZrbYUmt6/OTQWhFBeW3wuC2YyPEVM3vBzMYG5dOD/eVmdnsfe1HP8+ZklxVm9rSZrbTUehVXBHW+BUwPejXfDep+JXif1Wb2fzL43yjyNkooMhT9K3Cnu58LXAncG5SvB97j7meTmsX3H9KOOQ+41t0vDvbPBv4WmA2cDFzQy/sMA15w9zOBPwJ/lfb+/xq8/zHnRwrmlbqE1IwFAF3AR9z9HFLr73wvSGi3AJvd/Sx3/4qZXQbMBOYBZwFzzew9x3o/kROlySFlKHofMDttZtYqM6sEqoEHzGwmqZlVi9KOWebu6WtQvOTu9QBmtorU3Ep/6vE+Ud6cVHMFcGnw+DzeXFvip8A/HSHOsrTXXkFqrQpIza30D0FySJLquYzt5fjLgu3lYL+CVIL54xHeT6RflFBkKCoAznP3zvRCM/t/wO/d/SPB+Yhn0p5u7/Ea3WmPE/T+uxTzN09SHqnO0XS6+1lmVk0qMX0B+D6pdVBqgLnuHjOzrUBpL8cb8I/u/l/H+b4iJ0RDXjIUPUlqHREAzOzQtN/VwM7g8cIQ3/8FUkNtAFcdq7K7t5Ba7vdmMysiFWdDkEwuAqYEVVuByrRDnwA+G6yXgZlNMLMxGWqDyNsoochgV25m9Wnbl0h9OdcGJ6rXkVpyAOA7wD+a2bNAJMSY/hb4kpm9BIwHWo51gLu/TGom2atILUZVa2Z1pHor64M6+4Fng8uMv+vuT5IaUnvezF4FFvPWhCOSUbpsWGSABatIdrq7m9lVwCfc/YpjHSeS63QORWTgzQX+Lbgyq5ksLK0sEgb1UEREJCN0DkVERDJCCUVERDJCCUVERDJCCUVERDJCCUVERDJCCUVERDLi/wNQFX6b9/1SaQAAAABJRU5ErkJggg==\n", 614 | "text/plain": [ 615 | "
" 616 | ] 617 | }, 618 | "metadata": { 619 | "needs_background": "light" 620 | }, 621 | "output_type": "display_data" 622 | } 623 | ], 624 | "source": [ 625 | "learn = get_chexpert_learner(learn=learn, img_size=128, size=1)\n", 626 | "lr_find_no_cbs(learn)" 627 | ] 628 | }, 629 | { 630 | "cell_type": "code", 631 | "execution_count": null, 632 | "metadata": {}, 633 | "outputs": [ 634 | { 635 | "data": { 636 | "text/html": [ 637 | "\n", 638 | "
\n", 639 | " \n", 651 | " \n", 652 | " 20.00% [1/5 12:38<50:33]\n", 653 | "
\n", 654 | " \n", 655 | "\n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | "
epochtrain_lossvalid_lossavg_auc_metrictime
00.4407390.4583870.85447612:38

\n", 675 | "\n", 676 | "

\n", 677 | " \n", 689 | " \n", 690 | " 27.97% [488/1745 03:33<09:09 0.4376]\n", 691 | "
\n", 692 | " " 693 | ], 694 | "text/plain": [ 695 | "" 696 | ] 697 | }, 698 | "metadata": {}, 699 | "output_type": "display_data" 700 | } 701 | ], 702 | "source": [ 703 | "lr = 1e-3\n", 704 | "learn.fit_one_cycle(5,slice(lr))" 705 | ] 706 | }, 707 | { 708 | "cell_type": "code", 709 | "execution_count": 32, 710 | "metadata": {}, 711 | "outputs": [ 712 | { 713 | "name": "stdout", 714 | "output_type": "stream", 715 | "text": [ 716 | "Atelectasis \t auc: 0.799\t chexpert auc: 0.858\t difference: 0.0587\n", 717 | "Cardiomegaly \t auc: 0.811\t chexpert auc: 0.854\t difference: 0.0433\n", 718 | "Consolidation \t auc: 0.9\t chexpert auc: 0.939\t difference: 0.0393\n", 719 | "Edema \t auc: 0.908\t chexpert auc: 0.941\t difference: 0.0326\n", 720 | "Pleural Effusion\t auc: 0.91\t chexpert auc: 0.936\t difference: 0.0264\n", 721 | "\n", 722 | "Average auc: 0.866 \t CheXpert average auc 0.906\t Difference 0.0401\n" 723 | ] 724 | } 725 | ], 726 | "source": [ 727 | "validation_eval(learn)" 728 | ] 729 | }, 730 | { 731 | "cell_type": "code", 732 | "execution_count": 34, 733 | "metadata": {}, 734 | "outputs": [ 735 | { 736 | "data": { 737 | "text/html": [], 738 | "text/plain": [ 739 | "" 740 | ] 741 | }, 742 | "metadata": {}, 743 | "output_type": "display_data" 744 | }, 745 | { 746 | "name": "stdout", 747 | "output_type": "stream", 748 | "text": [ 749 | "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n", 750 | "Min numerical gradient: 2.75E-06\n" 751 | ] 752 | }, 753 | { 754 | "data": { 755 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8nWWd9/HPL0nTNM3SLV1ISjfaQi2F0tBhEWSxCMxYVEDRwaGoDzOjHWYE9AF9hnFgHBVURoWZR2RABBEFZJ4iO7Ipa9OW7pSme9rSptmXZjnJ7/nj3KGHNE3TnnPnnJN836/XefXc++/qOckv13Vf93WZuyMiIhKvjGQHICIiA4MSioiIJIQSioiIJIQSioiIJIQSioiIJIQSioiIJIQSioiIJIQSioiIJIQSioiIJERWmCc3swuBnwCZwD3u/v1u2xcBtwM7g1V3uvs9wbZjgXuAiYADF7v71kNda8yYMT558uQEl0BEZGBbtmzZPncvSsS5QksoZpYJ3AUsACqApWa2xN3Xddv1t+6+uIdT/Ar4rrs/b2Z5QGdv15s8eTJlZWWJCF1EZNAws22JOleYTV7zgXJ33+zubcDDwCV9OdDMZgFZ7v48gLs3untzeKGKiEi8wkwoxcCOmOWKYF13l5rZKjN71MwmButmALVm9nszW2Fmtwc1HhERSVFhJhTrYV33oY2fACa7+xzgBeD+YH0WcBZwA3AqMBVYdNAFzK4xszIzK6usrExU3CIichTCTCgVRG+odykBdsXu4O5V7t4aLP4CmBdz7IqguSwC/A9wSvcLuPvd7l7q7qVFRQm5pyQiIkcpzISyFJhuZlPMLBu4AlgSu4OZTYhZXAisjzl2pJl1ZYnzgO4380VEJIWE1svL3SNmthh4lmi34Xvdfa2Z3QKUufsS4FozWwhEgGqCZi137zCzG4A/mpkBy4jWYEREJEXZQJmxsbS01NVtWETkyJjZMncvTcS59KS8iEgae2xZBQ+9tT3ZYQBKKCIiae2x5RU8trwi2WEASigiImmtoSVCQU6oo2j1mRKKiEgaa2hpJz9nSLLDAJRQRETSWkNLhHzVUEREJF7RhKIaioiIxKGlvYO2jk7VUEREJD4NLREA3ZQXEZH4NLS0A6jJS0RE4tNVQ1GTl4iIxOVAQlENRURE4nCgyUs1FBERiYOavEREJCHqdVNeREQSoauGkjdUNRQREYlDQ0uEvKFZZGZYskMBlFBERNJWdGDI1KidgBKKiEjaSqWBIUEJRUQkbdWn0ND1oIQiIpK2VEMREZGESKXJtSDkhGJmF5rZBjMrN7Mbe9i+yMwqzeyd4PWVbtsLzGynmd0ZZpwiIuko1WoooUViZpnAXcACoAJYamZL3H1dt11/6+6LD3GaW4FXwopRRCSdpVpCCbOGMh8od/fN7t4GPAxc0teDzWweMA54LqT4RETSVtfkWgWDpMmrGNgRs1wRrOvuUjNbZWaPmtlEADPLAH4EfCPE+ERE0laqjeMF4SaUnh7d9G7LTwCT3X0O8AJwf7D+q8BT7r6DXpjZNWZWZmZllZWVcQcsIpIuukYaTqUaSpiprQKYGLNcAuyK3cHdq2IWfwH8IHh/OnCWmX0VyAOyzazR3W/sdvzdwN0ApaWl3ZOViMiAlYo1lDAjWQpMN7MpwE7gCuALsTuY2QR33x0sLgTWA7j7X8fsswgo7Z5MREQGs1SbXAtCTCjuHjGzxcCzQCZwr7uvNbNbgDJ3XwJca2YLgQhQDSwKKx4RkYEk1SbXgnBrKLj7U8BT3dbdHPP+JuCmw5zjl8AvQwhPRCRtpWKTl56UFxFJQ6k2uRYooYiIpKVUm1wLlFBERNJSqk2uBUooIiJpKdUm1wIlFBGRtJRq43iBEoqISFpqaE2toetBCUVEJC2phiIiIgkRTSiqoYiISJx0U15ERBKiXk1eIiISr5b2DtoiqTW5FiihiIiknVQcxwuUUERE0k4qjjQMSigiImnngxrKUDV5iYhIHNTkJSIiCdGQgkPXgxKKiEjaUQ1FREQSomtyLXUbFhGRuHwwuZZqKCIiEo9UnFwLlFBERNJOKo7jBSEnFDO70Mw2mFm5md3Yw/ZFZlZpZu8Er68E6082szfMbK2ZrTKzz4UZp4hIOknFoesBQovIzDKBu4AFQAWw1MyWuPu6brv+1t0Xd1vXDPyNu280s2OAZWb2rLvXhhWviEi6SMXJtSDcGsp8oNzdN7t7G/AwcElfDnT399x9Y/B+F7AXKAotUhGRNJKqNZQwE0oxsCNmuSJY192lQbPWo2Y2sftGM5sPZAObwglTRCS9pOLkWhBuQump+4F3W34CmOzuc4AXgPs/dAKzCcADwNXu3nnQBcyuMbMyMyurrKxMUNgiIqltMN6UrwBiaxwlwK7YHdy9yt1bg8VfAPO6tplZAfAk8H/c/c2eLuDud7t7qbuXFhWpRUxEBodUnFwLwk0oS4HpZjbFzLKBK4AlsTsENZAuC4H1wfps4HHgV+7+SIgxioikldZIak6uBSH28nL3iJktBp4FMoF73X2tmd0ClLn7EuBaM1sIRIBqYFFw+GeBs4HRZta1bpG7vxNWvCIi6SBVx/GCEBMKgLs/BTzVbd3NMe9vAm7q4bgHgQfDjE1EJB2lckLRk/IiImnkg6HrU2xyLVBCERFJK6qhiIhIQtTvT83JtUAJRUQkraiGIiIiCZGqk2uBEoqISFpJ1cm1QAlFRCStNLREGJ6dmXKTa4ESiohIWomO45V6zV2ghCIiklZSdeh6UEIREUkr0cm1lFBERCRO9fsjFAxTk5eIiMSpdn8bI3Ozkx1Gj5RQRETSSG1TOyNyVUMREZE4tHd00tAaYcQw1VBERCQOdcE4XiOHq4YiIiJxqG1uA2CE7qGIiEg8apqjNZQR6uUlIiLxqGmK1lDUy0tEROJSG9xDUS8vERGJy4F7KEooIiISh5rmdrIyjLyhg3DoFTO70Mw2mFm5md3Yw/ZFZlZpZu8Er6/EbLvKzDYGr6vCjFNEJB3UNrczIjcbs9Qbuh4gtDRnZpnAXcACoAJYamZL3H1dt11/6+6Lux07CvgXoBRwYFlwbE1Y8YqIpLra5raUbe6CcGso84Fyd9/s7m3Aw8AlfTz2E8Dz7l4dJJHngQtDilNEJC3UNLcxcpAmlGJgR8xyRbCuu0vNbJWZPWpmE4/wWBGRQaOryStVhZlQemrk827LTwCT3X0O8AJw/xEci5ldY2ZlZlZWWVkZV7AiIqmutrl90NZQKoCJMcslwK7YHdy9yt1bg8VfAPP6emxw/N3uXurupUVFRQkLXEQkFdU0tw3aGspSYLqZTTGzbOAKYEnsDmY2IWZxIbA+eP8scIGZjTSzkcAFwToRkUFpf1sHrZHOlL4pH1ovL3ePmNliookgE7jX3dea2S1AmbsvAa41s4VABKgGFgXHVpvZrUSTEsAt7l4dVqwiIqmudn9qD7sCISYUAHd/Cniq27qbY97fBNx0iGPvBe4NMz4RkXRR05TaA0OCnpQXEUkLqT50PfQxoZjZNDMbGrw/x8yuNbMR4YYmIiJdalN8ci3oew3lMaDDzI4D/huYAjwUWlQiIvIhNV01lBSd/hf6nlA63T0CfBr4D3f/OjDhMMeIiEiC1Dan9tD10PeE0m5mnweuAv4QrEvdUomIDDC1zW0MG5JJzpDMZIdySH1NKFcDpwPfdfctZjYFeDC8sEREJFZNij8lD33sNhyMEHwtQPCgYb67fz/MwERE5IDa5jYKU7iHF/S9l9fLZlYQDCu/ErjPzH4cbmgiItIl1cfxgr43eRW6ez3wGeA+d58HfDy8sEREJFZ06PoBUEMBsoJxtz7LgZvyIiLST2qb2ykcIDWUW4iOybXJ3Zea2VRgY3hhiYhIF3endn/qN3n19ab8I8AjMcubgUvDCkpERA5oaI3Q0ekDo8nLzErM7HEz22tme8zsMTMrCTs4ERGB2mBgyMIUHhgS+t7kdR/RuUyOIToV7xPBOhERCVnXsCsDooYCFLn7fe4eCV6/BDRFoohIP0iHgSGh7wlln5ldaWaZwetKoCrMwEREJKpr6PrCFB4YEvqeUL5EtMvw+8Bu4DKiw7GIiEjIapq6mrwGQA3F3be7+0J3L3L3se7+KaIPOYqISMi6mrwGyk35nlyXsChEROSQapvbKcjJIisztSfZjSc6S1gUIiJySDXNbSk99W+XeBKKJywKERE5pHQYuh4Ok1DMrMHM6nt4NRB9JqVXZnahmW0ws3Izu7GX/S4zMzez0mB5iJndb2arzWy9md10xCUTERkg6tKkhtLr0Cvunn+0JzazTOAuYAFQASw1syXB3Cqx++UTnWvlrZjVlwND3f1EM8sF1pnZb9x969HGIyKSrmqa25k8ZniywzisMO/wzAfK3X2zu7cBDwOX9LDfrcBtQEvMOgeGm1kWMAxoA+pDjFVEJGWlw9D1EG5CKQZ2xCxXBOs+YGZzgYnu3n1I/EeBJqLPvGwHfuju1SHGKiKSkiIdnTS0RBiR7vdQ4tRTL7APbuSbWQZwB3B9D/vNBzqI3qeZAlwfDJn/4QuYXWNmZWZWVllZmZioRURSSF3wDMqIFH8GBcJNKBXAxJjlEmBXzHI+MBt42cy2AqcBS4Ib818AnnH3dnffC7wGlHa/gLvf7e6l7l5aVKShxURk4Klp7hrHa3A3eS0FppvZFDPLBq4gOmIxAO5e5+5j3H2yu08G3gQWunsZ0Wau8yxqONFk826IsYqIpKS6/dFhV9Khl1doCcXdI8BiojM9rgd+5+5rzewWM1t4mMPvAvKANUQT033uviqsWEVEUlVNMBdKOjyH0qcZG4+Wuz8FPNVt3c2H2PecmPeNRLsOi4gMal1zoYxI8ZGGIdwmLxERiVNtcA9lRIrPhQJKKCIiKa12fxuZGUb+0FAblBJCCUVEJIXVNLczYtgQzFJ/PF4lFBGRFFbb3JYWDzWCEoqISEqrbW5Pi2FXQAlFRCSlVTephiIiInGKdHSyZV8Tk0an/kjDoIQiIpKyyisbaY10cmJxYbJD6RMlFBGRFLVmZ3TWjtnFBUmOpG+UUEREUtSanXXkZmcyZUxeskPpEyUUEZEUtWZnHbMmFJCZkfrPoIASiohISurodNbuqmd2mtw/ASUUEZGUtGVfI/vbO5RQREQkPl035NOlhxcooYiIpKTVO+vIGZLBtKL0eAYFlFBERFLSmp11nDChgKzM9Pk1nT6RiogMEp1dN+SPSZ/mLlBCERFJOduqm2lsjaTNA41dlFBERFLM6p11AGnVwwuUUEREUs7anXVkZ2YwfWx+skM5IkooIiIpZvXOOo6fkE92Vnr9ik6vaEVEBjh3Z83OOj6SZjfkIeSEYmYXmtkGMys3sxt72e8yM3MzK41ZN8fM3jCztWa22sxywoxVRCQV7KjeT31LJK0eaOySFdaJzSwTuAtYAFQAS81sibuv67ZfPnAt8FbMuizgQeCL7r7SzEYD7WHFKiKSKtbs6rohn149vCDcGsp8oNzdN7t7G/AwcEkP+90K3Aa0xKy7AFjl7isB3L3K3TtCjFVEJCWs3llHVoYxc3x63ZCHcBNKMbAjZrkiWPcBM5sLTHT3P3Q7dgbgZvasmS03s2/2dAEzu8bMysysrLKyMpGxi4gkxZqddcwYl8/QrMxkh3LEwkwoPQ3g7x9sNMsA7gCu72G/LOCjwF8H/37azM4/6GTud7t7qbuXFhUVJSZqEZEkqdvfzvJtNZw0cUSyQzkqYSaUCmBizHIJsCtmOR+YDbxsZluB04AlwY35CuAVd9/n7s3AU8ApIcYqIpJ0D7yxlaa2Dq487dhkh3JUwkwoS4HpZjbFzLKBK4AlXRvdvc7dx7j7ZHefDLwJLHT3MuBZYI6Z5QY36D8GrDv4EiIiA0NzW4R7X9vKecePTcsuwxBiQnH3CLCYaHJYD/zO3dea2S1mtvAwx9YAPyaalN4Blrv7k2HFKiKSbA+/vYPqpja+du60ZIdy1ELrNgzg7k8Rba6KXXfzIfY9p9vyg0S7DouIDGitkQ7ufnUzfzFlFPMmjUp2OEdNT8qLiCTZ48t38n59C4vPOy7ZocRFCUVEJIkiHZ381yubmFNSyEePG5PscOKihCIikkRPrt7NtqpmvnrOcZj19LRF+lBCERFJkvaOTu56qZzpY/O4YNa4ZIcTNyUUEZEkueulct7b08gNn5hJRkZ6105ACUVEJClWV9Rx54vlfHpuMZ/4yPhkh5MQSigiIv2spb2D6373DmPyhvKdhR9JdjgJE+pzKCIicrAfPbeBjXsb+dWX5lM4bEiyw0kY1VBERPrRW5uruOfPW7jytGM5e8bAGtRWCUVEpJ9sr2rmut+t5NhRuXzr4hOSHU7CqclLRKQfrNxRy5fvX0p7h/PAl+eTmz3wfv0OvBKJiKSYP67fw+KHVjA6L5vf/u18phXlJTukUCihiIiE6NdvbeOf/2cNs4sL+e+rTqUof2iyQwqNEoqISEhWVdTy7cfXcN7xY7nzC3MHZDNXLN2UFxEJye3PbmDU8Gx++vmBn0xACUVEJBRvbKriTxv38dVzppE3dOAnE1BCERFJOHfnh89tYFzBUK48bVKyw+k3SigiIgn28oZKlm2r4drzp5MzJDPZ4fQbJRQRkQTq7HRuf3YDx47K5bOlE5MdTr9SQhERSaCn1uxm3e56rlswgyGZg+tX7OAqrYhIiOpb2vnxc+8xY1wenzzpmGSH0+9CTShmdqGZbTCzcjO7sZf9LjMzN7PSbuuPNbNGM7shzDhFROK1dGs1F/3Hn9ha1cRNF59A5gCYMOtIhZZQzCwTuAu4CJgFfN7MZvWwXz5wLfBWD6e5A3g6rBhFROLV3tHJj57bwOd+/gaZGcYjf3cG584cm+ywkiLMztHzgXJ33wxgZg8DlwDruu13K3Ab8KFaiJl9CtgMNIUYo4jIUXtvTwPfeHQVK3fUctm8Er6z8COD5pmTnoTZ5FUM7IhZrgjWfcDM5gIT3f0P3dYPB/438K+9XcDMrjGzMjMrq6ysTEzUIiKH0dLewY+e28Bf/vRPbKtq4s4vzOWHl580qJMJhFtD6akB0T/YaJZBtElrUQ/7/Stwh7s3mh26HdLd7wbuBigtLfVD7igikiBvbKriW4+vZsu+Jj4zt5hv/+UJjM4buAM+HokwE0oFENsJuwTYFbOcD8wGXg6SxnhgiZktBP4CuMzMbgNGAJ1m1uLud4YYr4jIIW2rauK2Zzbw5OrdHDsqlwe+PJ+zpg+sGRfjFWZCWQpMN7MpwE7gCuALXRvdvQ4Y07VsZi8DN7h7GXBWzPrvAI1KJiKSDDVNbfzsxXIeeHMrWRkZ/OP50/m7j01jWPbgeQK+r0JLKO4eMbPFwLNAJnCvu681s1uAMndfEta1RUSOVnNbhNUVdazYUcuK7TW8Xl5FU1uEz5ZO5LoFMxhbkJPsEFOWuQ+MWw+lpaVeVlaW7DD6rLPTyRiE/dRFUlFlQyvPrXufp1e/z5ubq4h0Rn8vTh6dy7xJo7jm7KnMHJ+f5CjDYWbL3L308Hse3uDukpAEG/c0cMOjq2iLdPLb88dQ8J8/gwcfhMZGyMuDK6+E66+HadOSHarIgLWnvoXVFXWs2lnHW5urWLq1mk6HKWOG8+WzpvAXU0Zx8sSRjBqenexQ04oSSj/p6HTu/fMWbn9uA8OzM5m37k2yv/k93Duw9vboTg0NcM89cP/98OijcNFFyQ1aZACpb2nn569s4tFlFeypbwUgw2Dm+AIWnzedi08cz8xx+fTWs1R6p4SSAM1tETo6nfycIQdta+/oZN2uer775Hre3lrNglnj+P5JuRR+7/tktbUcfLL29ujrsstg1SrVVETi1Bbp5KG3tvHTF8upbmrjglnjOG3qaOaUFDLrmIJBMZNif9H/ZBzaOzq5//Wt/OSFjTS0RigeMYzp4/KYOS6f/e0drKqoY93uetoineQPzeKHl5/EpacUY1/7GnREDnPydrjjDrhTndtEjoa789Tq97n92XfZWtXM6VNH862LT+DEksJkhzZg6ab8Ufrzxn1854m1lO9t5GMzipg/ZRQb9zSwYU8jm/Y2MiTTmF1cyJySQk4sGcEZ00Yzpuvhp4KCaPPW4RQUQF1duAURGYDe3FzF955+l5U7apk5Lp8bLz6ec2YUqTmrB7opn2Q3/X4Vv3l7B8eOyuWevynl/BPGfuiLGunoJMPs0L24Ghv7dJ3Ohga+8//W0OlOQc4Qzj9hHHMnjlDvMJEedN1of+jt7bz47l4mFOZw+2Vz+MwpJYNy5N9kUEI5QpsqG/nN2zv4/Pxj+ZdPzupxes+sw02qk5fXpxpKU/Ywnli5iwwz6lva+c+XNzG+IIcLZ4/n4hMnUDpppJKLDFr72zp4bHkFL767l9U766hsiN5oz8/J4saLjmfRGZMH1fS7qUAJ5Qj9fnkFGQZf/3gcc0VfeWW0N1dX766eDBlC/leuZsXNFwDRHiovrt/LU6t389Db2/nl61sZX5DDX82ZwMKTj+HE4kJV52VQ2Fvfwq/e2MaDb22jtrmdqWOGc9ZxY5hdXMiJJYXMPqZQT7Enie6hHIGOTuejP3iRmePz+eXV84/+RJs2wZw50Nx86H1ycw/Zy6uxNcIf1+/hiZW7eOW9Sto7nGMKc5g0ejgTCnMYX5jDxFG5LJg17sB9mxTV3tHJqopa3thUxa66FrIyjMwMIyvDmFA4jL86aQJj8w9+MrmlvYP361oYmZtNwbAsJdNBYMu+Jv7r5XIeX7GTSKdzwaxxfOWsqZROGqnPPw6JvIeihHIE/rxxH1f+91v87PNz45/e8+mno12Du7oJdxkyJPrq43Modc3tPLN2N3/auI/361rYXdfCnvoWIp1OVoZx7vFjuXxeCecePzZl5rduao3w+IqdvLB+D29vqaa5rQOA0cOz6XCno9OJdDj72zvIMDhrehGfOaWYGePyea18H69u3Mdbm6tojXQCkJVhjBqezTEjhnH2jCLOP34sJxYXDrjmwJqmNrZVNzNlzHAKhx3cRX2gem9PA3e9VM4TK3cxJDODz506kS9/dAqTRg9PdmgDghJKD/ojoXz9t+/wwvo9LP32xxPTNrtpU7Rr8AMPHHhS/otfhK9/Pa7nTzo7nY17G/n98goeW76TfY2tjMnL5lMnF3N56cSDhpBoae+gfG8jGWbkZmeSOzST/KFDEt5ssKO6mV+9sZWHl+6goSXC1KLhnDltDKdPG81pU0cf9FTypspoGR5fvpNddQee2ZlWNJyzZxQxa0IB9S0RqhpbqWpso7yykRXba+h0KMofyrkzizjzuOj5e6rlpAt355GyCv7tyXXUt0S7mxflD2Va0XBOmFDA2dOLOG3q6LRq5mmNdPC7pTv4+aubGZM3lBsumMlHp4/50D6rK+q466Vynln7PrnZmXzx9El85aNTKcpP7Vp3ulFC6UHYCaWxNcKp//YCn5pbzPc+c2Jo10m09o5OXtlQySPLdvDH9XuJdDonlRRy8YkT2F3XwortNazbXU97x8Hfg8vmlXDzJ2dR0MMDm7E6O50X1u9h874mGlraaWyJ0NASobmtg/3t0VdTa4T1u+sxMy6aPZ6rz5zMKcf2ramis9N5c0sVO2v2c/q00ZSMzD3kvtVNbbzy3l5eWL+XP71X+cEv4Olj85hTMoL6lnb21EdrcTXN7YzNH8rEkbmUjBzGsaNyObGkkLnHjuy1BhDp6GR7dTNb9jVR2dBKVVMb1U1t1O1v58zjRrPwpOI+9Spyd1raO2lqi9Dc2kFTW4ScIZkUjxhGdla0Nrl1XxPfenw1r2+qYv6UUSw6YzI7qpvZVNlI+d5G1u6qpzXSSXZmBvOnjOKcmUV84iPjmTjq0P9HfYmrvcNp6+jEvecHdvuqfG8jrZEORg3PZmRuNlkZxu+X7+Qnf9zIztr9zJs0kvfrWthZu58zpo3mmxceT2t7B3e+VM6fNu6jICeLRWdM5uozpzBSw6CEQgmlB2EnlN+V7eCbj67isb8/g3mTRoZ2nTBVNbbyP+/s4pGyHbz7fgPDhmQyJ/gFemJxIZkZRnNbhKa2DjbtbeRXb0Rv/N9++UmcedyYg87n7rz8XiW3PbOB9bvrgehQFvk5Q8gbmsXwoZkMG5JJTvCaXVzAladNYkLhsH4pb0ens25XPa9v2sdrm6p4d3c9o4ZnM7Ygh/EFQykcNoS9Da1U1OxnR3Uze4NeQmbRBHRSyQiGZGXQ0t5Ba6ST5tYI26ub2V7dfFACzs2OlrWqqY3jx+dzwwUzD+pO3hXT21uqeXrNbp5Z8/4H14xlBhMKcigZmcvKilqyMzO46eITuOLUiQc147W0d7B0azWvvlfJq+/tY8OeaO/BE4sLuXD2eEonjeT9+ha2VTWzraqZ3XX7qY9J+k1tH37AttOjf4TE/lo4dlQu86eMYv7kUZw0cQT7GlvZuKeB8spGtlfvZ9aEAhbMOtClvS3SydNrdnP/61tZvr32Q+fPyrDoHzUTR3DDBTP46HFjaOvo5Ndvbueul8qpamoDYExeNl/+6FSuPO3YuBKaHJ4SSg/CTiif/fkbVDa08uL1H0v7G4Duzp76aDNYb12cV2yv4fpHVrK5somrTp/Ex2eN+2BbU2sH9/55C29vrebYUblct2AGC2aNIzc7M23/fxpbI6zaUcuybTUs217Dmp3RJDk0K4OcIRkMy47WHqYW5TF1zHCmFuUxvjCH0cOzyRmSSWen8+Tq3fz4+ffYsq+JeZNGUjp5JC1tHTS3RWsgb2+pZl9jGzlDMjh35ljmlIwgb2gmudlZ5GZn0tTWwY7qZnYEiatk5DBuuvgExvVxyPTtVc08s3Y3T695nxXdfpmPKxhK8YhhFA4bQn7OEPJzotfMiP28DLIzM6KvrAwinc7KHbUs3VpNTfOHeyUW5GRRPDKXjXsaiHQ6Y/KGcsa00byxuYrKhlYmj87li6dPpnhEDjXN7VQ3tVG/v51TJ4/qMdk2tkZ4+O3t5AzJ5LJ5Jery20+UUHoQZkLZUd3MWbe9xA0XzGDxedNDuUaq2t/WwW3Pvst9r209aFtR/lCuPe84PnfqsR/5w88UAAAKsklEQVQ00Uj0L/xHyir42YsbqWpsY1h2ZrQGk53JCeMLuPjECZx7fFHoY0jtqt3Phj0NlIwYxsRRuXH9gnZ3yvc2smZXHePyczhuXB5FeUMxM+r2t/Pyhr08v24Pr2+qYk5JIVedMZmPTS8acB0jBiIllB6EmVD+44X3+MkfN/Ln/30exSP6p7km1WzZ10RV44HmGTM4YYIG1hNJdxp6JYHcozcfszMzPlQFr9vfztIt1by5uYrHlldwxrTRgzaZQHSeiClj1E1TRA5t0CeU2uZ25t76PJkZ0S6zw7OzyM7KYEdNM+6QnZXB3IkjuOmiE5IdqohIShv0CSU7K4NvfGJmtHdTawf7g66unzmlmNOmjubkiSN0c1BEpA8GfUIZPjSLr517XLLDEBFJe+qaIyIiCRFqQjGzC81sg5mVm9mNvex3mZm5mZUGywvMbJmZrQ7+PS/MOEVEJH6hNXmZWSZwF7AAqACWmtkSd1/Xbb984FrgrZjV+4BPuvsuM5sNPAsUhxWriIjEL8waynyg3N03u3sb8DBwSQ/73QrcBnww+p+7r3D3XcHiWiDHzDQinIhICgszoRQDO2KWK+hWyzCzucBEd/9DL+e5FFjh7gcNemRm15hZmZmVVVZWJiJmERE5SmEmlJ7GXPjgsXwzywDuAK4/5AnMPgL8APjbnra7+93uXurupUVFRXGGKyIi8QgzoVQAE2OWS4BdMcv5wGzgZTPbCpwGLIm5MV8CPA78jbtvCjFOERFJgDATylJguplNMbNs4ApgSddGd69z9zHuPtndJwNvAgvdvczMRgBPAje5+2shxigiIgkSWi8vd4+Y2WKiPbQygXvdfa2Z3QKUufuSXg5fDBwH/LOZ/XOw7gJ333uoA5YtW7bPzLb1sKkQqDvMutjlnt7HrhtDtBfa0egplr7uc6Tl6L6czmU53Pt4ytFbnH3Znkpliecz6WnbYPl+dV/uXpawv1+97dMf369Jh4mt79x9QL+Auw+3Lna5p/fd1pUlMpa+7nOk5RhIZenD53PU5ehLWXrbnkplieczOdLv00D6fh2uLGF/vxJZlrB/Vg73GgxPyj/Rh3VPHOZ9T+dIVCx93edIy9F9OZ3L0pf38TjceXrbnkpliecz6WnbYPl+dV9O57KE/bPSqwEzH0p/MbMyT9DcAck2UMoyUMoBKksqGijlgPDLMhhqKIl2d7IDSKCBUpaBUg5QWVLRQCkHhFwW1VBERCQhVEMREZGEGNQJxczuNbO9ZrbmKI6dF4yGXG5mP7WY+YPN7B+CUZbXmtltiY26x1gSXg4z+46Z7TSzd4LXxYmPvMd4QvlMgu03BKNaj0lcxL3GE8bncquZrQo+k+fM7JjER35QLGGU43Yzezcoy+PBs2ehC6kslwc/651dD2aHKZ4yHOJ8V5nZxuB1Vcz6Xn+eehRmF7JUfwFnA6cAa47i2LeB04kOMfM0cFGw/lzgBWBosDw2TcvxHeCGgfCZBNsmEn0mahswJl3LAhTE7HMt8H/TtBwXAFnB+x8AP0jjz+QEYCbwMlCaqmUI4pvcbd0oYHPw78jg/cjeytvba1DXUNz9VaA6dp2ZTTOzZ4J5WP5kZsd3P87MJhD9wX7Do//zvwI+FWz+e+D7Hgxm6b08jJkoIZUjKUIsyx3AN4kZTy5sYZTF3etjdh1OP5QnpHI85+6RYNc3iQ7NFLqQyrLe3Tf0R/zB9Y6qDIfwCeB5d6929xrgeeDCo/3dMKgTyiHcDfyDu88DbgD+s4d9iomOVdYldiTlGcBZZvaWmb1iZqeGGu2hxVsOgMVBk8S9ZjYyvFAPK66ymNlCYKe7rww70D6I+3Mxs++a2Q7gr4GbQ4y1N4n4fnX5EtG/gJMlkWVJlr6UoSeHGhX+qMo76OeUj2VmecAZwCMxzYU9zcPS20jKWUSrjqcBpwK/M7OpQZbvFwkqx38RnavGg39/RPQHv1/FWxYzywW+TbSJJakS9Lng7t8Gvm1mNxEdpuhfEhxqrxJVjuBc3wYiwK8TGWNfJbIsydJbGczsauAfg3XHAU+ZWRuwxd0/zaHLdVTlVUL5sAyg1t1Pjl1p0dknlwWLS4j+so2toseOpFwB/D5IIG+bWSfR8XP6c8KWuMvh7ntijvsF0NucNWGKtyzTgCnAyuCHrQRYbmbz3f39kGPvLhHfr1gPER1EtV8TCgkqR3AD+K+A8/vzD65uEv2ZJEOPZQBw9/uA+wDM7GVgkbtvjdmlAjgnZrmE6L2WCo6mvGHfQEr1FzCZmJtbwOvA5cF7A046xHFLidZCum5YXRys/zvgluD9DKLVSUvDckyI2efrwMPp+pl022cr/XRTPqTPZXrMPv8APJqm5bgQWAcU9ddnEfb3i366KX+0ZeDQN+W3EG1VGRm8H9WX8vYYV39/mKn0An4D7AbaiWbkLxP9a/YZYGXwhb/5EMeWAmuATcCdHHhINBt4MNi2HDgvTcvxALAaWEX0L7QJYZcjrLJ022cr/dfLK4zP5bFg/Sqi4zMVp2k5yon+sfVO8Aq9t1qIZfl0cK5WYA/wbCqWgR4SSrD+S8HnUQ5cfbjy9vbSk/IiIpIQ6uUlIiIJoYQiIiIJoYQiIiIJoYQiIiIJoYQiIiIJoYQiA5qZNfbz9e4xs1kJOleHRUcVXmNmTxxuRF4zG2FmX03EtUWOhroNy4BmZo3unpfA82X5gUENQxUbu5ndD7zn7t/tZf/JwB/cfXZ/xCfSnWooMuiYWZGZPWZmS4PXmcH6+Wb2upmtCP6dGaxfZGaPmNkTwHNmdo6ZvWxmj1p0To9fd80VEawvDd43BgM5rjSzN81sXLB+WrC81Mxu6WMt6g0ODHaZZ2Z/NLPlFp2v4pJgn+8D04Jaze3Bvt8IrrPKzP41gf+NIgdRQpHB6CfAHe5+KnApcE+w/l3gbHefS3QU33+POeZ04Cp3Py9Yngv8EzALmAqc2cN1hgNvuvtJwKvA/4q5/k+C6x92fKRgXKnziY5YANACfNrdTyE6/86PgoR2I7DJ3U9292+Y2QXAdGA+cDIwz8zOPtz1RI6WBoeUwejjwKyYkVkLzCwfKATuN7PpREdWHRJzzPPuHjsHxdvuXgFgZu8QHVvpz92u08aBQTWXAQuC96dzYG6Jh4AfHiLOYTHnXkZ0rgqIjq3070Fy6CRacxnXw/EXBK8VwXIe0QTz6iGuJxIXJRQZjDKA0919f+xKM/sZ8JK7fzq4H/FyzOambudojXnfQc8/S+1+4CblofbpzX53P9nMCokmpq8BPyU6D0oRMM/d281sK5DTw/EGfM/df36E1xU5KmryksHoOaLziABgZl3DfhcCO4P3i0K8/ptEm9oArjjczu5eR3S63xvMbAjROPcGyeRcYFKwawOQH3Pos8CXgvkyMLNiMxuboDKIHEQJRQa6XDOriHldR/SXc2lwo3od0SkHAG4DvmdmrwGZIcb0T8B1ZvY2MAGoO9wB7r6C6EiyVxCdjKrUzMqI1lbeDfapAl4Luhnf7u7PEW1Se8PMVgOP8uGEI5JQ6jYs0s+CWST3u7ub2RXA5939ksMdJ5LqdA9FpP/NA+4MembVkoSplUXCoBqKiIgkhO6hiIhIQiihiIhIQiihiIhIQiihiIhIQiihiIhIQiihiIhIQvx/aUyxUA8moBkAAAAASUVORK5CYII=\n", 756 | "text/plain": [ 757 | "
" 758 | ] 759 | }, 760 | "metadata": { 761 | "needs_background": "light" 762 | }, 763 | "output_type": "display_data" 764 | } 765 | ], 766 | "source": [ 767 | "lr_find_no_cbs(learn)" 768 | ] 769 | }, 770 | { 771 | "cell_type": "code", 772 | "execution_count": 35, 773 | "metadata": {}, 774 | "outputs": [ 775 | { 776 | "data": { 777 | "text/html": [ 778 | "Total time: 12:38

\n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | "
epochtrain_lossvalid_lossavg_auc_metrictime
00.4298120.4439380.86893112:37
" 798 | ], 799 | "text/plain": [ 800 | "" 801 | ] 802 | }, 803 | "metadata": {}, 804 | "output_type": "display_data" 805 | } 806 | ], 807 | "source": [ 808 | "lr = 5e-3\n", 809 | "learn.fit_one_cycle(1,slice(lr))" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 36, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "name": "stdout", 819 | "output_type": "stream", 820 | "text": [ 821 | "Atelectasis \t auc: 0.81\t chexpert auc: 0.858\t difference: 0.0481\n", 822 | "Cardiomegaly \t auc: 0.802\t chexpert auc: 0.854\t difference: 0.0518\n", 823 | "Consolidation \t auc: 0.909\t chexpert auc: 0.939\t difference: 0.03\n", 824 | "Edema \t auc: 0.906\t chexpert auc: 0.941\t difference: 0.0355\n", 825 | "Pleural Effusion\t auc: 0.913\t chexpert auc: 0.936\t difference: 0.0234\n", 826 | "\n", 827 | "Average auc: 0.868 \t CheXpert average auc 0.906\t Difference 0.0377\n" 828 | ] 829 | } 830 | ], 831 | "source": [ 832 | "validation_eval(learn)" 833 | ] 834 | }, 835 | { 836 | "cell_type": "code", 837 | "execution_count": 41, 838 | "metadata": {}, 839 | "outputs": [ 840 | { 841 | "data": { 842 | "text/html": [], 843 | "text/plain": [ 844 | "" 845 | ] 846 | }, 847 | "metadata": {}, 848 | "output_type": "display_data" 849 | }, 850 | { 851 | "name": "stdout", 852 | "output_type": "stream", 853 | "text": [ 854 | "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n", 855 | "Min numerical gradient: 6.31E-07\n" 856 | ] 857 | }, 858 | { 859 | "data": { 860 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEKCAYAAAAiizNaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VeW59//PlZmQBAIEhDDK4ISKEhzq0aqtFvvrcThaq49jh+OvA/Wc09pTffWp7U9/fTqd1k62PbR1qNVqxbaiR4tDxdoKSlBEQJEQQRKGhDnjTvbO9fyxV2AbMkGyx3zfr9d6sda9hn3d7GRfude6932buyMiIpIoWckOQEREhhYlHhERSSglHhERSSglHhERSSglHhERSSglHhERSSglHhERSSglHhERSSglHhERSaicZAeQCGPGjPGpU6cmOwwRkbSycuXKne5eNtjXHRKJZ+rUqVRWViY7DBGRtGJmm+NxXd1qExGRhFLiERGRhFLiERGRhFLiERGRhFLiERGRhFLiERGRhFLiERGRhFLiERHJQG9v388Pn1lPfUMo2aEcQolHRCQDrdu6n5/8tYrGUDjZoRxCiUdEJAM1BQlneH52kiM5lBKPiEgGagxFACjKT72R0ZR4REQyUFMoTJbBsFy1eEREJAEaQ2GG5+VgZskO5RBKPCIiGagpFGZ4Ct5mAyUeEZGM1NQWTsmOBaDEIyKSkRpDEYoKcpMdRreUeEREMlBTKEyRWjwiIpIoja3RzgWpKK6Jx8zmm9l6M6sys1t7OOZKM1tnZmvN7KGY8hvMbEOw3BBTPtfM3gyu+RNLxS4bIiJJ1hgKp+R3eADiFpWZZQN3AxcANcAKM1vs7utijpkJ3Aac5e57zGxsUD4K+AZQATiwMjh3D/AL4CZgOfAUMB94Ol71EBFJR9HOBamZeOLZ4jkNqHL3andvAx4GLulyzL8CdwcJBXevC8o/Ajzr7ruDfc8C881sPFDi7svc3YHfApfGsQ4iImlpqHanLge2xGzXBGWxZgGzzOwfZrbczOb3cW55sN7bNQEws5vMrNLMKuvr6wdQDRGR9BIKR2iP+JDsXNDdsxfvsp0DzATOBa4Gfm1mI3s5tz/XjBa6L3T3CnevKCsr63fQIiLprikYp20otnhqgEkx2xOBrd0c87i7t7v7u8B6oomop3NrgvXerikiMqQdHJl66CWeFcBMM5tmZnnAVcDiLsf8GTgPwMzGEL31Vg0sAS40s1IzKwUuBJa4+zagwczOCHqzXQ88Hsc6iIiknc45eIpTNPHELSp3D5vZAqJJJBu4x93XmtkdQKW7L+ZgglkHRICvuPsuADO7k2jyArjD3XcH658D7gOGEe3Nph5tIiIxUr3FE9eo3P0pol2eY8tuj1l34EvB0vXce4B7uimvBGYPerAiIhmiIcUTj0YuEBHJMJ0tnlT9AqkSj4hIhknlaa9BiUdEJOOk8rTXoMQjIpJxUr1zgRKPiEiGaQqFycvJIjc7NT/iUzMqERE5Yo2hcMp+hweUeEREMk4qDxAKSjwiIhmnMRRR4hERkcRpDLWn7MjUoMQjIpJxmtTiERGRRNIzHhERSajGUJiiPCUeERFJELV4REQkYTo6nKa2iDoXiIhIYjS3B+O0FajFIyIiCZDq47RBnBOPmc03s/VmVmVmt3az/0YzqzezVcHymaD8vJiyVWbWamaXBvvuM7N3Y/bNiWcdRETSSWOKz8UDcZyB1MyygbuBC4AaYIWZLXb3dV0OfcTdF8QWuPsLwJzgOqOAKuCZmEO+4u6L4hW7iEi6amwNWjxDtFfbaUCVu1e7exvwMHDJEVznCuBpd28e1OhERDLQUL/VVg5sidmuCcq6utzMVpvZIjOb1M3+q4Dfdyn7VnDOXWaWP0jxioikvXS41RbPxGPdlHmX7SeAqe5+EvAccP/7LmA2HjgRWBJTfBtwLDAPGAV8tdsXN7vJzCrNrLK+vv7IaiAikmaa2lJ72muIb+KpAWJbMBOBrbEHuPsudw8Fm78C5na5xpXAn9y9PeacbR4VAu4lekvvEO6+0N0r3L2irKxsgFUREUkPqT7tNcQ38awAZprZNDPLI3rLbHHsAUGLptPFwFtdrnE1XW6zdZ5jZgZcCqwZ5LhFRNJWOjzjiVtk7h42swVEb5NlA/e4+1ozuwOodPfFwM1mdjEQBnYDN3aeb2ZTibaYXuxy6QfNrIzorbxVwGfjVQcRkXTTFApjBoV5qXurLa4p0d2fAp7qUnZ7zPptRJ/ZdHfuJrrpjODu5w9ulCIimaNzgNDoTaHUpJELREQySKoPEApKPCIiGaUxFE7pHm2gxCMiklEaQ5GU7tEGSjwiIhlFt9pERCShlHhERCShGkNh3WoTEZHEaVLnAhERSaSmUISi/Nxkh9ErJR4RkQzRFu6gLdJBkVo8IiKSCOkwThso8YiIZIxGJR4REUmkdJgEDpR4REQyhm61iYhIQh1s8ahzgYiIJEBTMPuoWjwiIpIQTXrGA2Y238zWm1mVmd3azf4bzazezFYFy2di9kViyhfHlE8zs1fMbIOZPRJMqy0iMuQN+c4FZpYN3A1cBBwPXG1mx3dz6CPuPidYfh1T3hJTfnFM+XeBu9x9JrAH+HS86iAikk7UuQBOA6rcvdrd24CHgUsGckGLzuV6PrAoKLofuHRAUYqIZIjGtjB5OVnkZqf2U5R4RlcObInZrgnKurrczFab2SIzmxRTXmBmlWa23Mw6k8toYK+7h/u4pojIkNOUBiNTQ3wTj3VT5l22nwCmuvtJwHNEWzCdJrt7BfC/gB+Z2fR+XjP64mY3BYmrsr6+/vCjFxFJM42tqT8yNcQ38dQAsS2YicDW2APcfZe7h4LNXwFzY/ZtDf6tBpYCpwA7gZFm1pnSD7lmzPkL3b3C3SvKysoGXhsRkRTXGIowPG9ot3hWADODXmh5wFXA4tgDzGx8zObFwFtBeamZ5QfrY4CzgHXu7sALwBXBOTcAj8exDiIiaSNdbrXFLUJ3D5vZAmAJkA3c4+5rzewOoNLdFwM3m9nFQBjYDdwYnH4c8N9m1kE0OX7H3dcF+74KPGxm/z/wOvCbeNVBRCSdNLWFKS1M/W+YxDU1uvtTwFNdym6PWb8NuK2b814GTuzhmtVEe8yJiEiMxlCYSaMKkx1Gn1K7z52IiPRbUyhM0RB/xiMiIgnUFIqk/JdHQYlHRCQjuDtNbeGUH5kalHhERDJCc1sE99QfLgeUeEREMkK6THsNSjwiIhkhXUamBiUeEZGMkC4jU4MSj4hIRjh4q02dC0REJAE6p70uzs9NciR9U+IREckATWrxiIhIIqlzgYiIJNS+lnYAigqUeEREJAHe3dlEWXE+hRqrTUREEqGqrpGZY4uSHUa/KPGIiKQ5d6eqrpEZSjwiIpIIO/aHaAyFlXgAzGy+ma03syozu7Wb/TeaWb2ZrQqWzwTlc8xsmZmtNbPVZvaJmHPuM7N3Y86ZE886iIikuqq6RgBmlKVH4onbUygzywbuBi4AaoAVZrY4ZgrrTo+4+4IuZc3A9e6+wcwmACvNbIm77w32f8XdF8UrdhGRdFJV1wCgFg/R6amr3L3a3duAh4FL+nOiu7/j7huC9a1AHVAWt0hFRNJYVX0jxQU5lBXnJzuUfoln4ikHtsRs1wRlXV0e3E5bZGaTuu40s9OAPGBjTPG3gnPuMrP0+J8WEYmTzh5tZpbsUPolnomnu/8B77L9BDDV3U8CngPuf98FzMYDDwCfdPeOoPg24FhgHjAK+Gq3L252k5lVmlllfX39kddCRCTFVdU1pc1tNuhn4jGz6Z0tCzM718xuNrORfZxWA8S2YCYCW2MPcPdd7h4KNn8FzI15zRLgf4D/7e7LY87Z5lEh4F6it/QO4e4L3b3C3SvKynSXTkQy097mNnY2hjIv8QCPAREzmwH8BpgGPNTHOSuAmWY2zczygKuAxbEHBC2aThcDbwXlecCfgN+6+6PdnWPRNuWlwJp+1kFEJOMc6NGWRomnv73aOtw9bGaXAT9y95+a2eu9nRAcvwBYAmQD97j7WjO7A6h098XAzWZ2MRAGdgM3BqdfCZwDjDazzrIb3X0V8KCZlRG9lbcK+Gx/KysikmkOdqUuTnIk/dffxNNuZlcDNwD/HJT1OemDuz8FPNWl7PaY9duIPrPpet7vgN/1cM3z+xmziEjGq6prJD8ni/LSYckOpd/6e6vtk8CZwLfc/V0zm0YPiUFERBKnqr6R6WVFZGelR4826GeLJ/jS580AZlYKFLv7d+IZmIiI9K2qrpFTJ5cmO4zD0t9ebUvNrMTMRgFvAPea2Q/jG5qIiPSmuS1M7d6WtOpYAP2/1TbC3fcD/wLc6+5zgQ/HLywREelLdX0T7unVow36n3hygm7MVwJPxjEeERHpp3TsSg39Tzx3EO0WvdHdV5jZ0cCG+IUlIiJ9qaprJDvLmDp6eLJDOSz97VzwKPBozHY1cHm8ghIRkb5V1TUyZXQheTnpNbVafzsXTDSzP5lZnZntMLPHzGxivIMTEZGeVdU3ps0cPLH6mybvJTrczQSiI0w/EZSJiEgStEc62LQzvQYH7dTfxFPm7ve6ezhY7kPz44iIJM3mXc2EOzyjE89OM7vWzLKD5VpgVzwDExGRnqXbrKOx+pt4PkW0K/V2YBtwBdFhdEREJAk6u1JPz9RnPO7+nrtf7O5l7j7W3S8l+mVSERFJgk27mhlXks/w/P6O9Zw6BtIH70uDFoWIiByW2j0tTCwtTHYYR2QgiSd9hkIVEckwtXtbKB+ZPlMhxBpI4vFBi0JERPqto8PZtq+FCWmaeHq9OWhmDXSfYAxIzxqLiKS5uoYQ7RFPq8nfYvXa4nH3Yncv6WYpdvc+n2iZ2XwzW29mVWZ2azf7bzSzejNbFSyfidl3g5ltCJYbYsrnmtmbwTV/Yma65SciQ0rt3mYAJqZpiyduA/yYWTZwN3ARcDxwtZkd382hj7j7nGD5dXDuKOAbwOnAacA3ggnoAH4B3ATMDJb58aqDiEgqqt3bCpCZLZ4BOg2ocvdqd28DHgYu6ee5HwGedffd7r4HeBaYH0zNUOLuy9zdgd8Cl8YjeBGRVFW7pwVgSHYu6Es5sCVmuyYo6+pyM1ttZovMbFIf55YH631dEzO7ycwqzayyvr7+SOsgIpJyavc2M7IwNy2/wwPxTTzdPXvp2lHhCWCqu58EPAfc38e5/blmtNB9obtXuHtFWZmGlRORzFG7J327UkN8E08NMClmeyKwNfYAd9/l7qFg81fA3D7OrQnWe7ymiEimq92bvl2pIb6JZwUw08ymmVkecBXRqRUOCJ7ZdLoYeCtYXwJcaGalQaeCC4El7r4NaDCzM4LebNcDj8exDiIiKcXd077FE7cbhO4eNrMFRJNINnCPu681szuASndfDNxsZhcDYWA3cGNw7m4zu5No8gK4w913B+ufA+4j+j2ip4NFRGRI2NfSTlNbhIlp2qMN4ph4ANz9KeCpLmW3x6zfBtzWw7n3APd0U14JzB7cSEVE0kNNmvdog/jeahMRkUG2dW+QeNK4xaPEIyKSRmr3qsUjIiIJVLunhYLcLEYNz0t2KEdMiUdEJI10dqVO52EqlXhERNJIOs/D00mJR0QkjURnHlXiERGRBGhpi7CrqU0tHhERSYyt+9K/KzUo8YiIpI2D0yEUJjmSgVHiERFJE7UZ8OVRUOIREUkbtXtayM4yxhXnJzuUAVHiERFJE7V7WziqpICc7PT+6E7v6EVEhpB0nw6hkxKPiEiaqN3bkvbPd0CJR0QkLYQjHWzf36oWj4iIJMaOhhCRDleLpy9mNt/M1ptZlZnd2stxV5iZm1lFsH2Nma2KWTrMbE6wb2lwzc59Y+NZBxGRVND5HZ4JGdDiidsMpGaWDdwNXADUACvMbLG7r+tyXDFwM/BKZ5m7Pwg8GOw/EXjc3VfFnHZNMBOpiMiQULu3GUjveXg6xbPFcxpQ5e7V7t4GPAxc0s1xdwLfA1p7uM7VwO/jE6KISHqozYAprzvFM/GUA1titmuCsgPM7BRgkrs/2ct1PsGhiefe4Dbb1y2dJ6UQEemn2r0tjB6ex7C87GSHMmDxTDzdJQQ/sNMsC7gL+HKPFzA7HWh29zUxxde4+4nA2cFyXQ/n3mRmlWZWWV9ffyTxi4ikhOr6Rp58Yxuzy0ckO5RBEc/EUwNMitmeCGyN2S4GZgNLzWwTcAawuLODQeAqurR23L02+LcBeIjoLb1DuPtCd69w94qysrIBVkVEJDmaQmE++7uV5GQb37psdrLDGRTxTDwrgJlmNs3M8ogmkcWdO919n7uPcfep7j4VWA5c3NlpIGgRfZzosyGCshwzGxOs5wIfA2JbQyIiGcPd+c/HVlNV18hPrz6ViaXpPSp1p7j1anP3sJktAJYA2cA97r7WzO4AKt19ce9X4Bygxt2rY8rygSVB0skGngN+FYfwRUSS7jd/f5f/Wb2Nr84/ln+aOSbZ4Qwac/e+j0pzFRUVXlmp3tcikj6WbdzFtb95hQuOG8cvrj2VZPSjMrOV7l7R95GHRyMXiIikmPZIB19Z9AZTRhfy/Y+flJSkE09KPCIiKWbxqq3U7Gnhax89juKC3GSHM+iUeEREUkhHh/PzpVUce1Qx5x+bmSOCKfGIiKSQJWu3s7G+iS+cNyPjbrF1UuIREUkR7s7PXqhi2pjhfPTE8ckOJ26UeEREUsSL79Szdut+PvfB6WRnZWZrB5R4RERSxt0vVDFhRAGXnlLe98FpTIlHRCQFvPrublZs2sNN5xxNXk5mfzRndu1ERNKAu/PTv25g9PA8PjFvcrLDiTslHhGRJHu0soaXNuzk8+fNyIhpD/qixCMikkTv7mzim0+s5QPTR/PJD0xNdjgJocQjIpIk7ZEO/u3h18nNzuIHV55MVgb3ZIsVt9GpRUSkdz967h1W1+zjF9ecyvgR6T+ldX+pxSMikgTLq3fx86Ub+UTFJC7K4C+LdkeJR0QkwdZt3c+/Pfw6U0YVcvs/H5/scBJOiUdEJIH+9HoN//KLfwDw82vmMjx/6D3xiGviMbP5ZrbezKrM7NZejrvCzNzMKoLtqWbWYmarguWXMcfONbM3g2v+xDJ1FD0RSUut7REeWLaJK/97Gd94fA3Pv7WDplCYtnAH33h8Df/xyBucPHEkT37xbI6fUJLscJMibqnWzLKBu4ELgBpghZktdvd1XY4rBm4GXulyiY3uPqebS/8CuAlYDjwFzAeeHuTwRUQOS2t7hIdffY9fvljN9v2tzBhbxB8qa7h/2WZys42xxQXU7m3hX8+exlfnH0tO9tC94RTPNt5pQJW7VwOY2cPAJcC6LsfdCXwPuKWvC5rZeKDE3ZcF278FLkWJR0SSJBzp4JHKLfz4uQ3UNYQ4beoofnjlyZw5fTRtkQ4qN+3hxXfqWVO7j9s+eiwfO2lCskNOungmnnJgS8x2DXB67AFmdgowyd2fNLOuiWeamb0O7Af+t7u/FFyzpss1M3s0PRFJSe7OX9+u49tPv01VXSMVU0r50VVzOPPo0Qfm0cnPyeasGWM4a8aYJEebWuKZeLp79uIHdpplAXcBN3Zz3DZgsrvvMrO5wJ/N7IS+rvm+Fze7iegtOSZPzvyxj0QkMZpCYV7asJP7Xn6X5dW7mTZmOP993VwuPH5cxk7cNtjimXhqgEkx2xOBrTHbxcBsYGnwZh0FLDazi929EggBuPtKM9sIzAquObGXax7g7guBhQAVFRXdJicRkf7Ytq+FZ9ft4Lm36li+cRdtkQ5GD8/jjktO4OrTJpM7hJ/XHIl4Jp4VwEwzmwbUAlcB/6tzp7vvAw60P81sKXCLu1eaWRmw290jZnY0MBOodvfdZtZgZmcQ7YxwPfDTONZBRIaonY0hnl6znSdWbeXVTbsBOHrMcK4/cwofOm4cFVNLlXCOUNwSj7uHzWwBsATIBu5x97VmdgdQ6e6Lezn9HOAOMwsDEeCz7r472Pc54D5gGNFOBepYICID1h7p4I0te/lH1S7+sXEnKzfvIdLhzBxbxJcvmMVHTxrP9LKiZIeZEcw98+9CVVRUeGVlZbLDEJEUsKepjZWb97B5dzPb97WwfX+IHftaWbN1H81tEczghAklfHBWGf988gSOGVc8ZJ/dmNlKd68Y7OsOva/MisiQ0hQK8/zbdSyv3sWKd3ezoa7xwL78nCyOGlHAuJICLj91ImfNGM0ZR49mZGFeEiPOfEo8IpJxwpEO/l61kz+/XsuStTtoaY9QnJ/D3KmlXHpKORVTSpk1rpiRhblDtjWTTEo8IpLWIh3Oso27WL+jgY31jWysa2T9jgb2NrczYlgul51azqVzypk7pZTsITLfTapT4hGRtPXiO/V8+6m3eHt7AwAjhuUyY2wRFx4/jg8dN45zjykjPyfzp5JON0o8IpLS3tvVzMsbdzK6KJ/xIwo4akQBdftDfPvpt3hpw04mjyrkx1fN4Z9mjGHU8DzdOksDSjwikpJa2iL8YmkVv/xbNW3hjkP2jxiWy9c/djzXnjFZrZo0o8QjIgnn7j22TNydJWu3c+eTb1G7t4VL50zgC+fNoLktwrZ9rezY30p7pIOPz53EiMLcBEcug0GJR0QSIhzp4LHXavjJ81W0RTo4Z2YZ5x5Txjkzy8jONl6p3sVLG3by0oZ6NtY3cexRxTxy0xmcfvToA9c4eVIvLyBpQ4lHZJBsrG9k/fYGmkJhWtojNLdFOG589IuIQ5m78/Sa7fzXM+uprm/ilMkjmVhayHNv7eCx12rIMsgyI9zhFORmcfq00Xzm7KP5+NyJQ3rOmkymxCMyQO7Ob5dt5s4n1xHuOHQkkPknHMU3Lz6Bo0YUJCG65Nm6t4UnV2/lj6/V8vb2BmaOLWLhdXO5IBjFOdLhrNqylxffqac90sHZM8Zw6pRSCnL1vCbTacgckQFoaYvwtT+9yR9fr+VDx47lyxceQ3FBDoV52eTlZPHA8s38+LkN5GZnccuFs7juzKkJ/S5Je6QDd8jLiU/Lwd3ZtKuZ+oYQ+1va2d/aTn1DiOffqjswsOZJE0dw3RlT+JdTJ+p7NGkmXkPmKPGkEXdn7db9FOZlc7QGK0yqzg/czz/4Gm9v38+/f2gWXzx/BlndfLC+t6uZr/35TV7asJOpows5e2YZZxw9mtOPHsWYovy4xbdoZQ3ffvptGlrbOfaoEmaXj+DE8hGcOX0008YMH9D19zW38/gbtfz+1S28tW3/Iftnji3i4pMn8LGTJwz4tSR5lHgGIJ0TT0tbhJc21PPXt+t4YX0dO/aHKC7I4fEvnNVn8gmFIzz95nberN3Hp/9pGhNGDktQ1L0LRzqo3dvC5FGFPfZsCkc6WL+jgdc272Hl5j2srt3HceNLuO6MKZw+bVTCvqvh7mysb+L5t3bw0oad7Njfyp7mdva1tNEecUoKcvjRVXM4/9hxfV7nydXbWLSyhhWbdtPcFgGiH9Dzpo1i3tRSKqaMYmLpsAN1i3Q47n7Yzzne3r6fr/95DSs27eHUySOZN3UUb9buY03tPva3hgGomFLKlRWT+OhJ4ynK7/uO++6mNt7Z0cA7OxpYuXkPf1mznVC4g9nlJVxZMYmjxxRRMiyHkoJcSoblUqqhaDKCEs8ApGvi2VjfyA33vErNnhaK83M4e9YYPjB9DD989h1KC3P58xfOorjg0O6kO/a38uDyzTz06hZ2NoYAKC3M5a5PzOHcY8b2+/XbwtEEsWV3M3UNIXY3hdjV1MaepjZysrOYMKKA8SOGMX5EAZNHF1I+clivHzbhSAd/er2Wu1+oYtOuZiqmlHLLR47hjJheS3X7W7n35U089Mp77GtpB2BscT6zy0ewcvMe9rW0M3NsEdedOYUTJowgFI4QCncQau/gqBEFnFg+YlBu51TXN/L7V9/j2XU72LSrGYBjjypm2pjhjCzMY2Rh9MP1otnjmTSq8LCu3R7pYE3tPpZX72Z59S5e27yHhlA0IZQU5NDh0T8a2iNObrZxxdxJLDh/BuV9/OFQ3xDily9u5L6XN1FSkMNtFx3HFXMnHmiFuTubdzXzl7Xb+UPlFqrrmyjMy+bUyaVANNFF3Il0OG3hjugS6WB/Szu7mtoOvM6IYbn888njuWreZGaXjzisukt6UeIZgHRMPCs37+HT968gJ8v4/hUnc9aMMQfu0y/buItrf/MK5x0zloXXzT3wwdLSFuG/nlnP/S9vIuLO+ceM5foPTKV85DAWPPQa63c0sOC8Gfz7h2eRnWXsaWrjlXd38/p7e9jd1EZTW5jGUITG1na272tl2/5Wuv545GVnUTo8l/aIszvmwwiiH5rHTyjhuPElHF1WRElBDkX50WXTribufmEj7+1u5vjxJXzkhKN46NXN7Ngf4uyZY7j+zKk8u247f359K+GODj5ywlHMn30Up04uPdAKaGmL8MTqrTywbDNv1u7r9v+ttDCXs4Nuuv80cwxji/v/QN/dWVa9i9+89C7Pv11Hbrbxgelj+PBxYzn/uHF9fvAfqUiHs357Ays27aaqrpHc7Czyc7MoyMlm+/4WHltZi+N8Yt4kvnDeDMaPeH8cNXuaWfi3ah5ZsYX2SAdXVkziq/OPpXR4zyMsuzuvvbeXRyujt8qys+zAkpOVRV5OFrnZRl5ONsPzspleVsSso4qZNa6Io0oK1JoZIpR4BiDdEs+z63bwxd+/xlElBdz/qdOYMvrQe+T3/eNdvvnEOm7+0Ey+dMEslm3cxa1/XM3mXc1cfdokPvvB6e87r6UtwjcWr+EPlTWcWD6CtnD0VhZEHzyPHp7H8PwchufnUJSfzbiSAiaVFjJ5VCGTRhUyriSfUcPzKMrPOfCh09oe/ULftr0tVO9sYt22/azbup/12xtoaY8cEvPs8hL+7UOz+PBxYzEzWtsjPLBsMz9fWsWe5nYKcrP4+NxJfObsad3WOdbarfvY2dhGfk4W+TlZ5GZnsbG+kRffqedv79SzszGaFKeNGc68qaXMmzqK8pHDqN7ZRFVdIxvrG9m+r/V9H/I7G0NsqGtk9PA8rjljCteeMfmwEle8bN3bwt0vVPGHyi24R1vpP5E4AAANT0lEQVSAo4ryGDU8n9ws48V36jGDfzllIp89d7qeqcigScvEY2bzgR8TnYH01+7+nR6OuwJ4FJgXTH19AfAdIA9oA77i7n8Njl0KjAdagtMvdPe63uJIl8TT0eH87pXNfHPxWk4sH8FvbpzX48Nnd+c/F63m0ZU1fPi4cTz31g6mjC7ku5ef9L5bV10tWlnDT57fwJTRhdEH3NNGcdLEkYPa6ynS4exsDNHQGqYxFKaxNUxBbhZzp5R2+5dyYyjM3zfUM2/qKEYPwsP2jo5oJ4xl1Tt59d09VG7ezd7m9gP7C4O/4MtHDiPc4dHbde0dZGXBZaeUc8mc8pTs0rtldzOPrNjCtn2t7G4Ksbupjf2tYT44q4ybzjk6ZZ7hSeZIu8RjZtnAO8AFQA2wArja3dd1Oa4Y+B+iSWZBkHhOAXa4+1Yzmw0scffy4PilwC3u3u9Mkg6J57X39nDHE+tYtWUv5x1Txt3XnEphXu8PfVvbI3xi4XJW1+zl02dN48sXHsOwvNT7wEy2jg6nqr6RHftbObqsiPElBd32PhOR90vHGUhPA6rcvRrAzB4GLgHWdTnuTuB7wC2dBe7+esz+tUCBmeW7eyiO8SbFtn0tfOfpt3l81VbGFufzg4+fzGWnlPfrg7EgN5sHP3M69Q0h3V7pRVaWMWtcMbPGFSc7FBEB4jkeRTmwJWa7Jig7IGjZTHL3J3u5zuXA612Szr1mtsrMvm7xeMq5cSN8/vNQUgJZWdF/P//5aPkgerlqJxf88G88vWY7C86bwQu3nMvlMb2Q+qMoP0dJR0TSSjxbPN19eh64r2dmWcBdwI09XsDsBOC7wIUxxde4e21wi+4x4Drgt92cexNwE8DkyZP7H/XTT8MVV0B7e3QBaGiAX/8a7r8fFi2Ciy7q//V68Jc127n5968zdUwhv7lh3mF3yRURSVfxbPHUALFjyU4EtsZsFwOzgaVmtgk4A1hsZhUAZjYR+BNwvbsfaGq4e23wbwPwENFbeodw94XuXuHuFWVl/RykcePGaNJpbj6YdDq1t0fLr7hiwC2fP6zYwucfXMkJ5SX84f89U0lHRIaUeCaeFcBMM5tmZnnAVcDizp3uvs/dx7j7VHefCiwHLg46F4wk2uHgNnf/R+c5ZpZjZmOC9VzgY8CaQYv4Bz84NOF01d4Od911WJft6HB2NYZ4a9t+fvr8Bv7zsdWcNWMMD37mdEYW9vxdCxGRTBS3W23uHjazBcASot2p73H3tWZ2B1Dp7ot7OX0BMAP4upl9PSi7EGgClgRJJxt4DvjVoAX9u9/1L/E88AD87Gc9HuLuvFm7jz++Vsuz63awY3/r+0Yt/n9OGs9dV86J28CNIiKpTF8gjZWVxSFf1e/puMihX5BsaG3nwVfe47GVNWyoayQvJ4vzjiljxtgiyoryGVsSnS9+zsSR6s4rIikvHbtTp5+iomhHgv4cF6NzAMg7n1xHXUOIUyeP5FuXzeZjJ07Q1LwiIl0o8cS69tpo77Vebrd5bi523XUHtqvrG7n98bX8vWonJ5aPYOH1FcyZNDIR0YqIpCUlnlhf/nK0y3QviaeVLP61+Ey2/mApADW7W8jPzeKOS07gmtOnaKIrEZE+KPHEmj49+j2drt/jAcjNJZKTwz0LvsuI2ccxwqJfVPrgrDI+d+70lBhMUkQkHSjxdHXRRbB6dbTL9AMPQGNj9JnOddeR/R//wRemT092hCIiaU292kREpFvx6tWmL5KIiEhCKfGIiEhCKfGIiEhCKfGIiEhCKfGIiEhCKfGIiEhCKfGIiEhCDYnv8ZhZPbC5S/EIYF8fZbHbfa2PAXYOIMzu4unvMYdbl67bneuxZQOpz0Dq0tM+vTd6bw4nzv4cl0p16S3O/hwTr5+z4e7ez5k0D4O7D8kFWNhXWex2X+tE5xga1Hj6e8zh1qWXOsSWHXF9BlIXvTd6bwazLoP53ujnbPCWoXyr7Yl+lD1xmOuDHU9/jzncunTdfqKHY47UQOrS0z69N4Mjk96b/l5jsN4b/ZwNkiFxqy0RzKzS4zC0RLJkUn0yqS6QWfVRXVJXPOszlFs8g21hsgMYZJlUn0yqC2RWfVSX1BW3+qjFIyIiCaUWj4iIJJQSTzfM7B4zqzOzNUdw7lwze9PMqszsJ2ZmMfu+aGbrzWytmX1vcKPuMZ5Br4uZfdPMas1sVbB8dPAj7zGmuLw3wf5bzMzNbMzgRdxrPPF4b+40s9XB+/KMmU0Y/Mh7jCke9fm+mb0d1OlPZpaQeeXjVJePB7/7HWYW92dBA6lDD9e7wcw2BMsNMeW9/l51K17d5dJ5Ac4BTgXWHMG5rwJnEp2g9GngoqD8POA5ID/YHpvGdfkmcEumvDfBvknAEqLf9xqTrnUBSmKOuRn4ZTq/N8CFQE6w/l3gu2lcl+OAY4ClQEWq1iGIb2qXslFAdfBvabBe2lt9e1vU4umGu/8N2B1bZmbTzewvZrbSzF4ys2O7nmdm44n+4i/z6DvyW+DSYPfngO+4eyh4jbr41iIqTnVJmjjW5y7gP4GEPfSMR13cfX/MocNJ//o84+7h4NDlwMT41iIqTnV5y93XJyL+4PWOqA49+AjwrLvvdvc9wLPA/CP9nFDi6b+FwBfdfS5wC/Dzbo4pB2pitmuCMoBZwNlm9oqZvWhm8+Iabe8GWheABcHtj3vMrDR+ofbLgOpjZhcDte7+RrwD7YcBvzdm9i0z2wJcA9wex1j7YzB+1jp9iuhf1MkymHVJlv7UoTvlwJaY7c56HVF9c/r5okOamRUBHwAejbl9md/dod2Udf7FmUO0iXoGMA/4g5kdHfyVkDCDVJdfAHcG23cCPyD6oZBwA62PmRUCXyN6SyepBum9wd2/BnzNzG4DFgDfGORQ+2Ww6hNc62tAGHhwMGPsr8GsS7L0Vgcz+yTwb0HZDOApM2sD3nX3y+i5XkdUXyWe/skC9rr7nNhCM8sGVgabi4l+IMfeCpgIbA3Wa4A/BonmVTPrIDoWUn08A+/GgOvi7jtizvsV8GQ8A+7DQOszHZgGvBH8Mk4EXjOz09x9e5xj72owfs5iPQT8D0lKPAxSfYIH2R8DPpToP9RiDPZ7kwzd1gHA3e8F7gUws6XAje6+KeaQGuDcmO2JRJ8F1XAk9Y33A650XYCpxDyUA14GPh6sG3ByD+etINqq6XzQ9tGg/LPAHcH6LKLNVkvTuoyPOeY/gIfT+b3pcswmEtS5IE7vzcyYY74ILErn9waYD6wDyhJZj3j+nJGgzgVHWgd67lzwLtG7NqXB+qj+1LfbuBL9ZqbDAvwe2Aa0E83onyb6V/FfgDeCX4Tbezi3AlgDbAR+xsEv6eYBvwv2vQacn8Z1eQB4E1hN9K+88YmoS7zq0+WYTSSuV1s83pvHgvLVRMfdKk/n9waoIvpH2qpgSUgvvTjV5bLgWiFgB7AkFetAN4knKP9U8H5UAZ/sq769LRq5QEREEkq92kREJKGUeEREJKGUeEREJKGUeEREJKGUeEREJKGUeGRIMrPGBL/er83s+EG6VsSio0+vMbMn+hqx2cxGmtnnB+O1RQaDulPLkGRmje5eNIjXy/GDg1nGVWzsZnY/8I67f6uX46cCT7r77ETEJ9IXtXhEAmZWZmaPmdmKYDkrKD/NzF42s9eDf48Jym80s0fN7AngGTM718yWmtkii84h82Dn3CRBeUWw3hgM5PmGmS03s3FB+fRge4WZ3dHPVtkyDg52WmRmz5vZaxadH+WS4JjvANODVtL3g2O/ErzOajP7/wbxv1GkT0o8Igf9GLjL3ecBlwO/DsrfBs5x91OIjvb8f2LOORO4wd3PD7ZPAf4dOB44Gjirm9cZDix395OBvwH/GvP6Pw5ev8/xroJxwj5EdPQIgFbgMnc/lej8Tz8IEt+twEZ3n+PuXzGzC4GZwGnAHGCumZ3T1+uJDBYNEipy0IeB42NG7i0xs2JgBHC/mc0kOvJubsw5z7p77Jwnr7p7DYCZrSI6Vtbfu7xOGwcHVl0JXBCsn8nBuUweAv6rhziHxVx7JdG5USA6Vtb/CZJIB9GW0Lhuzr8wWF4PtouIJqK/9fB6IoNKiUfkoCzgTHdviS00s58CL7j7ZcHzkqUxu5u6XCMUsx6h+9+xdj/4cLWnY3rT4u5zzGwE0QT2BeAnROffKQPmunu7mW0CCro534Bvu/t/H+brigwK3WoTOegZovPXAGBmncPHjwBqg/Ub4/j6y4ne4gO4qq+D3X0f0emtbzGzXKJx1gVJ5zxgSnBoA1Acc+oS4FPB/CyYWbmZjR2kOoj0SYlHhqpCM6uJWb5E9EO8Injgvo7oVBYA3wO+bWb/ALLjGNO/A18ys1eB8cC+vk5w99eJjjR8FdFJ0irMrJJo6+ft4JhdwD+C7tffd/dniN7KW2ZmbwKLeH9iEokrdacWSRHBbKgt7u5mdhVwtbtf0td5IulGz3hEUsdc4GdBT7S9JGk6cZF4U4tHREQSSs94REQkoZR4REQkoZR4REQkoZR4REQkoZR4REQkoZR4REQkof4vVrVb/LkGo5oAAAAASUVORK5CYII=\n", 861 | "text/plain": [ 862 | "

" 863 | ] 864 | }, 865 | "metadata": { 866 | "needs_background": "light" 867 | }, 868 | "output_type": "display_data" 869 | } 870 | ], 871 | "source": [ 872 | "learn = get_chexpert_learner(learn=learn, img_size=256, size=1)\n", 873 | "lr_find_no_cbs(learn)" 874 | ] 875 | }, 876 | { 877 | "cell_type": "code", 878 | "execution_count": 42, 879 | "metadata": {}, 880 | "outputs": [ 881 | { 882 | "data": { 883 | "text/html": [ 884 | "\n", 885 | "
\n", 886 | " \n", 898 | " \n", 899 | " 0.00% [0/15 00:00<00:00]\n", 900 | "
\n", 901 | " \n", 902 | "\n", 903 | " \n", 904 | " \n", 905 | " \n", 906 | " \n", 907 | " \n", 908 | " \n", 909 | " \n", 910 | " \n", 911 | " \n", 912 | " \n", 913 | " \n", 914 | "
epochtrain_lossvalid_lossavg_auc_metrictime

\n", 915 | "\n", 916 | "

\n", 917 | " \n", 929 | " \n", 930 | " 62.24% [4345/6981 31:03<18:50 0.4419]\n", 931 | "
\n", 932 | " " 933 | ], 934 | "text/plain": [ 935 | "" 936 | ] 937 | }, 938 | "metadata": {}, 939 | "output_type": "display_data" 940 | }, 941 | { 942 | "name": "stderr", 943 | "output_type": "stream", 944 | "text": [ 945 | "IOPub message rate exceeded.\n", 946 | "The notebook server will temporarily stop sending output\n", 947 | "to the client in order to avoid crashing it.\n", 948 | "To change this limit, set the config variable\n", 949 | "`--NotebookApp.iopub_msg_rate_limit`.\n", 950 | "\n", 951 | "Current values:\n", 952 | "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", 953 | "NotebookApp.rate_limit_window=3.0 (secs)\n", 954 | "\n" 955 | ] 956 | } 957 | ], 958 | "source": [ 959 | "lr = 5e-5\n", 960 | "learn.fit_one_cycle(15,slice(lr))" 961 | ] 962 | }, 963 | { 964 | "cell_type": "code", 965 | "execution_count": 45, 966 | "metadata": {}, 967 | "outputs": [ 968 | { 969 | "data": { 970 | "text/plain": [ 971 | "1" 972 | ] 973 | }, 974 | "execution_count": 45, 975 | "metadata": {}, 976 | "output_type": "execute_result" 977 | } 978 | ], 979 | "source": [ 980 | "1" 981 | ] 982 | }, 983 | { 984 | "cell_type": "code", 985 | "execution_count": 40, 986 | "metadata": {}, 987 | "outputs": [ 988 | { 989 | "name": "stdout", 990 | "output_type": "stream", 991 | "text": [ 992 | "Atelectasis \t auc: 0.801\t chexpert auc: 0.858\t difference: 0.0574\n", 993 | "Cardiomegaly \t auc: 0.82\t chexpert auc: 0.854\t difference: 0.0336\n", 994 | "Consolidation \t auc: 0.92\t chexpert auc: 0.939\t difference: 0.0194\n", 995 | "Edema \t auc: 0.914\t chexpert auc: 0.941\t difference: 0.0266\n", 996 | "Pleural Effusion\t auc: 0.918\t chexpert auc: 0.936\t difference: 0.0185\n", 997 | "\n", 998 | "Average auc: 0.875 \t CheXpert average auc 0.906\t Difference 0.0311\n" 999 | ] 1000 | } 1001 | ], 1002 | "source": [ 1003 | "validation_eval(learn)" 1004 | ] 1005 | }, 1006 | { 1007 | "cell_type": "markdown", 1008 | "metadata": {}, 1009 | "source": [ 1010 | "### Export entire model" 1011 | ] 1012 | }, 1013 | { 1014 | "cell_type": "code", 1015 | "execution_count": null, 1016 | "metadata": {}, 1017 | "outputs": [], 1018 | "source": [ 1019 | "# if learn is None:\n", 1020 | "# learn = load_learner(data_path/'models','naive_densenet.pkl')\n", 1021 | "# else:\n", 1022 | "# full_train_df['patient'] = full_train_df.Path.str.split('/',3,True)[2]\n", 1023 | "# learn.export(data_path/'models'/'size_{img_size}_auc_{self.avg_auc}_{datetime.datetime.now(): \"%Y-%m-%d %H:%M\"}.pkl')" 1024 | ] 1025 | }, 1026 | { 1027 | "cell_type": "markdown", 1028 | "metadata": {}, 1029 | "source": [ 1030 | "### Things to try to improve score\n", 1031 | "\n", 1032 | "- Building more sophisticated model structure to account for unknowns\n", 1033 | "- Curriculum learning\n", 1034 | "- Mixup\n", 1035 | "- (not really possible) Use the labelling tool from the ChexPert paper : https://github.com/stanfordmlgroup/chexpert-labeler\n", 1036 | "- Try SENet" 1037 | ] 1038 | }, 1039 | { 1040 | "cell_type": "code", 1041 | "execution_count": null, 1042 | "metadata": {}, 1043 | "outputs": [], 1044 | "source": [ 1045 | "list(DenseNet().modules())" 1046 | ] 1047 | }, 1048 | { 1049 | "cell_type": "code", 1050 | "execution_count": null, 1051 | "metadata": {}, 1052 | "outputs": [], 1053 | "source": [] 1054 | } 1055 | ], 1056 | "metadata": { 1057 | "kernelspec": { 1058 | "display_name": "Python 3", 1059 | "language": "python", 1060 | "name": "python3" 1061 | }, 1062 | "language_info": { 1063 | "codemirror_mode": { 1064 | "name": "ipython", 1065 | "version": 3 1066 | }, 1067 | "file_extension": ".py", 1068 | "mimetype": "text/x-python", 1069 | "name": "python", 1070 | "nbconvert_exporter": "python", 1071 | "pygments_lexer": "ipython3", 1072 | "version": "3.7.1" 1073 | } 1074 | }, 1075 | "nbformat": 4, 1076 | "nbformat_minor": 2 1077 | } 1078 | --------------------------------------------------------------------------------