├── .gitignore ├── images ├── Datasets.PNG ├── Samples.PNG └── UncertaintyFuseNet.png ├── evaluation.py ├── README.md ├── utils.py ├── grad_plot.py ├── Grad-CAM.ipynb ├── Uncertainty Plot.ipynb ├── plots.py ├── models.py ├── Fusion Model Results.ipynb └── Simple Models Results.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | -------------------------------------------------------------------------------- /images/Datasets.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moloud1987/UncertaintyFuseNet-for-COVID-19-Classification/HEAD/images/Datasets.PNG -------------------------------------------------------------------------------- /images/Samples.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moloud1987/UncertaintyFuseNet-for-COVID-19-Classification/HEAD/images/Samples.PNG -------------------------------------------------------------------------------- /images/UncertaintyFuseNet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moloud1987/UncertaintyFuseNet-for-COVID-19-Classification/HEAD/images/UncertaintyFuseNet.png -------------------------------------------------------------------------------- /evaluation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.metrics import accuracy_score, precision_score, recall_score 3 | 4 | 5 | def mode_robustness(x_test, y_test, model, std_coef, number_prediction=200): 6 | result = [] 7 | for std in std_coef: 8 | print("std: {}".format(std)) 9 | x_class = x_test 10 | mean = 0 11 | sigma = np.std(x_class) * std 12 | gaussian = np.random.normal(mean, sigma, x_class.shape) 13 | x_class = x_class + gaussian 14 | 15 | mc_predictions = [] 16 | 17 | for _ in range(number_prediction): 18 | y_p_class = model.predict(x_class) 19 | mc_predictions.append(y_p_class) 20 | 21 | mc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1) 22 | ensemble_acc = accuracy_score(y_test.argmax(axis=1), mc_ensemble_pred) 23 | ensemble_precision = precision_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted') 24 | ensemble_recall = recall_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted') 25 | ensemble_F1 = (2 * ensemble_precision * ensemble_recall) / (ensemble_precision + ensemble_recall) 26 | 27 | result = [ensemble_acc * 100, ensemble_precision * 100, ensemble_recall * 100, ensemble_F1 * 100] 28 | print("[{:0.5f}, {:0.5f}, {:0.5f}, {:0.5f}]\n".format(*result)) 29 | 30 | 31 | def evaluation(y_test, y_pred): 32 | acc = accuracy_score(y_test, y_pred) 33 | precision = precision_score(y_test, y_pred, average='weighted') 34 | recall = recall_score(y_test, y_pred, average='weighted') 35 | F1 = (2 * precision * recall) / (precision + recall) 36 | return acc, precision, recall, F1 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UncertaintyFuseNet 2 | This repository contains the code of the following paper: "UncertaintyFuseNet: Robust Uncertainty-aware Hierarchical Feature Fusion with Ensemble Monte Carlo Dropout for COVID-19 Detection" 3 | 4 | # Abstract 5 | The COVID-19 (Coronavirus disease 2019) pandemic has become a major global threat to human health and well-being. Thus, the development of computer-aided detection (CAD) systems that are capable of accurately distinguishing COVID-19 from other diseases using chest computed tomography (CT) and X-ray data is of immediate priority. Such automatic systems are usually based on traditional machine learning or deep learning methods. Differently from most of the existing studies, which used either CT scan or X-ray images in COVID-19-case classification, we present a new, simple but efficient deep learning feature fusion model, called __UncertaintyFuseNet__, which is able to classify accurately large datasets of both of these types of images. We argue that the uncertainty of the model's predictions should be taken into account in the learning process, even though most of the existing studies have overlooked it. We quantify the prediction uncertainty in our feature fusion model using effective Ensemble Monte Carlo Dropout (EMCD) technique. A comprehensive simulation study has been conducted to compare the results of our new model to the existing approaches, evaluating the performance of competing models in terms of Precision, Recall, F-Measure, Accuracy and ROC curves. The obtained results prove the efficiency of our model which provided the prediction accuracy of 99.08\% and 96.35\% for the considered CT scan and X-ray datasets, respectively. Moreover, our __UncertaintyFuseNet__ model was generally robust to noise and performed well with previously unseen data. 6 | # Packages Used 7 | Pandas, Tensorflow 2.0, Keras, Sklearn, Numpy, Matplotlib 8 | 9 | # Proposed Architecture of UncertaintyFuseNet 10 | 11 | ![Test Image 1](images/UncertaintyFuseNet.png) 12 | 13 | 14 | # Datasets 15 | Details of the COVID-19 datasets used in this study with some random samples: 16 | 17 | ![Test Image 1](images/Datasets.PNG) 18 | 19 | ![Test Image 1](images/Samples.PNG) 20 | 21 | 22 | ## Credits 23 | This work has been published by Information Fusion Journal, and you can find it in this [link](https://www.sciencedirect.com/science/article/pii/S1566253522001609?via%3Dihub). To access Arxiv version please see this [link](https://arxiv.org/abs/2105.08590). 24 | 25 | ## Reference 26 | 27 | ``` 28 | @article{abdar2023uncertaintyfusenet, 29 | title={UncertaintyFuseNet: robust uncertainty-aware hierarchical feature fusion model with ensemble Monte Carlo dropout for COVID-19 detection}, 30 | author={Abdar, Moloud and Salari, Soorena and Qahremani, Sina and Lam, Hak-Keung and Karray, Fakhri and Hussain, Sadiq and Khosravi, Abbas and Acharya, U Rajendra and Makarenkov, Vladimir and Nahavandi, Saeid}, 31 | journal={Information Fusion}, 32 | volume={90}, 33 | pages={364--381}, 34 | year={2023}, 35 | publisher={Elsevier} 36 | } 37 | ``` 38 | 39 | ---------- 40 | In case of any questions, bugs, suggestions or improvements, please feel free to contact us. 41 | 42 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import glob 4 | import matplotlib.pyplot as plt 5 | import cv2 6 | from os import listdir 7 | from tensorflow.keras.utils import to_categorical 8 | from sklearn.preprocessing import LabelEncoder 9 | from sklearn.model_selection import train_test_split 10 | import tensorflow as tf 11 | 12 | 13 | def rgb2gray(rgb): 14 | return np.dot(rgb[..., :3], [0.2989, 0.5870, 0.1140]) 15 | 16 | 17 | def transform_images(images: np.ndarray): 18 | """ 19 | Transform images to [-1, 1] 20 | """ 21 | images = 2 * images.astype(np.float32) - 1 22 | return images 23 | 24 | 25 | def load_covid_data(image_size=150, path='../Data/train', shuffle=False, class_frequency=False): 26 | size = image_size 27 | files = listdir(path) 28 | X = [] 29 | Y = [] 30 | 31 | for direct in files: 32 | files_in_folder = glob.glob(path + '/' + direct + '/*.jpg') 33 | for file in files_in_folder: 34 | data = plt.imread(file) 35 | data = cv2.resize(data, (size, size)) 36 | data = data.astype('float32') / 255 37 | if len(data.shape) > 2 and data.shape[2] == 3: 38 | data = rgb2gray(data) 39 | if len(data.shape) > 2 and data.shape[2] == 4: 40 | data = cv2.cvtColor(data, cv2.COLOR_BGRA2BGR) 41 | data = cv2.cvtColor(data, cv2.COLOR_BGR2RGB) 42 | data = rgb2gray(data) 43 | X.append(data) 44 | Y.append(direct) 45 | 46 | print(len(X)) 47 | X = np.array(X).astype(float) 48 | X = transform_images(X) 49 | X = X[:, :, :, None] 50 | 51 | le = LabelEncoder() 52 | Y = le.fit_transform(Y) 53 | Y = np.array(Y).astype(float) 54 | Y = to_categorical(Y, len(files)) 55 | 56 | if shuffle: 57 | idx = np.random.choice(len(X), size=len(X), replace=False) 58 | X = X[idx, :, :] 59 | Y = Y[idx, :] 60 | if class_frequency: 61 | classes = le.inverse_transform(np.argmax(Y, axis=1).astype(int)) 62 | unique, counts = np.unique(classes, return_counts=True) 63 | counts = np.array(counts) 64 | plt.bar(unique, counts) 65 | plt.title('Class Frequency(Percent)') 66 | plt.xlabel('Class') 67 | plt.ylabel('Frequency') 68 | plt.show() 69 | return X, Y 70 | 71 | 72 | def create_dataset_xray(x_train, y_train, x_test, y_test, batch_size=32): 73 | train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)) 74 | train_dataset = train_dataset.batch(batch_size) 75 | train_dataset = train_dataset.cache() 76 | train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE) 77 | 78 | validation_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test)) 79 | validation_dataset = validation_dataset.batch(batch_size) 80 | validation_dataset = validation_dataset.cache() 81 | validation_dataset = validation_dataset.prefetch(tf.data.experimental.AUTOTUNE) 82 | 83 | return train_dataset, validation_dataset 84 | 85 | 86 | def create_dataset_ct(X, Y, batch_size): 87 | np.random.seed(0) 88 | random.seed(0) 89 | idx = np.random.choice(len(X), size=len(X), replace=False) 90 | X = X[idx, :, :, :] 91 | Y = Y[idx, :] 92 | X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=42) 93 | 94 | train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)) 95 | train_dataset = train_dataset.batch(batch_size) 96 | train_dataset = train_dataset.cache() 97 | train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE) 98 | 99 | validation_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test)) 100 | validation_dataset = validation_dataset.batch(batch_size) 101 | validation_dataset = validation_dataset.cache() 102 | validation_dataset = validation_dataset.prefetch(tf.data.experimental.AUTOTUNE) 103 | 104 | return train_dataset, validation_dataset, X_train, X_test, y_train, y_test 105 | -------------------------------------------------------------------------------- /grad_plot.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import matplotlib.pyplot as plt 4 | import tensorflow as tf 5 | from tensorflow.keras.models import Model 6 | 7 | 8 | class GradCAM: 9 | def __init__(self, model, layer_name=None): 10 | self.model = model 11 | self.layer_name = layer_name 12 | if self.layer_name is None: 13 | self.layer_name = self.find_target_layer() 14 | 15 | def find_target_layer(self): 16 | for layer in reversed(self.model.layers): 17 | if len(layer.output_shape) == 4: 18 | return layer.name 19 | 20 | def compute_heatmap(self, image, class_idx, eps=1e-8): 21 | grad_model = Model( 22 | inputs=[self.model.inputs], 23 | outputs=[self.model.get_layer(self.layer_name).output, 24 | self.model.output]) 25 | grad_model.layers[-1].activation = tf.keras.activations.linear 26 | with tf.GradientTape() as tape: 27 | inputs = tf.cast(image, tf.float32) 28 | (conv_outputs, predictions) = grad_model(inputs) 29 | class_predictions = predictions[:, class_idx] 30 | 31 | grads = tape.gradient(class_predictions, conv_outputs) 32 | 33 | cast_conv_outputs = tf.cast(conv_outputs > 0, "float32") 34 | cast_grads = tf.cast(grads > 0, "float32") 35 | guided_grads = cast_conv_outputs * cast_grads * grads 36 | 37 | conv_outputs = conv_outputs[0] 38 | guided_grads = guided_grads[0] 39 | 40 | weights = tf.reduce_mean(guided_grads, axis=(0, 1)) 41 | cam = tf.reduce_sum(tf.multiply(weights, conv_outputs), axis=-1) 42 | 43 | (w, h) = (image.shape[1], image.shape[2]) 44 | # (w, h) = (150, 150) 45 | heatmap = cv2.resize(cam.numpy(), (w, h)) 46 | 47 | numer = heatmap - np.min(heatmap) 48 | denom = (heatmap.max() - heatmap.min()) + eps 49 | heatmap = numer / denom 50 | heatmap = (heatmap * 255).astype("uint8") 51 | 52 | return heatmap 53 | 54 | def overlay_heatmap(self, heatmap, image, alpha=0.5, colormap=cv2.COLORMAP_JET): 55 | # heatmap = cv2.resize(heatmap, (image.shape[0], image.shape[1])) 56 | heatmap = cv2.resize(heatmap, (150, 150)) 57 | heatmap = cv2.applyColorMap(heatmap, colormap) 58 | 59 | output = cv2.addWeighted(image, alpha, heatmap, 1 - alpha, 0) 60 | 61 | return heatmap, output 62 | 63 | 64 | def plot_GradCam(cam, model, image, true_label=None, name=None, uncertainty=False, dataset='Xray'): 65 | image = image[np.newaxis, :, :, :] 66 | all_heatmap = [] 67 | all_preds = [] 68 | if uncertainty: 69 | mc_iter = 200 70 | else: 71 | mc_iter = 1 72 | 73 | if dataset == 'Xray': 74 | class_text = ['Covid19', 'Normal', 'Pneumonia'] 75 | elif dataset == 'CT': 76 | class_text = ['nCT', 'NiCT', 'pCT'] 77 | 78 | for _ in range(mc_iter): 79 | preds = model.predict(image) 80 | class_idx = np.argmax(preds[0]) 81 | 82 | heatmap_p = cam.compute_heatmap(image, class_idx) 83 | all_heatmap.append(heatmap_p) 84 | all_preds.append(preds) 85 | 86 | preds_mean = np.mean(all_preds, axis=0) 87 | class_idx_mean = np.argmax(preds_mean[0]) 88 | 89 | heatmap_mean = np.mean(all_heatmap, axis=0) 90 | 91 | heatmap_mean = heatmap_mean.astype("uint8") 92 | 93 | image = (image + 1) / 2 94 | image1 = (image * 255).astype("uint8") 95 | image1 = np.squeeze(image1, axis=0) 96 | image1 = np.squeeze(image1, axis=2) 97 | image1 = cv2.cvtColor(image1, cv2.COLOR_GRAY2RGB) 98 | 99 | (heatmap_mean, output_p) = cam.overlay_heatmap(heatmap_mean, image1, alpha=0.65, 100 | colormap=cv2.COLORMAP_JET) 101 | 102 | print(preds_mean) 103 | fig = plt.figure(figsize=[15, 8]) 104 | 105 | plt.subplot(3, 1, 1) 106 | plt.imshow(image1) 107 | title = 'True: {} \n Prediction: {} ({}%)'.format(class_text[true_label], 108 | class_text[class_idx_mean], 109 | int(preds_mean[0][class_idx_mean] * 100)) 110 | plt.title(title, fontsize=16) 111 | plt.axis('off') 112 | 113 | plt.subplot(3, 1, 2) 114 | plt.imshow(np.flip(heatmap_mean, axis=2)) 115 | plt.axis('off') 116 | 117 | plt.subplot(3, 1, 3) 118 | plt.imshow(np.flip(output_p, axis=2)) 119 | plt.axis('off') 120 | 121 | fig.savefig('{}.pdf'.format(name), dpi=300) 122 | plt.show() 123 | -------------------------------------------------------------------------------- /Grad-CAM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "ztxzSKXoPD_s" 7 | }, 8 | "source": [ 9 | "# Covid Project\n", 10 | "# Grad Cam\n" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "id": "NcCA4mvbPD_t" 17 | }, 18 | "source": [ 19 | "## Imports" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "%load_ext autoreload\n", 29 | "%autoreload 2\n", 30 | "import numpy as np\n", 31 | "from utils import load_covid_data, create_dataset_ct\n", 32 | "from grad_plot import GradCAM, plot_GradCam\n", 33 | "from tensorflow.keras.models import load_model" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": { 39 | "id": "wd19wXJ816Cq" 40 | }, 41 | "source": [ 42 | "## Dataset1 (X-Ray)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": { 49 | "colab": { 50 | "base_uri": "https://localhost:8080/" 51 | }, 52 | "executionInfo": { 53 | "elapsed": 16053, 54 | "status": "ok", 55 | "timestamp": 1613498608591, 56 | "user": { 57 | "displayName": "Sina Qahremani", 58 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 59 | "userId": "15549406949129485910" 60 | }, 61 | "user_tz": -210 62 | }, 63 | "id": "Cg2wgS2R3dpv", 64 | "outputId": "a731a4c4-b538-4e83-bdb8-fd170ecb4bef", 65 | "scrolled": true 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "x_test,y_test = load_covid_data(path='Data/test')" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "## Dataset2 (CT)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "X = np.load('CT_X.npy')\n", 86 | "Y = np.load('CT_Y.npy')" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "BATCH_SIZE=64\n", 96 | "_, _, _, x_test, _, y_test = create_dataset_ct(X, Y, BATCH_SIZE)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": { 102 | "id": "9U1fWjlO2Ono" 103 | }, 104 | "source": [ 105 | "## Model: Fusion (Without MC)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "colab": { 113 | "base_uri": "https://localhost:8080/" 114 | }, 115 | "executionInfo": { 116 | "elapsed": 4515, 117 | "status": "ok", 118 | "timestamp": 1613504909685, 119 | "user": { 120 | "displayName": "Sina Qahremani", 121 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 122 | "userId": "15549406949129485910" 123 | }, 124 | "user_tz": -210 125 | }, 126 | "id": "vvckUvLr2Nuf", 127 | "outputId": "0ca61db4-4ed6-4ad6-87a7-6fd4142d7994" 128 | }, 129 | "outputs": [], 130 | "source": [ 131 | "model_F = load_model('path/model_covid_simple.h5')" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": { 138 | "colab": { 139 | "base_uri": "https://localhost:8080/", 140 | "height": 1000, 141 | "output_embedded_package_id": "1lYIdFR9zRJkPb__WbPYSis-H_kk4oxOO" 142 | }, 143 | "executionInfo": { 144 | "elapsed": 41396, 145 | "status": "ok", 146 | "timestamp": 1613511935512, 147 | "user": { 148 | "displayName": "Sina Qahremani", 149 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 150 | "userId": "15549406949129485910" 151 | }, 152 | "user_tz": -210 153 | }, 154 | "id": "AOaYObQZPDYg", 155 | "outputId": "28825e8f-f7fe-4f83-d242-7f5a8314338f" 156 | }, 157 | "outputs": [], 158 | "source": [ 159 | "cam_F = GradCAM(model_F, \"dropout_20\")\n", 160 | "name = 'Test'\n", 161 | "i = 1\n", 162 | "plot_GradCam(cam_F, model_F, x_test[i], np.argmax(y_test[i]), name, uncertainty=False, dataset='Xray')\n" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": { 168 | "id": "odhKjpHGcRF_" 169 | }, 170 | "source": [ 171 | "## Model: Fusion (With MC)" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "metadata": { 178 | "colab": { 179 | "base_uri": "https://localhost:8080/" 180 | }, 181 | "executionInfo": { 182 | "elapsed": 3621, 183 | "status": "ok", 184 | "timestamp": 1613506109887, 185 | "user": { 186 | "displayName": "Sina Qahremani", 187 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 188 | "userId": "15549406949129485910" 189 | }, 190 | "user_tz": -210 191 | }, 192 | "id": "O7vJ0dTTcWOy", 193 | "outputId": "5c5c7fd1-88ea-497d-ec8e-07509701a9c7" 194 | }, 195 | "outputs": [], 196 | "source": [ 197 | "model_F_mc=load_model('path/model_covid_mc.h5')" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": null, 203 | "metadata": { 204 | "colab": { 205 | "base_uri": "https://localhost:8080/", 206 | "height": 1000, 207 | "output_embedded_package_id": "1GGMJmjeenOqHrk4mIOfwx7d4tNwYO-mc" 208 | }, 209 | "executionInfo": { 210 | "elapsed": 2597756, 211 | "status": "ok", 212 | "timestamp": 1613515900790, 213 | "user": { 214 | "displayName": "Sina Qahremani", 215 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 216 | "userId": "15549406949129485910" 217 | }, 218 | "user_tz": -210 219 | }, 220 | "id": "q0GITPGJcY37", 221 | "outputId": "14af934f-1f34-4b7f-a626-a915b21187b9" 222 | }, 223 | "outputs": [], 224 | "source": [ 225 | "cam_F_mc = GradCAM(model_F_mc, \"dropout_61\")\n", 226 | "name = 'Test'\n", 227 | "i = 1\n", 228 | "plot_GradCam(cam_F_mc, model_F_mc, x_test[i], np.argmax(y_test[i]), name, uncertainty=True, dataset='Xray')\n" 229 | ] 230 | } 231 | ], 232 | "metadata": { 233 | "accelerator": "GPU", 234 | "colab": { 235 | "collapsed_sections": [], 236 | "name": "Xray_grad_cam.ipynb", 237 | "provenance": [] 238 | }, 239 | "kernelspec": { 240 | "display_name": "Python 3", 241 | "language": "python", 242 | "name": "python3" 243 | }, 244 | "language_info": { 245 | "codemirror_mode": { 246 | "name": "ipython", 247 | "version": 3 248 | }, 249 | "file_extension": ".py", 250 | "mimetype": "text/x-python", 251 | "name": "python", 252 | "nbconvert_exporter": "python", 253 | "pygments_lexer": "ipython3", 254 | "version": "3.8.5" 255 | } 256 | }, 257 | "nbformat": 4, 258 | "nbformat_minor": 1 259 | } -------------------------------------------------------------------------------- /Uncertainty Plot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "aulGDw7oRE28" 7 | }, 8 | "source": [ 9 | "# Covid Project\n", 10 | "# Uncertainty Plot" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "id": "lHFn2m3kRLXj" 17 | }, 18 | "source": [ 19 | "## Imports" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": { 26 | "executionInfo": { 27 | "elapsed": 2237, 28 | "status": "ok", 29 | "timestamp": 1615122907156, 30 | "user": { 31 | "displayName": "Sina Qahremani", 32 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhKpImYys1sQuY9zSkc3NW-sL3jIBU62aUGbslLoQ=s64", 33 | "userId": "15549406949129485910" 34 | }, 35 | "user_tz": -210 36 | }, 37 | "id": "8k-VAspORQAB" 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "%load_ext autoreload\n", 42 | "%autoreload 2\n", 43 | "import numpy as np\n", 44 | "from utils import load_covid_data, create_dataset_ct\n", 45 | "from plots import uncertainty_plot\n", 46 | "from tensorflow.keras.models import load_model" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "id": "vdcGQUT9Rg3q" 53 | }, 54 | "source": [ 55 | "## Dataset1 (X-Ray)" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": { 62 | "colab": { 63 | "base_uri": "https://localhost:8080/" 64 | }, 65 | "executionInfo": { 66 | "elapsed": 36676, 67 | "status": "ok", 68 | "timestamp": 1613834270204, 69 | "user": { 70 | "displayName": "Sina Qahramani", 71 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhxU3HQr7bqsOrjk8EX-dJumRRJJzjLFyYb_IGuiA=s64", 72 | "userId": "09975413313613087942" 73 | }, 74 | "user_tz": -210 75 | }, 76 | "id": "ozqgGQC9Rrj8", 77 | "outputId": "1c3b0490-8513-45b1-dfb8-6578c6b8d6b8" 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "x_test,y_test = load_covid_data(path='Data/test')" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": { 87 | "id": "qg1opzVga0UU" 88 | }, 89 | "source": [ 90 | "### Fusion" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": { 97 | "colab": { 98 | "base_uri": "https://localhost:8080/" 99 | }, 100 | "executionInfo": { 101 | "elapsed": 14692, 102 | "status": "ok", 103 | "timestamp": 1613834344991, 104 | "user": { 105 | "displayName": "Sina Qahramani", 106 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhxU3HQr7bqsOrjk8EX-dJumRRJJzjLFyYb_IGuiA=s64", 107 | "userId": "09975413313613087942" 108 | }, 109 | "user_tz": -210 110 | }, 111 | "id": "xAhT9AltazZt", 112 | "outputId": "3ad1e43e-cd9d-4ff9-97b0-60b40a90b36a" 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "model_F_mc = load_model('path/model_covid_mc.h5')" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": { 123 | "colab": { 124 | "base_uri": "https://localhost:8080/", 125 | "height": 494 126 | }, 127 | "executionInfo": { 128 | "elapsed": 44595, 129 | "status": "ok", 130 | "timestamp": 1613848071019, 131 | "user": { 132 | "displayName": "Sina Qahramani", 133 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhxU3HQr7bqsOrjk8EX-dJumRRJJzjLFyYb_IGuiA=s64", 134 | "userId": "09975413313613087942" 135 | }, 136 | "user_tz": -210 137 | }, 138 | "id": "eyx-EFLvjEpq", 139 | "outputId": "1bc36ede-feb3-42ca-8d26-1359ec6049ed" 140 | }, 141 | "outputs": [], 142 | "source": [ 143 | "name = 'Test'\n", 144 | "i=1\n", 145 | "uncertainty_plot(model_F_mc, x_test[i], y_test[i], save=True, name=name, mc_iter=1000, ylim=50)\n" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": { 151 | "id": "ozBmcTZS8p9b" 152 | }, 153 | "source": [ 154 | "## Dataset2 (CT)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": { 161 | "id": "oqUixThx9huw" 162 | }, 163 | "outputs": [], 164 | "source": [ 165 | "X = np.load('CT_X.npy')\n", 166 | "Y = np.load('CT_Y.npy')" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": { 173 | "id": "JD0UHv1C9izs" 174 | }, 175 | "outputs": [], 176 | "source": [ 177 | "BATCH_SIZE=64\n", 178 | "_, _, _, x_test, _, y_test = create_dataset_ct(X, Y, BATCH_SIZE)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": { 184 | "id": "9r66n4mj9He4" 185 | }, 186 | "source": [ 187 | "### Fusion" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": { 194 | "colab": { 195 | "base_uri": "https://localhost:8080/" 196 | }, 197 | "executionInfo": { 198 | "elapsed": 10728, 199 | "status": "ok", 200 | "timestamp": 1613850752821, 201 | "user": { 202 | "displayName": "Sina Qahramani", 203 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhxU3HQr7bqsOrjk8EX-dJumRRJJzjLFyYb_IGuiA=s64", 204 | "userId": "09975413313613087942" 205 | }, 206 | "user_tz": -210 207 | }, 208 | "id": "A9K55Y9h9vPJ", 209 | "outputId": "c2e003cc-056b-46a8-834e-4f40f04cea92" 210 | }, 211 | "outputs": [], 212 | "source": [ 213 | "model_F_mc_CT = load_model('path/model_covid_mc_CT.h5')" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": { 220 | "colab": { 221 | "base_uri": "https://localhost:8080/", 222 | "height": 1000 223 | }, 224 | "executionInfo": { 225 | "elapsed": 1838161, 226 | "status": "ok", 227 | "timestamp": 1613868814326, 228 | "user": { 229 | "displayName": "Sina Qahramani", 230 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhxU3HQr7bqsOrjk8EX-dJumRRJJzjLFyYb_IGuiA=s64", 231 | "userId": "09975413313613087942" 232 | }, 233 | "user_tz": -210 234 | }, 235 | "id": "LcwpRqr1_f61", 236 | "outputId": "c8665dd3-25c3-45ba-e560-79711bd8f496" 237 | }, 238 | "outputs": [], 239 | "source": [ 240 | "name = 'Test'\n", 241 | "i=1\n", 242 | "uncertainty_plot(model_F_mc_CT, x_test[i], y_test[i], save=True, name=name, mc_iter=1000, dataset='CT', ylim=25)\n" 243 | ] 244 | } 245 | ], 246 | "metadata": { 247 | "accelerator": "GPU", 248 | "colab": { 249 | "collapsed_sections": [], 250 | "name": "Uncertainty_Plot.ipynb", 251 | "provenance": [] 252 | }, 253 | "kernelspec": { 254 | "display_name": "Python 3", 255 | "language": "python", 256 | "name": "python3" 257 | }, 258 | "language_info": { 259 | "codemirror_mode": { 260 | "name": "ipython", 261 | "version": 3 262 | }, 263 | "file_extension": ".py", 264 | "mimetype": "text/x-python", 265 | "name": "python", 266 | "nbconvert_exporter": "python", 267 | "pygments_lexer": "ipython3", 268 | "version": "3.8.5" 269 | } 270 | }, 271 | "nbformat": 4, 272 | "nbformat_minor": 1 273 | } -------------------------------------------------------------------------------- /plots.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | from matplotlib.pyplot import figure 5 | from scipy import interp 6 | from itertools import cycle 7 | from sklearn.metrics import roc_curve, auc 8 | from sklearn.metrics import confusion_matrix 9 | 10 | matplotlib.rc('xtick', labelsize=22) 11 | matplotlib.rc('ytick', labelsize=22) 12 | plt.rcParams.update({'font.size': 22}) 13 | 14 | 15 | def plot_roc_handy(y_test, y_score, lw=2, name='Roc', class_name=None, zoom=False, 16 | axis=None): 17 | if axis is None: 18 | axis = [0.0, 0.12, 0.88, 1.0] 19 | if class_name is None: 20 | class_name = ['COVID19', 'Normal', 'Pneumonia'] 21 | fpr = dict() 22 | tpr = dict() 23 | roc_auc = dict() 24 | 25 | for i in range(int(y_test.shape[1])): 26 | fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) 27 | roc_auc[i] = auc(fpr[i], tpr[i]) 28 | fpr["micro"], tpr["micro"], _ = roc_curve(y_test.ravel(), y_score.ravel()) 29 | roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) 30 | all_fpr = np.unique(np.concatenate([fpr[i] for i in range(int(y_test.shape[1]))])) 31 | mean_tpr = np.zeros_like(all_fpr) 32 | for i in range(int(y_test.shape[1])): 33 | mean_tpr += interp(all_fpr, fpr[i], tpr[i]) 34 | mean_tpr /= int(y_test.shape[1]) 35 | fpr["macro"] = all_fpr 36 | tpr["macro"] = mean_tpr 37 | roc_auc["macro"] = auc(fpr["macro"], tpr["macro"]) 38 | 39 | f, ax = plt.subplots(figsize=[15, 15]) 40 | 41 | ax.plot(fpr["micro"], tpr["micro"], 42 | label='micro-average ROC curve (area = {0:0.2f})' 43 | ''.format(roc_auc["micro"]), 44 | color='deeppink', linestyle=':', linewidth=4) 45 | ax.plot(fpr["macro"], tpr["macro"], 46 | label='macro-average ROC curve (area = {0:0.2f})' 47 | ''.format(roc_auc["macro"]), 48 | color='navy', linestyle=':', linewidth=4) 49 | colors = cycle(['aqua', 'darkorange', 'cornflowerblue']) 50 | for i, color in zip(range(int(y_test.shape[1])), colors): 51 | ax.plot(fpr[i], tpr[i], color=color, lw=lw, 52 | label='ROC curve of class {0} (area = {1:0.2f})' 53 | ''.format(class_name[i], roc_auc[i])) 54 | ax.plot([0, 1], [0, 1], 'k--', lw=lw) 55 | ax.set_xlim([0.001, 1.0]) 56 | ax.set_ylim([0, 1.05]) 57 | ax.set_xlabel('False Positive Rate', fontsize=22) 58 | ax.set_ylabel('True Positive Rate', fontsize=22) 59 | ax.set_title(name, fontsize=22) 60 | ax.legend(loc="lower right") 61 | 62 | # inset axes.... 63 | if zoom: 64 | axins = ax.inset_axes([0.3, 0.4, 0.4, 0.4]) 65 | for i, color in zip(range(int(y_test.shape[1])), colors): 66 | if fpr[i].all() < 0.2 and tpr[i].all() < 0.95: 67 | axins.plot(fpr[i], tpr[i], color=color, lw=lw, 68 | label='ROC curve of class {0} (area = {1:0.2f})' 69 | ''.format(class_name[i], roc_auc[i])) 70 | 71 | axins.plot(fpr["micro"], tpr["micro"], 72 | label='micro-average ROC curve (area = {0:0.2f})' 73 | ''.format(roc_auc["micro"]), 74 | color='deeppink', linestyle=':', linewidth=4) 75 | 76 | axins.plot(fpr["macro"], tpr["macro"], 77 | label='macro-average ROC curve (area = {0:0.2f})' 78 | ''.format(roc_auc["macro"]), 79 | color='navy', linestyle=':', linewidth=4) 80 | 81 | x1, x2, y1, y2 = axis 82 | axins.set_xlim(x1, x2) 83 | axins.set_ylim(y1, y2) 84 | axins.set_xticklabels('') 85 | axins.set_yticklabels('') 86 | ax.indicate_inset_zoom(axins) 87 | 88 | plt.show() 89 | f.savefig('{}.pdf'.format(name)) 90 | ax.figure.savefig("{}.pdf".format(name), bbox_inches='tight') 91 | 92 | 93 | def plot_cm_handy(y_test, y_score, lw=2, name='Confusion Matrix of Fusion Model without Uncertainty (X-Ray)', 94 | class_name=None): 95 | 96 | if class_name is None: 97 | class_name = ['COVID19', 'Normal', 'Pneumonia'] 98 | CM = confusion_matrix(np.argmax(y_test, axis=1), np.argmax(y_score, axis=1)) 99 | cm = CM 100 | cmap = plt.cm.Blues 101 | fig, ax = plt.subplots(figsize=(6, 6), dpi=80, facecolor='w', edgecolor='k') 102 | im = ax.imshow(cm, interpolation='nearest', cmap=cmap) 103 | ax.set_title(name) 104 | ax.set(xticks=np.arange(cm.shape[1]), 105 | yticks=np.arange(cm.shape[0]), 106 | xticklabels=class_name, yticklabels=class_name, 107 | ylabel='True Label', 108 | xlabel='Predicted Label') 109 | ax.set_xticklabels(class_name, fontsize=15) 110 | ax.set_yticklabels(class_name, fontsize=15) 111 | ax.set_ylabel('True Label', fontsize=15) 112 | ax.set_xlabel('Predicted Label', fontsize=15) 113 | 114 | plt.setp(ax.get_xticklabels(), rotation=45, ha="right", 115 | rotation_mode="anchor") 116 | fmt = '.1f' 117 | thresh = cm.max() / 2. 118 | for i in range(cm.shape[0]): 119 | for j in range(cm.shape[1]): 120 | figure(num=None, figsize=(5, 5), dpi=80, facecolor='w', edgecolor='k') 121 | ax.text(j, i, format(cm[i, j], fmt), fontsize=12, 122 | ha="center", va="center", 123 | color="white" if cm[i, j] > thresh else "black") 124 | 125 | fig.tight_layout() 126 | fig.savefig('{}.pdf'.format(name), dpi=300) 127 | ax.figure.savefig("{}.pdf".format(name), bbox_inches='tight') 128 | 129 | 130 | def uncertainty_plot(model, image, label=None, save=True, name=None, mc_iter=200, dataset='Xray', ylim=20): 131 | image = image[np.newaxis, :, :, :] 132 | if label is not None: 133 | label_idx = np.argmax(label) 134 | 135 | if dataset == 'Xray': 136 | class_text = ['Covid19', 'Normal', 'Pneumonia'] 137 | elif dataset == 'CT': 138 | class_text = ['nCT', 'NiCT', 'pCT'] 139 | 140 | all_preds = [] 141 | for _ in range(mc_iter): 142 | preds = model.predict(image) 143 | all_preds.append(preds) 144 | 145 | preds_mean = np.mean(all_preds, axis=0) 146 | preds_mean = preds_mean[0] 147 | preds_std = np.std(all_preds, axis=0) 148 | preds_std = preds_std[0] 149 | class_idx_mean = np.argmax(preds_mean) 150 | 151 | preds = np.transpose(np.stack(all_preds), (1, 0, 2)) 152 | 153 | image = (image + 1) / 2 154 | image1 = (image * 255).astype("uint8") 155 | image1 = np.squeeze(image1, axis=0) 156 | image1 = np.squeeze(image1, axis=2) 157 | 158 | fig = plt.figure(figsize=[8, 6.5]) 159 | 160 | plt.subplot(2, 1, 1) 161 | plt.imshow(image1, cmap='gray') 162 | if label is not None: 163 | title = """ 164 | True: {} 165 | Prediction: {} 166 | Value Order: [{}, {}, {}] 167 | Mean: [{:0.2f}, {:0.2f}, {:0.2f}] 168 | STD: [{:0.2f}, {:0.2f}, {:0.2f}] 169 | """.format(class_text[label_idx], class_text[class_idx_mean], *class_text, 170 | *preds_mean, *preds_std) 171 | else: 172 | title = """ 173 | Prediction: {} 174 | Value Order: [{}, {}, {}] 175 | Mean: [{:0.2f}, {:0.2f}, {:0.2f}] 176 | STD: [{:0.2f}, {:0.2f}, {:0.2f}] 177 | """.format(class_text[class_idx_mean], *class_text, 178 | *preds_mean, *preds_std) 179 | 180 | plt.title(title, fontsize=12) 181 | 182 | plt.axis('off') 183 | 184 | for i in range(3): 185 | plt.subplot(2, 1, 2) 186 | plt.hist(preds[0][:, i], bins=50, alpha=0.3, label=class_text[i]) 187 | 188 | plt.axvline(np.quantile(preds[0][:, i], 0.5), color='red', linestyle='--', alpha=0.4) 189 | plt.axvline(0.5, color='green', linestyle='--') 190 | 191 | plt.xlabel('Uncertainty', fontsize=12) 192 | plt.ylabel('Density', fontsize=12) 193 | plt.ylim([0, ylim]) 194 | plt.legend() 195 | plt.tight_layout() 196 | if save: 197 | fig.savefig('{}.pdf'.format(name), dpi=300, bbox_inches='tight') 198 | plt.show() 199 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.models import Model 3 | from tensorflow.keras.layers import Input, Dense, Flatten, Dropout, BatchNormalization, Concatenate 4 | from tensorflow.keras.layers import Conv2D, SeparableConv2D, MaxPool2D 5 | from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping 6 | from abc import abstractmethod 7 | 8 | 9 | class ImageClassifierBase: 10 | 11 | def __init__(self, input_shape, lr, mc=True, metrics=True, trunc=False, trained_model=None, model_name="test"): 12 | self.input_shape = input_shape 13 | self.lr = lr 14 | self.mc = mc 15 | self.metrics = metrics 16 | self.trunc = trunc 17 | self.trained_model = trained_model 18 | self.model_name = model_name + "_with_mc" if self.mc else model_name + "_without_mc" 19 | 20 | def get_model(self): 21 | inputs = Input(shape=self.input_shape) 22 | feature_extraction_output = self._feature_extraction(inputs) 23 | output = self._fusion_layer(*feature_extraction_output) 24 | output = self._classifier(output) 25 | 26 | model = Model(inputs=inputs, outputs=output) 27 | 28 | if self.trained_model: 29 | for i, layer in enumerate(model.layers): 30 | layer.set_weights(self.trained_model.layers[i].get_weights()) 31 | 32 | callbacks = None if self.trunc else self._get_callbacks() 33 | 34 | return model, callbacks 35 | 36 | def _compile_model(self, model): 37 | adam = tf.keras.optimizers.Adam(lr=self.lr) 38 | 39 | compile_dict = { 40 | "optimizer": adam, 41 | "loss": "categorical_crossentropy" 42 | } 43 | 44 | if self.metrics: 45 | compile_dict["metrics"] = ['accuracy', self._get_metrics()] 46 | 47 | model.compile(**compile_dict) 48 | 49 | return model 50 | 51 | def _get_callbacks(self): 52 | model_checkpoint = ModelCheckpoint(f"{self.model_name}.h5", monitor='val_accuracy', mode='max', verbose=1, 53 | save_best_only=True) 54 | 55 | reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.8, verbose=1, patience=5) 56 | es = EarlyStopping(monitor='val_accuracy', mode='max', verbose=0, patience=30) 57 | return [reduce_lr, es, model_checkpoint] 58 | 59 | @abstractmethod 60 | def _feature_extraction(self, inputs): 61 | pass 62 | 63 | def _fusion_layer(self, *args): 64 | flattened = [Flatten(layer) for layer in args] 65 | concatenated_tensor = Concatenate(axis=1)(flattened) 66 | return concatenated_tensor 67 | 68 | @abstractmethod 69 | def _classifier(self, concatenated_features): 70 | pass 71 | 72 | def _get_metrics(self): 73 | return [ 74 | tf.keras.metrics.Precision(name='precision'), 75 | tf.keras.metrics.Recall(name='recall'), 76 | tf.keras.metrics.AUC(name='auc')] 77 | 78 | def _get_dropout(self, input_tensor, rate): 79 | if self.mc: 80 | return Dropout(rate=rate)(input_tensor, training=True) 81 | else: 82 | return Dropout(rate=rate)(input_tensor) 83 | 84 | 85 | # Our Proposed Fusion Model: 86 | class FusionModel(ImageClassifierBase): 87 | 88 | def __init__(self, input_shape=(150, 150, 1), lr=0.00005, mc=True, metrics=True, trunc=False, trained_model=None, model_name="test"): 89 | super().__init__(input_shape, lr, mc, metrics, trunc, trained_model, model_name) 90 | 91 | def _feature_extraction(self, inputs): 92 | input2 = tf.stack([inputs, inputs, inputs], axis=3)[:, :, :, :, 0] 93 | vgg_model = tf.keras.applications.VGG16(weights='imagenet', 94 | include_top=False, 95 | input_shape=(self.input_shape[0], self.input_shape[1], 3)) 96 | vgg_model.trainable = False 97 | 98 | vgg_feature = vgg_model(input2) 99 | # First conv block 100 | conv1 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(inputs) 101 | conv1 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(conv1) 102 | conv1 = MaxPool2D(pool_size=(2, 2))(conv1) 103 | 104 | # Second conv block 105 | conv2 = SeparableConv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same')(conv1) 106 | conv2 = SeparableConv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same')(conv2) 107 | conv2 = BatchNormalization()(conv2) 108 | conv2 = MaxPool2D(pool_size=(2, 2))(conv2) 109 | 110 | # Third conv block 111 | conv3 = SeparableConv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same')(conv2) 112 | conv3 = SeparableConv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same')(conv3) 113 | conv3 = BatchNormalization()(conv3) 114 | conv3 = MaxPool2D(pool_size=(2, 2))(conv3) 115 | 116 | # Fourth conv block 117 | conv4 = SeparableConv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same')(conv3) 118 | conv4 = SeparableConv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same', name='target_layer')( 119 | conv4) 120 | conv4 = BatchNormalization()(conv4) 121 | conv4 = MaxPool2D(pool_size=(2, 2))(conv4) 122 | conv4 = self._get_dropout(conv4, rate=0.2) 123 | 124 | # Fifth conv block 125 | conv5 = SeparableConv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same')(conv4) 126 | conv5 = SeparableConv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same')(conv5) 127 | conv5 = BatchNormalization()(conv5) 128 | conv5 = MaxPool2D(pool_size=(2, 2))(conv5) 129 | conv4 = self._get_dropout(conv4, rate=0.2) 130 | 131 | output_list = [conv3, conv4, conv5, vgg_feature] 132 | 133 | return output_list 134 | 135 | def _classifier(self, concatenated_features): 136 | x = Flatten()(concatenated_features) 137 | x = Dense(units=512, activation='relu')(x) 138 | 139 | if not self.trunc: 140 | 141 | x = self._get_dropout(x, rate=0.7) 142 | x = Dense(units=128, activation='relu')(x) 143 | x = self._get_dropout(x, rate=0.5) 144 | x = Dense(units=64, activation='relu')(x) 145 | x = self._get_dropout(x, rate=0.3) 146 | x = Dense(3, activation='softmax')(x) 147 | 148 | return x 149 | 150 | 151 | # Simple CNN Model: 152 | class SimpleCNNModel(ImageClassifierBase): 153 | 154 | def __init__(self, input_shape, lr, mc=True, metrics=True, trunc=False, trained_model=None, model_name="test"): 155 | super().__init__(input_shape, lr, mc, metrics, trunc, trained_model, model_name) 156 | 157 | def _feature_extraction(self, inputs): 158 | conv1 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(inputs) 159 | 160 | conv2 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(conv1) 161 | conv2 = MaxPool2D(pool_size=(2, 2))(conv2) 162 | 163 | conv3 = Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same')(conv2) 164 | 165 | conv4 = Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same')(conv3) 166 | conv4 = BatchNormalization()(conv4) 167 | conv4 = MaxPool2D(pool_size=(2, 2))(conv4) 168 | 169 | conv5 = Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same')(conv4) 170 | conv5 = BatchNormalization()(conv5) 171 | conv5 = self._get_dropout(conv5, rate=0.2) 172 | 173 | return [conv5] 174 | 175 | def _classifier(self, concatenated_features): 176 | x = Flatten()(concatenated_features) 177 | 178 | x = Dense(units=128, activation='relu')(x) 179 | 180 | if not self.trunc: 181 | x = self._get_dropout(x, rate=0.7) 182 | x = Dense(units=64, activation='relu')(x) 183 | x = self._get_dropout(x, rate=0.5) 184 | 185 | x = Dense(3, activation='softmax')(x) 186 | 187 | return x 188 | 189 | 190 | # Multi-headed Model: 191 | class MultiHeadedModel(ImageClassifierBase): 192 | 193 | def __init__(self, input_shape, lr, mc=True, metrics=True, trunc=False, trained_model=None, model_name="test"): 194 | super().__init__(input_shape, lr, mc, metrics, trunc, trained_model, model_name) 195 | 196 | def _feature_extraction(self, inputs): 197 | conv1 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(inputs) 198 | conv1 = Conv2D(filters=8, kernel_size=(3, 3), activation='relu', padding='same')(conv1) 199 | conv1 = BatchNormalization()(conv1) 200 | conv1 = MaxPool2D(pool_size=(2, 2))(conv1) 201 | conv1 = self._get_dropout(conv1, rate=0.2) 202 | 203 | conv2 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(inputs) 204 | conv2 = Conv2D(filters=8, kernel_size=(3, 3), activation='relu', padding='same')(conv2) 205 | conv2 = BatchNormalization()(conv2) 206 | conv2 = MaxPool2D(pool_size=(2, 2))(conv2) 207 | conv2 = self._get_dropout(conv2, rate=0.2) 208 | 209 | conv3 = Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same')(inputs) 210 | conv3 = Conv2D(filters=8, kernel_size=(3, 3), activation='relu', padding='same')(conv3) 211 | conv3 = BatchNormalization()(conv3) 212 | conv3 = MaxPool2D(pool_size=(2, 2))(conv3) 213 | conv3 = self._get_dropout(conv3, rate=0.2) 214 | 215 | output_list = [conv1, conv2, conv3] 216 | 217 | return output_list 218 | 219 | def _classifier(self, concatenated_features): 220 | x = Flatten()(concatenated_features) 221 | x = Dense(units=128, activation='relu')(x) 222 | 223 | if not self.trunc: 224 | x = self._get_dropout(x, rate=0.7) 225 | x = Dense(units=64, activation='relu')(x) 226 | x = self._get_dropout(x, rate=0.5) 227 | x = Dense(3, activation='softmax')(x) 228 | 229 | return x 230 | -------------------------------------------------------------------------------- /Fusion Model Results.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "8jHhXKOHNLbW", 7 | "pycharm": { 8 | "name": "#%%\n" 9 | } 10 | }, 11 | "source": [ 12 | "# Covid Chest Project" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": { 18 | "pycharm": { 19 | "name": "#%% md\n" 20 | }, 21 | "id": "6o5W2GAFyLq1" 22 | }, 23 | "source": [ 24 | "## Setup" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "id": "3--Z_VKgqJR4" 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import numpy as np\n", 36 | "from utils import load_covid_data, create_dataset_xray, create_dataset_ct\n", 37 | "from models import FusionModel\n", 38 | "from plots import plot_cm_handy, plot_roc_handy\n", 39 | "from evaluation import mode_robustness\n", 40 | "from sklearn.utils.class_weight import compute_class_weight\n", 41 | "from sklearn.metrics import accuracy_score, confusion_matrix,precision_score,recall_score,f1_score\n", 42 | "\n", 43 | "import warnings\n", 44 | "warnings.filterwarnings('ignore')" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "id": "YeZaGYcGRL8P", 51 | "pycharm": { 52 | "name": "#%% md\n" 53 | } 54 | }, 55 | "source": [ 56 | "## Dataset1 (X-Ray)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": { 63 | "id": "3f-gGg9hwscj" 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "x_train, y_train = load_covid_data(path='Data/train', shuffle=True, class_frequency=True)\n", 68 | "x_test,y_test = load_covid_data(path='Data/test')\n", 69 | "\n", 70 | "class_weights = compute_class_weight('balanced',np.unique(np.argmax(y_train,axis=1)), np.argmax(y_train,axis=1))\n", 71 | "class_weights = {0:class_weights[0],\n", 72 | " 1:class_weights[1],\n", 73 | " 2:class_weights[2]}\n", 74 | "BATCH_SIZE=256\n", 75 | "train_dataset, validation_dataset = create_dataset_xray(x_train, y_train, x_test, y_test, BATCH_SIZE)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "source": [ 81 | "## Dataset2 (CT)" 82 | ], 83 | "metadata": { 84 | "id": "3Nxi0CdmeQjv" 85 | } 86 | }, 87 | { 88 | "cell_type": "code", 89 | "source": [ 90 | "X = np.load('CT_X.npy')\n", 91 | "Y = np.load('CT_Y.npy')" 92 | ], 93 | "metadata": { 94 | "id": "bB_olTEZeUJ4" 95 | }, 96 | "execution_count": null, 97 | "outputs": [] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "source": [ 102 | "BATCH_SIZE=256\n", 103 | "train_dataset, validation_dataset, X_train, X_test, y_train, y_test = create_dataset_ct(X, Y, BATCH_SIZE)" 104 | ], 105 | "metadata": { 106 | "id": "Zy3M5-_DeVqA" 107 | }, 108 | "execution_count": null, 109 | "outputs": [] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": { 114 | "id": "GbVHdqnURj6j" 115 | }, 116 | "source": [ 117 | "## Fusion Model Without Monte-Carlo Dropout" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": { 124 | "id": "FDt1r6LPwshN" 125 | }, 126 | "outputs": [], 127 | "source": [ 128 | "fusion_model = FusionModel(mc=False, model_name=\"fusion_model\")\n", 129 | "model, callbacks = fusion_model.get_model()" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "source": [ 135 | "hist_mc = model.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n", 136 | " class_weight=class_weights, callbacks=callbacks)" 137 | ], 138 | "metadata": { 139 | "id": "FAUvwn7csyVX" 140 | }, 141 | "execution_count": null, 142 | "outputs": [] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "source": [ 147 | "## Results" 148 | ], 149 | "metadata": { 150 | "id": "5OaWSe7Ee6Zx" 151 | } 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "source": [ 156 | "In case you want to load model:" 157 | ], 158 | "metadata": { 159 | "id": "3kl5BNFFfIG5" 160 | } 161 | }, 162 | { 163 | "cell_type": "code", 164 | "source": [ 165 | "from tensorflow.keras.models import load_model\n", 166 | "\n", 167 | "model = load_model('path/fusion_model_without_mc.h5')\n" 168 | ], 169 | "metadata": { 170 | "id": "UXTB2r9JfLXW" 171 | }, 172 | "execution_count": null, 173 | "outputs": [] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "source": [ 178 | "\n", 179 | "preds = model.predict(x_test)\n", 180 | "acc = accuracy_score(np.argmax(y_test, axis=1), np.argmax(preds, axis=1))*100\n", 181 | "\n", 182 | "cm = confusion_matrix(np.argmax(y_test, axis=1)\n", 183 | ", np.argmax(preds, axis=1))\n", 184 | "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", 185 | "\n", 186 | "print('CONFUSION MATRIX ------------------')\n", 187 | "print(cm)\n", 188 | "\n", 189 | "print('\\nTEST METRICS ----------------------')\n", 190 | "\n", 191 | "precision = precision_score(np.argmax(y_test, axis=1),\n", 192 | " np.argmax(preds, axis=1), average='weighted')*100\n", 193 | "recall = recall_score(np.argmax(y_test, axis=1),\n", 194 | " np.argmax(preds, axis=1), average='weighted')*100\n", 195 | "\n", 196 | "print('Accuracy: {}%'.format(acc))\n", 197 | "print('Precision: {}%'.format(precision))\n", 198 | "print('Recall: {}%'.format(recall))\n", 199 | "print('F1-score: {}'.format( 2*precision*recall/(precision+recall) ))" 200 | ], 201 | "metadata": { 202 | "id": "PBleNY-je99u" 203 | }, 204 | "execution_count": null, 205 | "outputs": [] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "source": [ 210 | "y_p = model.predict(x_test, batch_size=BATCH_SIZE)\n", 211 | "\n", 212 | "\n", 213 | "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Fusion model without uncertainty',\n", 214 | " class_name=['COVID19','Normal','Pneumonia'])\n", 215 | "\n", 216 | "plot_cm_handy(y_test, y_p,\n", 217 | " lw=2, name='Confusion Matrix of Fusion model without uncertainty',\n", 218 | " class_name=['COVID19','Normal','Pneumonia'])" 219 | ], 220 | "metadata": { 221 | "id": "_Wq-BKQzfcer" 222 | }, 223 | "execution_count": null, 224 | "outputs": [] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "source": [ 229 | "### T-SNE" 230 | ], 231 | "metadata": { 232 | "id": "_znAZQUUfnMV" 233 | } 234 | }, 235 | { 236 | "cell_type": "code", 237 | "source": [ 238 | "trunc_model_object = FusionModel(mc=False, trunc=True,trained_model=model, model_name=\"fusion_model\")\n", 239 | "trunc_model, _ = trunc_model_object.get_model()\n", 240 | "hidden_features = trunc_model.predict(x_test)" 241 | ], 242 | "metadata": { 243 | "id": "B1gDNontfpJW" 244 | }, 245 | "execution_count": null, 246 | "outputs": [] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "source": [ 251 | "from sklearn.decomposition import PCA\n", 252 | "from sklearn.manifold import TSNE\n", 253 | "\n", 254 | "pca = PCA(n_components=120)\n", 255 | "pca_result = pca.fit_transform(hidden_features)\n", 256 | "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n", 257 | "\n", 258 | "tsne = TSNE(n_components=2, verbose = 1)\n", 259 | "tsne_results = tsne.fit_transform(pca_result)" 260 | ], 261 | "metadata": { 262 | "id": "oyntg2rDshCg" 263 | }, 264 | "execution_count": null, 265 | "outputs": [] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "source": [ 270 | "import matplotlib.pyplot as plt\n", 271 | "import matplotlib \n", 272 | "matplotlib.rc('xtick', labelsize=20) \n", 273 | "matplotlib.rc('ytick', labelsize=20) \n", 274 | "\n", 275 | "plt.rcParams.update({'font.size': 25})\n", 276 | "%matplotlib inline\n", 277 | "Name='T-SNE Visualization of Fusion model without uncertainty'\n", 278 | "fig = plt.figure(figsize=[15, 15])\n", 279 | "color_map = np.argmax(y_test, axis=1)\n", 280 | "classes=['COVID19','Normal','Pneumonia']\n", 281 | "for cl in range(3):\n", 282 | " indices = np.where(color_map==cl)\n", 283 | " indices = indices[0]\n", 284 | " plt.title(Name, fontsize=20)\n", 285 | " plt.ylabel('Dim_2', fontsize=20)\n", 286 | " plt.xlabel('Dim_1', fontsize=20)\n", 287 | " matplotlib.rc('xtick', labelsize=20) \n", 288 | " matplotlib.rc('ytick', labelsize=20) \n", 289 | " plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n", 290 | "\n", 291 | "plt.rcParams.update({'font.size': 20})\n", 292 | "\n", 293 | "plt.legend()\n", 294 | "plt.show()\n", 295 | "fig.savefig('{}.pdf'.format(Name),dpi=300)\n", 296 | "\n" 297 | ], 298 | "metadata": { 299 | "id": "VbIRwuywsiAO" 300 | }, 301 | "execution_count": null, 302 | "outputs": [] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "source": [ 307 | "## Fusion Model With Monte-Carlo Dropout" 308 | ], 309 | "metadata": { 310 | "id": "tMkQyYiftOto" 311 | } 312 | }, 313 | { 314 | "cell_type": "code", 315 | "source": [ 316 | "fusion_model = FusionModel(mc=True, lr=0.00001, model_name=\"fusion_model\")\n", 317 | "mc_model, mc_callbacks = fusion_model.get_model()\n" 318 | ], 319 | "metadata": { 320 | "id": "1KsaXrGVtQy_" 321 | }, 322 | "execution_count": null, 323 | "outputs": [] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "source": [ 328 | "mc_hist = mc_model.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n", 329 | " class_weight=class_weights, callbacks=mc_callbacks)" 330 | ], 331 | "metadata": { 332 | "id": "A4BK1MVktVUY" 333 | }, 334 | "execution_count": null, 335 | "outputs": [] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "source": [ 340 | "## Results" 341 | ], 342 | "metadata": { 343 | "collapsed": false 344 | } 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "source": [ 349 | "In case you want to load model:" 350 | ], 351 | "metadata": { 352 | "collapsed": false 353 | } 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "outputs": [], 359 | "source": [ 360 | "from tensorflow.keras.models import load_model\n", 361 | "\n", 362 | "mc_model = load_model('path/fusion_model_with_mc.h5')\n" 363 | ], 364 | "metadata": { 365 | "collapsed": false, 366 | "pycharm": { 367 | "name": "#%%\n" 368 | } 369 | } 370 | }, 371 | { 372 | "cell_type": "code", 373 | "execution_count": null, 374 | "outputs": [], 375 | "source": [ 376 | "import tqdm\n", 377 | "\n", 378 | "number_prediction=200\n", 379 | "mc_predictions = []\n", 380 | "for i in tqdm.tqdm(range(number_prediction)):\n", 381 | " y_p = mc_model.predict(x_test)\n", 382 | " mc_predictions.append(y_p)\n", 383 | "\n", 384 | "accs=recalls=precisions=F1s=[]\n", 385 | "for y_p in mc_predictions:\n", 386 | " acc = accuracy_score(y_test.argmax(axis=1), y_p.argmax(axis=1))\n", 387 | " recall=recall_score(y_test.argmax(axis=1), y_p.argmax(axis=1),average='weighted')\n", 388 | " precision=precision_score(y_test.argmax(axis=1), y_p.argmax(axis=1),average='weighted')\n", 389 | " F1=(2*precision*recall)/(precision+recall)\n", 390 | " accs.append(acc)\n", 391 | " recalls.append(recall)\n", 392 | " precisions.append(precision)\n", 393 | " F1s.append(F1)\n", 394 | "\n", 395 | "\n", 396 | "print(\"MC accuracy: {:.5%}\".format(sum(accs)/len(accs)))\n", 397 | "print(\"MC precision: {:.5%}\".format(sum(precisions)/len(precisions)))\n", 398 | "print(\"MC recall: {:.5%}\".format(sum(recalls)/len(recalls)))\n", 399 | "print(\"MC F1: {:.5%}\".format(sum(F1s)/len(F1s)))\n", 400 | "\n", 401 | "mc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1)\n", 402 | "ensemble_acc = accuracy_score(y_test.argmax(axis=1), mc_ensemble_pred)\n", 403 | "ensemble_precision=precision_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n", 404 | "ensemble_recall=recall_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n", 405 | "ensemble_F1=(2*ensemble_precision*ensemble_recall)/(ensemble_precision+ensemble_recall)\n", 406 | "\n", 407 | "print(\"MC-ensemble accuracy: {:.5%}\".format(ensemble_acc))\n", 408 | "print(\"MC-ensemble precision: {:.5%}\".format(ensemble_precision))\n", 409 | "print(\"MC-ensemble recall: {:.5%}\".format(ensemble_recall))\n", 410 | "print(\"MC-ensemble F1: {:.5%}\".format(ensemble_F1))\n" 411 | ], 412 | "metadata": { 413 | "collapsed": false, 414 | "pycharm": { 415 | "name": "#%%\n" 416 | } 417 | } 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": null, 422 | "outputs": [], 423 | "source": [ 424 | "Name = 'Histogram of ّFusion model with uncertainty'\n", 425 | "f, ax = plt.subplots(figsize=[10, 7])\n", 426 | "plt.hist(accs)\n", 427 | "plt.axvline(x=ensemble_acc, color=\"r\")\n", 428 | "ax.set_title(Name,fontsize=19)\n", 429 | "plt.show()\n", 430 | "f.savefig('{}.pdf'.format(Name))\n", 431 | "ax.figure.savefig(\"{}.pdf\".format(Name), bbox_inches='tight')" 432 | ], 433 | "metadata": { 434 | "collapsed": false, 435 | "pycharm": { 436 | "name": "#%%\n" 437 | } 438 | } 439 | }, 440 | { 441 | "cell_type": "code", 442 | "execution_count": null, 443 | "outputs": [], 444 | "source": [ 445 | "\n", 446 | "plot_roc_handy(y_test, mc_ensemble_pred, zoom=True, lw=2, name='Roc of Fusion model with uncertainty',\n", 447 | " class_name=['COVID19','Normal','Pneumonia'])\n", 448 | "\n", 449 | "plot_cm_handy(y_test, mc_ensemble_pred,\n", 450 | " lw=2, name='Confusion Matrix of Fusion model with uncertainty',\n", 451 | " class_name=['COVID19','Normal','Pneumonia'])" 452 | ], 453 | "metadata": { 454 | "collapsed": false, 455 | "pycharm": { 456 | "name": "#%%\n" 457 | } 458 | } 459 | }, 460 | { 461 | "cell_type": "markdown", 462 | "source": [ 463 | "#### Noise Robustness" 464 | ], 465 | "metadata": { 466 | "collapsed": false 467 | } 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": null, 472 | "outputs": [], 473 | "source": [ 474 | "std_coef=[1e-4,1e-3,1e-2,1e-1, 0.2,0.3,0.4,0.5,0.6]" 475 | ], 476 | "metadata": { 477 | "collapsed": false, 478 | "pycharm": { 479 | "name": "#%%\n" 480 | } 481 | } 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": null, 486 | "outputs": [], 487 | "source": [ 488 | "mode_robustness(x_test, y_test, mc_model, std_coef)" 489 | ], 490 | "metadata": { 491 | "collapsed": false, 492 | "pycharm": { 493 | "name": "#%%\n" 494 | } 495 | } 496 | }, 497 | { 498 | "cell_type": "markdown", 499 | "source": [ 500 | "### T-SNE" 501 | ], 502 | "metadata": { 503 | "collapsed": false 504 | } 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": null, 509 | "outputs": [], 510 | "source": [ 511 | "trunc_model_object = FusionModel(mc=True, trunc=True, trained_model=mc_model, model_name=\"fusion_model\")\n", 512 | "trunc_model, _ = trunc_model_object.get_model()" 513 | ], 514 | "metadata": { 515 | "collapsed": false, 516 | "pycharm": { 517 | "name": "#%%\n" 518 | } 519 | } 520 | }, 521 | { 522 | "cell_type": "code", 523 | "execution_count": null, 524 | "outputs": [], 525 | "source": [ 526 | "hidden_features=[]\n", 527 | "for i in range(200):\n", 528 | " hidden_features.append(trunc_model.predict(x_test))\n", 529 | "\n", 530 | "hidden_features=np.array(hidden_features).mean(axis=0)" 531 | ], 532 | "metadata": { 533 | "collapsed": false, 534 | "pycharm": { 535 | "name": "#%%\n" 536 | } 537 | } 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": null, 542 | "outputs": [], 543 | "source": [ 544 | "from sklearn.decomposition import PCA\n", 545 | "from sklearn.manifold import TSNE\n", 546 | "\n", 547 | "pca = PCA(n_components=120)\n", 548 | "pca_result = pca.fit_transform(hidden_features)\n", 549 | "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n", 550 | "\n", 551 | "tsne = TSNE(n_components=2, verbose = 1)\n", 552 | "tsne_results = tsne.fit_transform(pca_result)" 553 | ], 554 | "metadata": { 555 | "collapsed": false, 556 | "pycharm": { 557 | "name": "#%%\n" 558 | } 559 | } 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": null, 564 | "outputs": [], 565 | "source": [ 566 | "import matplotlib.pyplot as plt\n", 567 | "import matplotlib\n", 568 | "matplotlib.rc('xtick', labelsize=20)\n", 569 | "matplotlib.rc('ytick', labelsize=20)\n", 570 | "\n", 571 | "plt.rcParams.update({'font.size': 25})\n", 572 | "%matplotlib inline\n", 573 | "Name='T-SNE Visualization of Fusion model without uncertainty'\n", 574 | "fig = plt.figure(figsize=[15, 15])\n", 575 | "color_map = np.argmax(y_test, axis=1)\n", 576 | "classes=['COVID19','Normal','Pneumonia']\n", 577 | "for cl in range(3):\n", 578 | " indices = np.where(color_map==cl)\n", 579 | " indices = indices[0]\n", 580 | " plt.title(Name, fontsize=20)\n", 581 | " plt.ylabel('Dim_2', fontsize=20)\n", 582 | " plt.xlabel('Dim_1', fontsize=20)\n", 583 | " matplotlib.rc('xtick', labelsize=20)\n", 584 | " matplotlib.rc('ytick', labelsize=20)\n", 585 | " plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n", 586 | "\n", 587 | "plt.rcParams.update({'font.size': 20})\n", 588 | "\n", 589 | "plt.legend()\n", 590 | "plt.show()\n", 591 | "fig.savefig('{}.pdf'.format(Name),dpi=300)" 592 | ], 593 | "metadata": { 594 | "collapsed": false, 595 | "pycharm": { 596 | "name": "#%%\n" 597 | } 598 | } 599 | } 600 | ], 601 | "metadata": { 602 | "colab": { 603 | "provenance": [] 604 | }, 605 | "kernelspec": { 606 | "display_name": "Backyard", 607 | "language": "python", 608 | "name": "backyard" 609 | }, 610 | "language_info": { 611 | "codemirror_mode": { 612 | "name": "ipython", 613 | "version": 3 614 | }, 615 | "file_extension": ".py", 616 | "mimetype": "text/x-python", 617 | "name": "python", 618 | "nbconvert_exporter": "python", 619 | "pygments_lexer": "ipython3", 620 | "version": "3.6.13" 621 | } 622 | }, 623 | "nbformat": 4, 624 | "nbformat_minor": 0 625 | } -------------------------------------------------------------------------------- /Simple Models Results.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "kpkn3mPKEKCX" 7 | }, 8 | "source": [ 9 | "# Covid Chest Project\n", 10 | "## Simple Models" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "id": "G7_xE6-zEKCh" 17 | }, 18 | "source": [ 19 | "### Imports" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "import numpy as np\n", 29 | "from utils import load_covid_data, create_dataset_xray, create_dataset_ct\n", 30 | "from models import simple_cnn_model\n", 31 | "from plots import plot_cm_handy, plot_roc_handy\n", 32 | "from evaluation import mode_robustness, evaluation\n", 33 | "from sklearn.utils.class_weight import compute_class_weight\n", 34 | "\n", 35 | "import warnings\n", 36 | "warnings.filterwarnings('ignore')" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "id": "rvZSoeEVEKCi" 43 | }, 44 | "source": [ 45 | "## Dataset1 (X-Ray)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": { 52 | "colab": { 53 | "base_uri": "https://localhost:8080/", 54 | "height": 549 55 | }, 56 | "executionInfo": { 57 | "elapsed": 70795, 58 | "status": "error", 59 | "timestamp": 1612858364416, 60 | "user": { 61 | "displayName": "Sina Qahremani", 62 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 63 | "userId": "15549406949129485910" 64 | }, 65 | "user_tz": -210 66 | }, 67 | "id": "z3QHd4uKEKCj", 68 | "outputId": "b0e0a232-b138-4694-f626-27b9272fed55" 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "x_train, y_train = load_covid_data(path='Data/train', shuffle=True, class_frequency=True)\n", 73 | "x_test,y_test = load_covid_data(path='Data/test')\n", 74 | "\n", 75 | "class_weights = compute_class_weight('balanced',np.unique(np.argmax(y_train,axis=1)), np.argmax(y_train,axis=1))\n", 76 | "class_weights = {0:class_weights[0],\n", 77 | " 1:class_weights[1],\n", 78 | " 2:class_weights[2]}\n", 79 | "BATCH_SIZE=64\n", 80 | "train_dataset, validation_dataset = create_dataset_xray(x_train, y_train, x_test, y_test, BATCH_SIZE)" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "## Dataset2 (CT)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "X = np.load('CT_X.npy')\n", 97 | "Y = np.load('CT_Y.npy')" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": null, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "BATCH_SIZE=256\n", 107 | "train_dataset, validation_dataset, X_train, X_test, y_train, y_test = create_dataset_ct(X, Y, BATCH_SIZE)" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": { 113 | "id": "WIc0byuxGEn4" 114 | }, 115 | "source": [ 116 | "## Without Monte-Carlo Dropout" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": { 122 | "id": "WS-4sSQKIW8o" 123 | }, 124 | "source": [ 125 | "### Simple CNN Model" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": { 132 | "executionInfo": { 133 | "elapsed": 7026, 134 | "status": "ok", 135 | "timestamp": 1615809861845, 136 | "user": { 137 | "displayName": "Sina Qahremani", 138 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhKpImYys1sQuY9zSkc3NW-sL3jIBU62aUGbslLoQ=s64", 139 | "userId": "15549406949129485910" 140 | }, 141 | "user_tz": -210 142 | }, 143 | "id": "rPlSXAGrIVT2" 144 | }, 145 | "outputs": [], 146 | "source": [ 147 | "model, callbacks=simple_cnn_model(mc=False, lr=0.00001)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": { 154 | "colab": { 155 | "base_uri": "https://localhost:8080/" 156 | }, 157 | "executionInfo": { 158 | "elapsed": 853379, 159 | "status": "ok", 160 | "timestamp": 1607964880676, 161 | "user": { 162 | "displayName": "Sina Qahremani", 163 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 164 | "userId": "15549406949129485910" 165 | }, 166 | "user_tz": -210 167 | }, 168 | "id": "2_crYJJ1I5M-", 169 | "outputId": "e3b3280d-9679-4850-cab4-8d0abd9d6475" 170 | }, 171 | "outputs": [], 172 | "source": [ 173 | "hist = model.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n", 174 | " class_weight=class_weights, callbacks=callbacks)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": { 180 | "id": "fl3qTM7E_x5g" 181 | }, 182 | "source": [ 183 | "#### Results" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": { 190 | "colab": { 191 | "base_uri": "https://localhost:8080/" 192 | }, 193 | "executionInfo": { 194 | "elapsed": 11256, 195 | "status": "ok", 196 | "timestamp": 1611135065887, 197 | "user": { 198 | "displayName": "Sina Qahremani", 199 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 200 | "userId": "15549406949129485910" 201 | }, 202 | "user_tz": -210 203 | }, 204 | "id": "eoU0tkU0Lbnk", 205 | "outputId": "26a0c45a-de5a-44d4-d105-3bb2436acacc" 206 | }, 207 | "outputs": [], 208 | "source": [ 209 | "from sklearn.metrics import accuracy_score, confusion_matrix,precision_score,recall_score,f1_score\n", 210 | "from tensorflow.keras.models import load_model\n", 211 | "\n", 212 | "model = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/simple_cnn_model_covid_simple.h5')\n", 213 | "\n", 214 | "preds = model.predict(x_test)\n", 215 | "acc = accuracy_score(np.argmax(y_test, axis=1), np.argmax(preds, axis=1))*100\n", 216 | "\n", 217 | "cm = confusion_matrix(np.argmax(y_test, axis=1)\n", 218 | ", np.argmax(preds, axis=1))\n", 219 | "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", 220 | "\n", 221 | "print('CONFUSION MATRIX ------------------')\n", 222 | "print(cm)\n", 223 | "\n", 224 | "print('\\nTEST METRICS ----------------------')\n", 225 | "\n", 226 | "precision = precision_score(np.argmax(y_test, axis=1),\n", 227 | " np.argmax(preds, axis=1), average='weighted')*100\n", 228 | "recall = recall_score(np.argmax(y_test, axis=1),\n", 229 | " np.argmax(preds, axis=1), average='weighted')*100\n", 230 | "\n", 231 | "print('Accuracy: {}%'.format(acc))\n", 232 | "print('Precision: {}%'.format(precision))\n", 233 | "print('Recall: {}%'.format(recall))\n", 234 | "print('F1-score: {}'.format( 2*precision*recall/(precision+recall) ))" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": { 241 | "colab": { 242 | "base_uri": "https://localhost:8080/", 243 | "height": 1000 244 | }, 245 | "executionInfo": { 246 | "elapsed": 3120, 247 | "status": "ok", 248 | "timestamp": 1612627874978, 249 | "user": { 250 | "displayName": "Sina Qahremani", 251 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 252 | "userId": "15549406949129485910" 253 | }, 254 | "user_tz": -210 255 | }, 256 | "id": "o6TO6sghouGX", 257 | "outputId": "c114d070-526e-4cea-a4d0-12dc39e18e3f" 258 | }, 259 | "outputs": [], 260 | "source": [ 261 | "model = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/simple_cnn_model_covid_simple.h5')\n", 262 | "\n", 263 | "y_p = model.predict(x_test, batch_size=BATCH_SIZE)\n", 264 | "\n", 265 | "\n", 266 | "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Simple CNN model without uncertainty (X-Ray)',\n", 267 | " class_name=['COVID19','Normal','Pneumonia'])\n", 268 | "\n", 269 | "plot_cm_handy(y_test, y_p,\n", 270 | " lw=2, name='Confusion Matrix of Simple CNN model without uncertainty (X-Ray)',\n", 271 | " class_name=['COVID19','Normal','Pneumonia'])" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": { 277 | "id": "BZCN1ZW7_7qA" 278 | }, 279 | "source": [ 280 | "#### T-SNE" 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "metadata": { 287 | "colab": { 288 | "base_uri": "https://localhost:8080/" 289 | }, 290 | "executionInfo": { 291 | "elapsed": 1878, 292 | "status": "ok", 293 | "timestamp": 1612632381081, 294 | "user": { 295 | "displayName": "Sina Qahremani", 296 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 297 | "userId": "15549406949129485910" 298 | }, 299 | "user_tz": -210 300 | }, 301 | "id": "ROdgMjO3BE7S", 302 | "outputId": "785cf394-b056-44c8-e4e7-8713e9734702" 303 | }, 304 | "outputs": [], 305 | "source": [ 306 | "from models import simple_cnn_trunc_model\n", 307 | "model = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/simple_cnn_model_covid_simple.h5')\n", 308 | "trunc_model = simple_cnn_trunc_model(model, mc=False)\n", 309 | "hidden_features = trunc_model.predict(x_test)" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "metadata": { 316 | "colab": { 317 | "base_uri": "https://localhost:8080/" 318 | }, 319 | "executionInfo": { 320 | "elapsed": 9143, 321 | "status": "ok", 322 | "timestamp": 1612632444898, 323 | "user": { 324 | "displayName": "Sina Qahremani", 325 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 326 | "userId": "15549406949129485910" 327 | }, 328 | "user_tz": -210 329 | }, 330 | "id": "xt3pJy-pB27U", 331 | "outputId": "43d37fc8-488e-4c2d-e7ba-d05104513eb2" 332 | }, 333 | "outputs": [], 334 | "source": [ 335 | "from sklearn.decomposition import PCA\n", 336 | "from sklearn.manifold import TSNE\n", 337 | "\n", 338 | "pca = PCA(n_components=120)\n", 339 | "pca_result = pca.fit_transform(hidden_features)\n", 340 | "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n", 341 | "\n", 342 | "tsne = TSNE(n_components=2, verbose = 1)\n", 343 | "tsne_results = tsne.fit_transform(pca_result)" 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": null, 349 | "metadata": { 350 | "colab": { 351 | "base_uri": "https://localhost:8080/", 352 | "height": 917 353 | }, 354 | "executionInfo": { 355 | "elapsed": 1365, 356 | "status": "ok", 357 | "timestamp": 1612632448301, 358 | "user": { 359 | "displayName": "Sina Qahremani", 360 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 361 | "userId": "15549406949129485910" 362 | }, 363 | "user_tz": -210 364 | }, 365 | "id": "rmbdCrESCCYF", 366 | "outputId": "b5fc4bd6-1b67-4e6b-e2b7-bd1c4e81d408" 367 | }, 368 | "outputs": [], 369 | "source": [ 370 | "import matplotlib.pyplot as plt\n", 371 | "import matplotlib \n", 372 | "matplotlib.rc('xtick', labelsize=20) \n", 373 | "matplotlib.rc('ytick', labelsize=20) \n", 374 | "\n", 375 | "plt.rcParams.update({'font.size': 25})\n", 376 | "%matplotlib inline\n", 377 | "Name='T-SNE Visualization of Simple CNN model without uncertainty (X-Ray)'\n", 378 | "fig = plt.figure(figsize=[15, 15])\n", 379 | "color_map = np.argmax(y_test, axis=1)\n", 380 | "classes=['COVID19','Normal','Pneumonia']\n", 381 | "for cl in range(3):\n", 382 | " indices = np.where(color_map==cl)\n", 383 | " indices = indices[0]\n", 384 | " plt.title(Name, fontsize=20)\n", 385 | " plt.ylabel('Dim_2', fontsize=20)\n", 386 | " plt.xlabel('Dim_1', fontsize=20)\n", 387 | " matplotlib.rc('xtick', labelsize=20) \n", 388 | " matplotlib.rc('ytick', labelsize=20) \n", 389 | " plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n", 390 | "\n", 391 | "plt.rcParams.update({'font.size': 20})\n", 392 | "\n", 393 | "plt.legend()\n", 394 | "plt.show()\n", 395 | "fig.savefig('{}.pdf'.format(Name),dpi=300)\n", 396 | "\n" 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": { 402 | "id": "Tqr7C97x2qsZ" 403 | }, 404 | "source": [ 405 | "### Multi-headed Model" 406 | ] 407 | }, 408 | { 409 | "cell_type": "code", 410 | "execution_count": null, 411 | "metadata": { 412 | "executionInfo": { 413 | "elapsed": 996, 414 | "status": "ok", 415 | "timestamp": 1615812562739, 416 | "user": { 417 | "displayName": "Sina Qahremani", 418 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhKpImYys1sQuY9zSkc3NW-sL3jIBU62aUGbslLoQ=s64", 419 | "userId": "15549406949129485910" 420 | }, 421 | "user_tz": -210 422 | }, 423 | "id": "sb7CG05q2vRe" 424 | }, 425 | "outputs": [], 426 | "source": [ 427 | "model_mh, callbacks_mh = multi_headed_model(False)" 428 | ] 429 | }, 430 | { 431 | "cell_type": "code", 432 | "execution_count": null, 433 | "metadata": { 434 | "colab": { 435 | "base_uri": "https://localhost:8080/" 436 | }, 437 | "executionInfo": { 438 | "elapsed": 655818, 439 | "status": "ok", 440 | "timestamp": 1607965960940, 441 | "user": { 442 | "displayName": "Sina Qahremani", 443 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 444 | "userId": "15549406949129485910" 445 | }, 446 | "user_tz": -210 447 | }, 448 | "id": "kn0c2XOTB5EA", 449 | "outputId": "21850153-d14e-41e9-879f-18751ef7f885" 450 | }, 451 | "outputs": [], 452 | "source": [ 453 | "hist_mh = model_mh.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n", 454 | " class_weight=class_weights, callbacks=callbacks_mh)" 455 | ] 456 | }, 457 | { 458 | "cell_type": "markdown", 459 | "metadata": { 460 | "id": "wQIac5kbEY_4" 461 | }, 462 | "source": [ 463 | "#### Results" 464 | ] 465 | }, 466 | { 467 | "cell_type": "code", 468 | "execution_count": null, 469 | "metadata": { 470 | "colab": { 471 | "base_uri": "https://localhost:8080/" 472 | }, 473 | "executionInfo": { 474 | "elapsed": 9313, 475 | "status": "ok", 476 | "timestamp": 1611147157527, 477 | "user": { 478 | "displayName": "Sina Qahremani", 479 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 480 | "userId": "15549406949129485910" 481 | }, 482 | "user_tz": -210 483 | }, 484 | "id": "S52xhcUvpTmW", 485 | "outputId": "c3d4cc15-f585-4df7-9305-0b116965e0f7" 486 | }, 487 | "outputs": [], 488 | "source": [ 489 | "from sklearn.metrics import accuracy_score, confusion_matrix,precision_score,recall_score,f1_score\n", 490 | "from tensorflow.keras.models import load_model\n", 491 | "\n", 492 | "model_mh = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/multi_headed_model_covid_simple.h5')\n", 493 | "\n", 494 | "preds = model_mh.predict(x_test)\n", 495 | "acc = accuracy_score(np.argmax(y_test, axis=1), np.argmax(preds, axis=1))*100\n", 496 | "\n", 497 | "cm = confusion_matrix(np.argmax(y_test, axis=1)\n", 498 | ", np.argmax(preds, axis=1))\n", 499 | "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", 500 | "\n", 501 | "print('CONFUSION MATRIX ------------------')\n", 502 | "print(cm)\n", 503 | "\n", 504 | "print('\\nTEST METRICS ----------------------')\n", 505 | "\n", 506 | "precision = precision_score(np.argmax(y_test, axis=1),\n", 507 | " np.argmax(preds, axis=1), average='weighted')*100\n", 508 | "recall = recall_score(np.argmax(y_test, axis=1),\n", 509 | " np.argmax(preds, axis=1), average='weighted')*100\n", 510 | "\n", 511 | "print('Accuracy: {}%'.format(acc))\n", 512 | "print('Precision: {}%'.format(precision))\n", 513 | "print('Recall: {}%'.format(recall))\n", 514 | "print('F1-score: {}'.format( 2*precision*recall/(precision+recall) ))" 515 | ] 516 | }, 517 | { 518 | "cell_type": "code", 519 | "execution_count": null, 520 | "metadata": { 521 | "colab": { 522 | "base_uri": "https://localhost:8080/", 523 | "height": 1000 524 | }, 525 | "executionInfo": { 526 | "elapsed": 3393, 527 | "status": "ok", 528 | "timestamp": 1612627905296, 529 | "user": { 530 | "displayName": "Sina Qahremani", 531 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 532 | "userId": "15549406949129485910" 533 | }, 534 | "user_tz": -210 535 | }, 536 | "id": "4038KiUU2J-v", 537 | "outputId": "1778c1a3-08ff-4d23-a212-3760d6a95239" 538 | }, 539 | "outputs": [], 540 | "source": [ 541 | "model_mh = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/multi_headed_model_covid_simple.h5')\n", 542 | "\n", 543 | "y_p = model_mh.predict(x_test, batch_size=BATCH_SIZE)\n", 544 | "\n", 545 | "\n", 546 | "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Multi-headed model without uncertainty (X-Ray)',\n", 547 | " class_name=['COVID19','Normal','Pneumonia'], axis=[0.0, 0.18, 0.85, 1.0])\n", 548 | "\n", 549 | "plot_cm_handy(y_test, y_p,\n", 550 | " lw=2, name='Confusion Matrix of Multi-headed model without uncertainty (X-Ray)',\n", 551 | " class_name=['COVID19','Normal','Pneumonia'])" 552 | ] 553 | }, 554 | { 555 | "cell_type": "markdown", 556 | "metadata": { 557 | "id": "e3os807LEcwv" 558 | }, 559 | "source": [ 560 | "#### T-SNE" 561 | ] 562 | }, 563 | { 564 | "cell_type": "code", 565 | "execution_count": null, 566 | "metadata": { 567 | "colab": { 568 | "base_uri": "https://localhost:8080/" 569 | }, 570 | "executionInfo": { 571 | "elapsed": 7827, 572 | "status": "ok", 573 | "timestamp": 1612632482228, 574 | "user": { 575 | "displayName": "Sina Qahremani", 576 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 577 | "userId": "15549406949129485910" 578 | }, 579 | "user_tz": -210 580 | }, 581 | "id": "6UD3sXtaFAV0", 582 | "outputId": "5f3bc017-a539-465c-bd46-a4a5b892faae" 583 | }, 584 | "outputs": [], 585 | "source": [ 586 | "from models import multi_headed_trunc_model\n", 587 | "model_mh = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/multi_headed_model_covid_simple.h5')\n", 588 | "trunc_model = multi_headed_trunc_model(model_mh, mc=False)\n", 589 | "hidden_features = trunc_model.predict(x_test)" 590 | ] 591 | }, 592 | { 593 | "cell_type": "code", 594 | "execution_count": null, 595 | "metadata": { 596 | "colab": { 597 | "base_uri": "https://localhost:8080/" 598 | }, 599 | "executionInfo": { 600 | "elapsed": 9492, 601 | "status": "ok", 602 | "timestamp": 1612632501164, 603 | "user": { 604 | "displayName": "Sina Qahremani", 605 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 606 | "userId": "15549406949129485910" 607 | }, 608 | "user_tz": -210 609 | }, 610 | "id": "xqtX8FjEFhPK", 611 | "outputId": "ed832807-dc79-4bb6-a815-a1a3375a6608" 612 | }, 613 | "outputs": [], 614 | "source": [ 615 | "from sklearn.decomposition import PCA\n", 616 | "from sklearn.manifold import TSNE\n", 617 | "\n", 618 | "pca = PCA(n_components=120)\n", 619 | "pca_result = pca.fit_transform(hidden_features)\n", 620 | "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n", 621 | "\n", 622 | "tsne = TSNE(n_components=2, verbose = 1)\n", 623 | "tsne_results = tsne.fit_transform(pca_result)" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": null, 629 | "metadata": { 630 | "colab": { 631 | "base_uri": "https://localhost:8080/", 632 | "height": 917 633 | }, 634 | "executionInfo": { 635 | "elapsed": 1578, 636 | "status": "ok", 637 | "timestamp": 1612632508139, 638 | "user": { 639 | "displayName": "Sina Qahremani", 640 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 641 | "userId": "15549406949129485910" 642 | }, 643 | "user_tz": -210 644 | }, 645 | "id": "lNQ0sK47Flnv", 646 | "outputId": "79279490-5f11-492c-ae04-f95e2844993d" 647 | }, 648 | "outputs": [], 649 | "source": [ 650 | "import matplotlib.pyplot as plt\n", 651 | "import matplotlib \n", 652 | "matplotlib.rc('xtick', labelsize=20) \n", 653 | "matplotlib.rc('ytick', labelsize=20) \n", 654 | "\n", 655 | "plt.rcParams.update({'font.size': 25})\n", 656 | "%matplotlib inline\n", 657 | "Name='T-SNE Visualization of Multi-headed model without uncertainty (X-Ray)'\n", 658 | "fig = plt.figure(figsize=[15, 15])\n", 659 | "color_map = np.argmax(y_test, axis=1)\n", 660 | "classes=['COVID19','Normal','Pneumonia']\n", 661 | "for cl in range(3):\n", 662 | " indices = np.where(color_map==cl)\n", 663 | " indices = indices[0]\n", 664 | " plt.title(Name, fontsize=20)\n", 665 | " plt.ylabel('Dim_2', fontsize=20)\n", 666 | " plt.xlabel('Dim_1', fontsize=20)\n", 667 | " matplotlib.rc('xtick', labelsize=20) \n", 668 | " matplotlib.rc('ytick', labelsize=20) \n", 669 | " plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n", 670 | "\n", 671 | "plt.rcParams.update({'font.size': 20})\n", 672 | "\n", 673 | "plt.legend()\n", 674 | "plt.show()\n", 675 | "fig.savefig('{}.pdf'.format(Name),dpi=300)\n", 676 | "\n" 677 | ] 678 | }, 679 | { 680 | "cell_type": "markdown", 681 | "metadata": { 682 | "id": "9q6QoPz1PzaE" 683 | }, 684 | "source": [ 685 | "## With Monte-Carlo Dropout" 686 | ] 687 | }, 688 | { 689 | "cell_type": "markdown", 690 | "metadata": { 691 | "id": "zgJdbhoMQIJx" 692 | }, 693 | "source": [ 694 | "### Simple CNN Model" 695 | ] 696 | }, 697 | { 698 | "cell_type": "code", 699 | "execution_count": null, 700 | "metadata": { 701 | "id": "5XFLZ_DEPynC" 702 | }, 703 | "outputs": [], 704 | "source": [ 705 | "mc_model, mc_callbacks=simple_cnn_model(mc=True, lr=0.00001)" 706 | ] 707 | }, 708 | { 709 | "cell_type": "code", 710 | "execution_count": null, 711 | "metadata": { 712 | "colab": { 713 | "base_uri": "https://localhost:8080/" 714 | }, 715 | "executionInfo": { 716 | "elapsed": 1134160, 717 | "status": "ok", 718 | "timestamp": 1607970919286, 719 | "user": { 720 | "displayName": "Sina Qahremani", 721 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 722 | "userId": "15549406949129485910" 723 | }, 724 | "user_tz": -210 725 | }, 726 | "id": "bSIlot9nQaKR", 727 | "outputId": "2988cfe4-55ff-4243-ae82-f5678e5ebae8" 728 | }, 729 | "outputs": [], 730 | "source": [ 731 | "mc_hist = mc_model.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n", 732 | " class_weight=class_weights, callbacks=mc_callbacks)" 733 | ] 734 | }, 735 | { 736 | "cell_type": "markdown", 737 | "metadata": { 738 | "id": "tvxtdGOYHnTC" 739 | }, 740 | "source": [ 741 | "#### Results" 742 | ] 743 | }, 744 | { 745 | "cell_type": "code", 746 | "execution_count": null, 747 | "metadata": { 748 | "colab": { 749 | "base_uri": "https://localhost:8080/" 750 | }, 751 | "executionInfo": { 752 | "elapsed": 80458, 753 | "status": "ok", 754 | "timestamp": 1613650889182, 755 | "user": { 756 | "displayName": "Sina Qahremani", 757 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 758 | "userId": "15549406949129485910" 759 | }, 760 | "user_tz": -210 761 | }, 762 | "id": "KvHuWn993fOV", 763 | "outputId": "7e4f8f77-5384-4f57-c851-fa54914cc076" 764 | }, 765 | "outputs": [], 766 | "source": [ 767 | "from sklearn.metrics import accuracy_score,recall_score,precision_score\n", 768 | "import tqdm\n", 769 | "\n", 770 | "mc_model = load_model('path/simple_cnn_model_covid_mc.h5')\n", 771 | "\n", 772 | "number_prediction=200\n", 773 | "mc_predictions = []\n", 774 | "for i in tqdm.tqdm(range(number_prediction)):\n", 775 | " y_p = mc_model.predict(x_test)\n", 776 | " mc_predictions.append(y_p)\n", 777 | "\n", 778 | "accs=recalls=precisions=F1s=[]\n", 779 | "for y_p in mc_predictions:\n", 780 | " acc = accuracy_score(y_test.argmax(axis=1), y_p.argmax(axis=1))\n", 781 | " recall=recall_score(y_test.argmax(axis=1), y_p.argmax(axis=1),average='weighted')\n", 782 | " precision=precision_score(y_test.argmax(axis=1), y_p.argmax(axis=1),average='weighted')\n", 783 | " F1=(2*precision*recall)/(precision+recall)\n", 784 | " accs.append(acc)\n", 785 | " recalls.append(recall)\n", 786 | " precisions.append(precision)\n", 787 | " F1s.append(F1)\n", 788 | "\n", 789 | "\n", 790 | "print(\"MC accuracy: {:.5%}\".format(sum(accs)/len(accs)))\n", 791 | "print(\"MC precision: {:.5%}\".format(sum(precisions)/len(precisions)))\n", 792 | "print(\"MC recall: {:.5%}\".format(sum(recalls)/len(recalls)))\n", 793 | "print(\"MC F1: {:.5%}\".format(sum(F1s)/len(F1s)))\n", 794 | "\n", 795 | "mc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1)\n", 796 | "ensemble_acc = accuracy_score(y_test.argmax(axis=1), mc_ensemble_pred)\n", 797 | "ensemble_precision=precision_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n", 798 | "ensemble_recall=recall_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n", 799 | "ensemble_F1=(2*ensemble_precision*ensemble_recall)/(ensemble_precision+ensemble_recall)\n", 800 | "\n", 801 | "print(\"MC-ensemble accuracy: {:.5%}\".format(ensemble_acc))\n", 802 | "print(\"MC-ensemble precision: {:.5%}\".format(ensemble_precision))\n", 803 | "print(\"MC-ensemble recall: {:.5%}\".format(ensemble_recall))\n", 804 | "print(\"MC-ensemble F1: {:.5%}\".format(ensemble_F1))\n", 805 | "\n" 806 | ] 807 | }, 808 | { 809 | "cell_type": "code", 810 | "execution_count": null, 811 | "metadata": { 812 | "colab": { 813 | "base_uri": "https://localhost:8080/", 814 | "height": 449 815 | }, 816 | "executionInfo": { 817 | "elapsed": 1137, 818 | "status": "ok", 819 | "timestamp": 1612215394628, 820 | "user": { 821 | "displayName": "Sina Qahremani", 822 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 823 | "userId": "15549406949129485910" 824 | }, 825 | "user_tz": -210 826 | }, 827 | "id": "zINmKa3OheBj", 828 | "outputId": "bb0c18e8-3b5c-4984-9da1-e5f0996bd3d9" 829 | }, 830 | "outputs": [], 831 | "source": [ 832 | "Name = 'Histogram of Simple CNN model with uncertainty (X-Ray)'\n", 833 | "f, ax = plt.subplots(figsize=[10, 7])\n", 834 | "plt.hist(accs)\n", 835 | "plt.axvline(x=ensemble_acc, color=\"r\")\n", 836 | "ax.set_title(Name,fontsize=19)\n", 837 | "plt.show()\n", 838 | "f.savefig('{}.pdf'.format(Name))\n", 839 | "ax.figure.savefig(\"{}.pdf\".format(Name), bbox_inches='tight')" 840 | ] 841 | }, 842 | { 843 | "cell_type": "code", 844 | "execution_count": null, 845 | "metadata": { 846 | "colab": { 847 | "base_uri": "https://localhost:8080/", 848 | "height": 1000 849 | }, 850 | "executionInfo": { 851 | "elapsed": 2445, 852 | "status": "ok", 853 | "timestamp": 1612628062671, 854 | "user": { 855 | "displayName": "Sina Qahremani", 856 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 857 | "userId": "15549406949129485910" 858 | }, 859 | "user_tz": -210 860 | }, 861 | "id": "thp5jb-p3kIa", 862 | "outputId": "d65da74f-f6a8-43a0-951e-7d372792c0d5" 863 | }, 864 | "outputs": [], 865 | "source": [ 866 | "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Simple CNN model with uncertainty (X-Ray)',\n", 867 | " class_name=['COVID19','Normal','Pneumonia'], axis=[0.0, 0.21, 0.85, 1.0])\n", 868 | "\n", 869 | "plot_cm_handy(y_test, y_p,\n", 870 | " lw=2, name='Confusion Matrix of Simple CNN model with uncertainty (X-Ray)',\n", 871 | " class_name=['COVID19','Normal','Pneumonia'])\n" 872 | ] 873 | }, 874 | { 875 | "cell_type": "markdown", 876 | "metadata": { 877 | "id": "Bm0SUpOJHs32" 878 | }, 879 | "source": [ 880 | "#### Noise Robustness" 881 | ] 882 | }, 883 | { 884 | "cell_type": "code", 885 | "execution_count": null, 886 | "metadata": { 887 | "colab": { 888 | "base_uri": "https://localhost:8080/" 889 | }, 890 | "executionInfo": { 891 | "elapsed": 4463, 892 | "status": "ok", 893 | "timestamp": 1612105203067, 894 | "user": { 895 | "displayName": "Sina Qahremani", 896 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 897 | "userId": "15549406949129485910" 898 | }, 899 | "user_tz": -210 900 | }, 901 | "id": "o76otCkJ88v3", 902 | "outputId": "6948a184-f852-48bc-a7c1-9ed8fb12c169" 903 | }, 904 | "outputs": [], 905 | "source": [ 906 | "mc_model=load_model('path/simple_cnn_model_covid_mc.h5')\n", 907 | "std_coef=[1e-4,1e-3,1e-2,1e-1, 0.2,0.3,0.4,0.5,0.6]" 908 | ] 909 | }, 910 | { 911 | "cell_type": "code", 912 | "execution_count": null, 913 | "metadata": { 914 | "colab": { 915 | "base_uri": "https://localhost:8080/" 916 | }, 917 | "executionInfo": { 918 | "elapsed": 163152, 919 | "status": "ok", 920 | "timestamp": 1612105390317, 921 | "user": { 922 | "displayName": "Sina Qahremani", 923 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 924 | "userId": "15549406949129485910" 925 | }, 926 | "user_tz": -210 927 | }, 928 | "id": "Vx2E8JoI4IuT", 929 | "outputId": "3208144e-4b3f-436c-ef2f-37530bb991ac" 930 | }, 931 | "outputs": [], 932 | "source": [ 933 | "mode_robustness(x_test, y_test, mc_model, std_coef)" 934 | ] 935 | }, 936 | { 937 | "cell_type": "markdown", 938 | "metadata": { 939 | "id": "6LtcdA_2HxJi" 940 | }, 941 | "source": [ 942 | "#### T-SNE" 943 | ] 944 | }, 945 | { 946 | "cell_type": "code", 947 | "execution_count": null, 948 | "metadata": { 949 | "colab": { 950 | "base_uri": "https://localhost:8080/" 951 | }, 952 | "executionInfo": { 953 | "elapsed": 5745, 954 | "status": "ok", 955 | "timestamp": 1612632524315, 956 | "user": { 957 | "displayName": "Sina Qahremani", 958 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 959 | "userId": "15549406949129485910" 960 | }, 961 | "user_tz": -210 962 | }, 963 | "id": "4MgSCxJ6H0H3", 964 | "outputId": "80611d75-e482-44dc-eebe-31713d214507" 965 | }, 966 | "outputs": [], 967 | "source": [ 968 | "mc_model = load_model('path/simple_cnn_model_covid_mc.h5')\n", 969 | "trunc_model = simple_cnn_trunc_model(mc_model, mc=True)" 970 | ] 971 | }, 972 | { 973 | "cell_type": "code", 974 | "execution_count": null, 975 | "metadata": { 976 | "id": "kPKFW4cSJFmN" 977 | }, 978 | "outputs": [], 979 | "source": [ 980 | "hidden_features=[]\n", 981 | "for i in range(200):\n", 982 | " hidden_features.append(trunc_model.predict(x_test))\n", 983 | "\n", 984 | "hidden_features=np.array(hidden_features).mean(axis=0)" 985 | ] 986 | }, 987 | { 988 | "cell_type": "code", 989 | "execution_count": null, 990 | "metadata": { 991 | "colab": { 992 | "base_uri": "https://localhost:8080/" 993 | }, 994 | "executionInfo": { 995 | "elapsed": 8019, 996 | "status": "ok", 997 | "timestamp": 1612632613186, 998 | "user": { 999 | "displayName": "Sina Qahremani", 1000 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1001 | "userId": "15549406949129485910" 1002 | }, 1003 | "user_tz": -210 1004 | }, 1005 | "id": "pWFzWCHZIzq0", 1006 | "outputId": "ab05f0ed-2ffc-4cae-8551-3f5ccb3903bd" 1007 | }, 1008 | "outputs": [], 1009 | "source": [ 1010 | "from sklearn.decomposition import PCA\n", 1011 | "from sklearn.manifold import TSNE\n", 1012 | "\n", 1013 | "pca = PCA(n_components=120)\n", 1014 | "pca_result = pca.fit_transform(hidden_features)\n", 1015 | "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n", 1016 | "\n", 1017 | "tsne = TSNE(n_components=2, verbose = 1)\n", 1018 | "tsne_results = tsne.fit_transform(pca_result)" 1019 | ] 1020 | }, 1021 | { 1022 | "cell_type": "code", 1023 | "execution_count": null, 1024 | "metadata": { 1025 | "colab": { 1026 | "base_uri": "https://localhost:8080/", 1027 | "height": 917 1028 | }, 1029 | "executionInfo": { 1030 | "elapsed": 1774, 1031 | "status": "ok", 1032 | "timestamp": 1612632620393, 1033 | "user": { 1034 | "displayName": "Sina Qahremani", 1035 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1036 | "userId": "15549406949129485910" 1037 | }, 1038 | "user_tz": -210 1039 | }, 1040 | "id": "5IJq0K-hIzdP", 1041 | "outputId": "4de066da-43a5-47e9-f400-523b6132bf72" 1042 | }, 1043 | "outputs": [], 1044 | "source": [ 1045 | "from keras.utils import np_utils\n", 1046 | "import matplotlib.pyplot as plt\n", 1047 | "import matplotlib \n", 1048 | "matplotlib.rc('xtick', labelsize=20) \n", 1049 | "matplotlib.rc('ytick', labelsize=20) \n", 1050 | "\n", 1051 | "plt.rcParams.update({'font.size': 25})\n", 1052 | "%matplotlib inline\n", 1053 | "Name='T-SNE Visualization of Simple CNN model with uncertainty (X-Ray)'\n", 1054 | "fig = plt.figure(figsize=[15, 15])\n", 1055 | "color_map = np.argmax(y_test, axis=1)\n", 1056 | "classes=['COVID19','Normal','Pneumonia']\n", 1057 | "for cl in range(3):\n", 1058 | " indices = np.where(color_map==cl)\n", 1059 | " indices = indices[0]\n", 1060 | " plt.title(Name, fontsize=20)\n", 1061 | " plt.ylabel('Dim_2', fontsize=20)\n", 1062 | " plt.xlabel('Dim_1', fontsize=20)\n", 1063 | " matplotlib.rc('xtick', labelsize=20) \n", 1064 | " matplotlib.rc('ytick', labelsize=20) \n", 1065 | " plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n", 1066 | "\n", 1067 | "plt.rcParams.update({'font.size': 20})\n", 1068 | "\n", 1069 | "plt.legend()\n", 1070 | "plt.show()\n", 1071 | "fig.savefig('{}.pdf'.format(Name),dpi=300)\n", 1072 | "\n" 1073 | ] 1074 | }, 1075 | { 1076 | "cell_type": "markdown", 1077 | "metadata": { 1078 | "id": "ctVi8yViTTe0" 1079 | }, 1080 | "source": [ 1081 | "### Multi-headed Model" 1082 | ] 1083 | }, 1084 | { 1085 | "cell_type": "code", 1086 | "execution_count": null, 1087 | "metadata": { 1088 | "id": "4k6hlLSyTUO2" 1089 | }, 1090 | "outputs": [], 1091 | "source": [ 1092 | "mc_model_mh, mc_callbacks_mh = multi_headed_model(True)" 1093 | ] 1094 | }, 1095 | { 1096 | "cell_type": "code", 1097 | "execution_count": null, 1098 | "metadata": { 1099 | "colab": { 1100 | "base_uri": "https://localhost:8080/" 1101 | }, 1102 | "executionInfo": { 1103 | "elapsed": 1472938, 1104 | "status": "ok", 1105 | "timestamp": 1607975975007, 1106 | "user": { 1107 | "displayName": "Sina Qahremani", 1108 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1109 | "userId": "15549406949129485910" 1110 | }, 1111 | "user_tz": -210 1112 | }, 1113 | "id": "WgHcK7D1Terz", 1114 | "outputId": "7348ff07-6a2d-4415-c45f-c1132bdb6403" 1115 | }, 1116 | "outputs": [], 1117 | "source": [ 1118 | "mc_hist_mh = mc_model_mh.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n", 1119 | " class_weight=class_weights, callbacks=mc_callbacks_mh)" 1120 | ] 1121 | }, 1122 | { 1123 | "cell_type": "markdown", 1124 | "metadata": { 1125 | "id": "IoctPxwUIIx5" 1126 | }, 1127 | "source": [ 1128 | "#### Results" 1129 | ] 1130 | }, 1131 | { 1132 | "cell_type": "code", 1133 | "execution_count": null, 1134 | "metadata": { 1135 | "colab": { 1136 | "base_uri": "https://localhost:8080/" 1137 | }, 1138 | "executionInfo": { 1139 | "elapsed": 113061, 1140 | "status": "ok", 1141 | "timestamp": 1612628193425, 1142 | "user": { 1143 | "displayName": "Sina Qahremani", 1144 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1145 | "userId": "15549406949129485910" 1146 | }, 1147 | "user_tz": -210 1148 | }, 1149 | "id": "BQabg4To-SUP", 1150 | "outputId": "7c5c5af5-c294-4cc6-8580-275dfe4c3f65" 1151 | }, 1152 | "outputs": [], 1153 | "source": [ 1154 | "from sklearn.metrics import accuracy_score,recall_score,precision_score\n", 1155 | "import tqdm\n", 1156 | "\n", 1157 | "mc_model_mh = load_model('path/multi_headed_model_covid_mc.h5')\n", 1158 | "\n", 1159 | "number_prediction=200\n", 1160 | "mc_predictions = []\n", 1161 | "for i in tqdm.tqdm(range(number_prediction)):\n", 1162 | " y_p = mc_model_mh.predict(x_test)\n", 1163 | " mc_predictions.append(y_p)\n", 1164 | "\n", 1165 | "accs=recalls=precisions=F1s=[]\n", 1166 | "for y_p in mc_predictions:\n", 1167 | " acc = accuracy_score(y_test.argmax(axis=1), y_p.argmax(axis=1))\n", 1168 | " recall=recall_score(y_test.argmax(axis=1), y_p.argmax(axis=1), average='weighted')\n", 1169 | " precision=precision_score(y_test.argmax(axis=1), y_p.argmax(axis=1), average='weighted')\n", 1170 | " F1=(2*precision*recall)/(precision+recall)\n", 1171 | " accs.append(acc)\n", 1172 | " recalls.append(recall)\n", 1173 | " precisions.append(precision)\n", 1174 | " F1s.append(F1)\n", 1175 | "\n", 1176 | "\n", 1177 | "print(\"MC accuracy: {:.5%}\".format(sum(accs)/len(accs)))\n", 1178 | "print(\"MC precision: {:.5%}\".format(sum(precisions)/len(precisions)))\n", 1179 | "print(\"MC recall: {:.5%}\".format(sum(recalls)/len(recalls)))\n", 1180 | "print(\"MC F1: {:.5%}\".format(sum(F1s)/len(F1s)))\n", 1181 | "\n", 1182 | "mc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1)\n", 1183 | "ensemble_acc = accuracy_score(y_test.argmax(axis=1), mc_ensemble_pred)\n", 1184 | "ensemble_precision=precision_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n", 1185 | "ensemble_recall=recall_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n", 1186 | "ensemble_F1=(2*ensemble_precision*ensemble_recall)/(ensemble_precision+ensemble_recall)\n", 1187 | "\n", 1188 | "print(\"MC-ensemble accuracy: {:.5%}\".format(ensemble_acc))\n", 1189 | "print(\"MC-ensemble precision: {:.5%}\".format(ensemble_precision))\n", 1190 | "print(\"MC-ensemble recall: {:.5%}\".format(ensemble_recall))\n", 1191 | "print(\"MC-ensemble F1: {:.5%}\".format(ensemble_F1))\n", 1192 | "\n" 1193 | ] 1194 | }, 1195 | { 1196 | "cell_type": "code", 1197 | "execution_count": null, 1198 | "metadata": { 1199 | "colab": { 1200 | "base_uri": "https://localhost:8080/", 1201 | "height": 449 1202 | }, 1203 | "executionInfo": { 1204 | "elapsed": 1122, 1205 | "status": "ok", 1206 | "timestamp": 1612215642550, 1207 | "user": { 1208 | "displayName": "Sina Qahremani", 1209 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1210 | "userId": "15549406949129485910" 1211 | }, 1212 | "user_tz": -210 1213 | }, 1214 | "id": "nMPCIyVihzl-", 1215 | "outputId": "233d492c-c0b3-440e-a18d-8d5995ee1453" 1216 | }, 1217 | "outputs": [], 1218 | "source": [ 1219 | "Name = 'Histogram of Multi-headed model with uncertainty (X-Ray)'\n", 1220 | "f, ax = plt.subplots(figsize=[10, 7])\n", 1221 | "plt.hist(accs);\n", 1222 | "plt.axvline(x=ensemble_acc, color=\"r\")\n", 1223 | "ax.set_title(Name,fontsize=19)\n", 1224 | "plt.show()\n", 1225 | "f.savefig('{}.pdf'.format(Name))\n", 1226 | "ax.figure.savefig(\"{}.pdf\".format(Name), bbox_inches='tight')" 1227 | ] 1228 | }, 1229 | { 1230 | "cell_type": "code", 1231 | "execution_count": null, 1232 | "metadata": { 1233 | "colab": { 1234 | "base_uri": "https://localhost:8080/", 1235 | "height": 1000 1236 | }, 1237 | "executionInfo": { 1238 | "elapsed": 1903, 1239 | "status": "ok", 1240 | "timestamp": 1612628220225, 1241 | "user": { 1242 | "displayName": "Sina Qahremani", 1243 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1244 | "userId": "15549406949129485910" 1245 | }, 1246 | "user_tz": -210 1247 | }, 1248 | "id": "RbPuXZqX5IrZ", 1249 | "outputId": "a3d59402-a499-4c4d-a28e-3c076ed3ea68" 1250 | }, 1251 | "outputs": [], 1252 | "source": [ 1253 | "plot_roc_handy(y_test, np.array(mc_predictions).mean(axis=0) ,zoom=True, lw=2, name='Roc of Multi-headed model with uncertainty (X-Ray)',\n", 1254 | " class_name=['COVID19','Normal','Pneumonia'], axis=[0.0, 0.21, 0.85, 1.0])\n", 1255 | "\n", 1256 | "plot_cm_handy(y_test, np.array(mc_predictions).mean(axis=0),\n", 1257 | " lw=2, name='Confusion Matrix of Multi-headed model with uncertainty (X-Ray)',\n", 1258 | " class_name=['COVID19','Normal','Pneumonia'])" 1259 | ] 1260 | }, 1261 | { 1262 | "cell_type": "markdown", 1263 | "metadata": { 1264 | "id": "JDutmYYmIPrU" 1265 | }, 1266 | "source": [ 1267 | "#### Noise Robustness" 1268 | ] 1269 | }, 1270 | { 1271 | "cell_type": "code", 1272 | "execution_count": null, 1273 | "metadata": { 1274 | "colab": { 1275 | "base_uri": "https://localhost:8080/" 1276 | }, 1277 | "executionInfo": { 1278 | "elapsed": 9924, 1279 | "status": "ok", 1280 | "timestamp": 1612107340331, 1281 | "user": { 1282 | "displayName": "Sina Qahremani", 1283 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1284 | "userId": "15549406949129485910" 1285 | }, 1286 | "user_tz": -210 1287 | }, 1288 | "id": "AnWZdMkr_zHx", 1289 | "outputId": "673e2167-d9bc-4d72-9324-baeb4d276efd" 1290 | }, 1291 | "outputs": [], 1292 | "source": [ 1293 | "mc_model_mh = load_model('path/multi_headed_model_covid_mc.h5')\n", 1294 | "std_coef=[1e-4,1e-3,1e-2,1e-1, 0.2,0.3,0.4,0.5,0.6]" 1295 | ] 1296 | }, 1297 | { 1298 | "cell_type": "code", 1299 | "execution_count": null, 1300 | "metadata": { 1301 | "colab": { 1302 | "base_uri": "https://localhost:8080/" 1303 | }, 1304 | "executionInfo": { 1305 | "elapsed": 246515, 1306 | "status": "ok", 1307 | "timestamp": 1612107592091, 1308 | "user": { 1309 | "displayName": "Sina Qahremani", 1310 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1311 | "userId": "15549406949129485910" 1312 | }, 1313 | "user_tz": -210 1314 | }, 1315 | "id": "gGwv-LEEFVqk", 1316 | "outputId": "74a6ffac-e552-4fd7-bc45-e72b39c779f7" 1317 | }, 1318 | "outputs": [], 1319 | "source": [ 1320 | "mode_robustness(x_test, y_test, mc_model_mh, std_coef)" 1321 | ] 1322 | }, 1323 | { 1324 | "cell_type": "markdown", 1325 | "metadata": { 1326 | "id": "jq8OYXpaIUch" 1327 | }, 1328 | "source": [ 1329 | "#### T-SNE" 1330 | ] 1331 | }, 1332 | { 1333 | "cell_type": "code", 1334 | "execution_count": null, 1335 | "metadata": { 1336 | "colab": { 1337 | "base_uri": "https://localhost:8080/" 1338 | }, 1339 | "executionInfo": { 1340 | "elapsed": 1709, 1341 | "status": "ok", 1342 | "timestamp": 1612632666788, 1343 | "user": { 1344 | "displayName": "Sina Qahremani", 1345 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1346 | "userId": "15549406949129485910" 1347 | }, 1348 | "user_tz": -210 1349 | }, 1350 | "id": "wjPTphhEKQcu", 1351 | "outputId": "5db610ee-c78e-40e4-983a-f561e077e5c4" 1352 | }, 1353 | "outputs": [], 1354 | "source": [ 1355 | "mc_model_mh = load_model('path/multi_headed_model_covid_mc.h5')\n", 1356 | "trunc_model = multi_headed_trunc_model(mc_model_mh, mc=True)" 1357 | ] 1358 | }, 1359 | { 1360 | "cell_type": "code", 1361 | "execution_count": null, 1362 | "metadata": { 1363 | "id": "SXhCOKG2KU3T" 1364 | }, 1365 | "outputs": [], 1366 | "source": [ 1367 | "hidden_features=[]\n", 1368 | "for i in range(200):\n", 1369 | " hidden_features.append(trunc_model.predict(x_test))\n", 1370 | "\n", 1371 | "hidden_features=np.array(hidden_features).mean(axis=0)" 1372 | ] 1373 | }, 1374 | { 1375 | "cell_type": "code", 1376 | "execution_count": null, 1377 | "metadata": { 1378 | "colab": { 1379 | "base_uri": "https://localhost:8080/" 1380 | }, 1381 | "executionInfo": { 1382 | "elapsed": 8529, 1383 | "status": "ok", 1384 | "timestamp": 1612632796935, 1385 | "user": { 1386 | "displayName": "Sina Qahremani", 1387 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1388 | "userId": "15549406949129485910" 1389 | }, 1390 | "user_tz": -210 1391 | }, 1392 | "id": "uEGkXjZ3KUnJ", 1393 | "outputId": "ae9cc5f4-10fd-422f-e520-a0228b7fc38b" 1394 | }, 1395 | "outputs": [], 1396 | "source": [ 1397 | "from sklearn.decomposition import PCA\n", 1398 | "from sklearn.manifold import TSNE\n", 1399 | "\n", 1400 | "pca = PCA(n_components=120)\n", 1401 | "pca_result = pca.fit_transform(hidden_features)\n", 1402 | "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n", 1403 | "\n", 1404 | "tsne = TSNE(n_components=2, verbose = 1)\n", 1405 | "tsne_results = tsne.fit_transform(pca_result)" 1406 | ] 1407 | }, 1408 | { 1409 | "cell_type": "code", 1410 | "execution_count": null, 1411 | "metadata": { 1412 | "colab": { 1413 | "base_uri": "https://localhost:8080/", 1414 | "height": 917 1415 | }, 1416 | "executionInfo": { 1417 | "elapsed": 1745, 1418 | "status": "ok", 1419 | "timestamp": 1612632803522, 1420 | "user": { 1421 | "displayName": "Sina Qahremani", 1422 | "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64", 1423 | "userId": "15549406949129485910" 1424 | }, 1425 | "user_tz": -210 1426 | }, 1427 | "id": "i1a-kstDKUTq", 1428 | "outputId": "a63403e0-a38d-4986-9e09-643f7fc75c69" 1429 | }, 1430 | "outputs": [], 1431 | "source": [ 1432 | "import matplotlib.pyplot as plt\n", 1433 | "import matplotlib \n", 1434 | "matplotlib.rc('xtick', labelsize=20) \n", 1435 | "matplotlib.rc('ytick', labelsize=20) \n", 1436 | "\n", 1437 | "plt.rcParams.update({'font.size': 25})\n", 1438 | "%matplotlib inline\n", 1439 | "Name='T-SNE Visualization of Multi-headed model with uncertainty (X-Ray)'\n", 1440 | "fig = plt.figure(figsize=[15, 15])\n", 1441 | "color_map = np.argmax(y_test, axis=1)\n", 1442 | "classes=['COVID19','Normal','Pneumonia']\n", 1443 | "for cl in range(3):\n", 1444 | " indices = np.where(color_map==cl)\n", 1445 | " indices = indices[0]\n", 1446 | " plt.title(Name,fontsize=20)\n", 1447 | " plt.ylabel('Dim_2',fontsize=20)\n", 1448 | " plt.xlabel('Dim_1',fontsize=20)\n", 1449 | " matplotlib.rc('xtick', labelsize=20) \n", 1450 | " matplotlib.rc('ytick', labelsize=20) \n", 1451 | " plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n", 1452 | "\n", 1453 | "plt.rcParams.update({'font.size': 20})\n", 1454 | "\n", 1455 | "plt.legend()\n", 1456 | "plt.show()\n", 1457 | "fig.savefig('{}.pdf'.format(Name),dpi=300)\n", 1458 | "\n" 1459 | ] 1460 | } 1461 | ], 1462 | "metadata": { 1463 | "accelerator": "GPU", 1464 | "colab": { 1465 | "collapsed_sections": [], 1466 | "name": "xray_simple_models.ipynb", 1467 | "provenance": [], 1468 | "toc_visible": true 1469 | }, 1470 | "kernelspec": { 1471 | "display_name": "Python 3", 1472 | "language": "python", 1473 | "name": "python3" 1474 | }, 1475 | "language_info": { 1476 | "codemirror_mode": { 1477 | "name": "ipython", 1478 | "version": 3 1479 | }, 1480 | "file_extension": ".py", 1481 | "mimetype": "text/x-python", 1482 | "name": "python", 1483 | "nbconvert_exporter": "python", 1484 | "pygments_lexer": "ipython3", 1485 | "version": "3.8.5" 1486 | } 1487 | }, 1488 | "nbformat": 4, 1489 | "nbformat_minor": 1 1490 | } --------------------------------------------------------------------------------