├── MFT_Plots ├── HSILidar_channel_Confusionmatrix.png ├── HSILidar_pixel_Confusionmatrix.png ├── HSI_Confusionmatrix.png └── epoch_vs_train_loss.png ├── MFT_Transformer_HSI+LiDAR_Channel_Tokenization.ipynb ├── MFT_Transformer_HSI+LiDAR_Pixel_Tokenization.ipynb ├── MFT_Transformer_HSI_Only_Model.ipynb ├── README.md ├── dataset.py ├── mft_model.py └── record.py /MFT_Plots/HSILidar_channel_Confusionmatrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srinadh99/Transformer-Models-for-Multimodal-Remote-Sensing-Data/8e3bb88a88c545e5ac3300c82531a3073cc4bf3d/MFT_Plots/HSILidar_channel_Confusionmatrix.png -------------------------------------------------------------------------------- /MFT_Plots/HSILidar_pixel_Confusionmatrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srinadh99/Transformer-Models-for-Multimodal-Remote-Sensing-Data/8e3bb88a88c545e5ac3300c82531a3073cc4bf3d/MFT_Plots/HSILidar_pixel_Confusionmatrix.png -------------------------------------------------------------------------------- /MFT_Plots/HSI_Confusionmatrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srinadh99/Transformer-Models-for-Multimodal-Remote-Sensing-Data/8e3bb88a88c545e5ac3300c82531a3073cc4bf3d/MFT_Plots/HSI_Confusionmatrix.png -------------------------------------------------------------------------------- /MFT_Plots/epoch_vs_train_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srinadh99/Transformer-Models-for-Multimodal-Remote-Sensing-Data/8e3bb88a88c545e5ac3300c82531a3073cc4bf3d/MFT_Plots/epoch_vs_train_loss.png -------------------------------------------------------------------------------- /MFT_Transformer_HSI+LiDAR_Channel_Tokenization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "6be4a0ee", 6 | "metadata": {}, 7 | "source": [ 8 | "## Transformer Model for Hyperspectral Image Classification\n", 9 | "\n", 10 | "\n", 11 | "### The MFT model with both HSI and LiDAR images used for classification. The other multimodal data is from the LiDAR and is used to generate the external CLS through 'channel' tokenization. " 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "id": "91f97aaf", 18 | "metadata": { 19 | "ExecuteTime": { 20 | "end_time": "2022-08-11T11:17:16.979544Z", 21 | "start_time": "2022-08-11T11:17:16.971125Z" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "import sys\n", 27 | "sys.path.append(\"./../\")\n", 28 | "import matplotlib.pyplot as plt\n", 29 | "from torchsummary import summary\n", 30 | "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n", 31 | "import math\n", 32 | "from PIL import Image\n", 33 | "import time\n", 34 | "from scipy.io import loadmat as loadmat\n", 35 | "from scipy import io\n", 36 | "import random\n", 37 | "import numpy as np\n", 38 | "import os\n", 39 | "import torch \n", 40 | "import torch.utils.data as dataf\n", 41 | "import torch.nn as nn\n", 42 | "from operator import truediv\n", 43 | "import record\n", 44 | "import pandas as pd\n", 45 | "import seaborn as sns\n", 46 | "from mft_model import MFT, Transformer\n", 47 | "from dataset import Multimodal_Dataset_Train, Multimodal_Dataset_Test\n", 48 | "import torch.backends.cudnn as cudnn\n", 49 | "cudnn.deterministic = True\n", 50 | "cudnn.benchmark = False\n" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 2, 56 | "id": "896fad69", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "def get_confusion_matrix(y_test,y_pred, plt_name):\n", 61 | " df_cm = pd.DataFrame(confusion_matrix(y_test, y_pred), range(6),range(6))\n", 62 | " df_cm.columns = ['Buildings','Woods', 'Roads', 'Apples', 'ground', 'Vineyard']\n", 63 | " df_cm = df_cm.rename({0:'Buildings',1:'Woods', 2:'Roads', 3:'Apples', 4:'ground', 5:'Vineyard'})\n", 64 | " df_cm.index.name = 'Actual'\n", 65 | " df_cm.columns.name = 'Predicted'\n", 66 | " sns.set(font_scale=0.9)#for label size\n", 67 | " sns.heatmap(df_cm, cmap=\"Blues\",annot=True,annot_kws={\"size\": 16}, fmt='g')\n", 68 | " plt.savefig(''+str(plt_name)+'.eps', format='eps')\n", 69 | "\n", 70 | "def AA_andEachClassAccuracy(confusion_matrix):\n", 71 | " counter = confusion_matrix.shape[0]\n", 72 | " list_diag = np.diag(confusion_matrix)\n", 73 | " list_raw_sum = np.sum(confusion_matrix, axis=1)\n", 74 | " each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n", 75 | " average_acc = np.mean(each_acc)\n", 76 | " return each_acc, average_acc\n", 77 | "\n", 78 | "def reports (xtest,xtest2,ytest,name,model, HSIOnly, iternum):\n", 79 | " pred_y = np.empty((len(ytest)), dtype=np.float32)\n", 80 | " number = len(ytest) // testSizeNumber\n", 81 | " for i in range(number):\n", 82 | " temp = xtest[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 83 | " temp = temp.cuda()\n", 84 | " temp1 = xtest2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 85 | " temp1 = temp1.cuda()\n", 86 | " if HSIOnly:\n", 87 | " temp2 = model(temp)\n", 88 | " else:\n", 89 | " temp2 = model(temp,temp1)\n", 90 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 91 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 92 | " del temp, temp2, temp3,temp1\n", 93 | "\n", 94 | " if (i + 1) * testSizeNumber < len(ytest):\n", 95 | " temp = xtest[(i + 1) * testSizeNumber:len(ytest), :, :]\n", 96 | " temp = temp.cuda()\n", 97 | " temp1 = xtest2[(i + 1) * testSizeNumber:len(ytest), :, :]\n", 98 | " temp1 = temp1.cuda()\n", 99 | " if HSIOnly:\n", 100 | " temp2 = model(temp)\n", 101 | " else:\n", 102 | " temp2 = model(temp,temp1)\n", 103 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 104 | " pred_y[(i + 1) * testSizeNumber:len(ytest)] = temp3.cpu()\n", 105 | " del temp, temp2, temp3,temp1\n", 106 | "\n", 107 | " pred_y = torch.from_numpy(pred_y).long()\n", 108 | " \n", 109 | " oa = accuracy_score(ytest, pred_y)\n", 110 | " confusion = confusion_matrix(ytest, pred_y)\n", 111 | " each_acc, aa = AA_andEachClassAccuracy(confusion)\n", 112 | " kappa = cohen_kappa_score(ytest, pred_y)\n", 113 | " get_confusion_matrix(ytest, pred_y, 'test_'+str(iternum)+'')\n", 114 | " \n", 115 | " return confusion, oa*100, each_acc*100, aa*100, kappa*100\n" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 3, 121 | "id": "16b7982c", 122 | "metadata": { 123 | "ExecuteTime": { 124 | "end_time": "2022-08-11T11:18:10.978783Z", 125 | "start_time": "2022-08-11T11:18:01.894932Z" 126 | }, 127 | "scrolled": false 128 | }, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "---------------------------------- Dataset details for Trento ---------------------------------------------\n", 135 | "\n", 136 | "\n", 137 | "HSI Train data shape = torch.Size([819, 63, 11, 11])\n", 138 | "LIDAR Train data shape = 1\n", 139 | "Train label shape = torch.Size([819])\n", 140 | "HSI Test data shape = torch.Size([29395, 63, 11, 11])\n", 141 | "LIDAR Test data shape = 1\n", 142 | "Test label shape = torch.Size([29395])\n", 143 | "Number of Classes = 6\n", 144 | "\n", 145 | "\n", 146 | "---------------------------------- Model Summary ---------------------------------------------\n", 147 | "\n", 148 | "\n", 149 | "===============================================================================================\n", 150 | "Layer (type:depth-idx) Output Shape Param #\n", 151 | "===============================================================================================\n", 152 | "├─Sequential: 1-1 [-1, 8, 55, 11, 11] --\n", 153 | "| └─Conv3d: 2-1 [-1, 8, 55, 11, 11] 656\n", 154 | "| └─BatchNorm3d: 2-2 [-1, 8, 55, 11, 11] 16\n", 155 | "| └─ReLU: 2-3 [-1, 8, 55, 11, 11] --\n", 156 | "├─Sequential: 1-2 [-1, 64, 11, 11] --\n", 157 | "| └─HetConv: 2-4 [-1, 64, 11, 11] --\n", 158 | "| | └─Conv2d: 3-1 [-1, 64, 11, 11] 31,744\n", 159 | "| | └─Conv2d: 3-2 [-1, 64, 11, 11] 28,224\n", 160 | "| └─BatchNorm2d: 2-5 [-1, 64, 11, 11] 128\n", 161 | "| └─ReLU: 2-6 [-1, 64, 11, 11] --\n", 162 | "├─Sequential: 1-3 [-1, 64, 11, 11] --\n", 163 | "| └─Conv2d: 2-7 [-1, 64, 11, 11] 640\n", 164 | "| └─BatchNorm2d: 2-8 [-1, 64, 11, 11] 128\n", 165 | "| └─GELU: 2-9 [-1, 64, 11, 11] --\n", 166 | "├─Dropout: 1-4 [-1, 5, 64] --\n", 167 | "├─TransformerEncoder: 1-5 [-1, 64] --\n", 168 | "| └─ModuleList: 2 [] --\n", 169 | "| | └─Block: 3-3 [-1, 5, 64] 100,736\n", 170 | "| | └─Block: 3-4 [-1, 5, 64] 100,736\n", 171 | "| └─LayerNorm: 2-10 [-1, 5, 64] 128\n", 172 | "├─Linear: 1-6 [-1, 6] 390\n", 173 | "===============================================================================================\n", 174 | "Total params: 263,526\n", 175 | "Trainable params: 263,526\n", 176 | "Non-trainable params: 0\n", 177 | "Total mult-adds (M): 12.34\n", 178 | "===============================================================================================\n", 179 | "Input size (MB): 0.03\n", 180 | "Forward/backward pass size (MB): 1.12\n", 181 | "Params size (MB): 1.01\n", 182 | "Estimated Total Size (MB): 2.15\n", 183 | "===============================================================================================\n", 184 | "\n", 185 | "\n", 186 | "---------------------------------- Training started for Trento ---------------------------------------------\n", 187 | "\n", 188 | "\n", 189 | "Epoch: 0 | train loss: 1.9849 | test accuracy: 1.2723\n", 190 | "Epoch: 1 | train loss: 1.0730 | test accuracy: 33.4513\n", 191 | "Epoch: 2 | train loss: 0.5160 | test accuracy: 44.8376\n", 192 | "Epoch: 3 | train loss: 0.2624 | test accuracy: 49.1818\n", 193 | "Epoch: 4 | train loss: 0.1144 | test accuracy: 64.8920\n", 194 | "Epoch: 5 | train loss: 0.1248 | test accuracy: 63.9871\n", 195 | "Epoch: 6 | train loss: 0.0985 | test accuracy: 92.0837\n", 196 | "Epoch: 7 | train loss: 0.0711 | test accuracy: 96.3089\n", 197 | "Epoch: 8 | train loss: 0.0253 | test accuracy: 91.8081\n", 198 | "Epoch: 9 | train loss: 0.0309 | test accuracy: 97.4690\n", 199 | "Epoch: 10 | train loss: 0.0133 | test accuracy: 75.7170\n", 200 | "Epoch: 11 | train loss: 0.0158 | test accuracy: 97.4520\n", 201 | "Epoch: 12 | train loss: 0.0527 | test accuracy: 90.5664\n", 202 | "Epoch: 13 | train loss: 0.0106 | test accuracy: 88.2327\n", 203 | "Epoch: 14 | train loss: 0.0126 | test accuracy: 97.5574\n", 204 | "Epoch: 15 | train loss: 0.0191 | test accuracy: 95.7170\n", 205 | "Epoch: 16 | train loss: 0.0115 | test accuracy: 81.9085\n", 206 | "Epoch: 17 | train loss: 0.0277 | test accuracy: 97.5540\n", 207 | "Epoch: 18 | train loss: 0.0113 | test accuracy: 97.5642\n", 208 | "Epoch: 19 | train loss: 0.0104 | test accuracy: 97.0607\n", 209 | "Epoch: 20 | train loss: 0.0102 | test accuracy: 97.3703\n", 210 | "Epoch: 21 | train loss: 0.0074 | test accuracy: 97.6833\n", 211 | "Epoch: 22 | train loss: 0.0101 | test accuracy: 97.5268\n", 212 | "Epoch: 23 | train loss: 0.0089 | test accuracy: 97.2955\n", 213 | "Epoch: 24 | train loss: 0.0077 | test accuracy: 97.5506\n", 214 | "Epoch: 25 | train loss: 0.0064 | test accuracy: 97.7139\n", 215 | "Epoch: 26 | train loss: 0.0051 | test accuracy: 97.3465\n", 216 | "Epoch: 27 | train loss: 0.0207 | test accuracy: 97.6935\n", 217 | "Epoch: 28 | train loss: 0.0121 | test accuracy: 97.5574\n", 218 | "Epoch: 29 | train loss: 0.0110 | test accuracy: 97.6220\n", 219 | "Epoch: 30 | train loss: 0.0064 | test accuracy: 97.5234\n", 220 | "Epoch: 31 | train loss: 0.0074 | test accuracy: 97.7445\n", 221 | "Epoch: 32 | train loss: 0.0065 | test accuracy: 96.2170\n", 222 | "Epoch: 33 | train loss: 0.0079 | test accuracy: 97.6697\n", 223 | "Epoch: 34 | train loss: 0.0058 | test accuracy: 97.6935\n", 224 | "Epoch: 35 | train loss: 0.0060 | test accuracy: 96.1728\n", 225 | "Epoch: 36 | train loss: 0.0049 | test accuracy: 97.7275\n", 226 | "Epoch: 37 | train loss: 0.0081 | test accuracy: 95.7340\n", 227 | "Epoch: 38 | train loss: 0.0072 | test accuracy: 97.7683\n", 228 | "Epoch: 39 | train loss: 0.0053 | test accuracy: 97.5846\n", 229 | "Epoch: 40 | train loss: 0.0048 | test accuracy: 96.7682\n", 230 | "Epoch: 41 | train loss: 0.0065 | test accuracy: 97.9282\n", 231 | "Epoch: 42 | train loss: 0.0074 | test accuracy: 97.8704\n", 232 | "Epoch: 43 | train loss: 0.0057 | test accuracy: 97.5472\n", 233 | "Epoch: 44 | train loss: 0.0066 | test accuracy: 97.6731\n", 234 | "Epoch: 45 | train loss: 0.0066 | test accuracy: 91.7367\n", 235 | "Epoch: 46 | train loss: 0.0065 | test accuracy: 97.7343\n", 236 | "Epoch: 47 | train loss: 0.0113 | test accuracy: 97.1015\n", 237 | "Epoch: 48 | train loss: 0.0096 | test accuracy: 97.2853\n", 238 | "Epoch: 49 | train loss: 0.0084 | test accuracy: 97.7105\n", 239 | "Epoch: 50 | train loss: 0.0114 | test accuracy: 72.1653\n", 240 | "Epoch: 51 | train loss: 0.0114 | test accuracy: 94.2575\n", 241 | "Epoch: 52 | train loss: 0.0131 | test accuracy: 96.1796\n", 242 | "Epoch: 53 | train loss: 0.0070 | test accuracy: 97.5030\n", 243 | "Epoch: 54 | train loss: 0.0081 | test accuracy: 97.6220\n", 244 | "Epoch: 55 | train loss: 0.0117 | test accuracy: 98.0507\n", 245 | "Epoch: 56 | train loss: 0.0071 | test accuracy: 97.9350\n", 246 | "Epoch: 57 | train loss: 0.0108 | test accuracy: 97.7887\n", 247 | "Epoch: 58 | train loss: 0.0068 | test accuracy: 98.0031\n", 248 | "Epoch: 59 | train loss: 0.0113 | test accuracy: 97.6799\n", 249 | "Epoch: 60 | train loss: 0.0084 | test accuracy: 97.8500\n", 250 | "Epoch: 61 | train loss: 0.0090 | test accuracy: 97.8874\n", 251 | "Epoch: 62 | train loss: 0.0070 | test accuracy: 96.9689\n", 252 | "Epoch: 63 | train loss: 0.0084 | test accuracy: 97.6357\n", 253 | "Epoch: 64 | train loss: 0.0073 | test accuracy: 97.9452\n", 254 | "Epoch: 65 | train loss: 0.0079 | test accuracy: 96.3599\n", 255 | "Epoch: 66 | train loss: 0.0081 | test accuracy: 97.5132\n", 256 | "Epoch: 67 | train loss: 0.0095 | test accuracy: 97.9554\n", 257 | "Epoch: 68 | train loss: 0.0122 | test accuracy: 96.0027\n", 258 | "Epoch: 69 | train loss: 0.0182 | test accuracy: 97.3771\n", 259 | "Epoch: 70 | train loss: 0.0200 | test accuracy: 68.4164\n", 260 | "Epoch: 71 | train loss: 0.0167 | test accuracy: 97.4962\n", 261 | "Epoch: 72 | train loss: 0.0130 | test accuracy: 97.3329\n", 262 | "Epoch: 73 | train loss: 0.0093 | test accuracy: 97.5778\n", 263 | "Epoch: 74 | train loss: 0.0064 | test accuracy: 97.3125\n", 264 | "Epoch: 75 | train loss: 0.0062 | test accuracy: 94.9855\n", 265 | "Epoch: 76 | train loss: 0.0154 | test accuracy: 97.4758\n", 266 | "Epoch: 77 | train loss: 0.0093 | test accuracy: 96.8532\n", 267 | "Epoch: 78 | train loss: 0.0070 | test accuracy: 96.7954\n", 268 | "Epoch: 79 | train loss: 0.0096 | test accuracy: 97.0233\n", 269 | "Epoch: 80 | train loss: 0.0105 | test accuracy: 97.4996\n", 270 | "Epoch: 81 | train loss: 0.0083 | test accuracy: 96.6525\n", 271 | "Epoch: 82 | train loss: 0.0100 | test accuracy: 96.2102\n", 272 | "Epoch: 83 | train loss: 0.0096 | test accuracy: 95.8564\n", 273 | "Epoch: 84 | train loss: 0.0080 | test accuracy: 95.7170\n", 274 | "Epoch: 85 | train loss: 0.0085 | test accuracy: 96.7409\n", 275 | "Epoch: 86 | train loss: 0.0119 | test accuracy: 96.1456\n", 276 | "Epoch: 87 | train loss: 0.0103 | test accuracy: 95.1829\n", 277 | "Epoch: 88 | train loss: 0.0087 | test accuracy: 95.5809\n", 278 | "Epoch: 89 | train loss: 0.0078 | test accuracy: 96.3157\n", 279 | "Epoch: 90 | train loss: 0.0081 | test accuracy: 94.3766\n", 280 | "Epoch: 91 | train loss: 0.0130 | test accuracy: 96.6559\n" 281 | ] 282 | }, 283 | { 284 | "name": "stdout", 285 | "output_type": "stream", 286 | "text": [ 287 | "Epoch: 92 | train loss: 0.0178 | test accuracy: 97.1798\n", 288 | "Epoch: 93 | train loss: 0.0085 | test accuracy: 96.9995\n", 289 | "Epoch: 94 | train loss: 0.0098 | test accuracy: 96.8702\n", 290 | "Epoch: 95 | train loss: 0.0092 | test accuracy: 94.3528\n", 291 | "Epoch: 96 | train loss: 0.0107 | test accuracy: 97.2240\n", 292 | "Epoch: 97 | train loss: 0.0080 | test accuracy: 96.6253\n", 293 | "Epoch: 98 | train loss: 0.0098 | test accuracy: 95.9517\n", 294 | "Epoch: 99 | train loss: 0.0086 | test accuracy: 96.2919\n", 295 | "\n", 296 | "The train time (in seconds) is: 1246.8682582378387\n", 297 | "\n", 298 | "\n", 299 | "Overall Accuracy = 98.05068889266883\n", 300 | "\n", 301 | "\n", 302 | "----------Trento Training Finished -----------\n", 303 | "\n", 304 | "The Confusion Matrix on test data\n" 305 | ] 306 | }, 307 | { 308 | "data": { 309 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAEJCAYAAAA5Ekh8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABSCElEQVR4nO3dd3gU1dfA8W8qECBNOqEGOBTpHUWKiooCNsAK2LAjvhaUn4pdsQKCDUVQqQKCIlWa0rv0q6FJSOiQUBJIe/+YSUjZwCZks7vmfHj2YffulDOT2T1779y545OWloZSSinlTr7uDkAppZTSZKSUUsrtNBkppZRyO01GSiml3E6TkVJKKbfTZKSUUsrt/N0dgLcZt26/1/WF792kirtDyJNUL7zcwNfHx90hKA9U3J/LPjBKNH3K6Q9EwsaRXnsgajJSSilP5lM0GrA0GSmllCfz9XN3BIVCk5FSSnkyFzQBi8hooAcQY4xpYpddAUwBqgIG6G2MOSMiPsAooAtwCrjHGLPDnuch4CUgDRhsjJlql7cEvgOKAz8ZY16+VExFo/6nlFLeysfX+YfzvgduzFb2MvCrMaY2sBEYYJffDFQyxtQCngeGQUbyegloDlwNDBWREvY8nwN9gDpAOxFpe6mANBkppZQn8/Fx/uEkY8yfwPFsxd2BH+zn3wO3Zi83xiwE6otISeAGYK4xJt4YcxhYBXQUkUpAcWPMBmNMKvBjpmXlSpvplFLKk+WhxiMioUCog7dOGmNOXmL2K4wxx+zn0UBl+3kl4ECm6WKAig7K0+c54qC8w6Vi15qRUkp5srzVjAYCexw8BjqxJrdeU6E1I6WU8mR56003DBjroPykE/MeF5H02lEEF2o3MVyoJYFVI4q1yzOfC4oAljiYPvOycqXJSCmlPFkemunspriT+VzTLOB+rITWB5iZqfxBYJqIXAvssHvZzQfeFJFgrF5zbYGHjTEJInJeRJoBm4D7sDpHXJQmI6WU8mSu6dr9A3AdcIWIRGP1insPmCIiTwL/AL3tyWcBXUVkF1bX7nsBjDFHRWQosAGrie8lY0yCPc8TWJ0eigPTjDErLhWTj97pNW8yDwe0e/NaVv46maMH9pF45jRBpUOoXKc+7W/vS9mIahnz7DdbWTb9Bw79u4vk8+cIq1CZFtf3oHHHm7IsO/n8eZZO/Y6tyxdy7sxpyleLpNNdj1C1XqOMaY7FRrN+wUz2bd/EycOxBJYIolJN4Zo7+1G+WqTDmAtyOKCDsbF8OPQ9Vq1cTlpaGq3btuPFQYOpWKlSga3DmeGADh08yHdjRrN921b+MYbExER+m/c7lSpHZJnu3LlzfP7ZcGbP+pVTp+KpU7cuzzz7PM1btMyY5syZ07zx2ivs3L6dI0eP4O/vT7Vq1bn73vu5uVt3p2IuyOGACmMfFzSN2bECGQ7omtedHw7oj9e9djggTUZ5lDkZbVuxiIN7/6FSZD2CgkOIP3qYlb9OIv7YER55fzQhZctz+N/djH3tKSrVqkerG28noFhxdqz5g02LfuOGBwbQ/LoLX3YzR71L1KbVdL67P6HlKrJ+wS/s/msNfV8fQfnqtQBYN38GGxf9RqP2XShfvTbnzp5m1azJHNq3i/uHDKNijTo5Yi6oZJSQkECv23sQEBjIUwMG4uMDI0cMJzExgZ+m/0JQUFCBrMeZZLRuzWoGPf9/1GvQgNSUFFauWO4wGQ0e9Dx//rGUZ597gcoRVZgycQLLl/3BuPGTkLr1ADh58gRD332bVq3bUqlyZc6fP8/8ubOZ9ctMnnvxJe7r0++S8RRUMiqsfVyQNObcFUgy6vCm88lo6Wtem4xc1kwnIinAFvtlCvCkMWbVJeaZjVU1vAKYkX5lcLZpNmH1WT8PfGqM6Z19msLSoF1nGrTrnKWsUmRdvnrhAXau+YPWN/dk+8rFpKam0uv5twksbl0PVqNhc478u5utfy7ISEaH9u1i24pF3Nz/eRp3sK5Fq1avMV8Peog/po2j53NvAVC/bSeaX98Dn0xfftUaNOHzZ+5j7dzpdH/8JZdt7/SpU4iO3s/MWXOpWs2q+dWuI3TvegNTp0ymT78HXLbu7Jq1aMnCP5bbcf3EyhXLc0xjdu5kzm+zeP2td+hx2x0ANG/RkjtvvYXPR45g+MgvAAgNDeO9Dz7OMm/7azqwb99eZv48zalkVFA8aR87S2N2Mb+iMRyQK7t2nzLGNLETymDgnUvNYIzpaow55czCjTEx7kxEuSlROhgAH/sASklOxs/fH//AwCzTFQsqSeZa6T8bVuLr50/9Nh0zynz9/KjfphO7N68jOek8AEGlQ7IkIoDiQaUIrxjBqRPHcKUlixfRqFHjjA8vQEREFZo0bcaSxQtduu7sfH0vfeguXbIIf/8AutzYNaPM39+fG27sysrlyzh//vxF5w8NCcXPr3BPq3rSPnaWxuxiLrjo1RMV1nVGwUAcgIh0FJEZ6W+IyFgRudV+vte+aItM7weJyE8isl1EJgEBdnl1u5aEiPQTkakiMl9EokTk1UzzDxERIyJLReRnEelnl79vL3OziHx0ORuXmppCSnISxw9GM+fbTykZGk6Dtp0AaHhNFwDmjxvFqRNHSTxzmo2LfmPvto20vOmOjGUcid5LaLkKBBQrnmXZZSKqkZKcxIlDMbmuP+F0PEei91KmUtXL2YxL2hUVRWTtnM2AkZG12L0ryqXrzo9dUVFUjqhMiRIlspRH1qpNUlIS+//dl6U8LS2N5ORkTp48wbSfJrNyxXLuvb9vYYbsdfsYNGaXc81wQB7HlT/7StvJojhQAbg2n8t5AjhqjKkvIlcBy3KZrjHQDEgF/hGRz4DaWENZNAKCsJoNZ9pjKt0G1DXGpGVPgHk19rWnObjnbwDCylfm3sEfUjIkDIByVWpw7/8+Ytqnr7Ph918A8PXz58YHB2YkLIDE06coHlQqx7JLlCoNQMLp3CuM88eNJC0tjZY33n45m3FJcXFxBAcH5ygPCQkhPj7epevOj/i4kwQHh+QoDw6xyuLi4rKUT544nqHvvg2Av38AL7w0mG49bnV5nJl52z4GjdnlvLzG4yxXJqNTmUaDbQt8LyJX5mM5VwOfABhjlovInlym+z29ic/ughgBXIV17ukccE5E5tnTxgGJwLciMgur62K+dX98EOcSznLycCyrf/uJie8N4v4hwwgtW4HjB6OZPvwNykRU48YHB+IfGMg/61cwd8ww/AMCufKq/OZoy4qZE6xzTY88R3iFypeeQeWqy41dadioMSdPnmTp4kUMffdtfH19ubPXXe4OTRVlXl7jcVahNIgbY1aKSBmgLJBM1ubBYk4swpneJOcyPU/hIttmjEkWkVZYtbXbgaeAzrlNfyllKlvtzpVr1SOycStGDbyXlb9M4qaHBrJk8hh8/fzp9fw7+PlbIdW4shkJp+NZ8P0oGrTthI+vL8VLliLu2KEcy06vEaXXkDLb8PuvLJkyhg49H8jRTdwVgkOCHf5qzO1XprsFB4cQG5uzeTPerhGFhGStNYWHhxMeHg7AVVe3JzExgU8/+oAet91BQECA6wPG+/YxaMwuV0RqRoWSckWkLuAHHAP2AfVEJEBEwrn0AHrLsC++smtYNfKw6uVAdxEJFJEQrPtxICKlgFBjzGzgWaBhXrbnYoqXLEVY+cqcOGSNfnF4/x7KVY3MSETpKkbWJeF0PGfiTwJQJqI6Jw8fJOlcYpbpjh7Yh59/AGHls177sOXPBcwdO4LWXXty1a33FlT4FxUZWYtdUf/kKN+9exc1I2sVSgx5UbNWLQ5EHyAhISFL+e5dUQQEBFClarVc5rTUb3AlZ8+e5fgx13YMyczb9jFozC7n6+f8w4u5MhmVFpFN9nmjyUBfY0yKMWY/8DOwDWto8Q2XWM7nQAUR2Y51f43tzgZgjFkLzAG2Yg1tsRmIB0oDv4rIZqyE9XxeNuxiTsed4FjMv4TayaNUaBiH90WRkpyUZbqYqB34BwRm1HhqN2tDakoyO1b/kTFNakoKO1YtpUbD5vgHXOiNZ9YuY9bXH9Kk401ce++jBRX6JXXs1Jktm/8iev/+jLIDB6LZtHEDHTrlu2LpMh06diI5OYkF8+dmlCUnJzN/7hzatruKwGw9HLNbv24tQUFBhF8R7upQM3jbPgaN2eWKSAeG//xFryJSyhhz2h4/aRVwizFmd36Xl/mi16mfDqFC9dqUq1KDwBIlOX4wmjVzpnEm7jj93hzFFRUj2LH6D34e8SY1Gjan+XXd8Q8sxj8bVrJ+wUxa3XQH1933eMayf/7sbfZsXkfne/oTUrYCG37/lahNq+g7ZAQVatQG4N8dm5k4dBBlK1enS9+nsnTz9gsIoEL12jliLqiLXs+ePUuv23tQrHhxnhrwDD74MOqz4Zw5e4ap038hqGTJAlmPMxe9AhlJZs2qVUydMomXXxlCWHgYYWHhtGjZCoBBz/8fK1csY+D/vUDliAh+mjyRP5cuYeyPE6lXvwEAU6dMYsvmv2jdph3lypcn7uRJ5s+by/y5sxnw7HM88NAjl4yloC56Lax9XJA05twVyEWvN49w/qLX3wZ4bZteUUhGE4D6WOemPjfGfHY5y8ucjFb+Ookdq5Zy4nAMKcnJBF9Rlqr1GtOu+92Elq2QMc+uTWtYOWsSR6L3kZJ0ntBylWjauStNr70F30xV66Tz51g6ZQzbViwi8expyleNpNNdD1OtfpOMaf6YNo5l03/AkZAy5Xly+Pgc5QU5HFBsTEzWIVTatOWFlwZTOdvIB5fD2WTU9Mq6Dsubt2jJN2OtfZSYmMjIEZ8y97ffrOGApC7PPPscLVq1zph+08YNfPP1l5gdO4iLO0loWBg1akZy3/19ad+ho1OxFORwQIWxjwuaxuxYgSSjW0Y6n4xmPaXJqKjInIy8RUEmo8LgbDLyJAWZjNR/R4Eko26fO5+Mfn3Caw9EHbVbKaU8mZd3THCWJiOllPJkRaTWrclIKaU8mZf3knOWJiOllPJkWjNSSinlbtlH6f+v0mSklFIeTJORUkopt/Px1WSklFLKzbRmpJRSyu00GSmllHI7TUZKKaXcr2jkIk1GSinlybRmpBzytkFHAdbsPu7uEPKkVc3Cu3+QUp7O11dHYFBKKeVmWjNSSinlfkUjF2kyUkopT6Y1I6WUUm6nyUgppZTb6XBASiml3E5rRkoppdxOk5FSSim302SklFLK7VyVjETkWeBh++Uq4FGgGjARCAeWAQ8ZY1JEpDjwI9AEOAj0NMbE2st5BegHJAH9jTF/5ieeonFpr1JKeSufPDycJCIVgKeB5sCVQCXgRmAo8K4xphbgB/SyZ3kYiLbLvwNet5fTEOgB1ANuA77I72ZqzciNDsbG8uHQ91i1cjlpaWm0btuOFwcNpmKlSi5b5/rli1izdAF7o3ZyKu444WUr0KxtB7r27EvxoJIAjPn0LVYumu1w/gqVq/LWl5MB+GXCN/w68VuH0/kHBPLF9KVZyk4cO8zMH79my7qVnD19ipDwMrS65jpu7/tEAW5hTu7Yz5fD2+IF74t5wby5zJn9G9u3beX48WNUqFiRa6/rwsP9H6VkyVLuDi8LFw0H5IP1/V8cSLX/PwS050IC+h4rCU0EugMv2+UTgTfs592AScaYJGCniMSJSD1jzI68BqTJyE0SEhJ45MG+BAQG8ta7Q/HxgZEjhvPwg334afovBAUFuWS9836eQHiZ8tzW5zHCrijL/t1/88vEb9m5ZQMvffA1vr6+3HLXA3S46bYs8x07HMvoD1+jcev2GWVXd+lOg2Ztskx3/lwCw4c8S+NWV2cpP3oolqEv9qdM+Urc1f9ZgkPDOXY4lsOx0S7ZznTu2s/55W3xgnfGPG7sGCpWrMjTA5+lfPkK7NyxnS8/H8naNav5fvwkjxoPLi/NdCISCoQ6eOukMeZk+gtjTKyIfAzsB84Dk4A9wAljTKo9WTRQ2X5eCThgz3tWRPxEJMAuX5ZpPenzaDLyFtOnTiE6ej8zZ82larVqANSuI3TvegNTp0ymT78HXLLep1/9kNIhYRmvpWEzgkoH892nb2G2bKBe4xaUqxhBuYoRWebbsWkNAG07d80oCy9TjvAy5bJMt3LRHFJSUmh3bdcs5T9+PpTQK8ry3Luj8PcvvMPOXfs5v7wtXvDOmEeM+pLw8AsD8rZo2YqQkFBeGTyItWtW07pNWzdGl03eThkNBIY4KH8Du2kNQETCsGo11YHTwAysZjq38Zj0LyKfishTmV6vEpGPMr2eLCJ3XMbyB4rI65cZZoFZsngRjRo1zvjwAkREVKFJ02YsWbzQZevNnIjS1ahdH4CTx47kOt/KRXOoVqsulavVvOjyVy6aTXBoOA2atc4oOxwbzbYNq+l8S89CTUTgvv2cX94WL3hnzJkTUboGVzYE4PDhQ4UdzkX5+Pg4/QCGATUcPIZlW+x1wC5jzDFjzDngF6ABECYi6XkhArs2BMRg15JEpASQYjfNZZQ7mCdPPCYZASuBtgAiUgwIwDq5lq4tsMINcbnErqgoImvXyVEeGVmL3buiCjUWs3UDABWrVHf4ftT2vzgcG52lVuTI8SOH2LllA6073oCf34WkE7VjMwCBxYrxyasDePy2a3jmri58+8kbnI6PK5iNyIUn7WdneFu84J0xO7JunVX7r1kz0s2RZJWXZGSMOWmM2evgcTLbYv8F2opICRHxAToBO4HlWOeHAPoAM+3ns4D77ef3ALMzld8lIgEiUhcIzc/5IvCsZroVWD05wEpCy4GmdrtkOSAF6CQiL2NVXKcZY4YAiMg9WCfXspc/DAwCjgEGq00UERkAPAYkA9uNMXcVyhZmEhcXR3BwcI7ykJAQ4uPjCy2OE8cO88v4b6jXpCXVa9dzOM3KRXPw8/en1TXXX3RZq5bMJS01lXbZklbcsaMAjB3+Dm063chNd/bhSGw007//gtj9exn88bcua6P3lP3sLG+LF7wz5uwOHTrE5yNH0KZtu4wakqdwRdduY8xqEfkZ2ID13boKGI/1vTvJPp+0HJhizzIaGC8iUVgdHXray9ksIrOwElkSVvfwfPGYZGSMibZPilXAqgWtwqq5NcXq+74HeAdoCcQDS0VkKdZOyK38f1iJLcFe3h57dS8BNYwx5+wTfkVSYsJZRr09CF8/P/o984rDaZLOn2PdskU0ankVpUNCL7q8lYvmULVmHSJq1MpSnppmnQ+Vhs249/EXAKjXuAUlSpbi6w9eZduG1TRs4UFt9KpIOXvmDAOffhx/Pz/efPs9d4eTg6vGprN/tGc/vxQFtHAwbQJwey7LeYMLvevyzZOa6eBCU11b+/mqTK9nAouNMUeNMem9P9pjJSFH5a3s8uP2jpyWaT2bsbL8fVi1o0IXHBLs8Fdjbr8yC9r5c4mMfOsFjhw8wMA3Ps3RESHdptV/cvbMqRy1nez2/L2Ng9H7aHttzulKlQ4BoF6TVlnK6ze1Xv+72+RnE5zi7v2cV94WL3hnzOkSExN5+snHiN4fzRdff0v5ChXcHVIOeTxn5LU8MRm1w6q17MFKRm3sstB8LC8tl/KbgZFYta41IlLoNcTIyFrsivonR/nu3buoGVnLwRwFJzk5mS/f/x97o3YyYMgnRFTPfX0rF82hVHAoV7Zod9Flrlg4Gz9/f1p36JLjvUpVL97pwdfHdYehO/dzfnhbvOCdMQMkJSXx/LMD2L5tK6O+/JradcTdITmkycg9VgB3YPV9xxgThXVlbx1gDNBRRMJFJBDoDfwBrLlIeScRCbOHsrgNwO4pUsUYswTrfFJpoNCvcuvYqTNbNv9F9P79GWUHDkSzaeMGOnTq7LL1pqam8s1HQ9i5eR1P/u99Iutemeu08SeOs23Dalp36HLRXnDJSUms/fN3rmze1mFvvZp1GxASdgXbNqzKUr51vfW6eh3H56oKgrv2c355W7zgnTGnpqYyeNDzrFm9imGffU6jxk3cHVKufHycf3gzT0tGG4GKWDWidNHAZmPMfuAVYCnWSbcFxpjF9vhIjspjgHexktISrKY5sIa4+FFEtgCbgFEOepq43O139qJSpco88/QTLF70O0sWLWTgU09QvkIFevbs7bL1TvjyI9YvX8T1t95DYLES7Nq5NeNx/OjhLNOuWjqP1NSUS/ai27x2OWdOxefalOfn58/tfR9ny7oV/DBqKNs2rGbxb9OY8MVHSMNm1G2Uo4m6wLhrP+eXt8UL3hnzu2+/wfx5c+nT9wFKlCjB5r82ZTwOHTzo7vCyKCo1I5+0tNxaspQjicm5Nv3lWWxMTNYhVNq05YWXBlO5csSlZ86DNbuPZzx/6aHbOHbY8Yet290P0f2ehzNev/H0/aSlpfL6yPEXXf7It18kavtffDRuFv4BAblOt3LRHOZO+5HDMfspWTqYZld14vY+j1O8RNYr9FvVzHkNyOUorP1cULwtXvC+mG+6vjMxMY4vh3nsiad4/MmnC2Q9xf3zeMmqAzJontPfOWboDV6bkTQZ5VFBJqPCkjkZeYOCTkZKuUtBJKO6LzmfjHa+773JyGO6diullMrJV287rpRSyt28/FSQ0zQZKaWUB/P2jgnO0mSklFIerIjkIk1GSinlyTzp3kqupMlIKaU8mNaMlFJKuZ2eM1JKKeV2RSQXaTJSSilPpjUjpZRSbldEcpEmI6WU8mQ6AoNSSim302Y69Z/hbQOPnktKdXcIeVYsoGhcC6IKXxHJRZqMlFLKk2nNSCmllNsVkVykyUgppTyZdmBQSinldtpMp5RSyu00GSmllHK7IpKLNBkppZQn05qRUkoptysiuUiTkVJKeTLtTaeUUsrtfItI1UiTkRsdjI3lw6HvsWrlctLS0mjdth0vDhpMxUqV3B1arjwl5pUrlvH9d9+wZ/cuTsXHERYWTsPGTXnksSepGVkLgMce6sOG9Wsdzt+m3dWM+Hw0AGtWr2TWzJ/ZsnkTR48cpkzZcrRu247+jz9NePgVhbZN6TxlH+eFt8W8YN5c5sz+je3btnL8+DEqVKzItdd14eH+j1KyZCl3h5dFEclF+KSlpbk7Bq+SmEyB7LCEhAR63d6DgMBAnhowEB8fGDliOImJCfw0/ReCgoIKYjUFqrBidmZsunlzfsPs2E6Dho0ICwvj4MFYxo0ZzeFDB5nw00wqVqrM7l1RnDlzOst8W/7axLCPh/Liy69yZ+97AHjp+YEkJJzluutvpHJEBP/+u4/RX4wkIDCQCT/NICio5CXjKaix6fS4KBz33d2LihUr0rHztZQvX4GdO7bz5ecjqV6jJt+Pn4Svb8H8PYv7c9mp5IbPVzv9nTPvidZem7q0ZuQm06dOITp6PzNnzaVqtWoA1K4jdO96A1OnTKZPvwfcHGFOnhTzDTfdzA033ZylrMGVjeh5a1cW/T6fe/s8kFFDymzG9J8ICAjg+hu7ZpQNGvwaYeEXBpNt1qIVVatV57GH+vD7/Ll0v/UO121INp60j53ljTGPGPUl4Zn+5i1atiIkJJRXBg9i7ZrVtG7T1o3RZVVEThnhsUMNi0iKiGwSkW0i8quIhBbAMkNFZO/lR3f5lixeRKNGjTM+vAAREVVo0rQZSxYvdGNkufP0mENCQgHw8/Nz+H5iQgKLFsyjfYdOGdMCWRJRuvoNGgJw5PChAo/zYjx9HzvijTGHO/ibN7jS+psfLuS/+aX4+vo4/fBmnlwzOmWMaQIgIj8ATwLvuDWiArQrKoqOna/NUR4ZWYsF8+e6IaJL88SYU1JSSE1NITYmhlEjPuGKMmXokq3GlG7Jot85c+YMN3e79ZLL3Wifa6peI7Igw70kT9zHl+KNMTuybt0aAGrWLNy/+aX4XH5Ln0MiEg58C9QD0oCeQAIwEQgHlgEPGWNSRKQ48CPQBDgI9DTGxNrLeQXoByQB/Y0xf+YnHk9ORpktBxoDiEgT4CugBLADeNgYc0pEHgEeAYoB24G+xpjzIlILGA8EAbPTFygiDYDvgECsGuIdxph/CmuD4uLiCA4OzlEeEhJCfHx8YYWRJ54Y8wP392bn9m0AVKlSlc+/Hptrp4PfZs0kPPwK2l7V/qLLPHPmDJ98+B41akbSoVPOL1lX8sR9fCneGHN2hw4d4vORI2jTtl1GDclTuLDCMwKYYYy5TURKYOWD74B3jTEz7EpAL6zk9DAQbYy5U0QeAl4HHhWRhkAPrIQWCUwFrsxPMB7bTJdORPyA64Ff7aLvgf8zxjQCDgAv2OXTjDGtjDGNgVjgbrt8GPCJMaYhEJdp0Y8Bw+3aVwsg2pXboVzjjbeHMuaHSbz13keULFWKpx97iJgDB3JMd+TwYdauXskNXW/B3z/332DJycm88tJzHDl8iLff//ii06r/hrNnzjDw6cfx9/Pjzbffc3c4Ofj4+Dj9cJaIhABtjDHjAIwxCcBpoD3wiz3Z98Ct9vPuwA/284lAevNDN2CSMSbJGLMTiBORevnZTk/+pJUWkU1ABGCA+fYOLGWMWW5PMw4r2QA0FpG3gBD7kWSXt8bakWBVM/vbz1cC/xORKliJrNBqRQDBIcEOfzXm9ivTE3hizDXsJpUrGzam3VXt6dH1OsZ9N5qXX3k9y3RzfvuF1NTUizbRpaam8sarL7N29Uo++exLatcRF0bumCfu40vxxpjTJSYm8vSTjxG9P5ox436gfIUK7g4ph7x07bbPrYc6eOukMeZkptc1gCN27edKYAXwJnDCGJPenTUaqGw/r4T14x9jzFkR8RORALt8Wablps+zw/moLZ5cM0o/Z1QN8AGeuMT0Y7DaKxsCH2A11+XKGDMBK0mdAWaJSOfLjjgPIiNrsSsqZ/7bvXuXw15gnsDTYy4dHExE1apE79+X473ffp1B7Tp1qSN1c53//bdf5/f5c3j7/Y9p1do9vak8fR874o0xAyQlJfH8swPYvm0ro7782i0/Ppzh6+Pj9AMYCOxx8BiYbbH+WC1Cw4Bm9ut+hbE9ufHkZASAMeYMMAB4DitxnBKR9G+K+4E/7OelgMMiEsiFJjqA1UB639x70gtFpCaw2xjzGfAz0MhlG+FAx06d2bL5L6L3788oO3Agmk0bN9ChU6HmRad5eszHjh1l3549RERUzVK+fdtW9uzexc3deuQ677CPhzLz56m8+sY7dOx8natDzZWn72NHvDHm1NRUBg96njWrVzHss89p1LiJu0PKVR570w3DqvVkfwzLtthoYJ8xZr0xJg2YgfUdGCYi6XkhArs2BMRg15Ls80spxpikzOUO5skTT26my2CMWSciW7BOpvUFvhSRIGAn8JA92WvAWuAQsDHT7AOBCSLyKpk6MNjLul9Eztvz3EMhuv3OXkyaMJ5nnn6CpwY8gw8+jPpsOOUrVKBnz96FGYrTPCnmF559irr16lOrtlCyVCn+3beXiT+Ow8/Pj3v69Msy7exZM/Hz9+fGm7s5XNa470Yz4YexdLv1dqpUrcaWzZsy3gsLCyeiSlWH87mCJ+1jZ3ljzO++/Qbz583lkf6PUaJECTb/tSnjvfLlK3hUc11emunspriTTkx3UERiRKSWMSYK6IjV8asEVovRDKAPMNOeZRbWj//1WN+VszOVfysiI7A6MIQaY/LcRAc6AkOeFdQIDACxMTFZh1Bp05YXXhpM5coRBbWKAlcYMTszAsO470azcP5covfvJyk5ifLlK9C8RSv6PtifSpUv/FBLTkqia5cOXNmwMZ+M+MLhsi42bNDN3W5lyFuXPqldUCMwgB4XheGm6zsTE+P4B/xjTzzF408+XSDrKYgRGHqP2+j0d87kvk2dXp+INAVGY/Uo3gk8AFQEJgFhWL2YHzTGJNu1ofFYtadDWF27Y+zlDMFKXEnAo8aYpc7GkFmuyUhE9oPDL14fIM0YU3g/Fz1IQSYj5ZgzycjTFGQyUv8dBZGM7spDMpqUh2TkaS7WTHd1oUWhlFLKoSJ/cz1jTM4uSUoppQqVl4/y47RLdmAQkWpYXaUbAsXTy40xNV0Yl1JKKYrOzfWcaegeg9WzIg2ri/TvwFjXhaSUUiqdK0Zg8ETOJKMwY8xEINUYsxF4FLjFtWEppZQCq5nO2Yc3c+Y6o/P2/ydFpC7WuG9lXReSUkqpdN5e43GWM8lomj3U+PvAKqza1H/mVg5KKeXJikYqciIZGWM+tJ/+JiLlgOLGGO8YF14ppbycn7e3vznJmd50OQaXEhGMMYtcE5JSSql02kx3wauZnhfHutPfBkCTkVJKuVgRyUVONdN1yvzavkPq8y6LSCmlVAbfIpKN8jygljFmG9DUBbEopZTKxsfH+Yc3y+s5Iz+gDRfuoqqUUsqF9JzRBZnPGSUDUVj3AlLKJbxxBOzw3mPcHUKeHZ/8oLtDUE7w02SU4R5jTGzmAhHxnDtPKaXUf1gR6dnt1Dmj3xyUzXZQppRSqoAV+eGARCQUCAcCRaQGFy4EDgGCXB+aUkopPWcEfYGBQCWyXlMUB3ziwpiUUkrZvL3G46yL3VxvODBcRAYZY4YWYkxKKaVsRaRi5NQ5o00iEpL+QkRCRaSLC2NSSill8/fxcfrhzZxJRu8bY+IyvY7DGsFbKaWUixWVi16dSUZZNtEYk4ZzXcKVUkpdJl8fH6cf3syZZHRMRG5IfyEiNwHHXBeSUkqpdEWlZuRMDWcg8LOIJNqvA4AHXBaRUkqpDEW+N106Y8wW+3bj9YHuWGPTzQZCXRvaf8uhgwcZ8+1otm/byt9mJ4mJicyev5DKlSOyTNe4gTicf/LUGdStV68wQr2og7GxfDj0PVatXE5aWhqt27bjxUGDqVipkrtDy5U7Ym4j5RjcqymNaoRTItCfXbFxfDlnB98v+idjmmrlSvFun5Z0alSJAD9f1kUd5X/fr2HDrqwNDzu+6Em1cqVzrKP30N/5dc2/DtffWsqx8O2b8fX1oXTP70hJTSvYDcxGjwvX0Zvr2USkBdAP6AmUBB4B7nJtWP89//67j/nz5lC/fgOaNmvByhXLcp22+62307NX7yxl1apXd3GEl5aQkMAjD/YlIDCQt94dio8PjBwxnIcf7MNP038hKMjzroV2R8xXVgvjtyE3suafIzz1xXLOnkvmtrbV+fLJ9hQL8GP0vJ2ElyrGwrdv5lRiEk9/uYKE88k83e1K5rzRlWsG/YI5EJdlmQs2RvP25I1Zyv6JyTpNOn8/Hz57tB2H4xKoEOb6v4keF65VRHLRRUdgeAHrwtc0YBzWTfVWGGMmujIgEbkLmABUNcZE53MZe4EmxpiTBRjaZWneoiWL/1gBwPSpP100GZUvV45GjZsUUmTOmz51CtHR+5k5ay5Vq1UDoHYdoXvXG5g6ZTJ9+nle6607Yu55VU38fH24870FnElMBmDR5hiurBbOPR1qMXreTh65oS7lQktw/dOz2XPoFABLtsSy7fOevHJXM+7/eHGWZR6NT2TtP0ecWv+zPRri4+PD94v+4cU7Ghfsxjmgx4Vr+VA0stHFOjC8BxwE7jbGfGQPluraur7lLmAF0PtSE3oTX1/vG4k6uyWLF9GoUeOMDy9AREQVmjRtxpLFC90YWe7cEXOAvy9JKakknE/JUh5/9nxGj6dWdcoSFRufkYgAzp5LZsWOg9zUvEq+m2ZqlC/NoDubMHD0CpKSU/O/EXmgx4VrFZWx6S72DVkdWAxMF5H1IjIA635GLmNfXNsMeBI7GYlIPxH5WUT+EJG/ReQ5u7y6iGwTkckiskNEfhCRAAfLvF9E1orIXyLyiV1WUkR+s8u2iohHJb4pkyfRosmVtG7emIcf6MOG9evcHRIAu6KiiKxdJ0d5ZGQtdu+KckNEl+aOmH9cbJ0X+vjBNlQMK0FIUCAPXFeHjg0r8dmsrQCkpKZx3kGyOJeUSlAxf2pWyHqOqGuLqhyd0IcTk/qy5L1b6NaqqsN1j3i0HdNX7GH59kMFvFW50+PCtYpKMrrYcEDRwDvAOyJyFdZ5o2ARmQNMNMZ874J4bgNmGWP+EpEgEalpl7cCrsS6qd86EfkVOI/VqeJBY8xqEZmA1az4TfrCRKSevcy2xphkEflRRG4GigMxxpib7ekyRphwt5u7deeaDp0oV64cMTEHGDfmWx55sC9fjh5Dy1at3RpbXFwcwcHBOcpDQkKIj493Q0SX5o6Yt+8/yY2vzWHSoGt59Car08n5pBQGfL2cqcv3APB3TBydG1cmvFQxjp8+B1hdc1vULgNAWKliGcubvW4/66OOsvfwKcqFlOCxm+oxedB1PDh8KZP+2JUx3V3XRNK0ZhkeHDbNJduVGz0uXKuoDJTqVNuRMWa5MeYRrEFTxwP3uSieu4Ap9vPJXGiqm2eMOWGMOY3Vk6+dXb7HGLPafj4JuDrb8q4FWmMlsE1YSa0WsAW4XkQ+EJH22UaYcKt33/+QG2/qSrPmLbilWw/G/jiBsmXLMXLEMHeHppwUWTGYCS90Zvv+E9zx7gK6vj6Hb+bvZET/q+jd3vp99c18g68PfDPgGmqUL02F0BJ8/FAbqtu95tIyNYg/9+0qJiyNYsWOQ8xYtZeub8xlfdQR3ry3ecY0YaUCeb9fK16fsI4j8Ymo/w4/X+cf3ixPIykYYxKAH+1HgRKRMkAHoJ6IpI/ycBQYRtZzVWmZXudWns4XGGOMeTVbOSLSDOgKvCkii40xbxbEdhS0kiVL0f6aDvw8faq7QyE4JNjhr8bcfmV6AnfE/MY9zUlKSeWOdxeQnGIdkku2xBJeujgfPtiGKct2s/fQKR4YvpRhD7dl2+c9Adi46yifzdrGsz0aEnvibK7LT01NY/qKvbzTpyUVQktw8GQCQ+5uzsETCUxbsYeQoEAAigVareohQYEkJqVw9lyyS7ZXjwvX8vaRFZzlScP63ImVOJ5MLxCRzVhNal3s+yslYSWQr+xJaohIS2PMWqxaVNYuSLAQ65zXcGPMUREph3Xeywc4YYz5UUSOAo+5csMKgidU1SMja7Er6p8c5bt376JmZC03RHRp7oi5QbUwtuw9npGI0q2LOsJd10RSLqQEh04mMHPVPn5d8y+1KwZzPjmVPYdOMbx/W/YfOU300TNOrSt9DXUjQmlUPZwD43I2WkSPu5df1+yj91DXnJjX48K1vP1ckLM8qWJ3F/BztrKfgZeAdcBMYBPwjTHmb/v9HcBzIrLDfp3lPJYxZhvwJrDQTmyzsG4Y2BBYbTfdvQm8XdAbU1BOnz7NH0uXcGXDRu4OhY6dOrNl819E79+fUXbgQDSbNm6gQ6fObowsd+6I+dCJBBrVuIIA/6wfr5a1y5JwLjnjHBFYtRxzII49h05RMawEd7Sryeh5Oy+6fD9fH+68qgb/HjnNoZMJALz43WpueG12lscPdkeKrq/P4Y0J6wt4Ky/Q48K1XDkckIj42h28ZtivI0VkjYhEichYEfGzy4uLyFS7fJmIVMy0jFfs8h0i0j6/2+kxNSNjTEcHZUNEZA/WNUO3OZgtyRiT4wJcY0z1TM/HY53nymwbMO+yAs6HBfPmArB9u9WjavmffxAWFk5YeDgtWrZi3HffsnfPHlq2ak3ZcuWIjYlh3NgxHD16lPc++Kiww83h9jt7MWnCeJ55+gmeGvAMPvgw6rPhlK9QgZ49PapDYgZ3xPzlnB1MeKEzU1++jtFzd5JwPpmbW1ald/tIRvy6laTkVPz9fHjn/pYs236Q+LNJ1KsSygu3N2bH/hMM/3VrxrJ6Xl2TW1pWZd6GaKKPnaZcSAkevbEeTSPL0OeTCw0Bm/cezxHHNQ2s74s/tx106QgMely4lq9rrzN6HNiF1QIFMBR41xgzQ0R+AHoBE4GHgWhjzJ0i8hDwOvCoiDQEegD1gEhgKlZnszzzmGRUFDz/f89kef3OW28A0KJlK74d+wPVqtdg0e8LWLRwAadPn6ZkyVI0adqU1998h4aN3F8zCgoKYvSYcXw49D3+99KL1hAqbdrywkuDCSpZ0t3hOeSOmGes2sutb8/n/25tyKjHr6J4oD97DsYz8OsVfLPAAFYHhVoVg+nVPpLQkoEcOHaG7xf9zQfT/spyfdDeQ6coG1Kcd/q0JLxUMc6cS2LDrmN0f2sev2864JL480qPC9dyVQu9iJQHbsfqNT1ARHyA9lgJCKyWpoexklF34GW7fCLwhv28GzDJGJME7BSROBGpZ4xJb61ymk9aWmFcx/rfkZhcKBf+Ki8T3nuMu0PIs+OTH3R3CP95xf0vv1rz9ap9Tn/nfNy3SxiOxw09mX1EGhEZB3wJFMMaEPthYJkxpq79fj1gtDHmahHZClxnjDlovxcLVAU+teeZZJdPtuf5PS/bCJ51zkgppVQ2eTxnNBDY4+AxMPMyReQaIM0Ys7Iwt+VitJlOKaU8WB67dg8DxjooP5ntdTusay33Yp0vCgY+BsJExNcYkwpEAOltwTFAZeCgiJQAUowxSSKSXp4u8zx5oslIKaU8WF5ykd0Ud9KJ6d4H3gcQkY7AQGNMXxGZjnV+aAbQB6sXM1g9ke8H1gP3YA0+kF7+rYiMwOrAEJqf80WgzXRKKeXRfPPwKAAvAq+IyC6sy9jSR8QZDVQVkSjgQazedBhj0i+Z2YmVwJ7I74q1A0MeaQcG5Yh2YFCOFEQHhu/X7Xf6O6dPiypee4msNtMppZQH0+GAlFJKuV3RSEWajJRSyqMVkYqRJiOllPJknjBIcmHQZKSUUh6sqHR51mSklFIeTDswKKWUcjttplNKOc0br9kJa/mUu0PIkxNrR7o7BLfQZjqllFJupzUjpZRSblc0UpEmI6WU8mhFpGKkyUgppTyZXxHJRpqMlFLKg/kUkYY6TUZKKeXBikjFSJORUkp5Ml+tGSmllHI3rRkppZRyOx0OSCmllNv5Fo1cpMnInQ7GxvLh0PdYtXI5aWlptG7bjhcHDaZipUruDi1XGrPrFUa8/r6wZNxzNKxdmaASgUjX1/g39niWaYoF+jPkiVu4q2tLQkuXYPPfB/jf8Bks37ArY5pSQcX4csi9NKlbhQplg0lKTiFq32FGTVzKpNlrsyxvwH2duaZFbZrVr0rFsiG8/eVs3vlqdpZpqlYMx8x+M9e4z6dAqtM34b44bzkuikpvOp+0tAL6yxYRickUyA5LSEig1+09CAgM5KkBA/HxgZEjhpOYmMBP038hKCioIFZToDRm1yuMeH19IMAX5i3fhp+vL9e3q+cwGX33Tl9ubN+AwZ/OYM+BozzW6xq6XFWfjn0/ZvPfBwAIDynJJ4N6sniNYV/MMYoF+nNnl+bc1601L340jc/GL85Y3sZpr3DqTCIbd+6nf8/2DpNRYIA/TepG5Ih5yJO30KlVXc6lXPbmA4V3XBT3v/xMstgcc/o7p5Nc4bWZq8jXjERkE3CrMWZvYa53+tQpREfvZ+asuVStVg2A2nWE7l1vYOqUyfTp90BhhuMUjdn1CiPe1DQ4lwK3Pf0F/W5ry/Xt6uWYpmGdytzVtSX9h/zID7+sAuDP9VFsmPo/Xn3iFnoO/AqA43Fn6Dd4bJZ55y3bTu1q5ejTo22WZNTszndIS0vDz8+X/j3bO4ztfFIya7bszVJWongALRpUK7AaEXjXcVFUakZeMSCsiPi5O4aCtmTxIho1apzxQQCIiKhCk6bNWLJ4oRsjy53G7HqeEu/NHRpyPimZqfPXZ5SlpKTy07z1XN+2LoEBF/8de+zkGZJTslZj8tsK06NzE4JLlSClAJORp+xnZ/j6OP/wZh5RMxKRIcA9wEHgODATeB2YDHQBhohIKeBlrHEDpxljhtjznjTGhNrP+wFNjDEDRWQsEA+0BMoBTxlj5ohIEDAOaABsBgIKZyuz2hUVRcfO1+Yoj4ysxYL5c90Q0aVpzK7nKfHWj6zI3gPHSEhMylK+Y1csxQIDiKxShh27D2Z5z8/Pl5BSJbj12iZc37Yej705vkBiua9baw4diyckJLhAlgees5+dUVR607m9ZiQiLYHuQCPgVqzkkS7GGNMUWAe8A3QCmgFdRKSzE4svC7QD7gbessueAI4aY+oDnwH1C2Az8iwuLo7g4JwfrpCQEOLj490Q0aVpzK7nKfGGBQdxMv5sjvLjdll4SMks5Y/1vobT60ZwYMlQPn2pJ89/OJUJs9ZcdhyVyobQsWUdJs9Zd9nLysxT9rMzfPLw8GZuT0bAVcAMY8w5Y8wJYF6m936y/28JLDbGHDXGnAcmAY4bnbOaYYxJA9YD1e2yq4GJAMaY5cCey98EpYq2qfM3cNW9H9D9yVF89/MKPhnUk4fuuOqyl3vPLa3w8/PNOG9VFPn6+Dj98GYe0Ux3ETl/muWUuSW5WLb3zgEYY1JEJPO2ur0LYXBIsMNfYLn9YvMEGrPreUq8J+PPUrVieI7y8GCrl9nxuDNZyo+eOM3RE6cBWLBiB0HFA3nv2dsYN3Mlycmp+Y7jnltasWnnfrb+E5PvZTjiKfvZGd6dYpznCTWj5UB3EQkUkRCsc0TZrQE6iki4iAQCvYE/7PeOiUik3cmhuxPrW2bPj4i0BWpc9hbkQ2RkLXZF/ZOjfPfuXdSMrOWGiC5NY3Y9T4l3++5Yqle+ghLFs55SrVuzIufOJ7Fr/9GLzr9h+7+ULlmc8uH5/2JvXr8q9WpWZPyvq/O9jNx4yn52ShFpp3N7MjLGrAXmAFuxOi5sxup4kHmaWOAVYCmwAVhgjEnvM/oyMB/4E/jXiVV+DlQQke3AAGB7AWxGnnXs1Jktm/8iev/+jLIDB6LZtHEDHTo5czqs8GnMrucp8c5eupXAAH9uv75ZRpmfny93dmnG7yt3cj4p+aLzt29ei1NnEjl8/FS+Y7ivW2uSklIK/HwReM5+dkZRaabziIteRaSUMea0iAQDq4BbjDG73R2XIwV10evZs2fpdXsPihUvzlMDnsEHH0Z9NpwzZ88wdfovBJUseemFFDKN2fUKK15fH+j30jd0bCX079meAe9O4uiJ0xw5cZpl66MA+P79B7iubT0GD/uZvQeO0b9ne25qfyWd+n3Mpp3RADx0x1W0aliDxat3En34JFeElOSOLs3oeUNzXhk+g4/H/p6xzmb1q1KtUji+Pj78+MFDTJu/gWkLNgAwd9m2LD33Avz92D3/HVZt3pNxTdOJtSMLZNuh8PZzQVz0unZ3nNPfOS1rhnhtRvKUZDQBq1dbMeBzY8xnbg4pVwWVjABiY2KyDkfSpi0vvDSYypVzXoHuKTRm1yuMeIvncrb4j3X/cMMjw61pigXwxlPd6HVjC0JLl2DL3wf43/CZ/Ln+QvNWm8Y1GPTwjTSWCMJDgjh28gw79xzksx8XM3fZtizL/vqN+7i/exuH680+AkT3To2Y/El/7n7+G2Ys3AQUbDKCQtvPl5+M9uQhGdXQZFRkFGQyUsqdwlo+5e4Q8qSgk1FhKIhktG5PvNPfOS1qBHttMvL03nRKKVWkefmpIKdpMlJKKQ9WRHKRJiOllPJkPi6oGolIHWAMEAakAG8ZY34SkUisQQHCsS6Deci+TrM48CPQBGvYtp52L2dE5BWgH5AE9DfG/JmfmNzetVsppVTufHycf+TBOazE0QDr2s7hIlIaGAq8a4ypBfgBvezpHwai7fLvsMYORUQaAj2AesBtwBf53U5NRkop5cFccc2rMWafMWa7/fwgcAQogzXM2i/2ZN9jjRcK1oACP9jPJwI328+7AZOMMUnGmJ1AnIjkvCeJE7SZTimlPFkesoyIhAKhDt46aYw5mcs8LbHuXnAaOGGMSR+/KRqobD+vBBwAMMacFRE/EQmwy5dlWlz6PDucj9qiNSOllPJgPnn4BwzEGvw5+2Ogo2WLSDmsGtBDuHnMTk1GSinlwfJ4zmgY1nib2R/Dsi/XvrfbL8CbxpiVwDEgTETS80IEdm0IiMGuJYlICSDFGJOUudzBPHmizXRKKeXB8tIxwW6KO3mp6eyBpScBPxtj0m+pkyYiy7HOD80A+mCNFwowC7gf63Y89wCzM5V/KyIjgEgg1BiT5yY60GSklFIezcc1VxrdhNUJoaqI3G2X3Q+8CEwSkY+x7qgwxX5vNDBeRKKAQ0BPAGPMZhGZBezE6tr9aH4D0uGA8kiHA1L/FTockOsVxHBA22POOP2dU79SSa+9RlZrRkop5cG8NrvkkdaM8sgba0YJ51PcHUKelAj0c3cIygNt3HvS3SHkWdtaoZedS3bEOl8zqldRa0ZKKaVcwNtvmucsTUZKKeXBikYq0mSklFKerYhkI01GSinlwVzUtdvjaDJSSikPVkROGWkyUkopT1ZEcpEmI6WU8mSuuLmeJ9JkpJRSHqyI5CJNRkop5cmKSC7SZKSUUh6tiGQjTUZudOjgQcZ8O5rt27byt9lJYmIis+cvpHLlCHeHxsAn+7NqxTL6Pfwojz35DAAxMQe4/ebrHU6/4I9VlC4dnPH63LlzfP35CObO/pXTp05Ru05dnnzmOZo2b1Eo8Wd2MDaWD4e+x6qVy0lLS6N123a8OGgwFStVKvRYnOFt8YJ7Yl67bCGrls5n7z87iY87wRVly9O8XUdu6dWPEkEls0wbtXMLM8Z/wy6zlZTkZMpVqMwtvfvRpkOXjGnOnz/H9B++YuXiuZw9c5qqNWvT64GnkCubZlnW3J8nsGPzevb+s4O4E8focc/D3HbvIy7bTu3arVzu33/3MX/eHOrXb0DTZi1YuWLZpWcqBPPn/MY/f+/M9f2+Dz5C+w6ds5QFZfvwv/PGK6z48w+eevZ5KleOYOqUiQx88hFGj5tAHannkrgdSUhI4JEH+xIQGMhb7w7FxwdGjhjOww/24afpvxAUFFRosTjD2+IF98U8Z/p4rihbgTv6Pk54mXLs22WYMeEbdmxezysffYOvr3WPuE1rlvHZO4No0+EGHnvhTfz9A4jZv4ek8+ezLG/M8Hf4a+1yej/4NOUqVGbhrKl89OozvPLRN1SLrJMx3dK5MykRVJJmbTuwePZ0l2xbZnrOSLlc8xYtWfzHCgCmT/3JI5JRfHwcwz5+n4HPvcRrg19wOE2lylW4slHjXJfxj9nJ/Dm/8crrb3NLj9sBaNq8Jffc2Z2vPx/JR8NHuSR2R6ZPnUJ09H5mzppL1WrVAKhdR+je9QamTplMn34PFFoszvC2eMF9MQ8c8jHBIWEZr+s2bEap0iGM/uQNdm7ZQP3GLUg4e4Zvh71N55vv4N7+/5cxbYOmrbIs69/df7NqyTweGvgK7a/vBoA0bMr/Hr+bn3/8moFDPsqY9p0vJuLr60tKSnKhJCPfIpKMXHrbcRFZIiLXZit7S0TeFJFOrly3M0QkVET2umv96b/cPMmo4Z9QM7I2XW66Od/L+HPpYvz9/bmuy00ZZf7+/lx/w02sXrmM89l+kbrSksWLaNSoccaXJEBERBWaNG3GksULCy0OZ3lbvOC+mDMnonQ1alu17hPHDgNWU96puBPceNu9F13WxtV/4ufvT6v2F5qh/fz8aX3N9WzdsIqkpAvHbOF/bn3y8PBert6rk4Be2cp6AxOMMYtdvO4c7Fvtqlxs2rieObNm8sLLr150ui8++5SrWjTk2vateP6ZJ4n65+8s7+/eFUWlyhEUL1EiS3mNyFokJSURvX9fgceem11RUUTWrpOjPDKyFrt3RRVaHM7ytnjBs2LeuXUjAJWq1ADgn+1/UbJ0MNF7o3jliXt4sFs7/q9vN2ZM+IbUlAu3Vjnw727Klq9EseLFsyyvcrWaJCcncTgmuvA2IhsfH+cf3szVzXRTgSEi8qQxJllEmgIJwEsiMsMYM8OumYzDuu86wG3GmL0iUhb4CqiKdTvbJ4AoYCMgxpgUEakALDLG1BeRR4BHgGLAdqCvMea8iIwFEoFmwDwRGQeMB4K4cB/3Ii8p6TxD336de/o8QLXqNRxOExgQyG139KJV26sICwtj7949jPv2a/r3u4dvf5hMjZqRgNXUVzo4OMf8wcEh1vtxca7bkGzi4uIIdhBLSEgI8fHxhRaHs7wtXvCcmE8cPczPP35NgyatMtWQjnL+3Dm+/PA1ut/1INVr1WX7prX8MnEMZ0+f4p7+zwJw5lQ8QaVK51hmyVLWdp0+VXjHbHZenmOc5tKakTHmKPAXkN5U1xurtpTdQWNMU2AikH4v5GHA+8aYFkBf4CtjzCms+7LfaE9zH1ZiAZhmjGlljGkMxALp93UHKAe0Mca8ai/3E2NMQ8B9R5iH+WHsGM6dO0e/h3K/hX2ZsmUZ9MrrdLr2epo0a8Gtt/fky2+/Bx8fxn77VSFGq1RWiQlnGf7WC/j5+fHQs69klKelpZJ0/hw97n6Im26/l3qNmnNHn8focGMPFv42lbNnTrsxaucUlZpRYTR+Zm6q6wlMdjBN+lnA9UB1+/l1wNcisgmYApS3y78F+tnP7we+t583FpFlIrIFuBOon2n5U40xqfbz1sBP9vMf8745/z0HY2MY9+1X9H/iaZKSznPqVDynTlm/aJPOW69TUhzfLbZ8hYo0btKMHdu2ZpSVDg7mlINfxPHxVu4PDglxwVY4FhwS7PDXeW6/5t3N2+IF98d8/lwiw954jiMHY3jureGElymf8V6p0taxlr3DwpVNW5OSnMyBfbsBqwZ09vSpHMs+czo+y3LcwcfHx+mHNyuMZPQz0FVErgKOGWN2O5jmnP1/CheaDn2BVsaYJvajGoAx5g+grohcDxw2xuy3px8D9LdrPB9gNdelO1uwm/TfcuBANOfOneP1/w3i+mvaZDwAxn//Hddf04ZdUX9fdBmZPwg1a9Yi5kA0iQkJWabZs3sXAQEBRFSpln12l4mMrMWuqH9ylO/evYuakbUKLQ5neVu84N6Yk5OTGfnuy+yN2smzb3xClepZ11e5Ws2Lzu9jd1WrXLUGRw7FcC4xMcv7B/7dg79/AOUque/av6LRfaEQkpExJg5YDXyO41pRbhYCT6a/EJHMfYknYJ1nGpuprBRwWEQCydpEl91q4A77+T15iOc/q47UZdTosTkeADfe3I1Ro8cSUaWqw3kPxsbw16YN1G/QMKPs6g4dSU5OZuGCeRllycnJ/D5/Lq3aXkVgYKBLtyezjp06s2XzX0Tv359RduBANJs2bqBDp84XmdM9vC1ecF/MqampfPXha+zYvJ4Br3xArboNc0zTrE0HALauX5WlfMv6lQQEFiOimnWes0nr9qQkJ7N22YXefykpyaz543caNGtNQEDhHbPZFZVmusK6zmgSVgLJSzJ6GvhSRB7AivNXrPNPYJ1beokLzXsArwFrgUNYnRxyMxCYICKv4gEdGBbMmwvA9u1WM9fyP/8gLCycsPBwWrRsdbFZC0zp0sE0b+F4XRUqVsp4b/jHQ0lLS+PKRo0JCwtn3949fD9mNL4+vvR7+MK5Jqlbn+u63MSwj94nOTmZSpUrM/2nycQeiOaNdz4olG1Kd/udvZg0YTzPPP0ETw14Bh98GPXZcMpXqEDPnr0LNRZneFu84L6Yf/jiQ9YuW0i33g8QWLw4UTu3ZLwXXqYc4WXKE1E9kquvu5mfx39NWloa1SKFbZvWsnT+L3S/60GKl7AuyK0WKbS65jomfP0pKSnJlC1fiUWzp3HkUAyPvvBGlvXu+WcHRw/FkJaWBkDMv3syklijFlfl6JF3uYrKCAw+6TvUm4jIPUB7Y8zjhb3uxGQKdIc1biAOy1u0bMW3Y38okHUknHd8vudS2jStn2U4oF9nTGP6T5OJ3v8vZxPOEhISQouWbXjo0Sdy9MBLTEzky5HDmT93FqdPnaJWHeHJZ57LNellViKwYHvgx8bEZB2qpk1bXnhpsEcMu+SIt8ULhRPzxr0ns7x+7oFbOXY41uG0mYfoSU5KYubEb1m+8DfiTh6nTLmKXHvLnXTpcVeWec6fS2Ta91+ycsk8azigGrXp+cCT1GvUPMt0oz95k+ULf3O43g/H/EzZ8heGQGpbK/SyM8mR08lOf+eULeXvtZnL65KRiHyB1TuvizFmb2Gvv6CTUWHIbzJyl4JORuq/IXsy8gYFkYyO5iEZlfHiZOR1wwG5ozaklFLu4uvtJ4Oc5HXJSCmlipIikosKpWu3UkopdVFaM1JKKQ9WVGpGmoyUUsqDFZWu3ZqMlFLKg2nNSCmllNtpMlJKKeV22kynlFLK7bRmpJRSyu1clYtEpCvwKeAHDDfGfOaiVTlFrzNSSilP5oJ7SIiIPzAC675xjYAnRMStgx9qzUgppTxYXoYDEpFQINTBWyeNMSczvW4F7Ei/H5yITAe6AV/kN87Lpckoj4r7e9/ZxOL+OvCo8n5ta4W6OwS3yON3zuvAEAflb9jvpasEHMj0OhqonNfYCpImI6WU+u8YRtabjqY7WahR5IMmI6WU+o+wm+JOOjFpDFlrQhFYtSO30WSklFJFzxqgvohUAY4CtwNd3BmQ9qZTSqkixhiTDAwEFgJbgS/TOzO4i9fd6VUppdR/j9aMlFJKuZ0mI6WUUm6nyUgppZTbaTJSSinldtq1Ox9EJAXYYr9MAZ40xqy6xDyzgd7AFcAMY0wTB9NsAm4FzgOfGmN6F1zUDmP6FNhljBlpv14FLDPGPG+/ngxMMcZMy+fyBwKhxpjXCybiLMtO/xsEALuB+7MNd5KfZYYCm4wx1S83PgfLvguYAFQ1xuTreg4R2Qs0udzt9GTpnwFjzN5c3l8CvGWMWZip7C0gDVhsjFlcCGHmypXH0H+d1ozy55QxpomdUAYD71xqBmNMV2PMKWcWboyJcXUisq0E2gKISDGsL/bmmd5vC6wohDjyI/1v0ADrIr8n3RzPpdyFtS8L4+/qUiLizvGlJgG9spX1Bia4IxG5eV/8p2jN6PIFA3EAItIRGGiMudV+PRarFjQj/Vdt5hlFJAgYBzQANmMlA0Skuj1fExHpB9xir6cmMM4Y85Y93RDgHuAgcByYaYwZKyLvA92BZGB+ek3HgRXAUPt5c2A50FREAoByWLW+TiLyMtaYwNOMMUPsdd8DOCp/GBgEHAMMsMcuHwA8Zse03RhzlxP71lnLgcb2epoAXwElgB3Aw8aYUyLyCPAIUAzYDvQ1xpwXkVrAeCAImJ2+QBFpAHwHBGL9aLvDGPNPfoITkRCgGdZAlKOBj+2/aw+smnIF4CtjzMf23/43rGs/GgHrgAeNMUnZlnk/MMCOb6Ex5v9EpCQwBetqej+sGsTkfMSb47jCGtdsMtaFkUNEpBSO//4njTGh9vN+WDW5gfZnIR5oiXVsPWWMmZPbZ+Aiptrrf9IYkywiTYEE4CURyfxZG4f1GQC4zRizV0TKYh0bVYEk4AkgCtgIiDEmRUQqAIuMMfUvcsyMBRKx/qbzRGQcDo4hlTdaM8qf0iKySUR2Yn25XLJmlIsngKPGmPrAZ0D9XKZrDNxh//+4iISKSEusD1sjrKa9lgAicgVwG9DAGNMIeDu3ldvNRX72B7AtsAr4C2gKtMNKJO8AnbA+eF1EpLOIVLpI+f+A1vZ7TTKt7iWgqR3TY87uoEuxf5leD/xqF30P/J+9ngPAC3b5NGNMK2NMYyAWuNsuHwZ8YoxpiP2jwvYY1j1emgAtuLyhUm4DZhlj/gKCRKSmXd4KKyE1Ax4RkTp2eX07pnpYSaVvtm2uZy+zrb095UTkZuBGIMYY09gYcyUwN6+B5nZc2WKMMU2xEmSOv78Tiy+LdVzdDbxllzn7GQDAGHMU6xi91i7qjVVbyu6gHetE4Cm7bBjwvjGmBdY+/cpurViOte8A7sNKLJD7MQNWQm1jjHmV3I8hlQeajPInvYmoLnAT8L2I5Gc076uxPiwYY5Zj1yIc+N0Yc8oYcwbYhfXL9yqs2tM5Y8wJYJ49bRzWr7ZvReR24OwlYkhvqmtrP1+V6fVMrHb4o8aY81gf+vZYX1COylvZ5ceNMQlA5nNNm4HxInIfVu3ocpW2zy8cwqpZzLdrIKXsfQnWr+P29vPGIrJMRLYAd3LhS6818JP9/MdMy18JDBaRl4Bq9vbk111YNRawahfpTXXzjDEnjDGnsX5Rt7PL9xhjVtvPJ2EdJ5lda8e9zt4HrYBaWOfQrheRD0SkvTEmP1+MuR1XcGE/5fb3v5QZxpg0YD1Q3S5z9jOQWeamup5Y+zS76fb/mdd1HfC1vc+mAOXt8m+Bfvbz+7F+0EDuxwzAVGNMqv08t2NI5YEmo8tkjFkJlMH61ZdM1n1azIlFODMExrlMz1O4SPOqPcxHK6zmjK5c+tfxSqwvwRrGmD1YyaiNXRbqRGzZ5bY9NwMjsWpda+ybe12OU3atpRpWU9ETl5h+DNDf/vX6AZf42xhjJmDVEM4As5z85Z+DiJQBOgDj7OajR7mQjDLvq7RMr3MrT+cLjEk/b2mMqWOMGW6M+RurprIZeFNEXstPzBdxqR826fGmy76PzwEYY7Ifw3kdBuZnoKuIXAUcM8bsdjBN+mcm87p8gVaZ9ls1O54/gLoicj1wONOwOBc7ZpzZFyoPNBldJhGpi9WUcgzYB9QTkQARCcf6ErqYZdhfTCLSFqiRh1UvB7qLSKBdI+hiL6cUVg+22cCzQMNLLGcFVhPgfgBjTBRQD6iD9WHsKCLhIhJox/oH1iCLuZV3EpEwESmO1ZSEiPgCVYwxS7DOJ5UGSuVhW3Nl1xYHAM9hJY5T9r4E61fuH/bzUsBhO97MzS2r7e0H6zwJdsw1gd3GuhXzz1jNVvlxJ1biqGaMqW6MicD63BXHat4Ktc/1dMX6YQBQw24uA2vfLsu2zIVALzvRISLlRKSi3UyaYIz5EetcYLN8xOvwuMomt78/wDERibSbT7s7mDe7PH8G7BrfauBzHNeKcrOQTB1dRKRxpvcmYNWkx2Yqy+2Yyc7hMaTyRpNR/qSfM9qE9WHoa4xJsX9R/Qxsw6qub7jEcj4HKojIdqwv1O3OBmCMWQvMwTrRPRPr13A81hf9ryKyGeuLJbfOC+k2AhWxakTpooHN9va8Aiy1t2WBMWaxMSY2l/IY4F2sL6sldkxgJesf7eaOTcAoU4Ddk40x67CaqHphnQv41N7+qsBH9mSvAWuxvjQ3Z5p9IPCCPX1opvJewFYR2Yh17ut78ucurGMis5+xzqGtw/rbbQK+sWs2YHW8eE5Edtivs6zbGLMNeBNYaMc9CwjH+uGx2j4u3+Qi5wtzc5HjKvM0Dv/+9tsvA/OBP4F/nVhlfj8Dk7C2Ny/J6GngahHZbK8vc+KYCJTkQvMe5H7MZDcQx8eQygMdKNWLiUgpY8xpEQnGSia35NJkoTxM5p5m2cqrk8t1aIWlKB5Xdu/Q9saYx90dS1GlXbu929ciUh+rLfvz//oXhio0Req4EpEvsDqFuPV+PkWd1oyUUkq5nZ4zUkop5XaajJRSSrmdJiOllFJupx0YVJEgImlYw8j4AKnA8ybTyM/5XOZe4DpjTJRYo7I/mumCSUfTDwS+N8Ycz8e60oAA+6Jmpf5ztGakipIW9jhjrwOT7YtxM1zOqBDGGpU910RkG4h1PZBSKhutGamiaAHWaNlXiMhPWBf+tgOiRORB4H2sMdqKYY0QMMAe0bkh1kjexe3yjPEIs9WSGgLD7XWAdYFrY6AS8IuInMcaceEMMAJrxIviwE/mwojsHbGGT0oh50WzSv3naM1IFUW9gP3GmCP26wisEbDvxRquKNoY0wpr5IUSwEP2dOOA9+wRsedjjfCQhVi335gBfGTXwpoCq4wx7wMxQHd7XLQY4FPgF3tdzYD2ItJFrHtLjQcesJcRn309Sv3XaM1IFSXrRASs+/T0yFQ+PtMIzN2AUmLdLwisZHTMHqetlrHvemuMmS4iJx2sQ4Bz9tiA2Ms9kUs83YDmIvKq/boUUBdrJPLj9tA8YI0q/XGetlQpL6PJSBUlLXLpAHA603MfrJvZrcw8gZ2MCpoPcKM91lvmdTUm7yNZK+XVtJlOqax+Bf7PHqkZEblCRGrYI0VHiUj6SOS34nhQTAMEiEhXezpfEQmz34vHumNv5nUNSu9IISIR9o0OdwLhIpJ+C/gHCnIDlfJEmoyUyuo9YDew3h6FeT5Q2X6vL/A/e/TxG3AwKrWxbg9+G/CiPd0GrJuvAYwCJtojvlcCnsEaKfove9opWLf/OId1x9FxIvIX4IpamVIeRcemU0op5XZaM1JKKeV2moyUUkq5nSYjpZRSbqfJSCmllNtpMlJKKeV2moyUUkq5nSYjpZRSbqfJSCmllNv9PxaATz6jrAMAAAAAAElFTkSuQmCC\n", 310 | "text/plain": [ 311 | "
" 312 | ] 313 | }, 314 | "metadata": { 315 | "needs_background": "light" 316 | }, 317 | "output_type": "display_data" 318 | } 319 | ], 320 | "source": [ 321 | "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", 322 | "datasetNames = [\"Trento\"]\n", 323 | "data2Name = 'LIDAR'\n", 324 | "\n", 325 | "patchsize = 11\n", 326 | "batchsize = 64\n", 327 | "testSizeNumber = 500\n", 328 | "EPOCH = 100\n", 329 | "BandSize = 1\n", 330 | "LR = 5e-4\n", 331 | "FM = 16\n", 332 | "HSIOnly = False\n", 333 | "FileName = 'MFT_HSILidar_Channel'\n", 334 | "ntokens = 4\n", 335 | "token_type = 'channel' \n", 336 | "num_heads = 8\n", 337 | "mlp_dim = 512\n", 338 | "depth = 2\n", 339 | "train_loss = []\n", 340 | "\n", 341 | "def set_seed(seed):\n", 342 | " torch.manual_seed(seed)\n", 343 | " torch.cuda.manual_seed_all(seed)\n", 344 | " np.random.seed(seed)\n", 345 | "\n", 346 | "for BandSize in [1]:\n", 347 | " for datasetName in datasetNames:\n", 348 | " print(\"---------------------------------- Dataset details for \",datasetName,\" ---------------------------------------------\")\n", 349 | " print('\\n')\n", 350 | " try:\n", 351 | " os.makedirs(datasetName)\n", 352 | " except FileExistsError:\n", 353 | " pass\n", 354 | " \n", 355 | " train_dataset = Multimodal_Dataset_Train(Filename=datasetName, MM_Data=data2Name)\n", 356 | " test_dataset = Multimodal_Dataset_Test(Filename=datasetName, MM_Data=data2Name)\n", 357 | " NC = train_dataset.hs_ims.shape[1]\n", 358 | " NCLidar = train_dataset.lid_ims.shape[1]\n", 359 | " Classes = len(torch.unique(train_dataset.lbs))\n", 360 | "\n", 361 | " train_loader = dataf.DataLoader(train_dataset, batch_size=batchsize, shuffle=True, num_workers= 4)\n", 362 | " print(\"HSI Train data shape = \", train_dataset.hs_ims.shape)\n", 363 | " print(data2Name + \" Train data shape = \", train_dataset.lid_ims.shape[1])\n", 364 | " print(\"Train label shape = \", train_dataset.lbs.shape)\n", 365 | "\n", 366 | " print(\"HSI Test data shape = \", test_dataset.hs_ims.shape)\n", 367 | " print(data2Name + \" Test data shape = \", test_dataset.lid_ims.shape[1])\n", 368 | " print(\"Test label shape = \", test_dataset.lbs.shape)\n", 369 | "\n", 370 | " print(\"Number of Classes = \", Classes)\n", 371 | " \n", 372 | " TestPatch1 = test_dataset.hs_ims\n", 373 | " TestPatch2 = test_dataset.lid_ims\n", 374 | " TestLabel1 = test_dataset.lbs\n", 375 | " \n", 376 | " KAPPA = []\n", 377 | " OA = []\n", 378 | " AA = []\n", 379 | " ELEMENT_ACC = np.zeros((3, Classes))\n", 380 | "\n", 381 | " set_seed(42)\n", 382 | " for iterNum in range(1):\n", 383 | " print('\\n')\n", 384 | " print(\"---------------------------------- Model Summary ---------------------------------------------\")\n", 385 | " print('\\n')\n", 386 | " if HSIOnly: \n", 387 | " model = Transformer(FM=FM, NC=NC, Classes=Classes, ntokens=ntokens, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n", 388 | " summary(model, [(NC, patchsize**2)])\n", 389 | " \n", 390 | " else:\n", 391 | " model = MFT(FM=FM, NC=NC, NCLidar=NCLidar, Classes=Classes, ntokens=ntokens, token_type=token_type, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n", 392 | " summary(model, [(NC, patchsize**2),(NCLidar,patchsize**2)]) \n", 393 | " \n", 394 | " optimizer = torch.optim.Adam(model.parameters(), lr=LR,weight_decay=5e-3)\n", 395 | " loss_func = nn.CrossEntropyLoss() # the target label is not one-hotted\n", 396 | " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.9)\n", 397 | " BestAcc = 0\n", 398 | "\n", 399 | " torch.cuda.synchronize()\n", 400 | " print('\\n')\n", 401 | " print(\"---------------------------------- Training started for \",datasetName,\" ---------------------------------------------\")\n", 402 | " print('\\n')\n", 403 | " start = time.time()\n", 404 | " # train and test the designed model\n", 405 | " for epoch in range(EPOCH):\n", 406 | " for step, (b_x1, b_x2, b_y) in enumerate(train_loader):\n", 407 | "\n", 408 | " # move train data to GPU\n", 409 | " b_x1 = b_x1.cuda()\n", 410 | " b_y = b_y.cuda()\n", 411 | " if HSIOnly:\n", 412 | " out1 = model(b_x1)\n", 413 | " loss = loss_func(out1, b_y)\n", 414 | " else:\n", 415 | " b_x2 = b_x2.cuda()\n", 416 | " out= model(b_x1, b_x2)\n", 417 | " loss = loss_func(out, b_y)\n", 418 | "\n", 419 | " optimizer.zero_grad() # clear gradients for this training step\n", 420 | " loss.backward() # backpropagation, compute gradients\n", 421 | " optimizer.step() # apply gradients\n", 422 | "\n", 423 | " if step % 50 == 0:\n", 424 | " model.eval()\n", 425 | " pred_y = np.empty((len(TestLabel1)), dtype='float32')\n", 426 | " number = len(TestLabel1) // testSizeNumber\n", 427 | " for i in range(number):\n", 428 | " temp = TestPatch1[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 429 | " temp = temp.cuda()\n", 430 | " temp1 = TestPatch2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 431 | " temp1 = temp1.cuda()\n", 432 | " if HSIOnly:\n", 433 | " temp2 = model(temp)\n", 434 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 435 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 436 | " del temp, temp2, temp3\n", 437 | " else:\n", 438 | " temp2 = model(temp, temp1)\n", 439 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 440 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 441 | " del temp, temp1, temp2, temp3\n", 442 | "\n", 443 | " if (i + 1) * testSizeNumber < len(TestLabel1):\n", 444 | " temp = TestPatch1[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n", 445 | " temp = temp.cuda()\n", 446 | " temp1 = TestPatch2[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n", 447 | " temp1 = temp1.cuda()\n", 448 | " if HSIOnly:\n", 449 | " temp2 = model(temp)\n", 450 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 451 | " pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n", 452 | " del temp, temp2, temp3\n", 453 | " else:\n", 454 | " temp2 = model(temp, temp1)\n", 455 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 456 | " pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n", 457 | " del temp, temp1, temp2, temp3\n", 458 | "\n", 459 | " pred_y = torch.from_numpy(pred_y).long()\n", 460 | " accuracy = torch.sum(pred_y == TestLabel1).type(torch.FloatTensor) / TestLabel1.size(0)\n", 461 | "\n", 462 | " print('Epoch: ', epoch, '| train loss: %.4f' % loss.data.cpu().numpy(), '| test accuracy: %.4f' % (accuracy*100))\n", 463 | " train_loss.append(loss.data.cpu().numpy())\n", 464 | " # save the parameters in network\n", 465 | " if accuracy > BestAcc:\n", 466 | "\n", 467 | " BestAcc = accuracy\n", 468 | " \n", 469 | " torch.save(model.state_dict(), datasetName+'/net_params_'+FileName+'.pkl')\n", 470 | " \n", 471 | "\n", 472 | " model.train()\n", 473 | " scheduler.step()\n", 474 | " torch.cuda.synchronize()\n", 475 | " end = time.time()\n", 476 | " print('\\nThe train time (in seconds) is:', end - start)\n", 477 | " Train_time = end - start\n", 478 | "\n", 479 | " # load the saved parameters\n", 480 | " \n", 481 | " model.load_state_dict(torch.load(datasetName+'/net_params_'+FileName+'.pkl'))\n", 482 | "\n", 483 | " model.eval()\n", 484 | " \n", 485 | " confusion, oa, each_acc, aa, kappa = reports(TestPatch1,TestPatch2,TestLabel1,datasetName,model, HSIOnly, iterNum)\n", 486 | " KAPPA.append(kappa)\n", 487 | " OA.append(oa)\n", 488 | " AA.append(aa)\n", 489 | " ELEMENT_ACC[iterNum, :] = each_acc\n", 490 | " torch.save(model, datasetName+'/best_model_'+FileName+'_BandSize'+str(BandSize)+'_Iter'+str(iterNum)+'.pt')\n", 491 | " print('\\n')\n", 492 | " print(\"Overall Accuracy = \", oa)\n", 493 | " print('\\n')\n", 494 | " print(\"----------\" + datasetName + \" Training Finished -----------\")\n", 495 | " print(\"\\nThe Confusion Matrix on test data\")\n", 496 | " record.record_output(OA, AA, KAPPA, ELEMENT_ACC,'./' + datasetName +'/'+FileName+'_BandSize'+str(BandSize)+'_Report_' + datasetName +'.txt')\n", 497 | "\n" 498 | ] 499 | }, 500 | { 501 | "cell_type": "code", 502 | "execution_count": 5, 503 | "id": "c0ef6f47", 504 | "metadata": {}, 505 | "outputs": [], 506 | "source": [ 507 | "train_loss = np.asarray(train_loss)\n", 508 | "np.save('HSILiDAR_channel_train_loss.npy', train_loss)" 509 | ] 510 | }, 511 | { 512 | "cell_type": "code", 513 | "execution_count": null, 514 | "id": "77f5a7ac", 515 | "metadata": {}, 516 | "outputs": [], 517 | "source": [] 518 | } 519 | ], 520 | "metadata": { 521 | "kernelspec": { 522 | "display_name": "Python 3", 523 | "language": "python", 524 | "name": "python3" 525 | }, 526 | "language_info": { 527 | "codemirror_mode": { 528 | "name": "ipython", 529 | "version": 3 530 | }, 531 | "file_extension": ".py", 532 | "mimetype": "text/x-python", 533 | "name": "python", 534 | "nbconvert_exporter": "python", 535 | "pygments_lexer": "ipython3", 536 | "version": "3.8.10" 537 | }, 538 | "vscode": { 539 | "interpreter": { 540 | "hash": "26407125db06bdd9abe40e82cf041582bb19887fa16dd38638e528b7039723e8" 541 | } 542 | } 543 | }, 544 | "nbformat": 4, 545 | "nbformat_minor": 5 546 | } 547 | -------------------------------------------------------------------------------- /MFT_Transformer_HSI+LiDAR_Pixel_Tokenization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "94a576ee", 6 | "metadata": {}, 7 | "source": [ 8 | "## Transformer Model for Hyperspectral Image Classification\n", 9 | "\n", 10 | "\n", 11 | "### The MFT model with both HSI and LiDAR images used for classification. The other multimodal data is from the LiDAR and is used to generate the external CLS through 'pixel' tokenization. " 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "id": "73594467", 18 | "metadata": { 19 | "ExecuteTime": { 20 | "end_time": "2022-08-11T11:17:16.979544Z", 21 | "start_time": "2022-08-11T11:17:16.971125Z" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "import sys\n", 27 | "sys.path.append(\"./../\")\n", 28 | "import matplotlib.pyplot as plt\n", 29 | "from torchsummary import summary\n", 30 | "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n", 31 | "import math\n", 32 | "from PIL import Image\n", 33 | "import time\n", 34 | "from scipy.io import loadmat as loadmat\n", 35 | "from scipy import io\n", 36 | "import random\n", 37 | "import numpy as np\n", 38 | "import os\n", 39 | "import torch \n", 40 | "import torch.utils.data as dataf\n", 41 | "import torch.nn as nn\n", 42 | "from operator import truediv\n", 43 | "import record\n", 44 | "import pandas as pd\n", 45 | "import seaborn as sns\n", 46 | "from mft_model import MFT, Transformer\n", 47 | "from dataset import Multimodal_Dataset_Train, Multimodal_Dataset_Test\n", 48 | "import torch.backends.cudnn as cudnn\n", 49 | "cudnn.deterministic = True\n", 50 | "cudnn.benchmark = False\n" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 2, 56 | "id": "7e3667b8", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "def get_confusion_matrix(y_test,y_pred, plt_name):\n", 61 | " df_cm = pd.DataFrame(confusion_matrix(y_test, y_pred), range(6),range(6))\n", 62 | " df_cm.columns = ['Buildings','Woods', 'Roads', 'Apples', 'ground', 'Vineyard']\n", 63 | " df_cm = df_cm.rename({0:'Buildings',1:'Woods', 2:'Roads', 3:'Apples', 4:'ground', 5:'Vineyard'})\n", 64 | " df_cm.index.name = 'Actual'\n", 65 | " df_cm.columns.name = 'Predicted'\n", 66 | " sns.set(font_scale=0.9)#for label size\n", 67 | " sns.heatmap(df_cm, cmap=\"Blues\",annot=True,annot_kws={\"size\": 16}, fmt='g')\n", 68 | " plt.savefig(''+str(plt_name)+'.eps', format='eps')\n", 69 | "\n", 70 | "def AA_andEachClassAccuracy(confusion_matrix):\n", 71 | " counter = confusion_matrix.shape[0]\n", 72 | " list_diag = np.diag(confusion_matrix)\n", 73 | " list_raw_sum = np.sum(confusion_matrix, axis=1)\n", 74 | " each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n", 75 | " average_acc = np.mean(each_acc)\n", 76 | " return each_acc, average_acc\n", 77 | "\n", 78 | "def reports (xtest,xtest2,ytest,name,model, HSIOnly, iternum):\n", 79 | " pred_y = np.empty((len(ytest)), dtype=np.float32)\n", 80 | " number = len(ytest) // testSizeNumber\n", 81 | " for i in range(number):\n", 82 | " temp = xtest[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 83 | " temp = temp.cuda()\n", 84 | " temp1 = xtest2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 85 | " temp1 = temp1.cuda()\n", 86 | " if HSIOnly:\n", 87 | " temp2 = model(temp)\n", 88 | " else:\n", 89 | " temp2 = model(temp,temp1)\n", 90 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 91 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 92 | " del temp, temp2, temp3,temp1\n", 93 | "\n", 94 | " if (i + 1) * testSizeNumber < len(ytest):\n", 95 | " temp = xtest[(i + 1) * testSizeNumber:len(ytest), :, :]\n", 96 | " temp = temp.cuda()\n", 97 | " temp1 = xtest2[(i + 1) * testSizeNumber:len(ytest), :, :]\n", 98 | " temp1 = temp1.cuda()\n", 99 | " if HSIOnly:\n", 100 | " temp2 = model(temp)\n", 101 | " else:\n", 102 | " temp2 = model(temp,temp1)\n", 103 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 104 | " pred_y[(i + 1) * testSizeNumber:len(ytest)] = temp3.cpu()\n", 105 | " del temp, temp2, temp3,temp1\n", 106 | "\n", 107 | " pred_y = torch.from_numpy(pred_y).long()\n", 108 | " \n", 109 | " oa = accuracy_score(ytest, pred_y)\n", 110 | " confusion = confusion_matrix(ytest, pred_y)\n", 111 | " each_acc, aa = AA_andEachClassAccuracy(confusion)\n", 112 | " kappa = cohen_kappa_score(ytest, pred_y)\n", 113 | " get_confusion_matrix(ytest, pred_y, 'test_'+str(iternum)+'')\n", 114 | " \n", 115 | " return confusion, oa*100, each_acc*100, aa*100, kappa*100\n" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 3, 121 | "id": "dda72f4e", 122 | "metadata": { 123 | "ExecuteTime": { 124 | "end_time": "2022-08-11T11:18:10.978783Z", 125 | "start_time": "2022-08-11T11:18:01.894932Z" 126 | }, 127 | "scrolled": false 128 | }, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "---------------------------------- Dataset details for Trento ---------------------------------------------\n", 135 | "\n", 136 | "\n", 137 | "HSI Train data shape = torch.Size([819, 63, 11, 11])\n", 138 | "LIDAR Train data shape = 1\n", 139 | "Train label shape = torch.Size([819])\n", 140 | "HSI Test data shape = torch.Size([29395, 63, 11, 11])\n", 141 | "LIDAR Test data shape = 1\n", 142 | "Test label shape = torch.Size([29395])\n", 143 | "Number of Classes = 6\n", 144 | "\n", 145 | "\n", 146 | "---------------------------------- Model Summary ---------------------------------------------\n", 147 | "\n", 148 | "\n", 149 | "===============================================================================================\n", 150 | "Layer (type:depth-idx) Output Shape Param #\n", 151 | "===============================================================================================\n", 152 | "├─Sequential: 1-1 [-1, 8, 55, 11, 11] --\n", 153 | "| └─Conv3d: 2-1 [-1, 8, 55, 11, 11] 656\n", 154 | "| └─BatchNorm3d: 2-2 [-1, 8, 55, 11, 11] 16\n", 155 | "| └─ReLU: 2-3 [-1, 8, 55, 11, 11] --\n", 156 | "├─Sequential: 1-2 [-1, 64, 11, 11] --\n", 157 | "| └─HetConv: 2-4 [-1, 64, 11, 11] --\n", 158 | "| | └─Conv2d: 3-1 [-1, 64, 11, 11] 31,744\n", 159 | "| | └─Conv2d: 3-2 [-1, 64, 11, 11] 28,224\n", 160 | "| └─BatchNorm2d: 2-5 [-1, 64, 11, 11] 128\n", 161 | "| └─ReLU: 2-6 [-1, 64, 11, 11] --\n", 162 | "├─Sequential: 1-3 [-1, 64, 11, 11] --\n", 163 | "| └─Conv2d: 2-7 [-1, 64, 11, 11] 640\n", 164 | "| └─BatchNorm2d: 2-8 [-1, 64, 11, 11] 128\n", 165 | "| └─GELU: 2-9 [-1, 64, 11, 11] --\n", 166 | "├─Dropout: 1-4 [-1, 5, 64] --\n", 167 | "├─TransformerEncoder: 1-5 [-1, 64] --\n", 168 | "| └─ModuleList: 2 [] --\n", 169 | "| | └─Block: 3-3 [-1, 5, 64] 100,736\n", 170 | "| | └─Block: 3-4 [-1, 5, 64] 100,736\n", 171 | "| └─LayerNorm: 2-10 [-1, 5, 64] 128\n", 172 | "├─Linear: 1-6 [-1, 6] 390\n", 173 | "===============================================================================================\n", 174 | "Total params: 263,526\n", 175 | "Trainable params: 263,526\n", 176 | "Non-trainable params: 0\n", 177 | "Total mult-adds (M): 12.34\n", 178 | "===============================================================================================\n", 179 | "Input size (MB): 0.03\n", 180 | "Forward/backward pass size (MB): 1.12\n", 181 | "Params size (MB): 1.01\n", 182 | "Estimated Total Size (MB): 2.15\n", 183 | "===============================================================================================\n", 184 | "\n", 185 | "\n", 186 | "---------------------------------- Training started for Trento ---------------------------------------------\n", 187 | "\n", 188 | "\n", 189 | "Epoch: 0 | train loss: 2.6631 | test accuracy: 36.1660\n", 190 | "Epoch: 1 | train loss: 1.0541 | test accuracy: 36.3157\n", 191 | "Epoch: 2 | train loss: 0.8849 | test accuracy: 49.8146\n", 192 | "Epoch: 3 | train loss: 0.5145 | test accuracy: 61.1669\n", 193 | "Epoch: 4 | train loss: 0.2669 | test accuracy: 85.0519\n", 194 | "Epoch: 5 | train loss: 0.1643 | test accuracy: 92.5191\n", 195 | "Epoch: 6 | train loss: 0.1279 | test accuracy: 90.4474\n", 196 | "Epoch: 7 | train loss: 0.0688 | test accuracy: 82.4596\n", 197 | "Epoch: 8 | train loss: 0.0712 | test accuracy: 90.9134\n", 198 | "Epoch: 9 | train loss: 0.0702 | test accuracy: 94.1112\n", 199 | "Epoch: 10 | train loss: 0.0366 | test accuracy: 94.1589\n", 200 | "Epoch: 11 | train loss: 0.0423 | test accuracy: 82.5446\n", 201 | "Epoch: 12 | train loss: 0.0295 | test accuracy: 94.4242\n", 202 | "Epoch: 13 | train loss: 0.0377 | test accuracy: 94.5161\n", 203 | "Epoch: 14 | train loss: 0.0463 | test accuracy: 91.9510\n", 204 | "Epoch: 15 | train loss: 0.0255 | test accuracy: 93.3050\n", 205 | "Epoch: 16 | train loss: 0.0290 | test accuracy: 93.4989\n", 206 | "Epoch: 17 | train loss: 0.0229 | test accuracy: 90.8998\n", 207 | "Epoch: 18 | train loss: 0.0284 | test accuracy: 86.1303\n", 208 | "Epoch: 19 | train loss: 0.0342 | test accuracy: 94.4004\n", 209 | "Epoch: 20 | train loss: 0.0222 | test accuracy: 94.3562\n", 210 | "Epoch: 21 | train loss: 0.0268 | test accuracy: 94.1997\n", 211 | "Epoch: 22 | train loss: 0.0314 | test accuracy: 94.3052\n", 212 | "Epoch: 23 | train loss: 0.0221 | test accuracy: 93.7166\n", 213 | "Epoch: 24 | train loss: 0.1109 | test accuracy: 91.5768\n", 214 | "Epoch: 25 | train loss: 0.0546 | test accuracy: 82.8644\n", 215 | "Epoch: 26 | train loss: 0.0192 | test accuracy: 95.2577\n", 216 | "Epoch: 27 | train loss: 0.0636 | test accuracy: 93.5159\n", 217 | "Epoch: 28 | train loss: 0.0118 | test accuracy: 95.2339\n", 218 | "Epoch: 29 | train loss: 0.0137 | test accuracy: 95.2645\n", 219 | "Epoch: 30 | train loss: 0.0123 | test accuracy: 93.8527\n", 220 | "Epoch: 31 | train loss: 0.0195 | test accuracy: 94.9073\n", 221 | "Epoch: 32 | train loss: 0.0113 | test accuracy: 95.1420\n", 222 | "Epoch: 33 | train loss: 0.0210 | test accuracy: 95.1454\n", 223 | "Epoch: 34 | train loss: 0.0119 | test accuracy: 94.8563\n", 224 | "Epoch: 35 | train loss: 0.0235 | test accuracy: 94.6590\n", 225 | "Epoch: 36 | train loss: 0.0275 | test accuracy: 95.1046\n", 226 | "Epoch: 37 | train loss: 0.0287 | test accuracy: 95.2169\n", 227 | "Epoch: 38 | train loss: 0.0187 | test accuracy: 88.1851\n", 228 | "Epoch: 39 | train loss: 0.0107 | test accuracy: 95.2169\n", 229 | "Epoch: 40 | train loss: 0.0159 | test accuracy: 92.3184\n", 230 | "Epoch: 41 | train loss: 0.0130 | test accuracy: 95.1080\n", 231 | "Epoch: 42 | train loss: 0.0143 | test accuracy: 95.1216\n", 232 | "Epoch: 43 | train loss: 0.0128 | test accuracy: 94.1078\n", 233 | "Epoch: 44 | train loss: 0.0142 | test accuracy: 95.2441\n", 234 | "Epoch: 45 | train loss: 0.0109 | test accuracy: 94.5127\n", 235 | "Epoch: 46 | train loss: 0.0146 | test accuracy: 95.0978\n", 236 | "Epoch: 47 | train loss: 0.0142 | test accuracy: 95.2135\n", 237 | "Epoch: 48 | train loss: 0.0096 | test accuracy: 92.5021\n", 238 | "Epoch: 49 | train loss: 0.1106 | test accuracy: 93.6010\n", 239 | "Epoch: 50 | train loss: 0.0182 | test accuracy: 82.0922\n", 240 | "Epoch: 51 | train loss: 0.0158 | test accuracy: 94.2031\n", 241 | "Epoch: 52 | train loss: 0.0310 | test accuracy: 93.7847\n", 242 | "Epoch: 53 | train loss: 0.0172 | test accuracy: 92.1177\n", 243 | "Epoch: 54 | train loss: 0.0216 | test accuracy: 93.4138\n", 244 | "Epoch: 55 | train loss: 0.0114 | test accuracy: 93.7949\n", 245 | "Epoch: 56 | train loss: 0.0173 | test accuracy: 94.9992\n", 246 | "Epoch: 57 | train loss: 0.0131 | test accuracy: 94.6692\n", 247 | "Epoch: 58 | train loss: 0.0169 | test accuracy: 95.2781\n", 248 | "Epoch: 59 | train loss: 0.0226 | test accuracy: 93.1689\n", 249 | "Epoch: 60 | train loss: 0.0127 | test accuracy: 95.0332\n", 250 | "Epoch: 61 | train loss: 0.0124 | test accuracy: 93.3050\n", 251 | "Epoch: 62 | train loss: 0.0169 | test accuracy: 95.1692\n", 252 | "Epoch: 63 | train loss: 0.0113 | test accuracy: 94.1283\n", 253 | "Epoch: 64 | train loss: 0.0204 | test accuracy: 95.1897\n", 254 | "Epoch: 65 | train loss: 0.0093 | test accuracy: 91.4509\n", 255 | "Epoch: 66 | train loss: 0.0136 | test accuracy: 95.2679\n", 256 | "Epoch: 67 | train loss: 0.0133 | test accuracy: 95.3087\n", 257 | "Epoch: 68 | train loss: 0.0119 | test accuracy: 95.1795\n", 258 | "Epoch: 69 | train loss: 0.0217 | test accuracy: 95.0025\n", 259 | "Epoch: 70 | train loss: 0.0114 | test accuracy: 95.3257\n", 260 | "Epoch: 71 | train loss: 0.0138 | test accuracy: 92.7845\n", 261 | "Epoch: 72 | train loss: 0.0118 | test accuracy: 95.0162\n", 262 | "Epoch: 73 | train loss: 0.0157 | test accuracy: 95.1012\n", 263 | "Epoch: 74 | train loss: 0.0111 | test accuracy: 95.3461\n", 264 | "Epoch: 75 | train loss: 0.0081 | test accuracy: 94.6828\n", 265 | "Epoch: 76 | train loss: 0.0101 | test accuracy: 95.2815\n", 266 | "Epoch: 77 | train loss: 0.0109 | test accuracy: 95.2679\n", 267 | "Epoch: 78 | train loss: 0.0103 | test accuracy: 86.3956\n", 268 | "Epoch: 79 | train loss: 0.0177 | test accuracy: 94.4038\n", 269 | "Epoch: 80 | train loss: 0.0294 | test accuracy: 95.3291\n", 270 | "Epoch: 81 | train loss: 0.0162 | test accuracy: 94.9583\n", 271 | "Epoch: 82 | train loss: 0.0136 | test accuracy: 94.9719\n", 272 | "Epoch: 83 | train loss: 0.0116 | test accuracy: 95.1148\n", 273 | "Epoch: 84 | train loss: 0.0092 | test accuracy: 95.1386\n", 274 | "Epoch: 85 | train loss: 0.0084 | test accuracy: 94.4787\n", 275 | "Epoch: 86 | train loss: 0.0085 | test accuracy: 93.9548\n", 276 | "Epoch: 87 | train loss: 0.0103 | test accuracy: 95.0910\n", 277 | "Epoch: 88 | train loss: 0.0106 | test accuracy: 93.0600\n", 278 | "Epoch: 89 | train loss: 0.0105 | test accuracy: 90.7161\n", 279 | "Epoch: 90 | train loss: 0.0101 | test accuracy: 89.9269\n", 280 | "Epoch: 91 | train loss: 0.0108 | test accuracy: 94.7134\n" 281 | ] 282 | }, 283 | { 284 | "name": "stdout", 285 | "output_type": "stream", 286 | "text": [ 287 | "Epoch: 92 | train loss: 0.0112 | test accuracy: 93.9377\n", 288 | "Epoch: 93 | train loss: 0.0363 | test accuracy: 81.0614\n", 289 | "Epoch: 94 | train loss: 0.0215 | test accuracy: 95.0944\n", 290 | "Epoch: 95 | train loss: 0.0134 | test accuracy: 94.9923\n", 291 | "Epoch: 96 | train loss: 0.0131 | test accuracy: 95.2815\n", 292 | "Epoch: 97 | train loss: 0.0143 | test accuracy: 95.3223\n", 293 | "Epoch: 98 | train loss: 0.0087 | test accuracy: 95.4142\n", 294 | "Epoch: 99 | train loss: 0.0080 | test accuracy: 95.4720\n", 295 | "\n", 296 | "The train time (in seconds) is: 1240.328688621521\n", 297 | "\n", 298 | "\n", 299 | "Overall Accuracy = 95.47201905085899\n", 300 | "\n", 301 | "\n", 302 | "----------Trento Training Finished -----------\n", 303 | "\n", 304 | "The Confusion Matrix on test data\n" 305 | ] 306 | }, 307 | { 308 | "data": { 309 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAEKCAYAAAC/hjrSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABZwElEQVR4nO3dd3gU1dfA8W8qJZBGCy2UBA5FekdRwEYTBaWqNEF5RREril1+SLGBoiIdVLpIEykiSJEiHSkXEloqPQklkPr+MZOQsiGbkM1uzP3w7MPu3SlnJrN79t65c8cpOTkZTdM0TbMnZ3sHoGmapmk6GWmapml2p5ORpmmaZnc6GWmapml2p5ORpmmaZnc6GWmapml252rvADRN07T8JSLTgMeBcKVUQ7OsFLAI8AcU0EspdV1EnIBvgUeAq0BfpdRRc57ngLeBZGCUUmqJWd4MmAUUBRYrpd7JLiadjHJo5j9nC9yFWX0b+ds7hBxJKoDXvjk7Odk7BM0BFXXlrg+MYo1esvoDEbtvsrXrmwt8D8xMU/YOsFIpNVFERgPDgbFAZ6CCUipQRB4EJgKPmsnrbaAJRtLZLiK/KaVige+AfsB+YKOItFJKbb9TQLqZTtM0zZE5OVv/sJJSagtwOUNxV+BH8/lc4ImM5UqpDUAdEfEAHgXWKKVilFLngR1AWxGpABRVSu1VSiUBP6VZVpZ0zUjTNM2RObtYPamIeAPeFt6KUkpFZTN7KaXUJfN5KFDRfF4BCEszXThQ3kJ5yjwXLJQ/kF3sumakaZrmyJycrH/ACOCUhccIK9Zk1/ZxnYw0TdMcWc6a6SYC1Sw8JlqxpsvmeSCAStyu3YRzu5YERo0owkJ5yjxZld+RbqbTNE1zZDnoHGM2xUXlck2rgGcxElc/YHma8kHAL2YHhqNmL7t1wCci4onRgaEVMFgpFSsicSLSGKMDwzMYnSPuSCcjTdM0R5aDjgnWEpEfgYeAUiISitErbiywSESGASeAXubkq4BOIhKM0bX7aQCl1EURGQ/sxWjie9vsSQfwIkanh6LAL0qpv7OLyUnfQiJndNdu29Ndu7X/ijzp2t3qbeu7dm8fV2APRF0z0jRNc2Q56E1XkOlkpGma5shs0EzniHQy0jRNc2SFpAlYJ6O7cPLgP+xctYhLYWe4ef0axUp6UbFGHe57sh+lK1ZJnS70+L9sW/oj58+eJCHuFj7lKtL4kcep/0CHdMtLiItjy5LZHN62gVs3rlG2SgBtew+mcq36mdZ99fJFtiyZzckDu7h5/RolfEpRu2VbHuj1nE23OTIigs/Gj2XH9m0kJyfTolVr3ho5ivIVKth0vdbav3cvP3w/GaWOcevmTfyrVKVXn6d5ovuTqdOEhYby1RcT2LljOwkJCdxzTz1GvP4mde+pZ8fIb3P0fZyVLZv/Yub0qRw9cgRnZyeqVK3KiNfepEXLVvYOLVv/9/xz/L1tK0OeH8pLr7xq73DS0zUjLTs3r13Fr2oNGj30GMVLehFz6Tw7Vi7kxw+HM2jcVLxKl+P82ZMsHDuSCoG16fDcq7i6F0Ht2sLv074gMT6eRg89lrq836d/QfD+nbTt8zzeZfzY98cKFo1/h2c+mkS5KoGp00VfiOSnT0bgVcaPB58dhoeXN9EXznHlXLhNtzc2NpYhg/rj5u7O6E/H4+QEk7+exOBB/Vi8dAXFixe36fqzc1wphg4ZSL36DXj/o9EUK1qUP9av5eMP3iUuLo6evfsQFXWFgf364uHhwXsffEzRYkX5ac5snh/Unx/nL6Z6QIBdt8HR93FWFi9awLgxo+nd52meH/oiSUlJqGNHuXnzpr1Dy9bvv63iuFL2DiNrOhndHRFJBA6ZLxOBYUqpHdnMsxqjO2EpYFnKaLIZptmPMc5RHPCVUqpXxmnyS53W7anTun26svLVazH9rUGoXZtp3qkHR7dvJCkpiSdfH4170WIAVKvXhAshJ/l36/rUZHT+TDBH/v6TjkNeT60x+dduwIyRg9m6ZA5Pvj46dR1rZ06ipE9p+oz6HBdX809Y2/bbu3TJIkJDQ1i+ag3+VYyaX42aQtdOj7Jk0UL6DRho+yDuYO3vv5GYmMSkb7+neHEPAFq2vpfjxxWrVi6jZ+8+LF64gMuXLjFj9k9U9jd6GTZv3pIuHR5mynffMOGLiXbcAsffx5aEhYXy2bhPefX1N3mm34DU8nvva2O/oKwUEx3NZ+PH8ubId3j7rdftHY5lLoWjA4MtU+5VpVRDM6GMAsZkN4NSqpNS6qo1C1dKhdszEWWlWElPAJzNHjCJiQm4uLri6u6ebroixTxI263+xN7tOLu4Urtl29QyZxcXarVqy6lDe0iIjwPgyrlwTh3aTeNHHr+diPLJpo1/Ur9+g9QvSYBKlSrTsFFjNm3ckK+xWBIfH4+rmytFihRNV16yREmSk4x9fejAfvz9q6QmIoBixYvTqEkTtvy1iYSEhHyNOSNH38eWLFv6C07OzvTo1cfeoeTYxC8/J7BGDTp27mLvULKWs+GACqz8qv95AtEAItJWRJalvCEis0XkCfP5aXOgP9K8X1xEFovIERFZALiZ5VXNWhIiMkBElojIOhEJEpH308z/oYgoEflLRH4VkQFm+ThzmQdF5PO72bikpEQSE+K5HBnK2pkT8fDypXardgDUa/MIAH/M/ZarVy5y8/o19m9czZkj+2jaoXvqMi6GncG7jB9uGb5IS1esSmJCfGoTXNjxwwC4uhdhwbiRfD6gExOf78aqKeOJvRpzN5uRreCgIAJq1MxUHhAQyMngIJuu2xpdn+gGwISxYzh//hxXY2JYumQRu3bu4Oln+wNGgnd1c8s0r7u7Ozdv3iQ05Gy+xpyRo+9jS/bt3UO1atVZs/o3Ond4iMb169Clw8MsmPezvUO7o717drNyxTJGvfeBvUO5MxuM2u2IbPnTuqSZLIoCfsCDuVzOi8BFpVQdEbkX2JrFdA2AxkAScEJEvgFqYAx/Xh8ojtFsuNwcf6kbUEsplZwxAebUjx++TOSpEwD4lKtA71ET8PDyAaBM5Wr0GfU5Syd+xL4/VgLg7OLKIwNfoY6ZsABuXouhiEeJTMsuVqKk+b5RYbx2xRhU9/epX1D3vgdp9VhvrpwL569FM7gUdoZ+H0/Gydk2B2V0dDSenp6Zyr28vIiJsW0itEZgjZpMnzmX10a8xKIF8wBwdXVj1Psf0aFTZwCqVq3Gzu1/ExV1BW9v42+UlJTEv4eMFuXo6Gj7BG9y9H1syYUL57lw/jxffTGBl195jUqVK7N+3RrGjvmExMSE1B8CjiQ+Lo7RH39I/wGDqFqtur3DubMCXuOxli2T0dU0dxBsBcwVkXtysZz7gC8BlFLbRORUFtP9kdLEZw5bUQm4F+Pc0y3gloisNaeNBm4CM0RkFcZwF7nWeehI4mJvEHU+gl2rl7Bw3Ns888FXeJXx43JkKL9+/TGlK1Xh0UGv4OrmTtDe7aybNQlXN3fq3puzHJ2cnASAf+36PDJgOABV6jaiSHEPVkwew8lDuwlo0PxuNqfAOnPmNG+8OpyAgEDeff8jihQtyqY/N/Dp6I8oUqQInbo8xlM9ezP/5x95/523eeuddylarCgzpk4hPCwUAGcbJfL/sqSkZK5fv84nY8bx0MNGS0CLlq0IDwtjxrSp9H2mH04O9oU6a+Z0bt26yeAX/s/eoWSvgNd4rJUvW2ne4a80UAZIyLDeIlYswprhMG6leZ7IHRKtUioBaA4sAToBa6xYfpZKV6xChcDa1Gndnt7vTCD+Viw7Vi4AYPOimbi4uPLU6/8jsFFLqt7TmIf6DaNWiwfY8ON3JCcZyaWoR0luXb+WadmxZo2oqFlDKlbC+NVctV6TdNNVM1+fP227phxPL0+Lv86z+jWf3yZP+gpXV1cmfTuF+9u2o0XLVowc9R4PP9qBz8aNISkpiUqVKzNm/GccPXKYrp0e4ZF293PwwP7UX++ly5Sx6zY4+j62xNvbG4BWrVunK2/V+j4uXbrIhQvn7RBV1iLCw5k+dQrDXn6FuLg4YmJiUvd5XLzxOjEx0c5RpqHPGeUdEakFuACXgDNAbRFxExFfsr/p0lbMAfvMGla1HKx6G9BVRNxFxAvjHu6ISAnAWym1GngVyLMLTIp6lMCnXIXUczwXQk5T1r96ps4G5asLsddiuB4TBUDpSlWIuhBJ/K30XWEvhZ3BxdUNn3IVzOmq3nH9Tjb8FRUQEEhw0IlM5SdPBlM9INDCHPkr6MRxakot3DKcE7rnnvpERUVx+bLRxPnQw4+y9s+/+GX5b6xYvY55i5Zy48YN/PzKU768fa/lcfR9bElA4J3jcnawX/ahoSHcunWLUSPfpE2rZqkPgDmzZtKmVTNOHD9u5yjTcHax/lGA2fIoKSki+83zRguB/kqpRKVUCPArcBjjdrR7s1nOd4CfiBzBuCf7EWsDUEr9A/wO/IsxHPpBIAYoCawUkYMYCeuNnGzYnVyPvsKl8BC8yxpfah5ePpw7E0xiQny66cKDj+Hq5p56TiiwUSuSEhM4tnNz6jRJiYkc3bmJqvWa4Opm9MarEFgbDy9fTh3cnW55Jw/8A4Bf9cwnv/NK23btOXTwAKEhIallYWGh7N+3lwfatb/DnPmjVKnSKHWUeLPnYYpDhw5QpEgRvLy8UstcXFyoHhBAZX9/zp8/x7o1v/NUr975HXImjr6PLWn/4MMA/L01/encbVu3UM7Pz+61zYykVm2mz5qb6QHQ+bGuTJ81F39/BxpcuJB0YPjPj9otIiWUUtfMe27sALoopU7mdnlpR+1e+tVHlKsaSFn/6rgXK86ViFD+WbOU69GX6ffxZHzLV+LYrs0s/3o0Ves1ofFDXXF1N84Z7V2/gqYdnuTBZ4amLnv55DGcOribdn2G4FXGj30bVhG8fwfPfDAJv2o1Uqc7tHkdq6d+RsP2nanZ7D6uRIazefEsylUJoPeozzK1z+fVqN03btygZ/fHKVK0KC8NfwUnnPj2m0lcv3GdJUtXUNzDI0/Wk9tRu9evW8Nbr42gVet76dG7L0WLFOGvTX+ycP48nu7Xnzfeeof4+Hgmffk5TZo2w6NECYKDTjBz+lQqV/bnhxmzcHNzz35FFuTVqN35tY/zUnJyMkMG9ee4Urz0yggqVarM+rVrWPrLYj7531ge79Y9+4U4gAZ1Jc9HYMiTUbs7f239qN2/DS+wbXWFIRnNA+pgnJv6Tin1zd0sL20y2rFyAcd2bibqfDiJCQl4+pahcu0GtOraG68yfqnzBB/Yxc6VC7kYdobE+Di8y5anQbvONHywc+r1SADxcbfYvGgWR7f/yc0b1yjrH0DbXoPxr9MgUxz/bl3PzlWLuBIZRlGPkkjzNjzQ67nUC2vTystbSESEh6cfqqZlK958exQVK1bKs3XczS0ktm7ZzOwZ0wgODiLu1i0qVfbnyR49ebJHL1xcXEhISOC14cM4/O+/XL0aQ7lyfnTo1JlBQ16gWLHM+85aeXkLifzYx3nt2rVrfP3VF6xfv5aY6BiqVa/GoOeep1OXx7Kf2UE4bDLqMtn6ZLTqJZ2MCgt9PyPb0/cz0v4r8iQZPfad9clo5YsF9kDUY9NpmqY5sgLeMcFaOhlpmqY5skJS69bJSNM0zZEV8F5y1tLJSNM0zZHpmpGmaZpmb442lJKt6GSkaZrmwHQy0jRN0+zOyVknI03TNM3OdM1I0zRNszudjDRN0zS708lI0zRNs7/CkYt0MtI0TXNkumakWVTQBh0F2BF82d4h5EjLAF97h6BpDsPZWY/AoGmaptmZrhlpmqZp9lc4cpFORpqmaY5M14w0TdM0u9PJSNM0TbM7PRyQpmmaZne2qhmJyKvAYPPlDuAFoAowH/AFtgLPKaUSRaQo8BPQEIgEeiilIszlvAcMAOKB55VSW3ITT+HoM6hpmlZAOTk5Wf2wloj4AS8DTYB7gApAB2A88KlSKhBwAXqaswwGQs3yWcBH5nLqAY8DtYFuwPe53U6djDRN0xyYLZIRRh89V6Ao4Gb+fw5oA6wwp5kLPGE+7wr8aD6fD3Q2nz8GLFBKxSuljgHRIlI7N9upm+k0TdMcWA5rPN6At4W3opRSUSkvlFIRIvIFEALEAQuAU8AVpVSSOVkoUNF8XgEIM+e9ISIuIuJmlm9Ns56UeY5aHbRJ14w0TdMcmVMOHjACI6lkfIxIu0gR8cGo1VTFSCjVMZrp7EbXjOwoMiKCz8aPZcf2bSQnJ9OiVWveGjmK8hUq2Gyde7b9yT+b13Mm6BhXoy/jW8aPRi0foGOP/hQt7gHA7Imj2f7naovzl6vozyffL7T43polc/l17vcE1K7PW+N/SPfeqMHduHQ+MtM8/zdqHA1bPnCXW3Vn9tjPd6OgxQuOHfO2rVuYNWMaJ4ODiYmJxsfXl4YNGzH0xZcJCAy0ehp7yeFwQBOB2RbKozK8fggIVkpdAhCRFUBdwEdEnM3aUSXM2hAQjlHjiRSRYkCiUipeRFLKU6SdJ0d0MrKT2NhYhgzqj5u7O6M/HY+TE0z+ehKDB/Vj8dIVFC9e3CbrXf/rPHzLlOOJZ4fiU7oMZ4OPs2rBDNShvbw1YSrOzs507jWQ+zt0SzffpfMRTP/8Axo0b2NxuRciw1i9aDYlvXyyXHedRi14rM/gdGXlKtl2rD977efcKmjxguPHHBMdTZ26denVuy8+vr5ERIQzc/o0nu3bkyXLVlKhQkWrprGXnDTTmU1xUVZMehZoZSaWm0A74DdgG8b5oWVAP2C5Of0q4FlgD9AXWJ2mfIaIfA0EAN5KqRw30YFORnazdMkiQkNDWL5qDf5VqgBQo6bQtdOjLFm0kH4DBtpkvcPe/yxdwqh5T2M8Snoye+Jojh/aS60GTSlTvhJlyldKN9/R/bsAaPVgJ4vLnff9ZzR/4BHOhZ0lMTHR4jQlPL2pXuuePNoS69hrP+dWQYsXHD/mjp270LFzl3Rl9erV5/EuHVm/bi39Bwyyahq7sUHPbqXUThH5FdgLJGJ07f4ZIxktMM8nbQMWmbNMA34WkSCMjg49zOUcFJFVwDGMrt0v5DYmh0lGIvIVRrVxsvl6B7BVKfWG+XohsEgp9Usulz8CI2t/lDcR351NG/+kfv0GqR9egEqVKtOwUWM2bdxgsw+wpZpL1Rp1AIi6fCHL+XZs/B3/wFpU8K+e6b1df63lbLBi8BsfM2XsO3kXbB6w137OrYIWLxTMmL28vQFwdXG5q2nyg62uM1JKfQh8mKE4CGhqYdpYoHsWy/kY+Phu43GkDgzbgVYAIlIEo7thkzTvtwL+tkNcNhEcFERAjZqZygMCAjkZHJSvsRz/dy8AfpWqWnw/6MgBzkeE0qp95lrR9WsxLJo+ie4DhuFR0uuO6zn4z1Zeeqotw7rfz7g3BrN/x193HXt2HGk/W6OgxQsFJ+bExETi4+I4c+Y0oz/6kNKly9ChU5ccT5PfbNS12+E4TM0II9GMN583wagiNjK7D5bFqEq2E5F3MCquv5iZHRHpC1gqHwyMBC4BCqNXCSIyHBgKJABHlFK982UL04iOjsbT0zNTuZeXFzExMfkWx5VL51k5bzq1GzSjag3Llwfs2Pg7Lq6uNL//4Uzv/TJrMuUq+tP6wc4W5rytXrP7qFqjNqXLVSAm6jKbflvC95++zcBXP6RlO9t14nGU/WytghYvFJyYn+nTgyOHDwPg71+FaTPnUKpUqRxPk98KepKxlsMkI6VUqNl33Q+jFrQDo+bWCGOIilPAGKAZEAP8JSJ/YbRVZlX+LkZiizWXd8pc3dtANaXULbNffqF0M/YG340ZibOLC/1fec/iNPFxt9iz9U/qNb2XEp7e6d47cXg/Ozb+zrtfzc72A9PnhdfTvW7U8gHGvTmEZT9+b9NkpGkpxoz9jGvXrxEWEsKc2TN5YchAZv84j4oVK+VomvxWWMamc6RmOrjdVNfKfL4jzevlwEal1EWlVMpFWm0wkpCl8uZm+WWzvTPtuaaDGCfjnsGoHeU7Ty9Pi78as/qVmdfibt3k2/+9ycXIMF756Ct8Spe1ON2BXVu4cf2qxSa6n78bz70PPYZPqbLcuHaVG9eukpiYSFJSEjeuXSU+Pi7L9Tu7uNDk3vZcuXie6MsX82y7MrL3fs6pghYvFJyYqwcEUL9+Azp27sLUGbOJvXGDmdOn5nia/Kab6exjO9Aao9ZySkRcgNFANSA6F8tLzqK8M0bCegwYJSL1lVL5mpQCAgIJDjqRqfzkyWCqB9j2uobEhAR+GP8uZ4KOMeLjSVSsmvX6tv/5OyU8vanXtHWm9yJCThMRcprNa37N9N6rfR+hx3Ov8NDjVrSA2vBDZM/9nBsFLV4omDF7enpS2d+fkLNn72qa/FDQk4y1HK1m9DfwJMYQFSilgjAG4KsJzATaioiviLgDvYDNwK47lLcTER9zxNluACLiDFRWSm3COJ9UEiiRf5toaNuuPYcOHiA0JCS1LCwslP379vJAu/Y2W29SUhIzvvgQdXA3/zdq3B27WsdcucyRvTtp/sAjuLhm/t3y2phvMz0qVatBhSrVeW3MtzS5N+vtSExMYPfWP/At44eXj+3a5O21n3OroMULBTPmSxcvcurkKSpVzvo6N2umyQ9OTtY/CjJHqxntA8oDaevFoRjjKoWYQ5X/xe2OChshdQhzS+WfYiSlSxhNc2CMRPuTiHiZ03+bdsym/NL9qZ4smPczr7z8Ii8NfwUnnPj2m0mU8/OjR49eNlvv/Cmfs2fbn3TsOYAiRYpx8ti/qe/5lC6brrlu519rSUpKtNhEByD1GmcqK+5RgsTExHTv7fprHQd2beGeJq3wLV3O6MCw+hezO/gnebh1mdlrP+dWQYsXHD/mEcOHUbt2HWqK4OFRgjNnTvPT3Nm4urqkdju3Zhp7KSw1I6fk5KxasjRLbiZk2fSXYxHh4emHUGnZijffHpXnJ0t3BF9OfZ7VsDwAXXo/x2N9b4+QMHr4syQlJ/HhNz9bva4vRr1IYmJiuuGATh77l2U/TiH87EmuX4uhSNFiVAmsxSPdnqZu45aZltEywNfq9Vkjv/ZzXilo8YJjxzxz+lTWrV1DaMhZ4uPjKefnR9NmLXhuyPOp8VkzTW4Udb37S1Zl5Fqrv3PU+EcLbObSySiH8jIZ5Ze0yaggyOtkpGn2khfJqNbb1iejY+MKbjJytGY6TdM0LQ3nQtK1WycjTdM0B1ZIThnpZKRpmubICksHBp2MNE3THFghyUU6GWmapjmyHN5cr8DSyUjTNM2B6ZqRpmmaZnf6nJGmaZpmd4UkF+lkpGma5sh0zUjTNE2zu0KSi3Qy0jRNc2R6BAZN0zTN7nQznfafUdAGHo1PSLJ3CDnm5lo4rgXR8l8hyUU6GWmapjkyXTPSNE3T7K6Q5CKdjDRN0xyZ7sCgaZqm2Z1uptM0TdPsTicjTdM0ze4KSS7SyUjTNM2R6ZqRpmmaZneFJBfpZKRpmubIdG86TdM0ze6cC0nVSCejfLJt6xZmzZjGyeBgYmKi8fH1pWHDRgx98WUCAgMBOBcZycwZ0zhy+F+Oq2PcvHmT1es2ULFiJTtHb7BmG/LL9m1bmTNrOqdOmrH4+FK/YSOeHzqM6gHm/jwXyZyZ0zl65F+OH1fcunmTFav/oELFipmW17RBbYvr+XnhUqSW5fdsJTIigs/Gj2XH9m0kJyfTolVr3ho5ivIVKuRrHFmx9ji9desW334zkd9WruTq1RikVm1GvPYGTZo2c8h4G9QVi/MvXLKMWrXz9xhIq5DkIp2M8ktMdDR16talV++++Pj6EhERzszp03i2b0+WLFtJhQoVOXv2DOvW/k6dOnVp1Lgp2//eau+w07FmG/JLdEw0tevU5aleffDx8SEyIoI5M6cx8NneLFiynPIVKhJy9ix/rFtDrTp1aNSoCTu2b7vjMh/r2o3uPXqmK6tSpaoNtyKz2NhYhgzqj5u7O6M/HY+TE0z+ehKDB/Vj8dIVFC9ePF/jscTa4/Sj90exZfNfvPr6W1SqXJkF83/m/55/jrk/L8zXL/ecfK66PtGdHj17pSurUrWqjSO8M92BQctTHTt3oWPnLunK6tWrz+NdOrJ+3Vr6DxhEk6bN2Lj5bwCWLlnscMnImm3ILx06dqZDx87pyurWq89Tj3diw/p1PNN/II2bNGXdRmMfLlu6ONtkVKZcWerVb2irkK2ydMkiQkNDWL5qDf5VqgBQo6bQtdOjLFm0kH4DBto1PsCq41QdO8bq31bx8f8+5YluT6bO1/3xznw3eRJffzvFoeJNUa5sWeo3aJhPkVnHVqeMRMQXmAHUBpKBHkAsMB/wBbYCzymlEkWkKPAT0BCIBHoopSLM5bwHDADigeeVUltyE4/DDjUsIokisl9EDovIShHxzoNleovI6buPLm94eXsD4OriAoCzs8P+ObKUcRvsydvLGwAX14K7Pzdt/JP69RukJiKASpUq07BRYzZt3GDHyG6zZr9u2rgBV1c3Hu3QKbXM1dWVDh078/e2rcTFxdkyxHQK4nGQlrOzk9WPHPoaWKaUqgU0Bs4A44FPlVKBgAuQ0lQwGAg1y2cBHwGISD3gcYyE1g34PtfbmdsZ88FVpVRDpVRdIAoYZud48kRiYiLxcXGcOXOa0R99SOnSZejQqUv2MzoQR9qGxMRE4uPjOHvmNGNGf0ip0qV5tEPn7Ge04JdFC2jVtD73tmjE0MED2Ld3dx5Hm73goCACatTMVB4QEMjJ4KB8jye3goODqFipIsWKFUtXHhAYSHx8PGfPnrFTZHe2aOECmja8hxZNGjB4YD/27sn/YyAjpxz8s5aIeAEtlVJzAJRSscA1oA2wwpxsLvCE+bwr8KP5fD6Q8iF7DFiglIpXSh0DokUkV22wBaWZbhvQAEBEGgI/AMWAo8BgpdRVERkCDAGKAEeA/kqpOBEJBH4GigOrUxYoInUxMrw7RlJ+Uil1wtYb8kyfHhw5fBgAf/8qTJs5h1KlStl6tXnKkbZhwDO9OHrEiKWyvz9Tps3GNxexdOz8GG3ub0uZsmWJCA/nxzkzGTpkIN9OmUHTZs3zOuwsRUdH4+npmancy8uLmJiYfIvjbhnb4ZWp3MusvcZER+dzRNnr/FhX7n+gHWXLliU8PIw5M2cwZFB/pkybSbPmLewWV04qPGYLkreFt6KUUlFpXlcDLojIj8A9wN/AJ8AVpVTKDcVCgZQTwRWAMACl1A0RcRERN7M8bbtnyjxHrY/a4Mg1IwBExAV4GFhpFs0FXlNK1cfYOW+a5b8opZorpRoAEUAfs3wi8KVSqh6Q9hMwFJiklGoINMXYiTY3Zuxn/Dh/EeMmfIFHiRK8MGQgYWH5suo840jb8MmY8cz+cQH/G/c5Hh4lGPbCc4SHheV4OaM/ncAjHTrRqHFTOnXpyvTZP1OmTBm+nzzJBlFrjujTcZ/RoWMnGjdpSpfHHmf2T/MoU6Ysk7+eaNe4nJycrH4AI4BTFh4jMizWFeN7byJGE50rxnkfu3HkZFRSRPYD5wA/YJ1ZtSyhlEo5Ez0Ho1oJ0EBEtorIIeApoI5Z3gJYbD7/Kc3ytwOjRORtoIpZTbW56gEB1K/fgI6duzB1xmxib9xg5vSp+bHqPONI21CtegD31G9Ah46d+X7qLG7E3mD2zGl3vVwPDw/ua/MARw4fyoMorefp5WmxBpRVjclReXp6EhOTufYTHR1lvO+VudbkaDw8StDm/gc4/G/+HgMZOTlZ/8BILtUsPCZmWGwocEYptUcplQwsA+oDPiKSkhcqYdaGgHDMWpKIFAMSlVLxacstzJMjjpyMrpq1liqAE/BiNtPPxOjJUQ+YgNFclyWl1DyMdtDrwCoRaX/XEeeQp6cnlf39CTl7Nr9XnWccaRtKenpSubI/oSF5dz4iv7vVBgQEEhyUubX45Mng1OunCoKAgEDCQsOIjU3/G+9kcDBubm74+1fJYk7HY++u1c5OTlY/lFJRSqnTFh5RaZeplIoEws3TGABtMU5vbMP4XgToByw3n68CnjWf9+X2KY9VQG8RcRORWoC3UirHTXTg2MkIAKXUdWA48DpG4rgqIq3Mt58FNpvPSwDnRcSd2010ADuBJ83nfVMKRaQ6cFIp9Q3wK8avgnx16eJFTp08RaXK/vm96jzjSNtw6dJFTp86RcVKdx/LtWvX2LL5L+reUy8PIrNe23btOXTwAKEhIallYWGh7N+3lwfa5fvvpVx7oF17EhLiWb92TWpZQkICa9esplXr+3B3d7djdNa5du0am//axD318v2rIR0b9qZ7BVggIge5XXt6C3hPRIIxunsvMqedBviLSBAwCLM3nVLqIEZCOoZRu8qu0pClAtGBQSm122x+6wn0B6aISHGMHfCcOdkHwD8YzXr70sw+ApgnIu+TpgODuaxnRSTOnKcvNjRi+DBq165DTRE8PEpw5sxpfpo7G1dXl3TXjqR8eI8c+ReAbVs24+Pji4+vb76eSLfE2m3ID2+MeIlatesQWFMoYcYy76c5uLi68Ey/AanT/bF+LUBqJ4e/t23G28cXHx8fmjQ19uePc2Zy5vQpmjZrQekyZYmMMDowXLp4kf+NnZCv29X9qZ4smPczr7z8Ii8NfwUnnPj2m0mU8/OjR49e2S8gn2R3nNauXYdHO3ZiwvhPSUhIoGKlSixaMJ+w0FDGjv/c4eKdM2sGp0+dolnzFqmdWObMnsnFixcZOyH/403LVhUzpdQ+jPNGaQVZKEvpbdc9i+V8DHx8t/E4JScn3+0yCpWbCeRqh82cPpV1a9cQGnKW+Ph4yvn50bRZC54b8ny6YUmyGpKkabPmzJj9o8X38ou123C34hOSsp1m9sxp/LFuDaGhIUYs5fxo0rQ5A597Pt1wP1kN89O4aTOmzpgLwOZNG5k9axpnTp/i2rVrlPDwoH7Dxjw3ZKjVv4rdXPOukSEiPDz9cEAtW/Hm26McZlgosO44vXnzJt9M+orff1vF1asx1JRajHjtDbv0TMsu3k0b/2TW9KmcNo8BD48SNGzUiCEvvEi9+rmvGRV1zUF/6yz0mrPP6u+chf0bFdjhGrJMRiISAha/eJ2AZKWU/dtl7CC3yUiznjXJyNHkZTLS/jvyIhn1zkEyWlCAk9Gdmunuy7coNE3TNIvs3YEiv2SZjJRSjnmJtKZpWiFSSG5nlH0HBhGpgtFVuh5QNKVcKVXdhnFpmqZpFJ6b61nT0D0To8teMkYX6T+A2bYLSdM0TUuRwxEYCixrkpGPUmo+kGR2BXwBKFgje2qaphVQzk7WPwoya64zShnrPcq8wjYCKGO7kDRN07QUBb3GYy1rktEv5k2YxgE7MGpTY2walaZpmgZw933DC4hsk5FS6jPz6W8iUhYoqpQqOGPZa5qmFWAuBb39zUrW9KbLNCCWiKCU+tM2IWmapmkpdDPdbe+neV4U4x7oewGdjDRN02yskOQiq5rp2qV9bd4h9Q2bRaRpmqalci4k2SjHA2oppQ4DjWwQi6ZpmpZBDm+uV2Dl9JyRC9ASiLdZRJqmaVoqfc7otrTnjBIw7nfR0zbhaFrBHAHb56mCdet4gCtLnrd3CJoVXHQyStVXKRWRtkBE/GwUj6ZpmpZGIenZbdU5o98slK22UKZpmqblsUI/HJCIeAO+gLuIVOP2hcBeQHHbh6Zpmqbpc0bQHxgBVCD9NUXRwJc2jEnTNE0zFfQaj7XudHO9ScAkERmplBqfjzFpmqZppkJSMbLqnNF+EfFKeSEi3iLyiA1j0jRN00yuTk5WPwoya5LROKVUdJrX0RgjeGuapmk2VlguerUmGaXbRKVUMtZ1Cdc0TdPukrOTk9WPgsyaZHRJRB5NeSEiHYFLtgtJ0zRNS1FYakbW1HBGAL+KyE3ztRsw0GYRaZqmaakKfW+6FEqpQ+btxusAXTHGplsNeNs2tP+W9WvX8Pvq3zhy+F8uX76EX/nyPPjQIwx+/gU8PEqkThcTHc2XX0xg44Y/uHnrFg0aNOTNke9Qo6bYMfrbIiMi+Gz8WHZs30ZycjItWrXmrZGjKF+hgr1Dy5I9Ym5VqxyjejehftVSFCviQlB4DFNWH2buBpU6TZWyJRk7oAXtGlTEzcWZ3ScuMGr2DvYGX8xyuT3uC2DuGw8SdvEagYPnpXuvmLsLHz/TnCfvrU4pz6KcCI/mi1/2s2BzkM22M4U+LmynsNxczyk5OfmOE4hIU2AA0APwAIYAK5VS12wenQO6mcCdd1gWnunTk/Lly9O2/YOUK+fHsaNHmPLdZKpWq87cnxfg7OxMcnIyA57tS3h4GK+9/haenp7MmD6V4KATLPplOeX87DsKU2xsLD27P46buzsvDR+BkxNM/noSN2/GsnjpCooXd7xrofMr5rRj091TxZfNE55g1/HzTF55iBu3EujWujqDH63N8ClbmLbmKL4li/DPxKe4GhvP/+bv5kZcAsO71qdxYGnavLkMFRqVaR1eHu7sn9yT5GRISkrKlIyWf9CRFlKOj+f9w/GwKB5vWY0hHeow8Ks/WfBX5oSUV2PT6eMia0Vd7/6u4WM2BFn9nfPug4EFNnPdaQSGNzEufE0G5mDcVO9vpdR8WwYkIr2BeYC/Uio0l8s4DTRUSkXlYWh35etvp+Dr65v6ummz5nh5efPeqJH8s2snLVq2YtPGDezft5dpM+fQvEVLAOo3bESnRx5k1szpvD3qPXuFD8DSJYsIDQ1h+ao1+FepAkCNmkLXTo+yZNFC+g1wvNZbe8Tco00ALs5OPDlmDddvJgDw54Ew6lXx5em2NZm25ihDOtShrHcxHnp3BacirwKw6WA4R37ozft9mvDMZxsyLXdM/xYcOn2JyCs3aF+/Yrr3WtcuxyONKzPk60389OdxADbsD6NiKQ8+7d+CRVuCSUrK1e+obOnjwrac7j6fFQh36sAwFogE+iilPjcHS7XN0Zxeb+BvoFc+rCvfpE1EKereUw+A8+fPAbBp45+UKVs2NREBlCxZkgfatmPTn5m/nPLbpo1/Ur9+g9QPL0ClSpVp2KgxmzbaPz5L7BGzu6sz8YlJxMYlpiuPvhGHs9nk0rxmWYLCo1MTEcCNWwn8fSSSjk2rZGqaaVWrHH0eqMGIH7ZZXGfzmuUAWLcnJF35+n2hlPf1oEXNsne9XVnRx4VtFZax6e6UjKoCG4GlIrJHRIZj3M/IZsyLaxsDwzCTkYgMEJFfRWSziBwXkdfN8qoiclhEForIURH5UUTcLCzzWRH5R0QOiMiXZpmHiPxmlv0rInZJfLt37wKgevUAAIKDgggMrJlpuoDAQCIiwrlx/Xq+xpdRcFAQATUsxBcQyMlg25+XyA17xPyjWTP5YnBryvsUx8vDnYEP16Jd/Yp8s+IQAIlJycQlJGWa91Z8IsWLuFLdzzO1zNXFickvtuGrXw9wMjLG4joTzVpPXEL6BHgr3nhdxz/zj6G8oo8L2yr0yUgpFaqUGqOUqgkMB+oBniLyu4j0s1E83YBVSqkDQHERqW6WNwcex0hUQ0Qk5SiqA3yplKqNkSj7p12YiNQ2l9lKKdUAKCsinYEOQLhSqoFS6h5gjY22J0vnzp3ju8lf07JV69QaUnR0NJ5enpmm9fLyBiAmxvIXUX6Jjo7G09NSfF52jy0r9oj5yNkrPPreKh5rXoWTs54h8ucBTHz+Xl6esoXFW4MBOBEeTWAFL3xLFkmdz8kJmtYwajBpy1/v3pAibi589sv+LNd5PDwKgOZSLl15C/O1T5rl5TV9XNiWk5OT1Y+CzKq7mCmltimlhmAMmvoz8IyN4ukNLDKfL+R2U91apdQVs9PEaqC1WX5KKbXTfL4AuC/D8h4EWgC7RWQ/RlILBA4BD4vIBBFpk2GECZu7cf06I17+P1xdXPjkf2Pzc9VaPggo78n8kQ9zJOQK3f+3ho4frGL62qN8M7QNve8PBGDamiM4O8H0V9pRza8kfj7F+HLwvVQtVxIg9fxOdT9PRj7ViFenbkut5Vjyx75QjoZc4YvBrWkhZfH2cKf/Q0LPNkatO9lG54s023Nxtv5RkOVoJAWlVCzwk/nIUyJSGngAqC0iKaM8XAQmkv5cVXKa11mVp3AGZiql3s9Qjog0BjoBn4jIRqXUJ3mxHdm5efMmLw8bSmhIKDPn/Jiuh5ynpycx0Zl/lUVHR6W+b0+eXp4WfzVm9SvTEdgj5k+eaU58YhLd/7eGhETjkNx0MBzfkkX4bHBrFm4J4vS5qwz8aiNfPX8vR6b0AWBv8AW+WXGIV7s1IPLKDQC+GNKaTYfC2XX8PF4e7gC4u7rg5OSEl4c7t+ITuRmXSGJSMn3Hr2f26+3ZNP4JACKv3OCDn3bx2XOtiTCXZwv6uLCtgj6ygrUcaVifpzASx7CUAhE5CBQFHjHvrxSPkUB+MCepJiLNlFL/YNSiNmZY5gaMc16TlFIXRaQsRnOeE3BFKfWTiFwEhtpyw1LEx8fzxqvDOXL4X36YPivTtUMBgYFs/zvzCeqTwcGUL1+B4h4e+RFmlgICAgkOOpGp/OTJYKoHBNohouzZI+a6VXw5dOpSaiJKsfvEBXo/UIOyXsU4FxXLsu2nWLHzNDUqeBGXkMipyKtMeuE+Qi5cI+SicX6wdmUfqpQtSeTPAzKtJ/LnAUxeeYg3Z2wH4FhoFC1fXYp/2RJ4FHHjRHgUT7SsBsD2Y5E22VbQx4Wt2fJckIg4AzuBMKXUEyISAMzHuJfdVuA5pVSiiBTFqIQ0xOjY1iPlDuAi8h7G5T/xwPNKqS25icWRKna9gV8zlP0KvA3sBpYD+4HpSqnj5vtHgddF5Kj5em7amZVSh4FPgA1mYluFsZPrATvNprtPgP/l9cZklJSUxKiRb7Br5w4mfvMd9Rs0zDRN23YPcv7cOXb/syu17Nq1a/y1aSMPtGtv6xCz1bZdew4dPEBoyO0eW2Fhoezft9ch4rPEHjGfi7pB/WqlcHNN//FqVrMssbcSuHztVmpZUlIyKjSKU5FXKe9TnKfuq87UNUdS3+/3+QYeeW9luse6vSFciI7lkfdW8v1vhzOt/+z5axwNuQLA0M51Wb8vJF2vvbymjwvbsvFwQP8HBKd5PR74VCkViPHDvadZPhgINctnAR8BiEg9jPP5Kefnv89VFFhx0au9icgAjGuGRmQorwosU0o1zM94cnvR6/8++ZDFCxcw5Pmh3N+2Xbr3ypXzo5yfH0lJSQx4ti+RkRG8al70OnP6VI4rxeKly/ErXz5PtiG3bty4Qc/uj1OkaFFeGv4KTjjx7TeTuH7jOkuWrrB7zc2S/Io57UWv3VpVY97Ih1m/L4Spvx8hNi6BLs2qMrRzXb5efpCRs3bg6uLEp/1bsuVwODE34qnj78ObTzbkZGQMHT/4jXgLPe1STB3+AO3rV8x00esbTzYk5MJVwi/foHLpEgztVJdKpUvQ/p3lnD6XORnl1UWv+rjIWl5c9PrtttNWf+d8PehRHyyPjhOV8bpLESmHcU3nGIxOat0waj3llVJJIvIwMFgp1UtE1gHvKKX2iEhx4LhSqpKIjAJuKaW+MJe5zZznKDnkSM10/2nbthg112lTpzBt6pR07w198SX+b9jLODs78823U/ji8/F8Ovpj4uJuUb9BQ6bPmmv3RARQvHhxps2cw2fjx/Lu228ZQ6i0bMWbb49yyC8csE/Mv24/xeOf/M7r3Rvw3bD7KeruwsnIGF75YSvT1xqf0eRko6NDz/sD8PYoQtil68z5QzFhyb47JqI78SjiykdPN6O8rwdR12+xfm8ofSesJ/SibS8J0MeFbeWwxjMC+NBC+ceYtZk0JgDvASldLUthnL5IOQBDgZSrqysAYQBKqRsi4mJeSlMBozmPDPPkOBk5fM3I0eS2ZqT9t6WtGRUUeVUz0rKWFzWjqTvOWP2d80X/R6yqGYnI/cAgpdQAEWmLkcQGA1uVUrXMaWoD05RS94nIv8BDSqlI870IwB/4ypxngVm+0Jznj5xup64ZaZqmObCc1IzMhBNlxaStMS5vOY3RScwT+ALwERFns3ZUCbM2BIRj1HgiRaQYkKiUiheRlPIUaefJEUfqwKBpmqZlYIub6ymlximlKiqlqmJ0HlunlOoPbMO4OwNAP4yOY2B0/nrWfN4X43rPlPLeIuJm3t3BOzfni0AnI03TNIeWzzfXewt4T0SCMa7bTBmEYBrgLyJBwCDM809KqZReyseAZcCLuV2xPmeUQ/qckWaJPmekWZIX54xm/3PW6u+cAc38C+wVsvqckaZpmgPTIzBomqZpdqeTkaZpmmZ3hSMV6WSkaZrm0ApJxUgnI03TNEdW0O9TZC2djDRN0xxYYbn+RicjTdM0B6Y7MGiapml2p5vpNE2z2sVFQ+wdQo75NHvJ3iHkyJV/Jts7BLvQzXSapmma3emakaZpmmZ3hSMV6WSkaZrm0ApJxUgnI03TNEfmUkiykU5GmqZpDsypkDTU6WSkaZrmwApJxUgnI03TNEfmrGtGmqZpmr3pmpGmaZpmd3o4IE3TNM3unAtHLtLJyJ62bd3CrBnTOBkcTExMND6+vjRs2IihL75MQGCgvcOzKDIigs/Gj2XH9m0kJyfTolVr3ho5ivIVKtg7tCw5SsznIiOZPXMaRw7/y4njips3b7JqzR9UqFgpdZop333D1O+/tTi/u7s7O/YcBGDFsqV89P6oLNe1buMWSpcuY/E9J8DNBTbNeZ16NSpSvJg70ukDzkZcTjddEXdXPnyxC707NcO7ZDEOHg/j3UnL2LY3ON10w59pz/1Na9C4jj/ly3jxvymrGfPD6kzrHT28K4/eW5fK5X0o4ubK2YjLLFyzm4lzNxB7Mx4A//K+qNWfZLldcYmQlJzl2zniKMdFdnRvOs3mYqKjqVO3Lr1698XH15eIiHBmTp/Gs317smTZSipUqGjvENOJjY1lyKD+uLm7M/rT8Tg5weSvJzF4UD8WL11B8eLF7R1iJo4Uc0jIGdavXUPtOnVp2LgJO/7elmmabt170PreNunKYmNjefn/hvBA2/apZW3ub8vsnxakmy45OZlXX/4/KlaqnGUiAuMchKszXIm5wbZ9wTzcurbF6aZ8+DQd2tRl1FfLOBV2kaE972flt8No2/8LDh4PS51uYLfWXL1+k5WbDvJ8jzYWlwVQ0qMoP67YwfEz57gVl0DLBtUZ+dyjNKrtT89XpwIQeTGGB/p9nmneD4d1oV3zWnmWiBzpuMhOIWml08lIRPYDTyilTuf3ujt27kLHzl3SldWrV5/Hu3Rk/bq19B8wKL9DuqOlSxYRGhrC8lVr8K9SBYAaNYWunR5lyaKF9Bsw0M4RZuZIMTdu0ow//jIS0K+/LLaYjMr5+VHOzy9d2aqVy0lISKDL40+klvn4+uLj65tuur17dhMVFcULL758xziSkuFGPHR7+XsGdGtlMRnVq1mR3p2a8fyHP/Hjih0AbNkTxN4l7/L+i13oMeKH29v11BiSk5NxcXG+YzIaMXZRutebdh2neFF33hz0CKW8PbgUdZ24+AR2HTqdbrpiRd1oWrdKniUicKzjIjuFpWZUIAaEFREXe8eQX7y8vQFwdXG8Td608U/q12+Q+uEFqFSpMg0bNWbTxg12jCxrjhSzs3PuPm6rli+jVKnStGp9352nW7EMNzc3OnTqnKv1pNX5gXrExSewZN2e1LLExCQWr93Dw61q4e52+3dscnLus8TlqOsAJCQkZTnN4+0b4lmiGIl5mIwc6bjIjrOT9Y+CzCFqRiLyIdAXiAQuA8uBj4CFwCPAhyJSAngHo8n7F6XUh+a8UUopb/P5AKChUmqEiMwGYoBmQFngJaXU7yJSHJgD1AUOAm75s5VZS0xMJCkxkfCIcCZ9+QWlS5ehQ6cu2c+Yz4KDgmjb/sFM5QEBgaxft8YOEWWvIMacVmRkBLv/2UnfZ/rh6pr1x/XmzZv8sW4NbR5oi5eX912vt05AeU6HXUo9l5PiaHAERdzdCKhcmqMnI3O1bBcXZ4q6u9K8fjWGP9ue2cv+JvpabJbTP/NYC85disHLyzNX67OkIB0XujddPhGRZkBXoD5QHDiEkYwAwpVSjUSkArANI7HEAH+JyF9KqT+zWXwZoLU533fA78CLwEWlVB0RuRfYmtfblFPP9OnBkcOHAfD3r8K0mXMoVaqUnaPKLDo6Gk/PzF8IXl5exMTE2CGi7BXEmNNavWoFSUlJdOna7Y7TbfrzD65du8Zj2UxnLR/P4kTF3MhUftks8/XyyNVy6wSUZ8+Sd1Nf/7RyJ8NGz89y+gplvGjbrCbfzt/E873bZzldThWk46JwpCLHaKa7F1imlLqllLoCrE3z3mLz/2bARqXURaVUHLAAyLpx+rZlSqlkYA9Q1Sy7D5gPoJTaBpy6+024O2PGfsaP8xcxbsIXeJQowQtDBhIWFmrvsDQHsGrFcmrVrkNNkTtOt3LFMnx9S3Fvm/vzKbLcCQ65wL1PT+DhwRN5/5sVdG1Xnxmj+2U5fd8uzXFxcU49b1UYOTs5Wf0oyBwhGd1J5p9mmaVtSS6S4b1bAEqpRNLXAvOw9fnuVQ8IoH79BnTs3IWpM2YTe+MGM6dPtXdYmXh6eVr81ZjVr0xHUBBjTvHvoYOcPnWSLl2fuON0Fy6cZ9eO7XTo1OWOTXk5ERVzA2/PzD3KfM2yy9HXc7XcW3EJ7D1ylq17gvh85jpen7CE3p2a0bxeVYvT9+3SnP3HQvj3RHiu1peVgnRcOOXgUZA5QjLaBnQVEXcR8cI4R5TRLqCtiPiKiDvQC9hsvndJRALMTg5drVjfVnN+RKQVUO2utyAPeXp6Utnfn5CzZ+0dSiYBAYEEB53IVH7yZDDVAxzzuqiCGHOKlcuX4erqRsdszh+uXrWSxMREHkvT2+5uHTkZQdWKpShWNP0p1VrVy3MrLp7gkIt5sp69R4zjvHrlzF3Rm9Txp3b18vy8cmeerCutAnVcFJJsZPdkpJT6B+Nczr8Y54oOYpwXSjtNBPAe8BewF1ivlNpovv0OsA7YAljzDf4d4CciR4DhwJE82Iw8c+niRU6dPEWlyv72DiWTtu3ac+jgAUJDQlLLwsJC2b9vLw+0y7v2/LxUEGMGiI+PY92a37j3vjaZunBn9NuKZdSoKUgty9cL5cbqv/7F3c2V7g83Ti1zcXHmqUca88f2Y8TFJ+TJeto0qQHAqdDMye2Zx1oQH5/Iwt9358m60ipIx0VhaaazewcG0wSl1Aci4gnsAPYrpaqmnUApNQ+Yl3FGpdRibp9bSls+IMNrb/P/G8BTeRX43RgxfBi1zfMBHh4lOHPmND/NnY2rq4tDXeeQovtTPVkw72deeflFXhr+Ck448e03kyjn50ePHr3sHZ5FjhbzH2ZPraNHjA4r27ZuwcfHBx8fX5o0a5463ea/NhEdHZ3u2iJLjh45TFDQCV57Y2SO4nBxgm4PNaRRbeNHz6P31eHilWtcuHKNrXuCOKBCWbx2D5+98SRurs6cDrvE8z3aULViKQa+OzvdshrX8adKBd/UL8Pa1f3o9lBDANZsPUzszXjuqVGBsa92Y+n6fZwKu0gRd1fuaxzIsD5tWbP1MDsPpj916+bqwlOPNmHt30e4cOVajrbNGo52XNxJwU4x1nO6m2sE8oqIzAPqYJzz+U4p9Y2dQ8rSzYS8O980c/pU1q1dQ2jIWeLj4ynn50fTZi14bsjzVEwzRIwjiQgPTz+ESstWvPn2KIeNF/In5kQrr8hsXK+WxfImTZsxbdaPqa9ffflF9u/bw7qNW3Bzc89yeZ+NG8PihfP5ff0mSpUubXW8Hu6Wv+I27z7Bo0MmAVC0iBsfv/QYPTs0xbtkMQ4dD+PdScvZsid989bUj5/h2a4tLS4vZZihsr4lmfDGk7SoX5VypTy5cTOe02EX+XHFTmb9+nemmlbXdvVZ+OXz9HljOss27Afgyj+Trd4+a+THcVHU9e5zyT+noq3+zmlWzavA5i6HSEYFSV4mI+2/w9pk5EhKt7jzSA2OJq+TUX7Ii2S0+1SM1QdX02qeBTYZOUoznaZpmmZBAT8VZDWdjDRN0xxYIclFOhlpmqY5MicbVI1EpCYwE/ABEoHRSqnFIhKAMSiAL8ZlMM8ppRJFpCjwE9AQY9i2HmYvZ0TkPWAAEA88r5TakpuY7N61W9M0Tcuak5P1jxy4hZE46mJc2zlJREoC44FPlVKBgAvQ05x+MBBqls/CGDsUEakHPA7UBroB3+d2O3XNSNM0zYHlJMeIiDfgbeGtKKVUVMoLpdSZNM8jReQCUBpjmLWUBDQXIwnNxxhQ4B2zfD7wsfn8MWCBUioeOCYi0SJSWyl1NAdhA7pmpGma5thyNgLDCIzxNjM+RmS1eHOwajfgGnBFKZVyP49QIOUOnxWAMEi9VtNFRNzSlluYJ0d0zUjTNM2B5fDmehOB2RbKoyxNLCJlMWpAg7DzmJ06GWmapjmwnJwLMpvioqyZ1ry32wrgE6XUdhFxAnxExNmsHVXidq0nHKPGEykixYBEpVS8iKSUp0g7T47oZjpN0zQHZosODObA0guAX5VSKbfUScYcuNqcrB+37y23CnjWfN4XWJ2mvLeIuIlILcA7N+eLQNeMNE3THFoOm+ms1RHoDPiLSB+z7FngLWCBiHyBkZgWme9NA34WkSDgHNADQCl1UERWAccwuna/kNuA9HBAOaSHA9Is0cMB2V5hHQ7oSPh1qw+uOhU8Cuw1srpmpGma5sAKbHbJIV0zyqGCWDO6fC3O3iHkiG+JrEep1gqvo2FX7R1CjjWqUvKuc8nRCOtrRrXL65qRpmmaZgMF/aZ51tLJSNM0zYEVjlSkk5GmaZpjKyTZSCcjTdM0B2ajrt0ORycjTdM0B1ZIThnpZKRpmubICkku0slI0zTNkdni5nqOSCcjTdM0B1ZIcpFORpqmaY6skOQinYw0TdMcWiHJRjoZ2VFkRASfjR/Lju3bSE5OpkWr1rw1chTlK1TI91j27d7FzB8mc1wdoUiRIrRsfT9Dh7+Ob6nSqdOoo4eZMeUbTgWfIDo6ihIlSlJTavPscy9Qt17DdMsLOn6Mqd9O5NCBvTg7OdOwSTNefOVNKlb2z9ftOhcZycwZ0zhy+F+Oq2PcvHmT1es2ULFipXyNIycc6biwVn7HvGPzH/y9aS0njx8lOuoKpcuWo/m97Xmiz0CKFfcAIPbGdZb8NI2Tx49wOkgRe+M67382hboNmmZa3m9LfuLwgT2cPHGEqMuXePKZIfTol34A6hvXr/H7r/PZv3s7ESFnSEpKpGKV6nTt0Y9m97a1yXZC4enare9nZCexsbEMGdSfU6dOMvrT8YwZN4GzZ84weFA/bty4ka+xHNy3hzeHv0CJkiX5eOyXvPTq2xzcv4fXXxpCXNztce2uXbtKxUqV+b/hbzBh0hSGv/EO165dZcTQgRw9fCh1utCzZxj+Qn+uX7vKux+P4633RxMZEcYrQwdw5fKlfN22s2fPsG7t73h6etKoceYvIUfjSMeFtewR86olP+Hs7ELvgcN459OvebjLU6xftYQxbw8jKcm4a/bVmGg2rV2Bi4sr9Ro3v+PyNvy+jOioyzRt3TbLaS6dj2TdyiXUqdeYYSM/4ZV3x1K+oj9ffPwGa1csynK+u2WL+xk5Il0zspOlSxYRGhrC8lVr8K9SBYAaNYWunR5lyaKF9BswMN9imTPje8qVL8//JkzCxdU4JKpUq87QAb1ZvWIpTzzVG4AmzVrSpFnLdPM2b3kfTzzahvW/r6R23XoAzP9xJs7OLoyf+D0lSnoCUKduPZ5+qjMLf57D0Jdfy7dta9K0GRs3/w3A0iWL2f731nxbd2440nFhLXvE/NYnX+Hp7ZP6uk79JpQo6cl3n33EkQN7uKdRM8qUK8+MX/4E4NDenezaujHL5X0+bRHOzs4kJibwx6pfLE5Txq8iX89ZTpGiRVPLGjRtxaUL51ixcA6Pdu2ZR1uXnnMBTzLWsmnNSEQ2iciDGcpGi8gnItLOluu2hoh4i8hpe6x708Y/qV+/QeqHF6BSpco0bNSYTRs35GssR/49SNPmrVITEYDUrounlzdb/7pzLEWLFcPNzR0XF5d0y6tbr35qIgIoU86PatUD2bopf7fN2blgVf4d6biwlj1iTpuIUgRIXQAuXzoP5KxLtDXHSdFixdIlohTVa9bmyqWLVq8r55xy8Ci4bP1JXQBk/LnQC5inlMr6Z4qNmLfadQjBQUEE1KiZqTwgIJCTwUH5GouLswuurm6Zyt3d3TllIZakpCQSEuI5FxnBpM8/BaDz40+lvu/s7GxxeW7u7oSHhRB361YeRv/f4kjHhbUcJeYjB/cAUNG/Wr6tE+DYoX1UqFwl+wlzSTfT5Y0lwIciMkwplSAijYBY4G0RWaaUWmbWTOZw+77r3ZRSp0WkDPAD4I9xO9sXgSBgHyBKqUQR8QP+VErVEZEhwBCgCHAE6K+UihOR2cBNoDGwVkTmAD8Dxbl9H/d8Fx0djaenZ6ZyLy8vYmJi8jWWylWqcuTwwXRlkRHhXLp4AVfXzIfIx6PeYPPG9QD4+Pgy7qvvqFo9IN3yDh/cT0JCfGpSunH9OqdPBpOcnMzVqzGUKlLGhltUcDnScWEtR4j58sXzLJ7zA/UaNSegZp18WSfAH78t5cTRQ7w0crTN1lHAc4zVbFozUkpdBA4AKU11vTBqSxlFKqUaAfOBl8yyicA4pVRToD/wg1LqKsZ92TuY0zyDkVgAflFKNVdKNQAigJT7ugOUBVoqpd43l/ulUqoeEH3XG/kf0L3X0xw7fIgZU77myuVLnD19krEfvYOTszNOFpovXnj5Vb6fNZ+Px31F1YAavPP6S6ijh1Pff7Ln01y8cJ4vx43mwvlzREaEM370+8TGGiezC8sV5Vr+uBl7g88/fB0XFxeGvvFhvq338IHdzPnuc+5/qDP3PdjRZuspLDWj/GhQT9tU1wNYaGGapeb/e4Cq5vOHgKkish9YBJQzy2cAA8znzwJzzecNRGSriBwCngLS/jxaopRKMp+3ABabz3/K+ebkDU8vT4u/GrP6lWlLD3fowrMDn2fRvLl079iWAb2foHSZcrRs3YZSabp2p6hQsTK16tzD/e0eYvzE7/Hx8WXGlG9S36/XsDGvvPkum/9cT8/HHqLPE49y/fpVHu3UFTc3Nzy9vPJz8woURzourGXPmONu3WTCB69yLjKMd8ZOplSZctnPlAeC1WE+//B16jZsyvOvvW/TdTk5OVn9KMjyozfdr8BYEbkXuKSUOikiGadJOYmQmCYmZ6C5UirdPbOVUptF5FsReRg4r5QKMd+aCXRWSh0RkZeAwDSzOVyf2ICAQIKDTmQqP3kymOoBgRbmsK1BQ1+mT//niAgLxdvHF99Spenfqyv3NGh8x/nc3NyoHliToBPH0pU/8VRvOnXtTljoWTw8SlC2nB8jRwyldt16Fs8naQZHOy6sYa+YExIS+Gr0SE4eP8q7477Fv1r+7J+zp4L4dNTLVKlek9c+mGCxKTsvFewUYz2b14yUUtHATuA7LNeKsrIBGJbyQkQapHlvHsZ5ptlpykoA50XEnfRNdBntBJ40n/fNQTx5qm279hw6eIDQkJDUsrCwUPbv28sD7drbJaZixYpTPbAmvqVKs2v7Vs6ePkXX7nfurnrzZizq2GEqVKyc6T13d3eqVQ+kbDk/TgYdZ8+unXTt3stW4f8nOOJxkR17xJyUlMTkce/x7/7dvP7R59SoXc8m68koIuwsY94eRjm/irw1+ivci2TuXZfXCkszXX5dZ7QAI4HkJBm9DEwRkYEYca7EOP8Exrmlt7ndvAfwAfAPcA6jk0NWRgDzROR97NiBoftTPVkw72deeflFXhr+Ck448e03kyjn50ePHvn7hX1CHWXn9q3UlNoAHDqwl4U/zab3swO5p37D1Om+GPsxnp5e1KxdFy9vb85FRrBs8XwuX7zAqA8/TZ3uwrlIli9dRN16DXF3d0MdPcy8OTNo0+5BHny0U75uG8D6tWsAOHLkXwC2bdmMj48vPr6+NG1254sh85sjHRfWskfMMyePZ8fmP+jWZxBFixbjxNHbF137li6b2ly3b9c2bt2MJeR0MABHD+7lanQURYoWo1Hze1PnCT5+hAuR4SQnJwMQdvYUOzb/AUCj5vdRpGhRoq9cZszbw0hIiOepfi8QdvZUupiqBghu7u55vq2FZQQGp5SdX5CISF+gjVLq//J73TcTyLMdFhEenn4IlZatePPtUXk+VM3la3F3fP/UySC+HPsJp04GER8fR5Wq1enWow8dH+uWbrrVK35l9YpfCDlzmtibsZQpU5ZadevxdP/BVA+83bX38qWLjPnwbYKOG0OwVKhYmU5du/Nkr6fTXcuUFd8SefuBblA3U7MwAE2bNWfG7B/zdF15Ib+Oi7yUHzEfDbua+vylZx/j4rkIi9OlHconq+lKlyvP5B9Xpr7+7rOP2Lx+lcXlfT13BWX9KnD4wG5Gvzk0y/hSpkurUZWSd51JLlxLsPo7p0wJ1wKbuQpcMhKR7zF65z2ilDqd3+vPy2SUX7JLRo4mr5OR9t+QNhkVFHmRjC7mIBmVLsDJqMANB2SP2pCmaZq9OBf0k0FWKnDJSNM0rTApJLlIj9qtaZqm2Z+uGWmapjmwwlIz0slI0zTNgRWWrt06GWmapjkwXTPSNE3T7E4nI03TNM3udDOdpmmaZne6ZqRpmqbZna1ykYh0Ar4CXIBJSqlvspnFpvR1RpqmaY7MKQcPK4mIK/A1xn3j6gMviohdBz/UNSNN0zQHlpPhgETEG/C28FaUUioqzevmwNGU+8GJyFLgMeD73MZ5t3QyyqGirgXvbGIFbz3wqFbwNapS0t4h2EUOv3M+Aizde/1j870UFYCwNK9DgYo5jS0v6WSkaZr23zGR9DcdTRGVr1Hkgk5GmqZp/xFmU1yUFZOGk74mVAmjdmQ3OhlpmqYVPruAOiJSGbgIdAcesWdAujedpmlaIaOUSgBGABuAf4EpKZ0Z7KXA3elV0zRN++/RNSNN0zTN7nQy0jRN0+xOJyNN0zTN7nQy0jRN0+xOd+3OBRFJBA6ZLxOBYUqpHdnMsxroBZQClimlGlqYZj/wBBAHfKWU6pV3UVuM6SsgWCk12Xy9A9iqlHrDfL0QWKSU+iWXyx8BeCulPsqbiNMtO+Vv4AacBJ7NMNxJbpbpDexXSlW92/gsLLs3MA/wV0rl6noOETkNNLzb7XRkKZ8BpdTpLN7fBIxWSm1IUzYaSAY2KqU25kOYWbLlMfRfp2tGuXNVKdXQTCijgDHZzaCU6qSUumrNwpVS4bZORKbtQCsAESmC8cXeJM37rYC/8yGO3Ej5G9TFuMhvmJ3jyU5vjH2ZH39XmxIRFzuufgHQM0NZL2CePRKRnffFf4quGd09TyAaQETaAiOUUk+Yr2dj1IKWpfyqTTujiBQH5gB1gYMYyQARqWrO11BEBgBdzPVUB+YopUab030I9AUigcvAcqXUbBEZB3QFEoB1KTUdC/4GxpvPmwDbgEYi4gaUxaj1tRORdzDGBP5FKfWhue6+gKXywcBI4BKggFNm+XBgqBnTEaVUbyv2rbW2AQ3M9TQEfgCKAUeBwUqpqyIyBBgCFAGOAP2VUnEiEgj8DBQHVqcsUETqArMAd4wfbU8qpU7kJjgR8QIaYwxEOQ34wvy7Po5RU/YDflBKfWH+7X/DuPajPrAbGKSUis+wzGeB4WZ8G5RSr4mIB7AI42p6F4waxMJcxJvpuMIY12whxoWRH4pICSz//aOUUt7m8wEYNbkR5mchBmiGcWy9pJT6PavPwB0sMdc/TCmVICKNgFjgbRFJ+1mbg/EZAOimlDotImUwjg1/IB54EQgC9gGilEoUET/gT6VUnTscM7OBmxh/07UiMgcLx5CWM7pmlDslRWS/iBzD+HLJtmaUhReBi0qpOsA3QJ0spmsAPGn+/38i4i0izTA+bPUxmvaaAYhIKaAbUFcpVR/4X1YrN5uLXMwPYCtgB3AAaAS0xkgkY4B2GB+8R0SkvYhUuEP5u0AL872GaVb3NtDIjGmotTsoO+Yv04eBlWbRXOA1cz1hwJtm+S9KqeZKqQZABNDHLJ8IfKmUqof5o8I0FOMeLw2BptzdUCndgFVKqQNAcRGpbpY3x0hIjYEhIlLTLK9jxlQbI6n0z7DNtc1ltjK3p6yIdAY6AOFKqQZKqXuANTkNNKvjyhSulGqEkSAz/f2tWHwZjOOqDzDaLLP2MwCAUuoixjH6oFnUC6O2lFGkGet84CWzbCIwTinVFGOf/mC2VmzD2HcAz2AkFsj6mAEjobZUSr1P1seQlgM6GeVOShNRLaAjMFdEcjOa930YHxaUUtswaxEW/KGUuqqUug4EY/zyvRej9nRLKXUFWGtOG43xq22GiHQHbmQTQ0pTXSvz+Y40r5djtMNfVErFYXzo22B8QVkqb26WX1ZKxQJpzzUdBH4WkWcwakd3q6R5fuEcRs1inVkDKWHuSzB+HbcxnzcQka0icgh4ittfei2Axebzn9IsfzswSkTeBqqY25NbvTFqLGDULlKa6tYqpa4opa5h/KJubZafUkrtNJ8vwDhO0nrQjHu3uQ+aA4EY59AeFpEJItJGKZWbL8asjiu4vZ+y+vtnZ5lSKhnYA1Q1y6z9DKSVtqmuB8Y+zWip+X/adT0ETDX32SKgnFk+AxhgPn8W4wcNZH3MACxRSiWZz7M6hrQc0MnoLimltgOlMX71JZB+nxaxYhHWDIFxK83zRO7QvGoO89EcozmjE9n/Ot6O8SVYTSl1CiMZtTTLvK2ILaOstqczMBmj1rXLvLnX3bhq1lqqYDQVvZjN9DOB581frxPI5m+jlJqHUUO4Dqyy8pd/JiJSGngAmGM2H73A7WSUdl8lp3mdVXkKZ2BmynlLpVRNpdQkpdRxjJrKQeATEfkgNzHfQXY/bFLiTZFxH98CUEplPIZzOgzMr0AnEbkXuKSUOmlhmpTPTNp1OQPN0+y3KmY8m4FaIvIwcD7NsDh3Omas2RdaDuhkdJdEpBZGU8ol4AxQW0TcRMQX40voTrZifjGJSCugWg5WvQ3oKiLuZo3gEXM5JTB6sK0GXgXqZbOcvzGaAEMAlFJBQG2gJsaHsa2I+IqIuxnrZoxBFrMqbyciPiJSFKMpCRFxBiorpTZhnE8qCZTIwbZmyawtDgdex0gcV819Ccav3M3m8xLAeTPetM0tO83tB+M8CWbM1YGTyrgV868YzVa58RRG4qiilKqqlKqE8bkritG85W2e6+mE8cMAoJrZXAbGvt2aYZkbgJ5mokNEyopIebOZNFYp9RPGucDGuYjX4nGVQVZ/f4BLIhJgNp92tTBvRjn+DJg1vp3Ad1iuFWVlA2k6uohIgzTvzcOoSc9OU5bVMZORxWNIyxmdjHIn5ZzRfowPQ3+lVKL5i+pX4DBGdX1vNsv5DvATkSMYX6hHrA1AKfUP8DvGie7lGL+GYzC+6FeKyEGML5asOi+k2AeUx6gRpQgFDprb8x7wl7kt65VSG5VSEVmUhwOfYnxZbTJjAiNZ/2Q2d+wHvlV52D1ZKbUbo4mqJ8a5gK/M7fcHPjcn+wD4B+NL82Ca2UcAb5rTe6cp7wn8KyL7MM59zSV3emMcE2n9inEObTfG324/MN2s2YDR8eJ1ETlqvk63bqXUYeATYIMZ9yrAF+OHx07zuPyEO5wvzModjqu001j8+5tvvwOsA7YAZ61YZW4/Awswtjcnyehl4D4ROWiuL23imA94cLt5D7I+ZjIageVjSMsBPVBqASYiJZRS10TEEyOZdMmiyUJzMGl7mmUor0oW16Hll8J4XJm9Q9sopf7P3rEUVrprd8E2VUTqYLRlf/df/8LQ8k2hOq5E5HuMTiF2vZ9PYadrRpqmaZrd6XNGmqZpmt3pZKRpmqbZnU5GmqZpmt3pDgxaoSAiyRjDyDgBScAbKs3Iz7lc5mngIaVUkBijsr+Q5oJJS9OPAOYqpS7nYl3JgJt5UbOm/efompFWmDQ1xxn7CFhoXoyb6m5GhVDGqOxZJiLTCIzrgTRNy0DXjLTCaD3GaNmlRGQxxoW/rYEgERkEjMMYo60IxggBw80RnethjORd1CxPHY8wQy2pHjDJXAcYF7g2ACoAK0QkDmPEhevA1xgjXhQFFqvbI7K3xRg+KZHMF81q2n+OrhlphVFPIEQpdcF8XQljBOynMYYrClVKNccYeaEY8Jw53RxgrDki9jqMER7SEeP2G8uAz81aWCNgh1JqHBAOdDXHRQsHvgJWmOtqDLQRkUfEuLfUz8BAcxkxGdejaf81umakFSa7RQSM+/Q8nqb85zQjMD8GlBDjfkFgJKNL5jhtgcq8661SaqmIRFlYhwC3zLEBMZd7JYt4HgOaiMj75usSQC2Mkcgvm0PzgDGq9Bc52lJNK2B0MtIKk6ZZdAC4lua5E8bN7LanncBMRnnNCehgjvWWdl0NyPlI1ppWoOlmOk1LbyXwmjlSMyJSSkSqmSNFB4lIykjkT2B5UEwFuIlIJ3M6ZxHxMd+Lwbhjb9p1jUzpSCEilcwbHR4DfEUk5RbwA/NyAzXNEelkpGnpjQVOAnvMUZjXARXN9/oD75qjjz+KhVGplXF78G7AW+Z0ezFuvgbwLTDfHPG9AvAKxkjRB8xpF2Hc/uMWxh1H54jIAcAWtTJNcyh6bDpN0zTN7nTNSNM0TbM7nYw0TdM0u9PJSNM0TbM7nYw0TdM0u9PJSNM0TbM7nYw0TdM0u9PJSNM0TbM7nYw0TdM0u/t/xqDAj0xmbFEAAAAASUVORK5CYII=\n", 310 | "text/plain": [ 311 | "
" 312 | ] 313 | }, 314 | "metadata": { 315 | "needs_background": "light" 316 | }, 317 | "output_type": "display_data" 318 | } 319 | ], 320 | "source": [ 321 | "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", 322 | "datasetNames = [\"Trento\"]\n", 323 | "data2Name = 'LIDAR'\n", 324 | "\n", 325 | "patchsize = 11\n", 326 | "batchsize = 64\n", 327 | "testSizeNumber = 500\n", 328 | "EPOCH = 100\n", 329 | "BandSize = 1\n", 330 | "LR = 5e-4\n", 331 | "FM = 16\n", 332 | "HSIOnly = False\n", 333 | "FileName = 'MFT_HSILidar_Pixel'\n", 334 | "ntokens = 4\n", 335 | "token_type = 'pixel' \n", 336 | "num_heads = 8\n", 337 | "mlp_dim = 512\n", 338 | "depth = 2\n", 339 | "train_loss = []\n", 340 | "\n", 341 | "def set_seed(seed):\n", 342 | " torch.manual_seed(seed)\n", 343 | " torch.cuda.manual_seed_all(seed)\n", 344 | " np.random.seed(seed)\n", 345 | "\n", 346 | "for BandSize in [1]:\n", 347 | " for datasetName in datasetNames:\n", 348 | " print(\"---------------------------------- Dataset details for \",datasetName,\" ---------------------------------------------\")\n", 349 | " print('\\n')\n", 350 | " try:\n", 351 | " os.makedirs(datasetName)\n", 352 | " except FileExistsError:\n", 353 | " pass\n", 354 | " \n", 355 | " train_dataset = Multimodal_Dataset_Train(Filename=datasetName, MM_Data=data2Name)\n", 356 | " test_dataset = Multimodal_Dataset_Test(Filename=datasetName, MM_Data=data2Name)\n", 357 | " NC = train_dataset.hs_ims.shape[1]\n", 358 | " NCLidar = train_dataset.lid_ims.shape[1]\n", 359 | " Classes = len(torch.unique(train_dataset.lbs))\n", 360 | "\n", 361 | " train_loader = dataf.DataLoader(train_dataset, batch_size=batchsize, shuffle=True, num_workers= 4)\n", 362 | " print(\"HSI Train data shape = \", train_dataset.hs_ims.shape)\n", 363 | " print(data2Name + \" Train data shape = \", train_dataset.lid_ims.shape[1])\n", 364 | " print(\"Train label shape = \", train_dataset.lbs.shape)\n", 365 | "\n", 366 | " print(\"HSI Test data shape = \", test_dataset.hs_ims.shape)\n", 367 | " print(data2Name + \" Test data shape = \", test_dataset.lid_ims.shape[1])\n", 368 | " print(\"Test label shape = \", test_dataset.lbs.shape)\n", 369 | "\n", 370 | " print(\"Number of Classes = \", Classes)\n", 371 | " \n", 372 | " TestPatch1 = test_dataset.hs_ims\n", 373 | " TestPatch2 = test_dataset.lid_ims\n", 374 | " TestLabel1 = test_dataset.lbs\n", 375 | " \n", 376 | " KAPPA = []\n", 377 | " OA = []\n", 378 | " AA = []\n", 379 | " ELEMENT_ACC = np.zeros((3, Classes))\n", 380 | "\n", 381 | " set_seed(42)\n", 382 | " for iterNum in range(1):\n", 383 | " print('\\n')\n", 384 | " print(\"---------------------------------- Model Summary ---------------------------------------------\")\n", 385 | " print('\\n')\n", 386 | " if HSIOnly: \n", 387 | " model = Transformer(FM=FM, NC=NC, Classes=Classes, ntokens=ntokens, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n", 388 | " summary(model, [(NC, patchsize**2)])\n", 389 | " \n", 390 | " else:\n", 391 | " model = MFT(FM=FM, NC=NC, NCLidar=NCLidar, Classes=Classes, ntokens=ntokens, token_type=token_type, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n", 392 | " summary(model, [(NC, patchsize**2),(NCLidar,patchsize**2)]) \n", 393 | " \n", 394 | " optimizer = torch.optim.Adam(model.parameters(), lr=LR,weight_decay=5e-3)\n", 395 | " loss_func = nn.CrossEntropyLoss() # the target label is not one-hotted\n", 396 | " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.9)\n", 397 | " BestAcc = 0\n", 398 | "\n", 399 | " torch.cuda.synchronize()\n", 400 | " print('\\n')\n", 401 | " print(\"---------------------------------- Training started for \",datasetName,\" ---------------------------------------------\")\n", 402 | " print('\\n')\n", 403 | " start = time.time()\n", 404 | " # train and test the designed model\n", 405 | " for epoch in range(EPOCH):\n", 406 | " for step, (b_x1, b_x2, b_y) in enumerate(train_loader):\n", 407 | "\n", 408 | " # move train data to GPU\n", 409 | " b_x1 = b_x1.cuda()\n", 410 | " b_y = b_y.cuda()\n", 411 | " if HSIOnly:\n", 412 | " out1 = model(b_x1)\n", 413 | " loss = loss_func(out1, b_y)\n", 414 | " else:\n", 415 | " b_x2 = b_x2.cuda()\n", 416 | " out= model(b_x1, b_x2)\n", 417 | " loss = loss_func(out, b_y)\n", 418 | "\n", 419 | " optimizer.zero_grad() # clear gradients for this training step\n", 420 | " loss.backward() # backpropagation, compute gradients\n", 421 | " optimizer.step() # apply gradients\n", 422 | "\n", 423 | " if step % 50 == 0:\n", 424 | " model.eval()\n", 425 | " pred_y = np.empty((len(TestLabel1)), dtype='float32')\n", 426 | " number = len(TestLabel1) // testSizeNumber\n", 427 | " for i in range(number):\n", 428 | " temp = TestPatch1[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 429 | " temp = temp.cuda()\n", 430 | " temp1 = TestPatch2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 431 | " temp1 = temp1.cuda()\n", 432 | " if HSIOnly:\n", 433 | " temp2 = model(temp)\n", 434 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 435 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 436 | " del temp, temp2, temp3\n", 437 | " else:\n", 438 | " temp2 = model(temp, temp1)\n", 439 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 440 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 441 | " del temp, temp1, temp2, temp3\n", 442 | "\n", 443 | " if (i + 1) * testSizeNumber < len(TestLabel1):\n", 444 | " temp = TestPatch1[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n", 445 | " temp = temp.cuda()\n", 446 | " temp1 = TestPatch2[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n", 447 | " temp1 = temp1.cuda()\n", 448 | " if HSIOnly:\n", 449 | " temp2 = model(temp)\n", 450 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 451 | " pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n", 452 | " del temp, temp2, temp3\n", 453 | " else:\n", 454 | " temp2 = model(temp, temp1)\n", 455 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 456 | " pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n", 457 | " del temp, temp1, temp2, temp3\n", 458 | "\n", 459 | " pred_y = torch.from_numpy(pred_y).long()\n", 460 | " accuracy = torch.sum(pred_y == TestLabel1).type(torch.FloatTensor) / TestLabel1.size(0)\n", 461 | "\n", 462 | " print('Epoch: ', epoch, '| train loss: %.4f' % loss.data.cpu().numpy(), '| test accuracy: %.4f' % (accuracy*100))\n", 463 | " train_loss.append(loss.data.cpu().numpy())\n", 464 | " # save the parameters in network\n", 465 | " if accuracy > BestAcc:\n", 466 | "\n", 467 | " BestAcc = accuracy\n", 468 | " \n", 469 | " torch.save(model.state_dict(), datasetName+'/net_params_'+FileName+'.pkl')\n", 470 | " \n", 471 | "\n", 472 | " model.train()\n", 473 | " scheduler.step()\n", 474 | " torch.cuda.synchronize()\n", 475 | " end = time.time()\n", 476 | " print('\\nThe train time (in seconds) is:', end - start)\n", 477 | " Train_time = end - start\n", 478 | "\n", 479 | " # load the saved parameters\n", 480 | " \n", 481 | " model.load_state_dict(torch.load(datasetName+'/net_params_'+FileName+'.pkl'))\n", 482 | "\n", 483 | " model.eval()\n", 484 | " \n", 485 | " confusion, oa, each_acc, aa, kappa = reports(TestPatch1,TestPatch2,TestLabel1,datasetName,model, HSIOnly, iterNum)\n", 486 | " KAPPA.append(kappa)\n", 487 | " OA.append(oa)\n", 488 | " AA.append(aa)\n", 489 | " ELEMENT_ACC[iterNum, :] = each_acc\n", 490 | " torch.save(model, datasetName+'/best_model_'+FileName+'_BandSize'+str(BandSize)+'_Iter'+str(iterNum)+'.pt')\n", 491 | " print('\\n')\n", 492 | " print(\"Overall Accuracy = \", oa)\n", 493 | " print('\\n')\n", 494 | " print(\"----------\" + datasetName + \" Training Finished -----------\")\n", 495 | " print(\"\\nThe Confusion Matrix on test data\")\n", 496 | " record.record_output(OA, AA, KAPPA, ELEMENT_ACC,'./' + datasetName +'/'+FileName+'_BandSize'+str(BandSize)+'_Report_' + datasetName +'.txt')\n", 497 | "\n" 498 | ] 499 | }, 500 | { 501 | "cell_type": "code", 502 | "execution_count": 5, 503 | "id": "68dcf5c3", 504 | "metadata": {}, 505 | "outputs": [], 506 | "source": [ 507 | "train_loss = np.asarray(train_loss)\n", 508 | "np.save('HSILiDAR_pixel_train_loss.npy', train_loss)" 509 | ] 510 | }, 511 | { 512 | "cell_type": "code", 513 | "execution_count": null, 514 | "id": "10152686", 515 | "metadata": {}, 516 | "outputs": [], 517 | "source": [] 518 | } 519 | ], 520 | "metadata": { 521 | "kernelspec": { 522 | "display_name": "Python 3", 523 | "language": "python", 524 | "name": "python3" 525 | }, 526 | "language_info": { 527 | "codemirror_mode": { 528 | "name": "ipython", 529 | "version": 3 530 | }, 531 | "file_extension": ".py", 532 | "mimetype": "text/x-python", 533 | "name": "python", 534 | "nbconvert_exporter": "python", 535 | "pygments_lexer": "ipython3", 536 | "version": "3.8.10" 537 | }, 538 | "vscode": { 539 | "interpreter": { 540 | "hash": "26407125db06bdd9abe40e82cf041582bb19887fa16dd38638e528b7039723e8" 541 | } 542 | } 543 | }, 544 | "nbformat": 4, 545 | "nbformat_minor": 5 546 | } 547 | -------------------------------------------------------------------------------- /MFT_Transformer_HSI_Only_Model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "a7d3e1de", 6 | "metadata": {}, 7 | "source": [ 8 | "## Transformer Model for Hyperspectral Image Classification\n", 9 | "\n", 10 | "\n", 11 | "### The MFT model with only HSI images used for classification. The other multimodal data is NOT used as external CLS token, rather the external CLS token is taken as random " 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "id": "e5b2e976", 18 | "metadata": { 19 | "ExecuteTime": { 20 | "end_time": "2022-08-11T11:17:16.979544Z", 21 | "start_time": "2022-08-11T11:17:16.971125Z" 22 | } 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "import sys\n", 27 | "sys.path.append(\"./../\")\n", 28 | "import matplotlib.pyplot as plt\n", 29 | "from torchsummary import summary\n", 30 | "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n", 31 | "import math\n", 32 | "from PIL import Image\n", 33 | "import time\n", 34 | "from scipy.io import loadmat as loadmat\n", 35 | "from scipy import io\n", 36 | "import random\n", 37 | "import numpy as np\n", 38 | "import os\n", 39 | "import torch \n", 40 | "import torch.utils.data as dataf\n", 41 | "import torch.nn as nn\n", 42 | "from operator import truediv\n", 43 | "import record\n", 44 | "import pandas as pd\n", 45 | "import seaborn as sns\n", 46 | "from mft_model import MFT, Transformer\n", 47 | "from dataset import Multimodal_Dataset_Train, Multimodal_Dataset_Test\n", 48 | "import torch.backends.cudnn as cudnn\n", 49 | "cudnn.deterministic = True\n", 50 | "cudnn.benchmark = False\n" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 2, 56 | "id": "199d9d19", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "def get_confusion_matrix(y_test,y_pred, plt_name):\n", 61 | " df_cm = pd.DataFrame(confusion_matrix(y_test, y_pred), range(6),range(6))\n", 62 | " df_cm.columns = ['Buildings','Woods', 'Roads', 'Apples', 'ground', 'Vineyard']\n", 63 | " df_cm = df_cm.rename({0:'Buildings',1:'Woods', 2:'Roads', 3:'Apples', 4:'ground', 5:'Vineyard'})\n", 64 | " df_cm.index.name = 'Actual'\n", 65 | " df_cm.columns.name = 'Predicted'\n", 66 | " sns.set(font_scale=0.9)#for label size\n", 67 | " sns.heatmap(df_cm, cmap=\"Blues\",annot=True,annot_kws={\"size\": 16}, fmt='g')\n", 68 | " plt.savefig(''+str(plt_name)+'.eps', format='eps')\n", 69 | "\n", 70 | "def AA_andEachClassAccuracy(confusion_matrix):\n", 71 | " counter = confusion_matrix.shape[0]\n", 72 | " list_diag = np.diag(confusion_matrix)\n", 73 | " list_raw_sum = np.sum(confusion_matrix, axis=1)\n", 74 | " each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n", 75 | " average_acc = np.mean(each_acc)\n", 76 | " return each_acc, average_acc\n", 77 | "\n", 78 | "def reports (xtest,xtest2,ytest,name,model, HSIOnly, iternum):\n", 79 | " pred_y = np.empty((len(ytest)), dtype=np.float32)\n", 80 | " number = len(ytest) // testSizeNumber\n", 81 | " for i in range(number):\n", 82 | " temp = xtest[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 83 | " temp = temp.cuda()\n", 84 | " temp1 = xtest2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 85 | " temp1 = temp1.cuda()\n", 86 | " if HSIOnly:\n", 87 | " temp2 = model(temp)\n", 88 | " else:\n", 89 | " temp2 = model(temp,temp1)\n", 90 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 91 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 92 | " del temp, temp2, temp3,temp1\n", 93 | "\n", 94 | " if (i + 1) * testSizeNumber < len(ytest):\n", 95 | " temp = xtest[(i + 1) * testSizeNumber:len(ytest), :, :]\n", 96 | " temp = temp.cuda()\n", 97 | " temp1 = xtest2[(i + 1) * testSizeNumber:len(ytest), :, :]\n", 98 | " temp1 = temp1.cuda()\n", 99 | " if HSIOnly:\n", 100 | " temp2 = model(temp)\n", 101 | " else:\n", 102 | " temp2 = model(temp,temp1)\n", 103 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 104 | " pred_y[(i + 1) * testSizeNumber:len(ytest)] = temp3.cpu()\n", 105 | " del temp, temp2, temp3,temp1\n", 106 | "\n", 107 | " pred_y = torch.from_numpy(pred_y).long()\n", 108 | " \n", 109 | " oa = accuracy_score(ytest, pred_y)\n", 110 | " confusion = confusion_matrix(ytest, pred_y)\n", 111 | " each_acc, aa = AA_andEachClassAccuracy(confusion)\n", 112 | " kappa = cohen_kappa_score(ytest, pred_y)\n", 113 | " get_confusion_matrix(ytest, pred_y, 'test_'+str(iternum)+'')\n", 114 | " \n", 115 | " return confusion, oa*100, each_acc*100, aa*100, kappa*100\n" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 3, 121 | "id": "ac5c476f", 122 | "metadata": { 123 | "ExecuteTime": { 124 | "end_time": "2022-08-11T11:18:10.978783Z", 125 | "start_time": "2022-08-11T11:18:01.894932Z" 126 | }, 127 | "scrolled": true 128 | }, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "---------------------------------- Dataset details for Trento ---------------------------------------------\n", 135 | "\n", 136 | "\n", 137 | "HSI Train data shape = torch.Size([819, 63, 11, 11])\n", 138 | "LIDAR Train data shape = 1\n", 139 | "Train label shape = torch.Size([819])\n", 140 | "HSI Test data shape = torch.Size([29395, 63, 11, 11])\n", 141 | "LIDAR Test data shape = 1\n", 142 | "Test label shape = torch.Size([29395])\n", 143 | "Number of Classes = 6\n", 144 | "\n", 145 | "\n", 146 | "---------------------------------- Model Summary ---------------------------------------------\n", 147 | "\n", 148 | "\n", 149 | "===============================================================================================\n", 150 | "Layer (type:depth-idx) Output Shape Param #\n", 151 | "===============================================================================================\n", 152 | "├─Sequential: 1-1 [-1, 8, 55, 11, 11] --\n", 153 | "| └─Conv3d: 2-1 [-1, 8, 55, 11, 11] 656\n", 154 | "| └─GroupNorm: 2-2 [-1, 8, 55, 11, 11] 16\n", 155 | "| └─ReLU: 2-3 [-1, 8, 55, 11, 11] --\n", 156 | "├─Sequential: 1-2 [-1, 64, 11, 11] --\n", 157 | "| └─HetConv: 2-4 [-1, 64, 11, 11] --\n", 158 | "| | └─Conv2d: 3-1 [-1, 64, 11, 11] 31,744\n", 159 | "| | └─Conv2d: 3-2 [-1, 64, 11, 11] 28,224\n", 160 | "| └─GroupNorm: 2-5 [-1, 64, 11, 11] 128\n", 161 | "| └─ReLU: 2-6 [-1, 64, 11, 11] --\n", 162 | "├─Dropout: 1-3 [-1, 5, 64] --\n", 163 | "├─TransformerEncoder: 1-4 [-1, 64] --\n", 164 | "| └─ModuleList: 2 [] --\n", 165 | "| | └─Block: 3-3 [-1, 5, 64] 100,736\n", 166 | "| | └─Block: 3-4 [-1, 5, 64] 100,736\n", 167 | "| └─LayerNorm: 2-7 [-1, 5, 64] 128\n", 168 | "├─Linear: 1-5 [-1, 6] 390\n", 169 | "===============================================================================================\n", 170 | "Total params: 262,758\n", 171 | "Trainable params: 262,758\n", 172 | "Non-trainable params: 0\n", 173 | "Total mult-adds (M): 12.27\n", 174 | "===============================================================================================\n", 175 | "Input size (MB): 0.03\n", 176 | "Forward/backward pass size (MB): 1.00\n", 177 | "Params size (MB): 1.00\n", 178 | "Estimated Total Size (MB): 2.03\n", 179 | "===============================================================================================\n", 180 | "\n", 181 | "\n", 182 | "---------------------------------- Training started for Trento ---------------------------------------------\n", 183 | "\n", 184 | "\n", 185 | "Epoch: 0 | train loss: 2.4431 | test accuracy: 13.2846\n", 186 | "Epoch: 1 | train loss: 1.8137 | test accuracy: 39.9388\n", 187 | "Epoch: 2 | train loss: 1.2580 | test accuracy: 60.0204\n", 188 | "Epoch: 3 | train loss: 0.8837 | test accuracy: 77.5608\n", 189 | "Epoch: 4 | train loss: 0.5837 | test accuracy: 83.5006\n", 190 | "Epoch: 5 | train loss: 0.4990 | test accuracy: 85.8343\n", 191 | "Epoch: 6 | train loss: 0.4421 | test accuracy: 90.6175\n", 192 | "Epoch: 7 | train loss: 0.2673 | test accuracy: 87.5795\n", 193 | "Epoch: 8 | train loss: 0.2590 | test accuracy: 92.6450\n", 194 | "Epoch: 9 | train loss: 0.1682 | test accuracy: 88.9335\n", 195 | "Epoch: 10 | train loss: 0.1247 | test accuracy: 93.4343\n", 196 | "Epoch: 11 | train loss: 0.1216 | test accuracy: 92.0020\n", 197 | "Epoch: 12 | train loss: 0.1390 | test accuracy: 94.6862\n", 198 | "Epoch: 13 | train loss: 0.0527 | test accuracy: 93.8391\n", 199 | "Epoch: 14 | train loss: 0.0389 | test accuracy: 94.8018\n", 200 | "Epoch: 15 | train loss: 0.0494 | test accuracy: 93.2472\n", 201 | "Epoch: 16 | train loss: 0.0514 | test accuracy: 94.1759\n", 202 | "Epoch: 17 | train loss: 0.0343 | test accuracy: 93.9275\n", 203 | "Epoch: 18 | train loss: 0.0201 | test accuracy: 94.1453\n", 204 | "Epoch: 19 | train loss: 0.0972 | test accuracy: 93.7676\n", 205 | "Epoch: 20 | train loss: 0.0247 | test accuracy: 94.1929\n", 206 | "Epoch: 21 | train loss: 0.0136 | test accuracy: 93.7540\n", 207 | "Epoch: 22 | train loss: 0.0368 | test accuracy: 92.4545\n", 208 | "Epoch: 23 | train loss: 0.0227 | test accuracy: 91.0291\n", 209 | "Epoch: 24 | train loss: 0.0211 | test accuracy: 92.4885\n", 210 | "Epoch: 25 | train loss: 0.0148 | test accuracy: 83.6945\n", 211 | "Epoch: 26 | train loss: 0.1034 | test accuracy: 92.0463\n", 212 | "Epoch: 27 | train loss: 0.0476 | test accuracy: 94.9923\n", 213 | "Epoch: 28 | train loss: 0.0422 | test accuracy: 94.5569\n", 214 | "Epoch: 29 | train loss: 0.0385 | test accuracy: 94.9651\n", 215 | "Epoch: 30 | train loss: 0.0166 | test accuracy: 93.9752\n", 216 | "Epoch: 31 | train loss: 0.0089 | test accuracy: 94.5535\n", 217 | "Epoch: 32 | train loss: 0.0466 | test accuracy: 93.9718\n", 218 | "Epoch: 33 | train loss: 0.0093 | test accuracy: 94.3086\n", 219 | "Epoch: 34 | train loss: 0.0094 | test accuracy: 95.1420\n", 220 | "Epoch: 35 | train loss: 0.0138 | test accuracy: 94.6419\n", 221 | "Epoch: 36 | train loss: 0.0154 | test accuracy: 93.4343\n", 222 | "Epoch: 37 | train loss: 0.0150 | test accuracy: 94.2371\n", 223 | "Epoch: 38 | train loss: 0.0101 | test accuracy: 93.0668\n", 224 | "Epoch: 39 | train loss: 0.0066 | test accuracy: 94.5127\n", 225 | "Epoch: 40 | train loss: 0.0105 | test accuracy: 94.7338\n", 226 | "Epoch: 41 | train loss: 0.0088 | test accuracy: 94.0670\n", 227 | "Epoch: 42 | train loss: 0.0119 | test accuracy: 92.2266\n", 228 | "Epoch: 43 | train loss: 0.0084 | test accuracy: 94.3630\n", 229 | "Epoch: 44 | train loss: 0.0096 | test accuracy: 95.4550\n", 230 | "Epoch: 45 | train loss: 0.0120 | test accuracy: 93.9275\n", 231 | "Epoch: 46 | train loss: 0.0156 | test accuracy: 94.2439\n", 232 | "Epoch: 47 | train loss: 0.0098 | test accuracy: 93.7132\n", 233 | "Epoch: 48 | train loss: 0.0253 | test accuracy: 93.4138\n", 234 | "Epoch: 49 | train loss: 0.0075 | test accuracy: 94.2643\n", 235 | "Epoch: 50 | train loss: 0.2419 | test accuracy: 94.4412\n", 236 | "Epoch: 51 | train loss: 0.0185 | test accuracy: 94.8733\n", 237 | "Epoch: 52 | train loss: 0.0131 | test accuracy: 94.8937\n", 238 | "Epoch: 53 | train loss: 0.0102 | test accuracy: 93.3220\n", 239 | "Epoch: 54 | train loss: 0.0077 | test accuracy: 94.3222\n", 240 | "Epoch: 55 | train loss: 0.0136 | test accuracy: 94.3766\n", 241 | "Epoch: 56 | train loss: 0.0071 | test accuracy: 93.8527\n", 242 | "Epoch: 57 | train loss: 0.0112 | test accuracy: 94.6828\n", 243 | "Epoch: 58 | train loss: 0.0096 | test accuracy: 93.6622\n", 244 | "Epoch: 59 | train loss: 0.0084 | test accuracy: 94.5025\n", 245 | "Epoch: 60 | train loss: 0.0090 | test accuracy: 93.9956\n", 246 | "Epoch: 61 | train loss: 0.0090 | test accuracy: 92.7641\n", 247 | "Epoch: 62 | train loss: 0.0100 | test accuracy: 94.4616\n", 248 | "Epoch: 63 | train loss: 0.0088 | test accuracy: 94.6964\n", 249 | "Epoch: 64 | train loss: 0.0088 | test accuracy: 93.6554\n", 250 | "Epoch: 65 | train loss: 0.0083 | test accuracy: 92.5532\n", 251 | "Epoch: 66 | train loss: 0.0088 | test accuracy: 94.1351\n", 252 | "Epoch: 67 | train loss: 0.0120 | test accuracy: 93.8969\n", 253 | "Epoch: 68 | train loss: 0.0117 | test accuracy: 94.2133\n", 254 | "Epoch: 69 | train loss: 0.0102 | test accuracy: 94.2303\n", 255 | "Epoch: 70 | train loss: 0.0103 | test accuracy: 93.8731\n", 256 | "Epoch: 71 | train loss: 0.0103 | test accuracy: 93.1315\n", 257 | "Epoch: 72 | train loss: 0.0107 | test accuracy: 94.2303\n", 258 | "Epoch: 73 | train loss: 0.0099 | test accuracy: 94.6113\n", 259 | "Epoch: 74 | train loss: 0.0146 | test accuracy: 94.9379\n", 260 | "Epoch: 75 | train loss: 0.0137 | test accuracy: 94.9277\n", 261 | "Epoch: 76 | train loss: 0.0191 | test accuracy: 94.9889\n", 262 | "Epoch: 77 | train loss: 0.0072 | test accuracy: 94.6658\n", 263 | "Epoch: 78 | train loss: 0.0087 | test accuracy: 93.3458\n", 264 | "Epoch: 79 | train loss: 0.0103 | test accuracy: 93.5329\n", 265 | "Epoch: 80 | train loss: 0.0108 | test accuracy: 95.0400\n", 266 | "Epoch: 81 | train loss: 0.0121 | test accuracy: 95.0774\n", 267 | "Epoch: 82 | train loss: 0.0194 | test accuracy: 94.5365\n", 268 | "Epoch: 83 | train loss: 0.0158 | test accuracy: 94.9141\n", 269 | "Epoch: 84 | train loss: 0.0122 | test accuracy: 93.8731\n", 270 | "Epoch: 85 | train loss: 0.0093 | test accuracy: 93.8561\n", 271 | "Epoch: 86 | train loss: 0.0099 | test accuracy: 94.5365\n", 272 | "Epoch: 87 | train loss: 0.0090 | test accuracy: 94.5025\n", 273 | "Epoch: 88 | train loss: 0.0099 | test accuracy: 94.0602\n", 274 | "Epoch: 89 | train loss: 0.0125 | test accuracy: 94.3290\n", 275 | "Epoch: 90 | train loss: 0.0147 | test accuracy: 94.3834\n", 276 | "Epoch: 91 | train loss: 0.0137 | test accuracy: 94.6488\n", 277 | "Epoch: 92 | train loss: 0.0128 | test accuracy: 93.9820\n", 278 | "Epoch: 93 | train loss: 0.0105 | test accuracy: 92.6620\n", 279 | "Epoch: 94 | train loss: 0.0101 | test accuracy: 94.5841\n", 280 | "Epoch: 95 | train loss: 0.0101 | test accuracy: 94.0976\n", 281 | "Epoch: 96 | train loss: 0.0098 | test accuracy: 94.1555\n", 282 | "Epoch: 97 | train loss: 0.0080 | test accuracy: 94.6522\n" 283 | ] 284 | }, 285 | { 286 | "name": "stdout", 287 | "output_type": "stream", 288 | "text": [ 289 | "Epoch: 98 | train loss: 0.0115 | test accuracy: 94.0092\n", 290 | "Epoch: 99 | train loss: 0.0092 | test accuracy: 94.8699\n", 291 | "\n", 292 | "The train time (in seconds) is: 1223.482893705368\n", 293 | "\n", 294 | "\n", 295 | "Overall Accuracy = 95.45500935533255\n", 296 | "\n", 297 | "\n", 298 | "----------Trento Training Finished -----------\n", 299 | "\n", 300 | "The Confusion Matrix on test data\n" 301 | ] 302 | }, 303 | { 304 | "data": { 305 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAEJCAYAAAA5Ekh8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABgO0lEQVR4nO3dd3wURRvA8V8qECCNlkCoCQy9t4AoRVFAadJEpYOINJUmFhRERORFOghSlA4iXUAp0rvUwEBCDUnoKUACae8fuzlSLuQSLrmLmS+f+3A3t+XZy949O7Ozszbx8fEoiqIoiiXZWjoARVEURVHJSFEURbE4lYwURVEUi1PJSFEURbE4lYwURVEUi1PJSFEURbE4e0sHkN0sOR6Y7frCd6jmZekQ0iUuLtt9xNja2lg6hHSLy2aXddjaZL/POLc9Lxx0nhoDTf5DRf47I/t9SDqVjBRFUayZjfkbsIQQ84A2QJCUsrpeVgBYBZQAJNBZSvlICGEDzASaAxFAVynleX2e3sAoIB4YLaVco5fXARYCuYHVUsrP0opJNdMpiqJYM1s70x+m+xV4I1nZZ8BGKWVZ4F9gsF7eCigqpfQBhgE/gSF5jQJqAS8BE4UQefR5ZgHdgHJAAyGEb1oBqZqRoiiKNUtH86QQwhVwNfJWqJQyNOGFlHKvEKJUsmlaAwlJ41dgCTBBL/9Nn2+HEGKRECIv8DqwVUoZDoQLIQ4BjYUQp4DcUsoTekxLgLbAwefFrmpGiqIo1szG1vQHDAWuGHkMNWFNBaSU9/TngUAx/XlR4Gai6YIATyPlCfOkVv5cqmakKIpizdLXceMnYJGR8lAT5rVojxaVjBRFUaxZOjow6E1xoRlc030hRELtyItntZsgktZsigLBennic0FewG4j0ydeVqpUM52iKIo1s7Ex/fFiNgHv68+7AeuTlwshmgHnpZSPgO1ACyGEsxCiMFpi2i2lDAKeCiFqCiFsgfcSLStVqmakKIpizdLXS84kQojfgFeBAkKIQLRecROAVUKIj4BLQGd98k1ASyFEAFrX7ncBpJR3hRATgRNoTXyjpJSR+jwD0Do95AZ+l1IeSCsmG3U/o/RRF71mPnXRa9ZQF71mPrNc9PrSl6Zf9LpvXPb7kHSqZqQoimLNsmESzgiVjF5QwKmjHNi4gjs3rxH16CFOzi54la3EK293o5BXqSTTXvr3MAc2Lif4yiVsbG0p4OFFs679KF2pBgDr50zk9J7tRtdTwLM4AyYvMryOefqU3asXcmb/30Q9ekiRkt40e6cfJStUzaxNBSAkOJhJEydw6OB+4uPjqefbgBEjR+NZtGimrje5WyEhLFwwD79zZ7l0URIVFcXmrX9TtNizWmBQ0E1+mDAeKS/w4P498uTJQxlvH3r06kujl19Jsrzg4CBmzZjKsSNHePDgPkU8PGjevAW9+vQjj5NTlm6btXzGqTl65BCzpk/jvN85cuXKTaOXX+HjYSMoULBgqvN8+80Yfl+9kpat3mL8xElZGK1x+/ftZeEv87gcEEB4eBhu7u5Ur16D/gMG4e3jY+nwksqEERiskUpGLyjyUQSepctR67XW5HV2JezubfZvWM6CMYP44Pv5uBYqAsDxHRvZumg6dZq3pVG794iPiyfkmj/RT6IMy2rU7n1qNXsryfJD74Twx4zxlKuV9ALmjT//yKWTh3i16we4Ffbk6F/rWfb9SHp+Mx2PUpnzZYqMjKRvr+44ODoy7ruJ2NjAjGlT6dOrG6vXbsApC3+0b9y4xl/btlKhYiVq1KzFwQP7U8b7+DGubm58NGgIRYoU4eHDR/zx+yoGf/QBP06ZRrNXmxum69+3JzExMXw4cDCenp6cO3uWObOmc/36NSb+OCXLtsuaPmNjThw/xoB+ffBt0JBJU6YRFhrKzOlT+aBPT5at+h1HR8cU85w8cYItmzaSL18+C0RsXHhYGBUrVaJzl664ubsTHBzEgvnzeL9rJ9as20jRomleFpN1VDJ6MUKIWOCM/jIW+EhKeSiNebagnTQrAKxLGDMp2TQn0a7mfQpMkVJ2Tj5NVqrcoCmVGzRNUlbUuzyzh/Xg/JF/8G3VidA7IWz/dRavdv2Aei3eNkznXa1OkvncixTFvUjSo9/LZ44DUPXl1w1lIdcCOHtgB2/1G071xtqIHiUrVGP2iF7sXrOILsO+Nes2Jli7ZhWBgTdYv2krJUqWBKBsOUHrlq+zZtVKuvXomSnrNaZmrTrs+EdLQGt/X200GXn7lOXrseOTlDV6+RXefONVNqxba0hGJ0+e4Pq1a8yaOx/fBi8BUKdufcLCwvht8QIiIyPJkydPiuVnBmv6jI2ZO3smnp5F+d+0mdjbaz8fpcuU4b0uHVm3dg2dunRNMn10dDTfjh1D734f8PvqlZYI2agWrd6kRas3k5RVqVKVNm+24K/t2+jeo5eFIjPCzvwdGKxRZqbcCClldT2hjAbGpzE9UsqWUsoIUxYupQyydCJKjVM+ZwBs9V4wJ3f/iY2tbYpajynO7P0Lz9LlKJyoye/i8QPY2tlTybexoczWzo5Kvk24fPoYMdFPXyj+1OzetZOqVasZfiQBvLyKU71GTXbv2pEp60yNrW3Gdl17e3vy5c+Pnd2z47Do6GgA8uZNeuSeP39+4uLiIAtP9FvTZ2zMmVOnqOfbwJCIACpVroKrqys7//47xfS/LvyFuNhYulnTj3sqXFxdAbC3th//rOvabVFZ1UznDIQBCCEaA0OllG3114vQakHrhBBXgeqJZxRCOAGLgUrAacBBLy+lz1ddCNEDeFNfTxlgsZRynD7dGKArEALcB9ZLKRcJIb5HG3MpBtgupRz2IhsYFxdLfFwcoXdusXPFfPK5uhtqTDfkWQp4FufcwZ3s/WMJoXdv4VrIg3ot3qZO87apLvOGPMv9Wzd5vfvAJOV3Aq/iWtgDh1y5k5QX8ipFbEw0928FJUle5hLg70/jps1SlHt7+/DX9q1mX5+5xMXFERcXR2joA35fvYprV68yfORow/v16jegRMmSTJ3yI6O//BpPT0/OnjnD8qW/0aFTlyw9Z2Ttn7GdnS0ODg4pyh0cHQnwv5Sk7Pr1a8z/eQ7TZs0xOo81iI2NJS42lqDgIKb+bzIFCxbijZZvpj1jVlLNdC8sv96klhvwAFJ+w0wzALgrpawohGgI7EtlumpATSAOuCSEmA6URUs4VQEntGbD9fpos+2A8lLKeH1wwRey4MuBBF+5CIB7kWK89/mP5HVxAyDiwT0iQu/x97KfadK5N25FinL+0D9sXTSduNjYJE13iZ3eux1bO3sq+yZtBox6FEGevPlTTJ9QFvUw/EU3x6iwsDCcnZ1TlLu4uBAenjnrNIef/jeJ3xYvBMDJyYnvJ02mXv1n5+By5crFwsXLGPbJYDq0ffZD1O7tjowa/WWWxmrtn3HJUqU5c/pUkrKgoJvcvXMnSW0J4Lux39C02WvUqVs/K0NMl/fe6YjfuXMAlChRknkLFlOgQAELR5VMNq/xmCormunKAy2AX/X7YqTXS8ByACnlfrRB/4z5W0oZoV8ZHIA2BEVDtNrTEynlA2CbPm0YEAX8IoRoDzzOQFxJtBkwil5jZ9Bu4Oc45nFi6YQRhN4JASA+Po6nkY9p2ftjajZtRelKNWjZeyje1eqwf8NyjF3rFfP0KX6H/qFszfo4Obu8aHg52rvvdWfJitVMnTGbhi+9zOiRw9jzzy7D+0+ePGHk8I+5f/8e3373A/MX/sbHnw5n+9YtTBg/1oKRW5+u73Xj7JnTzJz2E/fv3ePK5ct88dlIbG1tkzSdbt64gXPnzvDJ8JEWjDZt4ydM4rflq/j+h8nkzZePD/r25ObNQEuHlVT6BkrNtrIkeinlQaAgUAitWSzxenOZsAhTGu2fJHoey3NqfVLKGKAusAZoCbxw+0ehYiUp5lOByg2a8v7nP/I0KpL9G5YDkEc/h1SmSq0k85SpUptHYQ94GHovxfLkiQNEPX5ItUbNU7yXO29+Ih+lPLWWUJY7X8oja3NwdnE2enSe2tG8tSji4UGlSlV4+ZUm/DD5J6pUrcaUH38wvL9u7RqOHT3C9Fk/0+qt1tSqXYduPXrzybCRrFm1AikvZFms1v4Zt3zzLfp88CG/LV5Is1ca8nabVhQuXJiGjV6mYKFCADx+/IjJk76nR68+ODo6EhEeTkR4OPFx8cTExBARHm44T2dpZby9qVq1Gi1avcnPvywi8vFjFsz/2dJhJZVDzhllSTISQpQH7IB7wDWgghDCQQjhDrzy3Jm1ZrnO+nJ8gdLpWPV+oLUQwlEI4YJ2p0KEEPkAVynlFuBjoEp6tictufPmw71IMR6EBAGkuN4oORsjRzSn92zHKb8LPtXrpXivkFdJQm+HJOkWDnA38Bp29g4peuSZi7e3T4rzAgCXLwdQxtvKrs14joqVKnPjxnXD60uXLuLs7ELx4iWSTFepinbN1pXLAVkWW3b4jD8aNIRdew+yau16/tq1l+8n/Y8b165RvYZ2sBX64AEP7t9nxtQpvNygruEREhLM9m1/8nKDuuzb84+FtyIlZ2dnipcowY3r19OeOCtlzs31rE5mJqP8QoiT+nmjlUB3KWWslPIG8AdwDu3mTSfSWM4swEMI4Yd250E/UwOQUh4F/gTOog3UdxoIB/IDG4UQp9ES1gt1XkjuYdh97gZdx62IJwDl62jdhQNOH00yXcCpozi7FyKfq3uK+QNOH6Vyg6bY2aes4JWt6UtcbAx+h599oeNiYzl3aDdlqtTC3iHltR7m0LhJU86cPkXgjRuGsps3Azn57wleadL0OXNaj7i4OE7+ewKv4sUNZQULFiQ8PIzr168lmfasfm6kcOEiWRZfdvmM8zg5UbacoEDBguzft5crVy7ToZPWubVAwULMW7A4xaNAgYLUq9+AeQsWU71mrTTWkPXu3b3LlctX8Ep2UGJxOaSZLtM6MEgpU03TUsoRwAgj5aX0p6HoveqklI+BDqksKmGaRcmW0zjRyx+klF8JIZyBQ8BJKWUwWjPdC1v1v6/wKFWWIiXKkCuPE/dCAjm85Xds7eyo36ojAD7V61GqYnW2/DKFyIhwXAt7cv7wP1w+c4zWHwxPscyz+3YQHxdH1ZdTNtEBeJYqS8X6jdn+6yziYmNwLeTJsb83EHonmHYfpXmr+Qxr36ETK5YtZcigAQwcPAQbbJg5fSpFPDzo2DHre9kn9C4776edgN63by9ubm64ublTu05d5syaTlhYGNVr1KRAgYLcu3eXdWvXcPbMab6b+KNhOa3btGPJr4sYNKAfvfv2x9PTE79z55g3dxYVKlaieo2aWbZN1vYZJ3fhvB/79+6hfMVKAJw8cZzFC3+hR68+hs8pV65c1K6bskbvmMuRAgUKGH0vqw0d/BEVKlSknBDkzZuPa9eusuTXRdjb21n8Wq4Usnnzm6n+8wOlCiGWARXRzk3NklJOf5HlJR8odf+G5fgd+ocHt4OIjYnBuUAhSlWoRsM2XXEt5GGY7snjR+xcOZ/zh/cQ+eghBYsWp0Hrd6jSMGUnw7mj+hIfH0//ifNTjSP66RN2rfyFswd2EvX4IUVKeNPsnb6Uqlg9xbTmHCg1OCgo6VA19X0ZPmo0xYqZbx2mDpRao0p5o+W1atdh/sLf2L1rJ8uWLMbf/xIPIyIoULAQ5YSgZ6++KRJMQIA/c2fN4PSpk4SGPqCIhwevNG5Kn779cXZJuwOJOQdKzYrPGDI2UGqA/yW+/WYM/v6XiH76lNJlvOnS9V3atDPeIzSxls2bUqNGrQwPB2TOgVIXzP+Z7du2EnjjOtHR0RTx8KB2nXr07tvPrJ+zWQZKfXOG6QOlbhqYbTPXfz4ZmZsatTvzqVG7s4YatTvzmSUZvTXL9GS0cUD2+5B0amw6RVEUa5bNOyaYSiUjRVEUa5YNa4QZoZKRoiiKNcvmveRMpZKRoiiKNVM1I0VRFMXSbFQyUhRFUSxNJSNFURTF4myy4WUDGaGSkaIoihVTNSNFURTF4lQyUhRFUSxOJSNFURTF8nJGLlLJSFEUxZqpmpFiVHYbdBTg5NVQS4eQLtVLuVo6hBwhuw08GhObvQZ2BcD+xT/jxLdz/y9TyUhRFMWKqZqRoiiKYnk5IxepZKQoimLNVM1IURRFsTiVjBRFURSLU8MBKYqiKBanakaKoiiKxWVWMhJCfAz00V8eAj4ASgLLAXdgH9BbShkrhMgNLAGqAyFARyllsL6cL4AeQDTQT0q5NyPx5IwO7IqiKNmUjY2NyQ9TCSE8gEFALaAyUBR4A5gIfCel9AHsgE76LH2AQL18IfC1vpwqQBugAtAOmJ3R7VTJSFEUxYplRjJC6zBuD+QGHPT/bwGNgA36NL8CbfXnrYHf9OfLgVb687eAFVLKaCnlBSBMCFEhI9upmukURVGsWTpyjBDCFXA18laolDI04YWUMlgIMRm4ATwFVgBXgAdSyjh9skCgmP68KHBTn/exEMJOCOGgl+9LtJ6Eec6bHrVGJSMLCgkOZtLECRw6uJ/4+Hjq+TZgxMjReBYtmmnrPLpvB4f+2c6VSxcID3tAgUJFqNWgMW916kEep7wA3LkVxLBe7YzOP2vl3+TNl9/wevXiWVy5dJ6r/hd4FBFOn6Ff0ui1N58bwyW/04wf0Y/4+HgWbNiPnZ15dsO/tm3lzy2b8Tt3lvv37+Hh6UmzV5vTp98H5M2bzzCdv/8lZk6fyplTJ4l4+JCiRYvRtl173n2/O/b2lv9KWGK/eFHWEvPf27ey7c/N+Pmd48H9e3h4eNLk1dfo1efZPnDe7ywzp/2Ev/9FwkJDyZ/fmfIVKtLngw+pWq1GimWeOXWSubNncObMKWKiYyjm5UXvvv15vUWrFNNmhnQOBzQUGGOk/Bv0pjUAIYQbWq2mFPAQWIfWTGcxlv/m5VCRkZH07dUdB0dHxn03ERsbmDFtKn16dWP12g04OTllynr/XLsU90IedOj+Ie4FC3MtQLJu2XwunD7OFz/OT7Ljv9mpOzXqNUoyf548SeP6e+NqSpQpS/W6L7F/x5Y01x8TE8OiGd/j7OpO2IN75tko3eJFC/D09GTQ0I8pUsSDC+f9mDNrBkePHObXpSuwtbXl9u1b9OnxPoULF2H4qNG4urpx5PAhpkyexP379/n40+FmjSm9LLVfvAhrivm3xQvx8PTko8EfU6RIEeT588ydM4NjR46w8Lfl2NraEhEeQfESJXirTTsKFirE/fv3WfbbIvr27MYvi5dSuUpVw/L27tnNsKGDeKNlK8Z//yMODg5cDvDnydMnWbZN6Wx++wlYZKQ8NNnrV4EAKeU9ACHEBqAS4CaEsNVrR17otSEgCK3GEyKEyAPESimjhRAJ5QkSz5MuKhlZyNo1qwgMvMH6TVspUbIkAGXLCVq3fJ01q1bSrUfPTFnv0DGTcXZxM7wuX6UmefO7MO9/33DhzAkqVqtteK+QRzF8yld57vJmr9qBra0tt4JumJSM/vx9CfHE8/Jrb7Fx1aIMb4cx02bOwd3d3fC6dp26uLi48sXokRw9cph69X3Zs3s3Dx48YNGS5ZQqVRqAevV9uXHjOps2rLd4MrLUfvEirCnmn6bPxi3RPlCrdl2cXVwY88Uojh09Qt169alb35e69X2TzNeg4Us0e9mXLZvWG5LRo0cP+ebL0XTs/A7DRo42TFuvfoOs2ZgE6chFelNcqAmTXgd89cQSBTQBNgP70c4PrQO6Aev16TcB7wPHga7AlkTlvwghpgHegKuUMt1NdGBFHRiEEFOEEAMTvT4khPgx0euVQoi3X2D5Q4UQX79gmGaze9dOqlatZvjyAnh5Fad6jZrs3rUj09abOBElKFNWO9/44N7tdC8vPU0It4ID2bByAd0GjMAuE5rDEieiBJUqa8n09u1bAERHRwOQL1GzHUD+/PmJi4/D0iy1X7wIa4rZ7Tn7wB19HzAmTx4nHB0dkzQZ/719Gw8e3Oe97pY9AMiMDgxSysPAH8AJ4AwQDiwFRgBfCCECgHhglT7LPKCEEMIf6IXe5CelPI2WkC6gJbABGd1Oa6oZHUTrIjhDCJELrYdHrUTv+6K1h/4nBPj707hpsxTl3t4+/LV9a5bGcuHsvwAULV46SfmaRbNYPGMiuXLnRlSuSYfu/SleyifD61k8YyJ1XmpG+co1OH/q2AvFbKpjx44AUKaMNwDNX3+DubNnMGH8OD4eNhxXVzcOHzrIpo0b6P/hR1kS0/NY035hKmuP+fixowCULlMmSXlcXByxsbHcvXuHRb/MA6Dd2x0N75/89zguLi74X7rI4AH9uHrlMgULFqJt+w707vchdnZ2WRJ/Zl1nJKUcQ8rzS/5AbSPTRgLtU1nON2jnpF6INSWjA2h93EFLQvuBGnqPjcJALNBECPEZWsX1d/3DRAjRFTBW3gcYCdwDJFpvEYQQg4H+QAzgJ6XskiVbmEhYWBjOzs4pyl1cXAgPD8+yOO7fvc3aJT9TqXpdSus1JAcHR5q0aEflGvXI7+JKcOA1Nq5axLfD+jLmfwsoWqJ0GktNaf/OP7nqf4Hvh6809yak6tatW8yaMY36vg0MR8cFChbk16UrGTJoAK1efxXQvuz9BwykZ+++WRZbaqxlv0gPa4759q1bzJk5jXr1G1CxUtIm51HDhrLj7+0AuLsXYOrMnynj/exg687t20RFRfH5qGH06fchFSpW4sihg8z/eTYRERF8OuKzLNkGNQJDFpNSBurdBT3QakGH0JoRa6BdFXwFGA/UQatS/iOE+Aetepha+edoiS1SX94VfXWjgNJSyid6V8gcKSryMVPHDcfOzo4+H39hKHd1L0iPgaMMr0XlGlSp5cvoD7uwYeUi+g9P30HQw4gwls+fSofuH+LsmrIZJTM8fvSIoYM+xN7OjrHfTjCU379/n0+GDiRPnjxMnjINF1dXjhw+xLy5c3B0dKRXn35ZEp+S+R4/fsQnQwZgZ2/HmLHfpXh/8CfD6d6rL7dCglm1YhlDB/Vn9s8LDEkrLj6OJ0+eMGDQUN7rpjXV1a5Tj9DQUFatWEa/DweSP3/+FMs1NzU2nWUcREtEvsBwtGTki5aM1gPVpJR3AYQQK9Au0MoP7DJS7qyX39fLf+fZObLTwFIhxDq0ds4s5+zibPSoMbWjTHN7+iSKKd98yp2QID6bOBv3gkWeO32BQkUoV6kaVy75pXtdv/86F1f3AtRt1IxHDyMAiH76FIDIR49wcHQkV+486d+IVERFRTHoo/4E3ghkweLfKOLhYXhv0YJ5BN28yda/duHs4gJAnbr1iIuLY+b0abR7uwNublmTMI2x9H6REdYYc1RUFEMHfcjNwEB+XvBrkn0ggZdXcby8ilOpchUavdKYTu3fYtb0qcyYMx8AVxdXIGWHhfoNGvL76hVcDrhEteo1M31bVM3IMg4CDdBqLVeEEHbAOKA0EJaB5aV2n+JWaAnrLWC0EKKqlDImIwFnlLe3DwH+l1KUX74ckKSpIDPExMQw47vPuOp/geHfTkvXeaCMfDGCblzhxhV/PurSPMV7H73TnJr1X2bIl5PSvVxjoqOjGfbxYPzOnWXu/IWULSeSvH/p4kWKlyhpSEQJKlepQkxMNDeuX7doMrLkfpFR1hZzdHQ0Iz4dwvlzZ5k1d0GKfcAYBwdHypYVaIMIaMr4lH3uPLY2WdP/K6ckI6vpTac7ALyNdlUwUkp/tDGPygELgMZCCHchhCPQGdgDHHlOeRMhhJs+yF87ACGELVBcSrkb7XxSfiBp16os0LhJU86cPkXgjRuGsps3Azn57wleadI009YbFxfHnElf4Xf6OIO/+CHNrtsJ7t0O4eK5U5QuVzHd6+za92NGTZiV5PFSM+2CwRHjZ/D2+/3TvUxj4uLiGD1yGEcOH+Kn6bOoWq16imkKFizEjevXCA9Lemxz5vRpAAoXfn4NMbNZar94EdYUc1xcHF98NpxjRw4x+aeZVDGyDxgTGRmJn99ZvIoXN5Q1bqJ1yjh4YF+SaQ/u30uuXLnwLvv8ZGUuNjamP7Iza6sZ/Qt4Aj8nKgtEG8rihj467D8866iwCwyjxhor/w4tKd1Da5oDbfC/JUIIF336mYmHycgq7Tt0YsWypQwZNICBg4dggw0zp0+liIcHHTt2zrT1/jp7Ekf37eCtzj3JlTs3/hfOGN5zL1gY94JFWD5/KnFxcfhUqIKzsyvBN6+zadVibG1tad05aTfXC2dOEBH2gNAH9wG44n+e3Hm0Jrc6L2lf5pLe5VLEceHMCQDKV6lhthEYvvv2G7Zv20rffv3JkycPp0+dNLxXpIiH9tl27sKWzRvp37cX3Xv1xtXVjaNHDrN44QKavvoaHp6eZokloyy1X7wIa4r5+/Fj+Xv7Vnr31faBM4n2gcL6PjB+7Fc4u7hQsWJlXN3cCA4KYtWKpdy9c4dx438wTO9TthxvtWnHnFnTiY+Lp3zFihw+dJB1a9fQp9+HOOkjlmS2nFIzsomPT60lSzEmKibVpr90Cw4KSjqESn1fho8aTbFiXuZaBQAnr4Yann/asy13bwcbna5t1z60e7cve7ZvYOeWtdwKCuRJ1GPy5XehQrXatO3aB0+vkknmmTDqQ0NiSW7x5sOpxvTH0nmsWzbf6HBA1Uu5mrZhybR4rSlBQcYv/u4/YCAffjQIgNOnTjJ39kwunD/Pw0facEAtWraiW49e5M6dO0PrNqes2i/MKStijolN+6v35htNCQ4KMvpev/4f8cGAQaz/43fWrV3NtatXiIyMpFDhIlSuUpWevfulaNKLjn7KvDmz2LRhHffu3aNosaJ07PwuXd/rZlLM+XK9eCYRI7eZ/JsjJ76ebTOXSkbpZM5klFUSJ6PsIKPJSPlvMyUZWRtzJKPyo0xPRhe+z77JyNqa6RRFUZREbFXXbkVRFMXScsgpI5WMFEVRrFlO6cCgkpGiKIoVyyG5SCUjRVEUa5bOm+tlWyoZKYqiWDFVM1IURVEsTp0zUhRFUSwuh+QilYwURVGsmaoZKYqiKBaXQ3KRSkaKoijWTI3AoCiKolicaqZT/jOy28CjT6LjLB1CuuVyyBnXgliSvV3O+FFOLofkIpWMFEVRrJmqGSmKoigWl0NykUpGiqIo1kx1YFAURVEsTjXTKYqiKBankpGiKIpicTkkF6lkpCiKYs1UzUhRFEWxuBySi1QyUhRFsWaqN52iKIpicbY5pGqkkpEFhQQHM2niBA4d3E98fDz1fBswYuRoPIsWtXRoqbKWmA8e2MevC+dz5XIAEeFhuLm5U6VaDfr2/4gy3j5Jpt2/9x8WL5yHPH8eW1sbipcsxaChw6hTtz4A5/3OMXvGTwRcukhYWCj58jtTvnwFevX7kKrVamTpdoH1fMbpYc0x3woJYcEv8/A7d5aL8gJRUVFs2b6DYsW8kkw37af/ce7cWc6fO0dYWChjv51Am3btLRT1MzkkF2ETHx9v6RiylagYzPKBRUZG0ql9GxwcHRk4eCg2NjBj2lSioiJZvXYDTk5O5liNWWVVzKaMTbftz83I835UqlIVNzc3QkKCWbxgHrdvhbBs9Xo8ixYDYO2alUz6/ls6du5Kw5deJi4+novyPGW8fWj0chMAjhw+yD87/6ZajZoULFiI+/fvs3zJYs77nWPewiVUqlI1zXjMNTad2i/M7+iRw4wY9jEVK1YiNjaOgwf2GU1GvnVqIMpXwMurOBs3rDNLMsptzwunktdnHTb5N2fbgHrZNnWpmpGFrF2zisDAG6zftJUSJUsCULacoHXL11mzaiXdevS0cIQpWVPMr7doxestWiUpq1S5Kh3btmTn39t5t1tPgm7eZMqkCQweOox33utumM63wUtJ5qtbz5e69XyTlPk2bETzxr5s2bzBpGRkLtb0GZvK2mOuVbsOu/YcAGDtmtUcPLDP6HT7Dx/H1taW69eusXHDuiyM8PlyyCkj601GQohY4AzgAFwG3pdShr7gMl2Bk1LKUi8a34vavWsnVatWM3x5Aby8ilO9Rk1279ph8S+wMdYes4uLKwB2dnYAbFz/OzY2trTv2CXdy8qTJw+Ojo6GZWUVa/+MjbH2mG1tTau1mjpdVsusDgxCCHfgF6ACEA90BCKB5YA7sA/oLaWMFULkBpYA1YEQoKOUMlhfzhdADyAa6Cel3JuReKzz09dESCmrSykrAaHARxaOx6wC/P3xLlsuRbm3tw+XA/wtEFHarDHm2NhYoqOfcv3aVSZ8O4YCBQvSXK8xnfz3BKVKl+avrVto92ZzfGtVpv1br7N6xVKjy4qLiyMmOpqQ4CAmTRgHQNv2HbNsW8A6P+O0ZMeYsxObdPxLp2nAOilleaAmcA2YCHwnpfQB7IBO+rR9gEC9fCHwNYAQogrQBi2htQNmZ3Q7rbZmlMx+oBqAEKI6MBfIA5wH+kgpI4QQfYG+QC7AD+gupXwqhPABlgJOwJaEBQohKqF9qI5oSfltKeWlrNqgsLAwnJ2dU5S7uLgQHh6eVWGkizXG3PP9zlzwOwdA8eIlmPXzItzdCwBw985t7t65zbSfJjFg4FCKFS/Bjr+2Mun7b4mNjaXLu92SLGv0iI/Z+fd2ANzdCzBlxtwUnSEymzV+xmnJjjFnJ+mpGOmtP65G3gpN3LIkhHAB6ksp3wOQUkYKIWyARjxLQL+iJaHlQGvgM718OfCN/vwtYIWUMhq4IIQIE0JUkFKeNz1qjTXXjAAQQtgBrwEb9aJfgU+klFWBm8Bwvfx3KWVdKWU1IBh4Ry//CfiflLIKEJZo0f2BqVLK6kBtIDAzt0PJHN98O5EFv61g3IQfyZsvH4P69ybo5k1Aq+k8evSIz774mrZvd6JO3fqM+vxrfBs2YtGCeSTvvDNo6DAWLVnFxMlTKeNTlk8GfYjfubOW2CxFMbCxsTH5AQwFrhh5DE222NLAHSHEb0KIf4UQM4HCwAMpZUIPokCgmP68KNrvLVLKx4CdEMIhcbmRedLFmpNRfiHESeAW4AFs17N5Pinlfn2axWiZHKCaEGKfEOIM0AGoqJfXA1brz5ckWv5BYLQQYhRQUkoZmXmbkpKzi7PRo8bUjjKtgTXGXLqMN5WrVOP1Fq2YOXchjx8/ZvHCeQC4uLoCULd+wyTz1KvfgPv37nL3zp0k5cW8ilOxchWaNGvO1JlzcXN3Z87MqVmyHQms8TNOS3aMOTuxsTH9gXbwXdrI46dki7VHOwj/Ca2Jzh7tvI/FWHMyitBrLSUBG2BAGtMvQDt5VgX4Aa25LlVSymVoVc9HwCYhRNMXjjgdvL19CPBP2Sp4+XJAljcNmcraY87v7IxXiRIE3rgGkGZMNs9p/3BwcKRsOUHgjetmjTEt1v4ZG5MdY85ObG1sTH5IKUOllFeNPEKTLTYQuCalPC6ljAfWAVUBNyFEQl7w4lmtJwi9xiOEyAPE6k1zhnIj86RvOzMyU1aSUj4CBgOfoiWOCCFEQj/c94E9+vN8wG0hhCPPmugADgNv68+7JhQKIcoAl6WU04E/0P4QWaZxk6acOX2KwBs3DGU3bwZy8t8TvNIkS/Oiyaw95nv37nLtyhW8vEoA0LjJqwAcStaV9+CBfRQu4kHBgoVSXVZUZCTnz52lmFfxzAvYCGv/jI3JjjFnJ7a2NiY/TCWlDAGC9HPqAI3RzrXvRztIB+gGrNefb0L7vQXtd3RLovIuQggHIUR5wDUj54sgm3RgkFIe05vfOgHdgTlCCCfgAtBbn+wr4Chas96/iWYfCiwTQnxJog4M+rLeF0I81efpShZq36ETK5YtZcigAQwcPAQbbJg5fSpFPDzo2LFzVoZiMmuKefjHAylfoSI+ZQV58+Xj+rWrLF+yGDs7O7p26wFAw0avUKtOPSZ8O4bQ0AcUK1acHX9t5fDB/Xz1zXeGZU0YNwZnFxcqVKyMq6srwcFBrF6xjLt37/D1+IlZul3W9BmbKjvE/Ne2rQD4+WnnAPfv3YObmztu7u7UrlMXgGNHj/Dg/n3u3r0LwLlzZw0X7L72+hsWiFqTiSMwDAFW6AfwF4CegKdeNhktMa3Sp50HLBVC+KP9XnYEkFKeFkJs0uePBj7IaDBqBIZ0MtcIDADBQUFJh1Cp78vwUaNTXBluTbIiZlNGYFi8cB47tm8l8MYNomOiKVLEg1q169K9Vz+KFnvWavDw4UNmTfsfO//eTnh4OKVKl6Zbz7680fJNwzQb1v3O+rVruHbtClGRkRQqXIRKlavSo3c/fIx0WTbGXCMwgNovMkO1SsJoee06dfll0W8A9O7xPseOHjE63alzMkPrNccIDJ0X/2vyb87K7jWy7SWyqSYjIcQNMPrDawPESylLZGZg1sqcyUgxzpRkZG3MmYyU/w5zJKMu6UhGK7JxMnpeM91Lz3lPURRFyQI5/uZ6UsprWRmIoiiKkpIam04nhCiJ1lW6CpA7oVxKWSYT41IURVHIOTfXM6WhewFaH/R4tC7SfwOLMi8kRVEUJUE6R2DItkxJRm5SyuVAnJTyX7Sue2+mMY+iKIpiBrY2pj+yM1OuM3qq/x+qX9QUDKR+taCiKIpiNtm9xmMqU5LR7/p9L74HDqHVpsZnalSKoigKwIv3Dc8m0kxGUspJ+tPNQojCQG4ppRoXXlEUJQvYZff2NxOZ0psuxeBSQgiklDszJyRFURQlgWqme+bLRM9zo9129gSgkpGiKEomyyG5yKRmuiaJX+t3SB2WaREpiqIoBrY5JBule0AtKeU5oEYmxKIoiqIkk86b62Vb6T1nZAfURxsqXFEURclk6pzRM4nPGcUA/mj3AlKUTJEdR8B26zjf0iGk24PVfSwdgmICO5WMDLpKKYMTFwghPDIpHkVRFCWRHNKz26RzRpuNlG0xUqYoiqKYWY4fDkgI4Qq4A45CiNI8uxDYBXDK/NAURVEUdc4IugNDgaIkvaYoDPhfJsakKIqi6LJ7jcdUz7u53lRgqhBipJRyYhbGpCiKouhySMXIpHNGJ4UQLgkvhBCuQojmmRiToiiKorO3sTH5kZ2Zkoy+l1KGJXodhjaCt6IoipLJcspFr6YkoySbKKWMx7Qu4YqiKMoLsrWxMfmRnZmSjO4JIV5PeCGEaAHcy7yQFEVRlAQ5pWZkSg1nKPCHECJKf+0A9My0iBRFURSDHN+bLoGU8ox+u/GKQGu0sem2AK6ZG9p/362QEBb8Mg+/c2e5KC8QFRXFlu07KFbMy9KhpSokOJhJEydw6OB+4uPjqefbgBEjR+NZtKilQ0uVJWL2LV+E0Z1rULV0AfI42uEfFM6cP/34dcdFwzQlC+djQvd6NKlWFAc7W45dusPoxUc4EXA3xfKKujvxVddavF6zOG75chF8/zGr9wXw1ZJjhmnyONrxaftqdGrkjVeBvNyLiOKfM8GMXX6c63ceZtq2gtovMlNOubmeTXx8/HMnEELUBnoAHYG8QF9go5Qyc/duKxUVw/M/sHQ4euQwI4Z9TMWKlYiNjePggX1WnYwiIyPp1L4NDo6ODBw8FBsbmDFtKlFRkaxeuwEnJ+u7FjqrYk48Nl3lku7smdiaIxdvM2PTWR4/iaWdbyn6vF6BwXP2M2/bedzz5+LolPZEREbz7YoTPH4Sw+DWlanpU5BGIzYgA0MNyytRKB+7JrzF1dsRzNp0jlthkZQsnB9vD2fGLj9umG7Rx415q14pvl1xnOP+dyleKB9fdqlJbFw8dT9ey6OomCQxm2tsOrVfpC63/YvfNXz8Dn+Tf3M+b+aTbTPX80ZgGI524Ws8sBjtpnoHpJTLMzMgIUQXYBlQQkoZmMFlXAWqSylDzRia2dWqXYddew4AsHbNag4e2GfhiJ5v7ZpVBAbeYP2mrZQoWRKAsuUErVu+zppVK+nWw/paby0Rc8eXymBna8Pb3203JICdp25SpZQ77zbxYd628/R9vQKFXfPw6hebuBISAcDuM0H4zenMl11q8t6Pz64zn96/IUH3H/H6l5uJidV+l/adC0myzjyOdrzdsAz/++M0U9adMZTfDo1kw1dv4Fu+CH+fvGn2bQW1X2Q2mxfPZ9nC8zowTABCgHeklD/qg6WarVbwHF2AA0DnLFiXRdnaZq/RqXfv2knVqtUMX14AL6/iVK9Rk927dlgwstRZImZHe1uiY+OIfBqbpDzs0VNDj6e6ojD+weGGRATw+EkMB/xCaFG7hKFpprRHfprXLM6szX6GRGSMna0t9na2REQ+TbbOJwDYZmJTj9ovMleOH5sOKIVWM1orhIhAqx3ZZWYw+sW1NYG3gHnAZCFED6ANUADwAOZKKScLIUqhDeJ6FqgKHAN6SSmjky3zfWAw4AjskFJ+IoTIC6wCvPRtGielXJmZ2/ZfEODvT+OmzVKUe3v78Nf2rRaIKG2WiPm3XZfo+0YFJvfx5YfVJ3n8JIb2DUvTpGoxek/dDUBsXDxPo2NTzPskOhanXPaU8XDmUlAYvuWLABD1NIZNY1rwUiUPHj+JYcvR64xYeIj7EVqyeRgVzdJdlxjQqhJHLt7huP8dShTKx3fd63Hqyj12nQ7KlG0FtV9ktuyeZEz1vOGAAoHxwHghREO080bOQog/geVSyl8zIZ52wCYp5SkhhJMQooxeXheojHZTv2NCiI3AU7ROFb2klIeFEMvQkqeh8V4IUUFfpq+UMkYIsUQI0QrIDQRJKVvp0xlGmFBSFxYWhrOzc4pyFxcXwsPDLRBR2iwRs9/1B7z+5WZWjnyN/i0qAvA0OpZBc/axet9lAC7dDKNZtWK4589lSCg2NlC7bCEA3PPnArSOCwBzBr7Msn/8mbT2pHau6P06lC/uSqMR60k47dtvxh4m9/Fl27hWhliOyNu8+fWfRMfEZcq2gtovMltOGSjVpHYiKeV+KWVftEFTlwLvZVI8XdBqLAAredZUt01K+UDvNLEFaKCXX5FSHtafrwBeSra8ZkA9tAR2Ei2p+QBngNeEED8IIRolG2FCUV6It6czy0e8it+NB7Qfv40WX21h/rYLTO//El1e9gZg3rbz2NrA/MGvUNojPx5uefhfH19KFckPQFyclmESfoj2nA3m458P8M+ZYBb8JRk69wC1fArxWo1nnV2+7lqLd172YdTCw7z6+SZ6/rQb9/y5WPfl6zjlUtepZ1d2tqY/srN07aFSykhgif4wKyFEQeAVoIIQImGUh7vATyQ9VxWf6HVq5QlsgQVSyi+TlSOEqAm0BMYKIXZJKceaYzv+y5xdnI0eNaZ2lGkNLBHz2PdqEx0bR/vx2wzneXafCcI9fy4m9fZl5d4Art6KoOeU3Uzp1wC/2dox14mAu0zfeJaP21Yl5MFjAEOtaeeppJ0P/j6p9e2pVroA208EUqG4K8Pfrk7/GXtYnKj7+NGLtzk7qxM9XxPM3HQuU7ZX7ReZK7uPrGAqazpc6oCWOD5KKBBCnEZrUmuu318pGi2BzNUnKS2EqCOlPIpWi9qVbJk70M55TZVS3hVCFEY7R2QDPJBSLhFC3AX6Z+aG/Vd4e/sQ4H8pRfnlywGU8faxQERps0TMlUq4c+bq/RQdDo7536HLKz4UdsnDrdBI1h26yoYj1yhb1IWnMbFcCYlg6gcNuXHnITfuPgLg/I0Hz11XwqUZlUu6A3DcP+k1SgHB4Tx4+ATh5WqmrUtJ7ReZKzPPGQkhbIHDwE0pZVshhDewHO1edvuA3lLKWCFEbrRKSHW0jm0dE+4ALoT4Au00TjTQT0q5NyOxWFPFrgvwR7KyP4BRaJ0T1gMngflSyoRDv/PAp0KI8/rrJOexpJTngLHADj2xbUL7kKsAh/Wmu7HAt+bemP+ixk2acub0KQJv3DCU3bwZyMl/T/BKk6YWjCx1loj5VmgkVUu542Cf9OtVp2whIp/EcP/hE0NZXFw8MjCUKyEReLo50aFhaX7eet7w/mF5m+D7j3m1RtJrz5rrr49d0pJPyINI4Nk5pwQ+RZ1xy5eLoHuPzLeByaj9InNl8nBAHwIBiV5PBL6TUvqgHbh30sv7AIF6+ULgawAhRBW0DmYJ5+dnZygKTLjo1dL03nTVpZRDk5WXAtZJKatnZTzmvOgV4K9tWs+dw4cPsnrlCj7/cgxubu64ubtTu05dc67qhT1+/JhO7duQK3duBg4egg02zJw+lUePH7Fm7Qac8ua1dIgpZFXMiS96bedbimUjXuWvfwP5easfkU9iebNuCfq3rMS0DWcYufAw9nY2fNetLnvPhRAe+ZSKxd0Y/nY1LodE0GLMliQdDt5tUpb5g19h3rbzrD94FW9PZ75+tzanr9zjja+2AFrX7QM/tqVU4fxMXPOv4aLXUR2qU9AlN3WHrjXUthKY66JXtV+kzhwXvc7cf9Xk35yPGpYyeX1CiCJo13SOR+tx3A6t1uMppYwTQrwG9JFSdhZCbAc+k1IeF0I4ARellF5CiNHAEynlZH2Z+/V5zhtd6XNYUzNdjjTskyFJXo8f9w0AtevU5ZdFv1kipFQ5OTkxb8FiJk2cwOejRmhDqNT3Zfio0Vb5gwOWifmPg1dpM24rn7arxqwBjcjtaMflkAiGzN3P/O0XAIiPB++iLnR62QfXvI7cvPeIxTsu8sOakyl6vi3ddYm4uHg+bV+Vbk3LcT/iCSv+8efLJUcN08TFxdNyzBZGvF2dXq+V58su2nBAhy7cZuzy4ykSkTmp/SJzpafGo5/OcDXyVqiRQQB+AL4AcumvC6CdvkjYAQOBYvrzosBNACnlYyGEnRDCQS9PfLV+wjzpTkZWXzOyNuauGSn/DYlrRtmFuWpGSurMUTP6+dA1k39zJndv/g0wxshb30gpv054IYR4Ge2ymB5CiMZoA2L3AfZJKcvr01QA5kkpXxJCnAVelVKG6O8FAyWAKfo8K/Tylfo8f6d3O1XNSFEUxYql81zQT8AiI+WhyV43QLu85SpaJzFnYDLgJoSw1WtHXui1ISAIrcYTIoTIA8RKKaOFEAnlCRLPky4qGSmKolix9HTt1pviQk2Y7nv0O3Yn1IyklN2FEGvR7s6wDuiG1nEMtM5f7wPHga5o13smlP8ihJgGeAOuGTlfBNbVm05RFEVJJotvrjcC+EIIEYB23WbCIATzgBJCCH+gF3pvOillQi/lC2gJbEBGV6zOGaWTOmekGKPOGSnGmOOc0aKj103+zelRp0S2vUJWNdMpiqJYMTUCg6IoimJxKhkpiqIoFpczUpFKRoqiKFYth1SMVDJSFEWxZjnlfkYqGSmKolixnHL9jUpGiqIoVkx1YFAURVEsTjXTKUbFZcOLhHPKkZUlZccLSN3qDLR0COny4OgMS4dgEaqZTlEURbE4VTNSFEVRLC5npCKVjBRFUaxaDqkYqWSkKIpizexySDZSyUhRFMWK2eSQhjqVjBRFUaxYDqkYqWSkKIpizWxVzUhRFEWxNFUzUhRFUSwup1y0rpKRoiiKFbPNGblIJaPMcCskhIUL5uF37iyXpCQqKorN2/6maDGvJNM9efKEWdOnsmXTRiIiwilXvjxDPh5Grdp1DNNcu3qFlcuXcezIYQIDA8mbNy8VK1dmwMAhiPLls3rTCAkOZtLECRw6uJ/4+Hjq+TZgxMjReBYtmuWxmCq7xXzk8CFmTp/Keb9z5MqVm0avvMKnw0ZSoGBBs63D3hZ2L/6UKmWL4ZTHEdHyK64H308yTS5He8YMeJMuLevgmj8Ppy/e5POp69h/IsAwjU+JwvTv3IiXa5ejtFdBIh5FcdzvOmNnbeLMxZsp1tuzXQOGvN+UUsUKcC3oPtOX7mL+mn2pxlmqWAGOr/4cpzyOPIkBcw3G9de2rfy5ZTN+585y//49PDw9afZqc/r0+4C8efOZaS3mkVN60+WUYY+y1I3r1/hr61acnV2oUbNWqtN989XnrP19NR8OHMTUmXMoVLAwH33QB3nhvGGagwf2c+zIYd5s05apM2fx2Rdf8eDBfbq/2xm/c2ezYnMMIiMj6durO1euXGbcdxMZ//0PXL92jT69uvH48eMsjcVU2S3mE8eP8WG/3uR3dmbyT9MZ8dloThw7Rt/ePXj69KlZ1mFrA3Y28CD8Mfv/DUh1ujlj3qVn+waMm72Z9kPmEHInjI0zP6JquWKGaV71Lc/LtcuxdNNh3h4yh6ETVlLILR//LP6UGhWKJ1lez3YNmPFFF9btOEXrj2ax9q9/mfpZJ/p2fCnVGKZ+1pmwh5EvvtHJLF60ADs7WwYN/ZhZc+fTqfM7rF65nA/69CIuLs7s63sRNjamP7Izm/hsOPCnOQkhTgJtpZRXTZn+cXTaH1hcXBy2tlqeX7tmNeO+/jJFzUheuECXDm35etx42rR7G4CYmBg6tH2TkqVKM3XGbAAePHiAq6trkvGpIiIiaPV6M15+pQnfTpiYZszmanNe+ttifvzhe9Zv2kqJkiUBCAy8QeuWrzP0k+F069HTLOsxp+wWc7/ePQi6eZN1m/7E3l5ruDh39gxdO3dg9Bdf0fmdd822Lrc6A+nRzpfZX72bomZUpVwxjqz8jH5jlvDbhkMA2NnZcmLN51y8dpuOQ+cCUMA1L/dCHyVZrnO+3FzYPJYte87Q58vfDPNe3j6e7fv96PvVb4Zp54x5l1avVKF089HExCRNAp3fqM3EYe35ccF2Jg3vYNaa0f3793F3d09StnH9Or4YPZKff1lEvfq+ZllPbvsXr9bslvdN3uzGwj3bpqRsUTMSQthZOob0SEhEz/PP7p3Y2zvQ/I2WhjJ7e3tef6MlB/fvMxwFu7m5pRgoMX/+/JQsWYo7t2+ZN/A07N61k6pVqxl+1AG8vIpTvUZNdu/akaWxmCq7xXz61CnqN2hgSEQAlSpXwdXVlZ07/s6yOFq9UoWn0TGs2X7cUBYbG8fqbcd5zbc8jg5afMkTEUD4wyj8r92maCFXQ1n9qqUp7J6f5ZuPJpl22eYjFHTLR4Pq3knKXfPn4ftP2/PZlD8IjTB/zSh5IgLtcwa4ncXfq7TY2pj+yM6s4pyREGIM0BUIAe4D64GvgZVAc2CMECIf8BnauIG/SynH6POGSild9ec9gOpSyqFCiEVAOFAHKAwMlFL+KYRwAhYDlYDTgEPWbGVSAf7+FPMqRp48eZKUe/uUJTo6mhvXr+HtU9bovGFhofj7X6JN23ZZEapBgL8/jZs2S1Hu7e3DX9u3ZmkspspuMdvZ2eLgkHKXdHB0xP/SpSyLo6K3J1dv3iMyKjpJ+fmAYHI5OuBdvCDnL4cYndfN2YmKPp78tv6QoayCtycAfgFBKZYHUKGMJ3uOPdu+8UPbcvHqLZZvPsp7b9Uzyzal5dixIwCUKeOdxpRZK6f0prN4zUgIUQdoDVQF2qIljwRBUsoawDFgPNAEqAk0F0I0NWHxhYAGwDvAOL1sAHBXSlkRmA5UNMNmpFt4WCjOzi4pyp1dtLKwsLBU55343bcQH0/X97tnWnzGhIWF4ezsnKLcxcWF8PDwLI3FVNkt5pKlSnP61KkkZUFBN7l75w5hYaFZFoebsxOh4SnPqd3Xy9xd8qY67/9GdsQGG6Yv25VkeaCdpzK2PDcXJ0NZwxrevPtmXYZMWJnxDUinW7duMWvGNOr7NjDUkKyFTToe2ZnFkxHQEFgnpXwipXwAbEv03mr9/zrALinlXSnlU2AF0MiEZa+TUsYDx4FSetlLwHIAKeV+4MqLb0LW+WXeXP7cvImRo7+kRImSac+gZCvvvt+Ns2dOM2PqFO7du8eVywF8PmoEtra2JjX/WtqwXs3p0rIOH09cxeUbd9M9v4O9HdO/6ML0pbu4kErNy9weP3rE0EEfYm9nx9hvJ2TJOtPD1sbG5Ed2Zu17tyndnRKf3MuV7L0nAFLKWJI2SVq814azswvh4SlrP+F6jcjFJWWtafXKFcyYOoWPBg2lbfu3Mz3G5JxdnI3WJlKrfViD7BZzqzdb0/eDD/l18UKavtyAdq1bUbhwEV5q9DIFCxXKsjhCwx/j6uyUotxdL7sflvJcUZ8OLzFuUGvGzNjIr4ma6ABCI/QaULJlJizvQZj2/qB3m+Ca34lZy//BJV8eXPLlwSm344tvUCqioqIY9FF/Am8EMvvnXyji4ZFp68qonFIzsoZzRvuBWUKIiUAetHNEe5NNcwSYIoRwBx4CnYHP9ffuCSG8gatozX1pNazv0+ffK4TwBUqbYyPSq4yPDzt3/E1kZGSS80aXA/xxcHCgeLJaz6YN65nw7Te8370nfT7on9XhAtp5lgD/lB/v5csBlPH2sUBEacuOMQ8cPJReffpxM/AG7u4FKFCwIG3favHcywTMze9yMK2bViNPbock543Kl/HkydNoApLVet5pVYepn3Xip1938MMv25IvDr+Ec0PenoTcfXZwUL6Mdi7p/OVg/bUHnoVcuLx9fIpl5LKHuHh4Gvvi2wcQHR3NsI8H43fuLHPnL6RsOWGeBZtbds8yJrJ4zUhKeRT4EziL1nHhNFrHg8TTBANfAP8AJ4C/pJQJDdKfAdvREth1E1Y5C/AQQvgBgwE/M2xGur3SuAkxMdFJTqLHxMSwfeuf+DZoiKPjs6PBnX//xddfjqbd2x34ZPhIS4QLQOMmTTlz+hSBN24Yym7eDOTkvyd4pYkpp/CyXnaMGcDJyYmy5QQFChZk/949XLl8mY6dumTZ+rf8cxZHB3vav1bTUGZnZ0uH5jX5++AFnkbHGMpbN6nKz1+/x8I/DvLZlD+MLu/w6SvceRBBlxZ1kpS/06oO90IfcfDkZQB+XPgXzftMTfL4ceF2QEtC0WZKRHFxcYweOYwjhw/x0/RZVK1W3TwLzgQ5pZnOGmpGAD9IKb8SQjgDh4CTUspSiSeQUi4DliWfUUq5mmfnlhKX90j22lX//zHQwVyBpyYhyZz3OwfAvr17cXN3w83Nndp16lK+QkWav9GSHydOICY6hmJeXqxeuZybNwMZP3GSYTnHjx3lsxGfUk4I3mrTjtOnThrec3R0pHyFrOt/0b5DJ1YsW8qQQQMYOHgINtgwc/pUinh40LFj5yyLIz2yW8znz/uxf+8eKuh/139PHGfRwl/o0asP1WvUTGNu09naQLtXq1OjQgkAXn+pIncfPOTOg4fsO+7PKRnI6m3HmTTsbRzsbbl68x79OjaiVLEC9Px8kWE5DWt6s3hCT05fvMmSjYeoW6WU4b0nT2M4JQMBiImJY+yszUz9rBNBt0PZeVjSuG45urepzycT1xAdo2WZi1dvcfFq0q7VJYtq3bDj483Xvv7dt9+wfdtW+vbrT548eZJ8r4oU8bCq5rrsnWJMZxUXvQohlqH1assFzJJSTrdwSKky5aJXgBqVjQ/VU6t2HeYv0i76i4qKYsa0KWzdvFkbDkiUZ8jHn1K77rOurHNmTmfu7JlGl+VZtChbtu9MMxZzHjEFBwUlHVqnvi/DR42mWLKhjqxJdorZ3/8S477+igD/Szx9+pTSZbx55933aNvOvOcIc6dyGLrn2CVe7ztVmyaXA98MfItOb9TGNX8ezly8yedT17P3+LNmz88/aMkX/VsaXda1oHuUbzUmSVnvtxsy5P1mlPB040bIA6Yv2cXPq5O3yif13lv1mDf2fbNe9NritaYEBaUcrgig/4CBfPjRILOsxxwXvR69EmbyZtcp7ZJtc5dVJKPsxNRkZE2ye/VdyRxudQZaOoR0eXB0hqVDSDdzJKNjV8JN/s2pXdo5237ZraWZTlEURTEipxxLqmSkKIpixXJILlLJSFEUxZolH5vSHIQQ5YAFgBsQC4yTUq7WL5NZDrijXQbTW0oZK4TIDSwBqqMN29ZR7+WMEOILoAcQDfSTUj7/JGAqLN61W1EURUldJt1C4gla4qiEdm3nVCFEfmAi8J2U0gewAzrp0/cBAvXyhWhjhyKEqAK0ASoA7YDZGd1OlYwURVGsWGaMwCClvCal9NOfhwB3gIJow6xt0Cf7FW28UNAGFEi498dyoJX+/C1ghZQyWkp5AQgTQlRI5yYCqplOURTFuqUjywghXAFXI2+FSilDU5mnDtrdCx4CD6SUCTeWCgQS7qRYFLgJ2rWaQgg7IYSDXp74Vr0J85wnnVTNSFEUxYrZpOMfMBRt8Ofkj6HGli2EKIxWA+qNhcfsVMlIURTFiqXznNFPaONtJn/8lHy5+r3dNgBjpZQHgXuAmxAiIS94odeGgCD0WpIQIg8QK6WMTlxuZJ50Uc10iqIoViw9HRP0prjQtKbT7569AvhDSplwS514IcR+tPND64BuaOOFAmwC3ke7HU9XYEui8l+EENMAb8BVSpnuJjpQyUhRFMWq2WTOlUYt0DohlBBCvKOXvQ+MAFYIISaj3VFhlf7ePGCpEMIfuAV0BJBSnhZCbAIuoHXt/iCjAanhgNJJDQek/Feo4YAynzmGA/ILemTyb07Fonmz7Zdd1YwURVGsWLbNLumkakbpFBVj+bvEpldEZEzaE1mR/HnUMZKS0pkbKe+MbO3MMYr2+WDTa0YVPFXNSFEURckEOaWZXSUjRVEUK5YzUpFKRoqiKNYth2QjlYwURVGsWCZ17bY6KhkpiqJYsRxyykglI0VRFGuWQ3KRSkaKoijWLDNurmeNVDJSFEWxYjkkF6lkpCiKYs1ySC5SyUhRFMWq5ZBspJKRBYUEBzNp4gQOHdxPfHw89XwbMGLkaDyLFs3SOAb268HJE0eNvlfXtyH/m/4zAOHhYcyaOpm9u3fw5MkTKlWtxuBPRuLtUy7JPE+ePGH+nOls37KRiIcRlC1Xng8HfUL1mrUzdTtuhYSw4Jd5+J07y0V5gaioKLZs30GxYl6Gac6dPcPvq1dx/PhRQoKDcXV1o2atWnw0eCheXsUzNT5TWct+kR6WiPnI3h0c3L2dy5fOEx76gAKFilCnYRNad+lBHqe8AFy5dJ5Vi2YTeDWAh+FhOOXLRymf8rR9pxdlK1ZNddkLpk1g55Y/aNDkDQaMHJvkvadPn7Bm8Vz27/yTx48eUrJMWbr0Hkj5KjUzZTtzStduNTZdOplrbLrIyEg6tW+Dg6MjAwcPxcYGZkybSlRUJKvXbsDJyckcqwHSHpvuymV/Hj16lKTs3OmTTJ/yA5+M/IL2Hd8hPj6eAX3eJyQ4iAGDPyW/szNLFs3nSoA/C5f9TuEiHoZ5v/liBAf37WHAkE8pWqw4a1cv59CBvcxdsJSyokKa8WZ0bLqjRw4zYtjHVKxYidjYOA4e2JciGU2eNJHTJ/+l5Ztv4e1Tltu3bvHz3Fncv3efVb+vw8PTM0PrNpes3C/MJatiTj423ZihvShQqAi1fF/BvWBhrgVI1i6Zh2fxUoz533xsbW05++8Rjh/4h3KVquHqXpDw0Pts/WM5ly+d56vJ8/AWlVKs5+K5U0z8fDC2trbUqNcoRTKaNfFLTh7Zzzt9BlPIoyh/b1zDqWMH+XrKL5T0TnpgZo6x6a7ff2Lyb04J91zZNnOpmpGFrF2zisDAG6zftJUSJUsCULacoHXL11mzaiXdevTMslhKl/FJUbbxjzU4ODjwavMWAOz7ZxdnTv3LtDkLqFm7HgCVq1anY+vmLPt1AUOHjwbg0sUL/LV1M5999S2tWrcDoHrN2rzfqQ3z58xg4pSZmbYdtWrXYdeeAwCsXbOagwf2pZimZ+++uLu7JymrXrMmLZs34/c1q/ho0JBMi88U1rRfmMpSMX/69WScXd0MrytUrUne/M7M/fEbzp8+TqXqdahcoy6Va9RNMl/V2r582Lk5+3ZsSZGMYmJiWDBtAm269GTnlj9SrPPa5Ysc2LWNvp98ySvN3zKsd2S/Lqz5dS6ffjPZ7Ntpm23TS/pk6m3HhRC7hRDNkpWNE0KMFUI0ycx1m0II4SqEuGqJde/etZOqVasZvrwAXl7FqV6jJrt37bBESAZRUZHs2rGNho0a4+ziCsC+PbsoWKiwIREB5MuXn4aNGrP3n52Gsv17dmFvb0+z5m8Yyuzt7Xn19RYcObSfp0+fZlrctrZp787JExFA0aLFcHN35/btW5kRVrpY836RGkvFnDgRJShTriIAD+7eSXW+XLnzYO/giJ2dXYr3Nq/5jbi4OFp2eM/ovCcO7sXO3p76L79mKLOzs8e3cXPOnDhEdKbs3zbpeGRfmZqM0G5r2ylZWWdgmZRyVyavOwX9VrtWIcDfH++y5VKUe3v7cDnA3wIRPbNn1w4eP3rEG2+2MZRduexPGe+UNajSZXy4FRLM48daM9+VgAA8i3qRO3eeJNOVKuNDdHQ0gTeuZ27wGXA5IID79+5Rpoy3pUOx6v0iNdYU84UzJwAoWqJUkvK4uDhiYmK4ezuExbMmAdDkjbZJpgkJusH65QvoMXAE9vbGG41uXr9MoSJFyZU7d5LyYiVLExMdza2gG+bZkERsbEx/ZGeZ3Uy3BhgjhPhIShkjhKgBRAKjhBDrpJTr9JrJYrT7rgO0k1JeFUIUAuYCJdBuZzsA8Af+BYSUMlYI4QHslFJWFEL0BfoCuQA/oLuU8qkQYhEQBdQEtgkhFgNLASee3cc9y4WFheHs7Jyi3MXFhfDwcAtE9MzWzetxcy9A/QaNDGXh4WF4ehZLMa2ziwsAEeHhODnlJTw8jPxGtsvZOWE667onTUxMDN+OHYObuzvt2newdDhWvV+kxlpivn/3Nr//+jOVa9Q11JASTP9uNEf3aTV4Z1d3ho+dQrGSZZJMs2j6RGo3aELFaql3tHkYEU7e/Cm3NV8+bf9++ND825vNc4zJMrVmJKW8C5wCEprqOqPVlpILkVLWAJYDCfdC/gn4XkpZG+gOzJVSRqDdlz2hDeg9tMQC8LuUsq6UshoQDCTc1x2gMFBfSvmlvtz/SSmrANb1y2gF7t65zbEjh2j+RqtUjw7/SyaMH8upk//y3feTDIlVyX6iIh8z5Zth2NrZ0feTL1O8/07vQXwzdRFDvpiIV6kyTB7zCZcv+hne37fjTy5f9OPdfpY9Z2hMTqkZZXYzHSRtqusIrDQyzVr9/+NAKf35q8DPQoiTwCqgiF7+C9BDf/4+8Kv+vJoQYp8Q4gzQAUh8aLRGShmnP68HrNafL0n/5piHs4uz0aPG1I4ys8q2LRuJi4tL0kQHkD+/MxERKXN3eJhWllAbyu/sTISR7QoPT5jOen7wf/rfj/y+ehVfjxtPg4YvWTocwHr3i+exdMxPn0Qxecyn3A4OYuT4aRQoVCTFNIU9i+EtKlLnpSaMGDcVZ1c3Vi+eA2iJbNnPP/Fmx27YOzjy6GEEjx5GEB8fR2xsDI8eRhATo/VIzZsvP48iUm7rw4fa/p0vn/m318bGxuRHdpYVyegPoKUQoiFwT0p52cg0T/T/Y3nWdGgL1JVSVtcfJQGklHuA8kKI14DbUsqERtoFQD+9xvMDWnNdgsfm3aQX5+3tQ4D/pRTlly8HGD03k1X+3Lwen3KCsuXKJykvXcaHK5cDUkx/9UoARTw8cdKv6yhdxpvgoECioiKTTnc5AAcHB7yKl8i84NNh3tzZLPxlHiM/+4K3Wre1dDgG1rpfPI8lY46JiWHqt59x5dJ5ho+bQvHSaa/P3sGB4qXLcisoEICIsFDCwx6watEsPujQzPC4d+cWh/f8zQcdmnHyiNYz06tkGe7cCuJJVFSSZd68dgV7BweKFDX/tWo5o/tCFiQjKWUYcBiYhfFaUWp2AB8lvBBCVEv03jK080yLEpXlA24LIRxJ2kSX3GHgbf1513TEY1aNmzTlzOlTBN54dsLz5s1ATv57gleaNLVITBf8znL1cgAtWrVJ8d5LrzThzu1b/Hv82cWxjx4+ZP/e3bz08rOOkQ0bNSEmJoZdf28zlMXExLDzr63Uqd8AR0fHTN0GUyxd8iszpv3EoCEf8867xntNWYo17hdpsVTMcXFxzJr4JX6njjH0qx/wqVDFpPmeREVx5eJ5iujnQF3cCzB64uwUDxc3dyrXqMvoibMRlaoDUKNeI2JjYji892/D8mJjYzi852+q1KyHQybs3zmlmS6rTgqsQEsg6UlGg4A5QoieaHFuRDv/BNq5pVE8a94D+Ao4CtxC6+SQmqHAMiHEl1iwA0P7Dp1YsWwpQwYNYODgIdhgw8zpUyni4UHHjp0tEtPWzRuws7OneYs3U7z30stNqFy1OuO+GmW46PW3hfOJj4+na7dehunKla9As9daMHXyRGJiYvAs6sW6NSsIDgrkq28nZvo2/LVtKwB+fmcB2L93D25u7ri5u1O7Tl3+3LKZSd9/R8OXGlG3Xn1OnzppmDdv3nx4+1i29mGN+0VaLBXz4pk/cGTvDtp06Unu3HnwP3/G8J5bwcIUKFSEX6ZOIF9+Z0qXq0B+Z1fu3g7mrw2rCX1wl/4jvgbA0TEXFavVSrF8B4dcOLu6J3mvlI+g/iuvsWTuFGJjYijkUZQdm9dyJySID0eMTbEMc1AjMFgxIURXoJGU8sOsXre5RmAACA4KSjqESn1fho8anWTEAHNIawQGgJiYaNq+0YSKVaryw5RZRqcJDwtlxk8/svefHTx98pTKVasx8OMRKZr0nkRF8fOsqfy1bTMPIyLwLiv4cNAn1Kxd1+hyk8voCAwA1SoJo+W169Tll0W/8eXoUWxYn/JixsTTWFpW7RfmlBUxJx+BYWi3Nty9HWx02nbv9uHt9/vxz7YN7N66nuDAazyJisKtYCG8RSVad+6RZpPe0G5tKFepWsrhgJ5EsWrRbA7u3sbjhw8pUaYsnXsNNJrQzDECw52HMSb/5hTKZ59tM1e2S0ZCiNlovfOaSymvZvX6zZmMsoopyciavEgyUv67kiej7MAcyehuOpJRwWycjLLdt94StSFFURRLsc3uJ4NMlO2SkaIoSk6SQ3JRlnTtVhRFUZTnUjUjRVEUK5ZTakYqGSmKolixnNK1WyUjRVEUK6ZqRoqiKIrFqWSkKIqiWJxqplMURVEsTtWMFEVRFIvLrFwkhGgJTAHsgKlSyumZtCqTqOuMFEVRrFkm3ENCCGEPTEO7b1xVYIAQwqKDH6qakaIoihVLz3BAQghXwNXIW6FSytBEr+sC5xPuByeEWAu8BczOaJwvSiWjdMptn/3OJubOr/7MSvZXp7T13CU4K6XzN+drYIyR8m/09xIUBW4meh0IFEtvbOakfqUURVH+O34i6U1HE4RmaRQZoJKRoijKf4TeFBdqwqRBJK0JeaHVjixGJSNFUZSc5whQUQhRHLgLtAeaWzIg1ZtOURQlh5FSxgBDgR3AWWBOQmcGS8l2d3pVFEVR/ntUzUhRFEWxOJWMFEVRFItTyUhRFEWxOJWMFEVRFItTXbszQAgRC5zRX8YCH0kpD6UxzxagM1AAWCelrG5kmpNAW+ApMEVK2dl8URuNaQoQIKWcob8+BOyTUg7TX68EVkkpf8/g8ocCrlLKr80TcZJlJ/wNHIDLwPvJhjvJyDJdgZNSylIvGp+RZXcBlgElpJQZup5DCHEVqP6i22nNEr4DUsqrqby/GxgnpdyRqGwcEA/sklLuyoIwU5WZ+9B/naoZZUyElLK6nlBGA+PTmkFK2VJKGWHKwqWUQZmdiHQHAV8AIUQutB/2Wone9wUOZEEcGZHwN6iEdpHfRxaOJy1d0D7LrPi7ZiohhJ0FV78C6JSsrDOwzBKJyMKfxX+Kqhm9OGcgDEAI0RgYKqVsq79ehFYLWpdwVJt4RiGEE7AYqAScRksGCCFK6fNVF0L0AN7U11MGWCylHKdPNwboCoQA94H1UspFQojvgdZADLA9oaZjxAFgov68FrAfqCGEcAAKo9X6mgghPkMbE/h3KeUYfd1dAWPlfYCRwD1AAlf08sFAfz0mPyllFxM+W1PtB6rp66kOzAXyAOeBPlLKCCFEX6AvkAvwA7pLKZ8KIXyApYATsCVhgUKISsBCwBHtoO1tKeWljAQnhHABaqINRDkPmKz/Xdug1ZQ9gLlSysn6334z2rUfVYFjQC8pZXSyZb4PDNbj2yGl/EQIkRdYhXY1vR1aDWJlBuJNsV+hjWu2Eu3CyDFCiHwY//uHSild9ec90GpyQ/XvQjhQB23fGiil/DO178BzrNHX/5GUMkYIUQOIBEYJIRJ/1xajfQcA2kkprwohCqHtGyWAaGAA4A/8CwgpZawQwgPYKaWs+Jx9ZhEQhfY33SaEWIyRfUhJH1Uzypj8QoiTQogLaD8uadaMUjEAuCulrAhMByqmMl014G39/w+FEK5CiDpoX7aqaE17dQCEEAWAdkAlKWVV4NvUVq43F9npX0Bf4BBwCqgBNEBLJOOBJmhfvOZCiKZCiKLPKf8cqKe/Vz3R6kYBNfSY+pv6AaVFPzJ9DdioF/0KfKKv5yYwXC//XUpZV0pZDQgG3tHLfwL+J6Wsgn5QoeuPdo+X6kBtXmyolHbAJinlKcBJCFFGL6+LlpBqAn2FEOX08op6TBXQkkr3ZNtcQV+mr749hYUQrYA3gCApZTUpZWVga3oDTW2/0gVJKWugJcgUf38TFl8Ibb96Bxinl5n6HQBASnkXbR9tphd1RqstJReix7ocGKiX/QR8L6WsjfaZztVbK/ajfXYA76ElFkh9nwEtodaXUn5J6vuQkg4qGWVMQhNReaAF8KsQIiOjeb+E9mVBSrkfvRZhxN9Syggp5SMgAO3ItyFa7emJlPIBsE2fNgztqO0XIUR74HEaMSQ01fnqzw8ler0erR3+rpTyKdqXvhHaD5Sx8rp6+X0pZSSQ+FzTaWCpEOI9tNrRi8qvn1+4hVaz2K7XQPLpnyVoR8eN9OfVhBD7hBBngA48+9GrB6zWny9JtPyDwGghxCigpL49GdUFrcYCWu0ioalum5TygZTyIdoRdQO9/IqU8rD+fAXafpJYMz3uY/pnUBfwQTuH9poQ4gchRCMpZUZ+GFPbr+DZ55Ta3z8t66SU8cBxoJReZup3ILHETXUd0T7T5Nbq/yde16vAz/pntgooopf/AvTQn7+PdkADqe8zAGuklHH689T2ISUdVDJ6QVLKg0BBtKO+GJJ+prlMWIQpQ2A8SfQ8luc0r+rDfNRFa85oSdpHxwfRfgRLSymvoCWj+nqZqwmxJZfa9rQCZqDVuo7oN/d6ERF6raUkWlPRgDSmXwD0049efyCNv42UchlaDeERsMnEI/8UhBAFgVeAxXrz0Qc8S0aJP6v4RK9TK09gCyxIOG8ppSwnpZwqpbyIVlM5DYwVQnyVkZifI60Dm4R4EyT/jJ8ASCmT78PpHQbmD6ClEKIhcE9KednINAnfmcTrsgXqJvrcSurx7AHKCyFeA24nGhbnefuMKZ+Fkg4qGb0gIUR5tKaUe8A1oIIQwkEI4Y72I/Q8+9B/mIQQvkDpdKx6P9BaCOGo1wia68vJh9aDbQvwMVAljeUcQGsCvAEgpfQHKgDl0L6MjYUQ7kIIRz3WPWiDLKZW3kQI4SaEyI3WlIQQwhYoLqXcjXY+KT+QLx3bmiq9tjgY+BQtcUTonyVoR7l79Of5gNt6vImbWw7r2w/aeRL0mMsAl6V2K+Y/0JqtMqIDWuIoKaUsJaX0Qvve5UZr3nLVz/W0RDswACitN5eB9tnuS7bMHUAnPdEhhCgshPDUm0kjpZRL0M4F1sxAvEb3q2RS+/sD3BNCeOvNp62NzJtcur8Deo3vMDAL47Wi1OwgUUcXIUS1RO8tQ6tJL0pUlto+k5zRfUhJH5WMMibhnNFJtC9DdyllrH5E9QdwDq26fiKN5cwCPIQQfmg/qH6mBiClPAr8iXaiez3a0XA42g/9RiHEabQfltQ6LyT4F/BEqxElCARO69vzBfCPvi1/SSl3SSmDUykPAr5D+7HarccEWrJeojd3nARmSjN2T5ZSHkNrouqEdi5gir79JYAf9cm+Ao6i/WieTjT7UGC4Pr1rovJOwFkhxL9o575+JWO6oO0Tif2Bdg7tGNrf7iQwX6/ZgNbx4lMhxHn9dZJ1SynPAWOBHXrcmwB3tAOPw/p+OZbnnC9MzXP2q8TTGP37629/BmwH9gLXTVhlRr8DK9C2Nz3JaBDwkhDitL6+xIljOZCXZ817kPo+k9xQjO9DSjqogVKzMSFEPinlQyGEM1oyeTOVJgvFyiTuaZasvBSpXIeWVXLifqX3Dm0kpfzQ0rHkVKprd/b2sxCiIlpb9qz/+g+GkmVy1H4lhJiN1inEovfzyelUzUhRFEWxOHXOSFEURbE4lYwURVEUi1PJSFEURbE41YFByRGEEPFow8jYAHHAMJlo5OcMLvMq8KqU0l9oo7J/kOiCSWPTDwV+lVLez8C64gEH/aJmRfnPUTUjJSeprY8z9jWwUr8Y1+BFRoWQ2qjsqSYi3VC064EURUlG1YyUnOgvtNGyCwghVqNd+NsA8BdC9AK+RxujLRfaCAGD9RGdq6CN5J1bLzeMR5isllQFmKqvA7QLXKsBRYENQoinaCMuPAKmoY14kRtYLZ+NyN4YbfikWFJeNKso/zmqZqTkRJ2AG1LKO/prL7QRsN9FG64oUEpZF23khTxAb326xcAEfUTs7WgjPCQhtNtvrAN+1GthNYBDUsrvgSCgtT4uWhAwBdigr6sm0EgI0Vxo95ZaCvTUlxGefD2K8l+jakZKTnJMCAHafXraJCpfmmgE5reAfEK7XxBoyeiePk6bj9TveiulXCuECDWyDgE80ccGRF/ug1TieQuoJYT4Un+dDyiPNhL5fX1oHtBGlZ6cri1VlGxGJSMlJ6mdSgeAh4me26DdzO5g4gn0ZGRuNsAb+lhviddVjfSPZK0o2ZpqplOUpDYCn+gjNSOEKCCEKK2PFO0vhEgYibwtxgfFlICDEKKlPp2tEMJNfy8c7Y69idc1MqEjhRDCS7/R4QXAXQiRcAv4nubcQEWxRioZKUpSE4DLwHF9FObtQDH9ve7A5/ro469jZFRqqd0evB0wQp/uBNrN1wBmAsv1Ed+LAkPQRoo+pU+7Cu32H0/Q7ji6WAhxCsiMWpmiWBU1Np2iKIpicapmpCiKolicSkaKoiiKxalkpCiKolicSkaKoiiKxalkpCiKolicSkaKoiiKxalkpCiKolicSkaKoiiKxf0fPB/BcAHDoKoAAAAASUVORK5CYII=\n", 306 | "text/plain": [ 307 | "
" 308 | ] 309 | }, 310 | "metadata": { 311 | "needs_background": "light" 312 | }, 313 | "output_type": "display_data" 314 | } 315 | ], 316 | "source": [ 317 | "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", 318 | "datasetNames = [\"Trento\"]\n", 319 | "data2Name = 'LIDAR'\n", 320 | "\n", 321 | "patchsize = 11\n", 322 | "batchsize = 64\n", 323 | "testSizeNumber = 500\n", 324 | "EPOCH = 100\n", 325 | "BandSize = 1\n", 326 | "LR = 5e-4\n", 327 | "FM = 16\n", 328 | "HSIOnly = True\n", 329 | "FileName = 'MFT_HSI'\n", 330 | "ntokens = 4\n", 331 | "token_type = 'channel' \n", 332 | "num_heads = 8\n", 333 | "mlp_dim = 512\n", 334 | "depth = 2\n", 335 | "train_loss = []\n", 336 | "\n", 337 | "def set_seed(seed):\n", 338 | " torch.manual_seed(seed)\n", 339 | " torch.cuda.manual_seed_all(seed)\n", 340 | " np.random.seed(seed)\n", 341 | "\n", 342 | "for BandSize in [1]:\n", 343 | " for datasetName in datasetNames:\n", 344 | " print(\"---------------------------------- Dataset details for \",datasetName,\" ---------------------------------------------\")\n", 345 | " print('\\n')\n", 346 | " try:\n", 347 | " os.makedirs(datasetName)\n", 348 | " except FileExistsError:\n", 349 | " pass\n", 350 | " \n", 351 | " train_dataset = Multimodal_Dataset_Train(Filename=datasetName, MM_Data=data2Name)\n", 352 | " test_dataset = Multimodal_Dataset_Test(Filename=datasetName, MM_Data=data2Name)\n", 353 | " NC = train_dataset.hs_ims.shape[1]\n", 354 | " NCLidar = train_dataset.lid_ims.shape[1]\n", 355 | " Classes = len(torch.unique(train_dataset.lbs))\n", 356 | "\n", 357 | " train_loader = dataf.DataLoader(train_dataset, batch_size=batchsize, shuffle=True, num_workers= 4)\n", 358 | " print(\"HSI Train data shape = \", train_dataset.hs_ims.shape)\n", 359 | " print(data2Name + \" Train data shape = \", train_dataset.lid_ims.shape[1])\n", 360 | " print(\"Train label shape = \", train_dataset.lbs.shape)\n", 361 | "\n", 362 | " print(\"HSI Test data shape = \", test_dataset.hs_ims.shape)\n", 363 | " print(data2Name + \" Test data shape = \", test_dataset.lid_ims.shape[1])\n", 364 | " print(\"Test label shape = \", test_dataset.lbs.shape)\n", 365 | "\n", 366 | " print(\"Number of Classes = \", Classes)\n", 367 | " \n", 368 | " TestPatch1 = test_dataset.hs_ims\n", 369 | " TestPatch2 = test_dataset.lid_ims\n", 370 | " TestLabel1 = test_dataset.lbs\n", 371 | " \n", 372 | " KAPPA = []\n", 373 | " OA = []\n", 374 | " AA = []\n", 375 | " ELEMENT_ACC = np.zeros((3, Classes))\n", 376 | "\n", 377 | " set_seed(42)\n", 378 | " for iterNum in range(1):\n", 379 | " print('\\n')\n", 380 | " print(\"---------------------------------- Model Summary ---------------------------------------------\")\n", 381 | " print('\\n')\n", 382 | " if HSIOnly: \n", 383 | " model = Transformer(FM=FM, NC=NC, Classes=Classes, ntokens=ntokens, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n", 384 | " summary(model, [(NC, patchsize**2)])\n", 385 | " \n", 386 | " else:\n", 387 | " model = MFT(FM=FM, NC=NC, NCLidar=NCLidar, Classes=Classes, ntokens=ntokens, token_type=token_type, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n", 388 | " summary(model, [(NC, patchsize**2),(NCLidar,patchsize**2)]) \n", 389 | " \n", 390 | " optimizer = torch.optim.Adam(model.parameters(), lr=LR,weight_decay=5e-3)\n", 391 | " loss_func = nn.CrossEntropyLoss() # the target label is not one-hotted\n", 392 | " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.9)\n", 393 | " BestAcc = 0\n", 394 | "\n", 395 | " torch.cuda.synchronize()\n", 396 | " print('\\n')\n", 397 | " print(\"---------------------------------- Training started for \",datasetName,\" ---------------------------------------------\")\n", 398 | " print('\\n')\n", 399 | " start = time.time()\n", 400 | " # train and test the designed model\n", 401 | " for epoch in range(EPOCH):\n", 402 | " for step, (b_x1, b_x2, b_y) in enumerate(train_loader):\n", 403 | "\n", 404 | " # move train data to GPU\n", 405 | " b_x1 = b_x1.cuda()\n", 406 | " b_y = b_y.cuda()\n", 407 | " if HSIOnly:\n", 408 | " out1 = model(b_x1)\n", 409 | " loss = loss_func(out1, b_y)\n", 410 | " else:\n", 411 | " b_x2 = b_x2.cuda()\n", 412 | " out= model(b_x1, b_x2)\n", 413 | " loss = loss_func(out, b_y)\n", 414 | "\n", 415 | " optimizer.zero_grad() # clear gradients for this training step\n", 416 | " loss.backward() # backpropagation, compute gradients\n", 417 | " optimizer.step() # apply gradients\n", 418 | "\n", 419 | " if step % 50 == 0:\n", 420 | " model.eval()\n", 421 | " pred_y = np.empty((len(TestLabel1)), dtype='float32')\n", 422 | " number = len(TestLabel1) // testSizeNumber\n", 423 | " for i in range(number):\n", 424 | " temp = TestPatch1[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 425 | " temp = temp.cuda()\n", 426 | " temp1 = TestPatch2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n", 427 | " temp1 = temp1.cuda()\n", 428 | " if HSIOnly:\n", 429 | " temp2 = model(temp)\n", 430 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 431 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 432 | " del temp, temp2, temp3\n", 433 | " else:\n", 434 | " temp2 = model(temp, temp1)\n", 435 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 436 | " pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n", 437 | " del temp, temp1, temp2, temp3\n", 438 | "\n", 439 | " if (i + 1) * testSizeNumber < len(TestLabel1):\n", 440 | " temp = TestPatch1[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n", 441 | " temp = temp.cuda()\n", 442 | " temp1 = TestPatch2[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n", 443 | " temp1 = temp1.cuda()\n", 444 | " if HSIOnly:\n", 445 | " temp2 = model(temp)\n", 446 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 447 | " pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n", 448 | " del temp, temp2, temp3\n", 449 | " else:\n", 450 | " temp2 = model(temp, temp1)\n", 451 | " temp3 = torch.max(temp2, 1)[1].squeeze()\n", 452 | " pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n", 453 | " del temp, temp1, temp2, temp3\n", 454 | "\n", 455 | " pred_y = torch.from_numpy(pred_y).long()\n", 456 | " accuracy = torch.sum(pred_y == TestLabel1).type(torch.FloatTensor) / TestLabel1.size(0)\n", 457 | "\n", 458 | " print('Epoch: ', epoch, '| train loss: %.4f' % loss.data.cpu().numpy(), '| test accuracy: %.4f' % (accuracy*100))\n", 459 | " train_loss.append(loss.data.cpu().numpy())\n", 460 | " # save the parameters in network\n", 461 | " if accuracy > BestAcc:\n", 462 | "\n", 463 | " BestAcc = accuracy\n", 464 | " \n", 465 | " torch.save(model.state_dict(), datasetName+'/net_params_'+FileName+'.pkl')\n", 466 | " \n", 467 | "\n", 468 | " model.train()\n", 469 | " scheduler.step()\n", 470 | " torch.cuda.synchronize()\n", 471 | " end = time.time()\n", 472 | " print('\\nThe train time (in seconds) is:', end - start)\n", 473 | " Train_time = end - start\n", 474 | "\n", 475 | " # load the saved parameters\n", 476 | " \n", 477 | " model.load_state_dict(torch.load(datasetName+'/net_params_'+FileName+'.pkl'))\n", 478 | "\n", 479 | " model.eval()\n", 480 | " \n", 481 | " confusion, oa, each_acc, aa, kappa = reports(TestPatch1,TestPatch2,TestLabel1,datasetName,model, HSIOnly, iterNum)\n", 482 | " KAPPA.append(kappa)\n", 483 | " OA.append(oa)\n", 484 | " AA.append(aa)\n", 485 | " ELEMENT_ACC[iterNum, :] = each_acc\n", 486 | " torch.save(model, datasetName+'/best_model_'+FileName+'_BandSize'+str(BandSize)+'_Iter'+str(iterNum)+'.pt')\n", 487 | " print('\\n')\n", 488 | " print(\"Overall Accuracy = \", oa)\n", 489 | " print('\\n')\n", 490 | " print(\"----------\" + datasetName + \" Training Finished -----------\")\n", 491 | " print(\"\\nThe Confusion Matrix on test data\")\n", 492 | " record.record_output(OA, AA, KAPPA, ELEMENT_ACC,'./' + datasetName +'/'+FileName+'_BandSize'+str(BandSize)+'_Report_' + datasetName +'.txt')\n", 493 | "\n" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": 4, 499 | "id": "8d7a95c2", 500 | "metadata": {}, 501 | "outputs": [], 502 | "source": [ 503 | "train_loss = np.asarray(train_loss)\n", 504 | "np.save('HSIOnly_train_loss.npy', train_loss)" 505 | ] 506 | }, 507 | { 508 | "cell_type": "code", 509 | "execution_count": null, 510 | "id": "0e19924b", 511 | "metadata": {}, 512 | "outputs": [], 513 | "source": [] 514 | } 515 | ], 516 | "metadata": { 517 | "kernelspec": { 518 | "display_name": "Python 3", 519 | "language": "python", 520 | "name": "python3" 521 | }, 522 | "language_info": { 523 | "codemirror_mode": { 524 | "name": "ipython", 525 | "version": 3 526 | }, 527 | "file_extension": ".py", 528 | "mimetype": "text/x-python", 529 | "name": "python", 530 | "nbconvert_exporter": "python", 531 | "pygments_lexer": "ipython3", 532 | "version": "3.8.10" 533 | }, 534 | "vscode": { 535 | "interpreter": { 536 | "hash": "26407125db06bdd9abe40e82cf041582bb19887fa16dd38638e528b7039723e8" 537 | } 538 | } 539 | }, 540 | "nbformat": 4, 541 | "nbformat_minor": 5 542 | } 543 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Transformer-Models-for-Multimodal-Remote-Sensing-Data 2 | 3 | For any queries, please contact at srinadhml99@gmail.com 4 | 5 | # Description 6 | In this work, I studied the performance of a new Multimodal Fusion Transformer (MFT) for Remote Sensing (RS) Image Classification proposed recently. The link for the original MFT paper is here https://arxiv.org/pdf/2203.16952.pdf. 7 | 8 | Transformer-based models are widely used in several image-processing applications due to their promising performance over Convolutional Neural Networks (CNNs). However, there are certain challenges while adapting transformer models for the Hyperspectral Image (HSI) classification tasks. In RS image classification, using images from multiple sources and exploiting complementary information is getting more attention with the increased availability of multi-sensor data. However, considering the patch-level processing in the transformer models and the number of spectral bands in the HSI data or the fused data (HSI+other modalities), the number of model parameters is becoming a major issue. A new transformer-based architecture with novel attention and tokenization mechanisms is proposed in the MFT work to obtain complementary information from other modalities through an external CLS token. They showed that the CLS token extracted from other multimodal data generalises well compared to the random CLS token used in general transformer models. 9 | 10 | In this work, I specifically studied the performance of the MFT model with varying the CLS token. Specifically, I used the CLS token as 'random', processed through 'channel' and 'pixel' tokenization methods. I considered the 'Trneto' dataset to validate the performance of the MFT model. 11 | The performance is studied using Overall Accuracy (OA), Average Accuracy (AA) and KAPPA Scores. The training loss and confusion matrix are also studied. 12 | 13 | # Code updates 14 | I have made some changes to the original codes. 15 | 16 | 1. Two transformer models named 'MFT' and 'Transformer' are written to train or test the model performance with (HSI + other multimodal data) and without (HSI only) multimodal data, respectively. 17 | 2. Transformer parameters such as 'number of HSI tokens', 'heads', 'depth', and 'mlp_dimension' are easily tuned. 18 | 3. I have also updated the codes with the 'token_type' to call 'channel' or 'pixel' instead of writing two separate codes like in the original work. 19 | 20 | # Results 21 | Using Trento data 22 | CLS Token | Overall Accuracy (OA) | Average Accuracy (AA) | KAPPA Score | Number of Parameters 23 | --- | --- | --- | --- | --- 24 | Random (HSI) | 95.45 | 92.85 | 93.91 | 262758 25 | Channel (HSI+LiDAR) | **98.05** | **96.96** | **97.38** | 263526 26 | Pixel (HSI+LiDAR) | 95.47 | 91.28 | 93.93 | 263526 27 | 28 | #### The confusion matrix for CLS random, channel, pixel tokenizations and the train loss plots are here (from top to bottom). 29 | 30 | ![plot](./MFT_Plots/HSI_Confusionmatrix.png) 31 | 32 | ![plot](./MFT_Plots/HSILidar_channel_Confusionmatrix.png) 33 | 34 | ![plot](./MFT_Plots/HSILidar_pixel_Confusionmatrix.png) 35 | 36 | ![plot](./MFT_Plots/epoch_vs_train_loss.png) 37 | 38 | # Observations 39 | The MFT model is studied with only HSI images and with both HSI and LiDAR images for the land-cover classification task using Trento data. The metrics indicate that the MFT model with 'channel' tokenization performs better than the 'random' CLS and 'pixel' tokenized CLS. Moreover, the gain in performance is achieved with a few additional trainable parameters. 40 | 41 | # Our Next Contributions 42 | Even though the MFT model is doing better in fusing multimodal data and obtaining complimentary information, there are several limitations concerning speed, etc. 43 | 1. Please check our latest work https://github.com/srinadh99/VISION-TRANSFORMER-DRIVEN-LIDAR-DATA-FUSION-FOR-ENHANCED-HYPERSPECTRAL-IMAGE-CLASSIFICATION 44 | 45 | # Acknowledgment 46 | We considered part of our codes from the following source. 47 | 1. https://github.com/AnkurDeria/MFT (The Trento dataset also can be downloaded from here) 48 | -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import numpy as np 4 | from torch.utils.data import Dataset, DataLoader 5 | import torch 6 | import torch.nn as nn 7 | from scipy import io 8 | 9 | class Multimodal_Dataset_Train(Dataset): 10 | def __init__(self, Filename='Trento', MM_Data='LIDAR'): 11 | 12 | HSI = io.loadmat('./'+str(Filename)+'11x11/HSI_Tr.mat') 13 | LIDAR = io.loadmat('./'+str(Filename)+'11x11/'+str(MM_Data)+'_Tr.mat') 14 | label = io.loadmat('./'+str(Filename)+'11x11/TrLabel.mat') 15 | 16 | #self.hs_ims = torch.from_numpy(HSI['Data'].astype(np.float32)).permute(0,3,1,2) 17 | self.hs_ims = (torch.from_numpy(HSI['Data'].astype(np.float32)).to(torch.float32)).permute(0,3,1,2) 18 | self.lid_ims = (torch.from_numpy(LIDAR['Data'].astype(np.float32)).to(torch.float32)).permute(0,3,1,2) 19 | self.lbs = ((torch.from_numpy(label['Data'])-1).long()).reshape(-1) 20 | 21 | def __len__(self): 22 | return self.hs_ims.shape[0] 23 | 24 | def __getitem__(self, i): 25 | return self.hs_ims[i], self.lid_ims[i], self.lbs[i] 26 | 27 | class Multimodal_Dataset_Test(Dataset): 28 | def __init__(self, Filename='Trento', MM_Data='LIDAR'): 29 | 30 | HSI = io.loadmat('./'+str(Filename)+'11x11/HSI_Te.mat') 31 | LIDAR = io.loadmat('./'+str(Filename)+'11x11/'+str(MM_Data)+'_Te.mat') 32 | label = io.loadmat('./'+str(Filename)+'11x11/TeLabel.mat') 33 | 34 | self.hs_ims = (torch.from_numpy(HSI['Data'].astype(np.float32)).to(torch.float32)).permute(0,3,1,2) 35 | self.lid_ims = (torch.from_numpy(LIDAR['Data'].astype(np.float32)).to(torch.float32)).permute(0,3,1,2) 36 | self.lbs = ((torch.from_numpy(label['Data'])-1).long()).reshape(-1) 37 | 38 | def __len__(self): 39 | return self.hs_ims.shape[0] 40 | 41 | def __getitem__(self, i): 42 | return self.hs_ims[i], self.lid_ims[i], self.lbs[i] 43 | 44 | -------------------------------------------------------------------------------- /mft_model.py: -------------------------------------------------------------------------------- 1 | # Multimodal Fusion Transformer (MFT) PyTorch Implementation (modified, original code is from https://github.com/AnkurDeria/MFT) 2 | # This is an updated version of MFT PyTorch code for both serial and distributed training 3 | # Link for the original paper is: https://arxiv.org/abs/2203.16952 4 | 5 | # All the changes are commented and are mainly to facilitate hypaer parameters to be passed through the main function 6 | # Added both 'Channel' and 'Pixel' tokenization for the other multimodal data (like the LiDAR data stream) in the same code so that we can call it using the parameter 'LiDAR_token_type' from the main function 7 | 8 | # Import all the desired packages 9 | 10 | from torch.nn import LayerNorm, Linear, Dropout, Softmax 11 | from einops import rearrange, repeat 12 | import copy 13 | from torchsummary import summary 14 | import math 15 | import time 16 | import torchvision.transforms.functional as TF 17 | from torch.nn.parameter import Parameter 18 | import torch.utils.data as dataf 19 | import torch.nn as nn 20 | import torch 21 | import torch.nn.functional as F 22 | from torch import einsum 23 | import random 24 | import numpy as np 25 | import os 26 | import torch.backends.cudnn as cudnn 27 | cudnn.deterministic = True 28 | cudnn.benchmark = False 29 | cudnn.enabled = False 30 | 31 | #random_seed = 42 32 | #random.seed(random_seed) 33 | #torch.manual_seed(random_seed) 34 | #torch.cuda.manual_seed_all(random_seed) 35 | 36 | # HetConv layer for the HSI data processing 37 | class HetConv(nn.Module): 38 | def __init__(self, in_channels, out_channels, p = 64, g = 64): 39 | super().__init__() 40 | # Groupwise Convolution 41 | self.gwc = nn.Conv2d(in_channels, out_channels, kernel_size=3,groups=g,padding = 1) 42 | # Pointwise Convolution 43 | self.pwc = nn.Conv2d(in_channels, out_channels, kernel_size=1,groups=p) 44 | def forward(self, x): 45 | return self.gwc(x) + self.pwc(x) 46 | 47 | # Attention Module in the Tramsformer Encoder 48 | class MCrossAttention(nn.Module): 49 | def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0.1, proj_drop=0.1): 50 | super().__init__() 51 | self.num_heads = num_heads 52 | head_dim = dim // num_heads 53 | self.scale = qk_scale or head_dim ** -0.5 54 | 55 | self.wq = nn.Linear(head_dim, dim , bias=qkv_bias) 56 | self.wk = nn.Linear(head_dim, dim , bias=qkv_bias) 57 | self.wv = nn.Linear(head_dim, dim , bias=qkv_bias) 58 | # self.attn_drop = nn.Dropout(attn_drop) 59 | self.proj = nn.Linear(dim * num_heads, dim) 60 | self.proj_drop = nn.Dropout(proj_drop) 61 | 62 | def forward(self, x): 63 | 64 | B, N, C = x.shape 65 | q = self.wq(x[:, 0:1, ...].reshape(B, 1, self.num_heads, C // self.num_heads)).permute(0, 2, 1, 3) # B1C -> B1H(C/H) -> BH1(C/H) 66 | k = self.wk(x.reshape(B, N, self.num_heads, C // self.num_heads)).permute(0, 2, 1, 3) # BNC -> BNH(C/H) -> BHN(C/H) 67 | v = self.wv(x.reshape(B, N, self.num_heads, C // self.num_heads)).permute(0, 2, 1, 3) # BNC -> BNH(C/H) -> BHN(C/H) 68 | attn = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale 69 | # attn = (q @ k.transpose(-2, -1)) * self.scale # BH1(C/H) @ BH(C/H)N -> BH1N 70 | attn = attn.softmax(dim=-1) 71 | # attn = self.attn_drop(attn) 72 | x = torch.einsum('bhij,bhjd->bhid', attn, v).transpose(1, 2) 73 | # x = (attn @ v).transpose(1, 2) 74 | x = x.reshape(B, 1, C * self.num_heads) # (BH1N @ BHN(C/H)) -> BH1(C/H) -> B1H(C/H) -> B1C 75 | x = self.proj(x) 76 | x = self.proj_drop(x) 77 | return x 78 | 79 | # MLP Module in the Tramsformer Encoder 80 | class Mlp(nn.Module): 81 | def __init__(self, dim, mlp_dim): 82 | super().__init__() 83 | self.fc1 = Linear(dim, mlp_dim) 84 | self.fc2 = Linear(mlp_dim, dim) 85 | self.act_fn = nn.GELU() 86 | self.dropout = Dropout(0.1) 87 | 88 | self._init_weights() 89 | 90 | def _init_weights(self): 91 | 92 | nn.init.xavier_uniform_(self.fc1.weight) 93 | nn.init.xavier_uniform_(self.fc2.weight) 94 | 95 | nn.init.normal_(self.fc1.bias, std=1e-6) 96 | nn.init.normal_(self.fc2.bias, std=1e-6) 97 | 98 | def forward(self, x): 99 | x = self.fc1(x) 100 | x = self.act_fn(x) 101 | x = self.dropout(x) 102 | x = self.fc2(x) 103 | x = self.dropout(x) 104 | return x 105 | 106 | # Single Tramsformer Encoder Block that combines the Attention and Mlp layers 107 | class Block(nn.Module): 108 | def __init__(self, dim, num_heads, mlp_dim): 109 | super().__init__() 110 | self.hidden_size = dim 111 | self.hidden_dim_size = mlp_dim 112 | self.attention_norm = LayerNorm(dim, eps=1e-6) 113 | self.ffn_norm = LayerNorm(dim, eps=1e-6) 114 | self.ffn = Mlp(dim, mlp_dim) 115 | self.attn = MCrossAttention(dim, num_heads) 116 | def forward(self, x): 117 | h = x 118 | x = self.attention_norm(x) 119 | x= self.attn(x) 120 | x = x + h 121 | 122 | h = x 123 | x = self.ffn_norm(x) 124 | x = self.ffn(x) 125 | x = x + h 126 | 127 | return x 128 | 129 | # Transformer Encoder Block with repetition 130 | class TransformerEncoder(nn.Module): 131 | 132 | def __init__(self, dim, num_heads=8, mlp_dim=512, depth=2): 133 | super().__init__() 134 | self.layer = nn.ModuleList() 135 | self.encoder_norm = LayerNorm(dim, eps=1e-6) 136 | for _ in range(depth): 137 | layer = Block(dim, num_heads, mlp_dim) 138 | self.layer.append(copy.deepcopy(layer)) 139 | 140 | def forward(self, x): 141 | for layer_block in self.layer: 142 | x= layer_block(x) 143 | 144 | encoded = self.encoder_norm(x) 145 | 146 | return encoded[:,0] 147 | 148 | # The Final MFT Implementation with cls from other modalities 149 | class MFT(nn.Module): 150 | def __init__(self, FM, NC, NCLidar, Classes, ntokens, token_type, num_heads, mlp_dim, depth): 151 | super().__init__() 152 | #self.HSIOnly = HSIOnly 153 | self.ntokens = ntokens 154 | self.FM = FM 155 | 156 | self.conv5 = nn.Sequential( 157 | nn.Conv3d(1, 8, (9, 3, 3), padding=(0,1,1), stride = 1), 158 | nn.BatchNorm3d(8), 159 | #nn.GroupNorm(4,8), 160 | nn.ReLU() 161 | ) 162 | 163 | self.conv6 = nn.Sequential( 164 | HetConv(8 * (NC - 8), FM*4, 165 | p = 1, 166 | g = (FM*4)//4 if (8 * (NC - 8))%FM == 0 else (FM*4)//8, 167 | ), 168 | nn.BatchNorm2d(FM*4), 169 | #nn.GroupNorm(4,FM*4), 170 | nn.ReLU() 171 | ) 172 | 173 | self.lidarConv = nn.Sequential( 174 | nn.Conv2d(NCLidar,FM*4,3,1,1), 175 | nn.BatchNorm2d(FM*4), 176 | nn.GELU() 177 | ) 178 | self.ca = TransformerEncoder(FM*4, num_heads, mlp_dim, depth) 179 | self.out3 = nn.Linear(FM*4 , Classes) 180 | #self.cls_token = nn.Parameter(torch.zeros(1, 1, FM*4)) 181 | 182 | self.position_embeddings = nn.Parameter(torch.randn(1, ntokens + 1, FM*4)) 183 | self.dropout = nn.Dropout(0.1) 184 | 185 | torch.nn.init.xavier_uniform_(self.out3.weight) 186 | 187 | torch.nn.init.normal_(self.out3.bias, std=1e-6) 188 | self.token_wA = nn.Parameter(torch.empty(1, ntokens, FM*4), 189 | requires_grad=True) # Tokenization parameters 190 | 191 | torch.nn.init.xavier_normal_(self.token_wA) 192 | self.token_wV = nn.Parameter(torch.empty(1, FM*4, FM*4), 193 | requires_grad=True) # Tokenization parameters 194 | torch.nn.init.xavier_normal_(self.token_wV) 195 | 196 | if token_type == "pixel": 197 | 198 | self.token_wA_L = nn.Parameter(torch.empty(1, 1, 1), 199 | requires_grad=True) # Tokenization parameters 200 | 201 | torch.nn.init.xavier_normal_(self.token_wA_L) 202 | self.token_wV_L = nn.Parameter(torch.empty(1, 1, FM*4), 203 | requires_grad=True) # Tokenization parameters 204 | 205 | torch.nn.init.xavier_normal_(self.token_wV_L) 206 | 207 | elif token_type == "channel": 208 | 209 | self.token_wA_L = nn.Parameter(torch.empty(1, 1, FM*4), 210 | requires_grad=True) # Tokenization parameters 211 | 212 | torch.nn.init.xavier_normal_(self.token_wA_L) 213 | self.token_wV_L = nn.Parameter(torch.empty(1, FM*4, FM*4), 214 | requires_grad=True) # Tokenization parameters 215 | 216 | torch.nn.init.xavier_normal_(self.token_wV_L) 217 | 218 | else: 219 | raise ValueError( 220 | print("unknown Lidar_token_type {token_type}, acceptable pixel, channel") 221 | ) 222 | 223 | def forward(self, x1, x2): 224 | x1 = x1.reshape(x1.shape[0],-1,11,11) 225 | x1 = x1.unsqueeze(1) 226 | x1 = self.conv5(x1) 227 | x1 = x1.reshape(x1.shape[0],-1,11,11) 228 | x1 = self.conv6(x1) 229 | 230 | x1 = x1.flatten(2) 231 | x1 = x1.transpose(-1, -2) 232 | wa = self.token_wA.expand(x1.shape[0],-1,-1) 233 | wa = rearrange(wa, 'b h w -> b w h') # Transpose 234 | A = torch.einsum('bij,bjk->bik', x1, wa) 235 | A = rearrange(A, 'b h w -> b w h') # Transpose 236 | A = A.softmax(dim=-1) 237 | wv = self.token_wV.expand(x1.shape[0],-1,-1) 238 | VV = torch.einsum('bij,bjk->bik', x1, wv) 239 | T = torch.einsum('bij,bjk->bik', A, VV) 240 | 241 | x2 = x2.reshape(x2.shape[0],-1,11,11) 242 | x2 = self.lidarConv(x2) 243 | x2 = x2.reshape(x2.shape[0],-1,11**2) 244 | x2 = x2.transpose(-1, -2) 245 | 246 | wa_L = self.token_wA_L.expand(x2.shape[0],-1,-1) 247 | wa_L = rearrange(wa_L, 'b h w -> b w h') # Transpose 248 | A_L = torch.einsum('bij,bjk->bik', x2, wa_L) 249 | A_L = rearrange(A_L, 'b h w -> b w h') # Transpose 250 | A_L = A_L.softmax(dim=-1) 251 | wv_L = self.token_wV_L.expand(x2.shape[0],-1,-1) 252 | VV_L = torch.einsum('bij,bjk->bik', x2, wv_L) 253 | L = torch.einsum('bij,bjk->bik', A_L, VV_L) 254 | 255 | x = torch.cat((L, T), dim = 1) #[b,n+1,dim] 256 | x = x + self.position_embeddings 257 | x = self.dropout(x) 258 | x = self.ca(x) 259 | x = x.reshape(x.shape[0],-1) 260 | out3 = self.out3(x) 261 | return out3 262 | 263 | # The Final MFT Implementation with cls from random 264 | class Transformer(nn.Module): 265 | def __init__(self, FM, NC, Classes, ntokens, num_heads, mlp_dim, depth): 266 | super().__init__() 267 | #self.HSIOnly = HSIOnly 268 | self.ntokens = ntokens 269 | self.FM = FM 270 | 271 | self.conv5 = nn.Sequential( 272 | nn.Conv3d(1, 8, (9, 3, 3), padding=(0,1,1), stride = 1), 273 | #nn.BatchNorm3d(8), 274 | nn.GroupNorm(4,8), 275 | nn.ReLU() 276 | ) 277 | 278 | self.conv6 = nn.Sequential( 279 | HetConv(8 * (NC - 8), FM*4, 280 | p = 1, 281 | g = (FM*4)//4 if (8 * (NC - 8))%FM == 0 else (FM*4)//8, 282 | ), 283 | #nn.BatchNorm2d(FM*4), 284 | nn.GroupNorm(4,FM*4), 285 | nn.ReLU() 286 | ) 287 | 288 | self.last_BandSize = NC//2//2//2 289 | 290 | self.ca = TransformerEncoder(FM*4, num_heads, mlp_dim, depth) 291 | self.out3 = nn.Linear(FM*4 , Classes) 292 | self.cls_token = nn.Parameter(torch.zeros(1, 1, FM*4)) 293 | 294 | 295 | self.position_embeddings = nn.Parameter(torch.randn(1, ntokens + 1, FM*4)) 296 | self.dropout = nn.Dropout(0.1) 297 | 298 | torch.nn.init.xavier_uniform_(self.out3.weight) 299 | 300 | torch.nn.init.normal_(self.out3.bias, std=1e-6) 301 | self.token_wA = nn.Parameter(torch.empty(1, ntokens, FM*4), 302 | requires_grad=True) # Tokenization parameters 303 | 304 | torch.nn.init.xavier_normal_(self.token_wA) 305 | self.token_wV = nn.Parameter(torch.empty(1, FM*4, FM*4), 306 | requires_grad=True) # Tokenization parameters 307 | 308 | torch.nn.init.xavier_normal_(self.token_wV) 309 | 310 | 311 | def forward(self, x1): 312 | x1 = x1.reshape(x1.shape[0],-1,11,11) 313 | x1 = x1.unsqueeze(1) 314 | x1 = self.conv5(x1) 315 | x1 = x1.reshape(x1.shape[0],-1,11,11) 316 | x1 = self.conv6(x1) 317 | 318 | x1 = x1.flatten(2) 319 | x1 = x1.transpose(-1, -2) 320 | wa = self.token_wA.expand(x1.shape[0],-1,-1) 321 | wa = rearrange(wa, 'b h w -> b w h') # Transpose 322 | A = torch.einsum('bij,bjk->bik', x1, wa) 323 | A = rearrange(A, 'b h w -> b w h') # Transpose 324 | A = A.softmax(dim=-1) 325 | wv = self.token_wV.expand(x1.shape[0],-1,-1) 326 | VV = torch.einsum('bij,bjk->bik', x1, wv) 327 | T = torch.einsum('bij,bjk->bik', A, VV) 328 | 329 | L = self.cls_token.repeat(x1.shape[0], 1, 1) 330 | 331 | x = torch.cat((L, T), dim = 1) #[b,n+1,dim] 332 | embeddings = x + self.position_embeddings 333 | embeddings = self.dropout(embeddings) 334 | x = self.ca(embeddings) 335 | x = x.reshape(x.shape[0],-1) 336 | out3 = self.out3(x) 337 | return out3 338 | 339 | -------------------------------------------------------------------------------- /record.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from operator import truediv 4 | 5 | def evaluate_accuracy(data_iter, net, loss, device): 6 | acc_sum, n = 0.0, 0 7 | with torch.no_grad(): 8 | for X, y in data_iter: 9 | test_l_sum, test_num = 0, 0 10 | #X = X.permute(0, 3, 1, 2) 11 | X = X.to(device) 12 | y = y.to(device) 13 | net.eval() 14 | y_hat = net(X) 15 | l = loss(y_hat, y.long()) 16 | acc_sum += (y_hat.argmax(dim=1) == y.to(device)).float().sum().cpu().item() 17 | test_l_sum += l 18 | test_num += 1 19 | net.train() 20 | n += y.shape[0] 21 | return [acc_sum / n, test_l_sum] # / test_num] 22 | 23 | 24 | def aa_and_each_accuracy(confusion_matrix): 25 | list_diag = np.diag(confusion_matrix) 26 | list_raw_sum = np.sum(confusion_matrix, axis=1) 27 | each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum)) 28 | average_acc = np.mean(each_acc) 29 | return each_acc, average_acc 30 | 31 | 32 | 33 | def record_output(oa_ae, aa_ae, kappa_ae, element_acc_ae, path): 34 | f = open(path, 'w') 35 | sentence0 = 'OAs for each iteration are:' + str(oa_ae) + '\n' 36 | f.write(sentence0) 37 | sentence1 = 'AAs for each iteration are:' + str(aa_ae) + '\n' 38 | f.write(sentence1) 39 | sentence2 = 'KAPPAs for each iteration are:' + str(kappa_ae) + '\n' + '\n' 40 | f.write(sentence2) 41 | sentence3 = 'mean_OA ± std_OA is: ' + str(np.mean(oa_ae)) + ' ± ' + str(np.std(oa_ae)) + '\n' 42 | f.write(sentence3) 43 | sentence4 = 'mean_AA ± std_AA is: ' + str(np.mean(aa_ae)) + ' ± ' + str(np.std(aa_ae)) + '\n' 44 | f.write(sentence4) 45 | sentence5 = 'mean_KAPPA ± std_KAPPA is: ' + str(np.mean(kappa_ae)) + ' ± ' + str(np.std(kappa_ae)) + '\n' + '\n' 46 | f.write(sentence5) 47 | 48 | element_mean = np.mean(element_acc_ae, axis=0) 49 | element_std = np.std(element_acc_ae, axis=0) 50 | sentence8 = "Mean of all elements in confusion matrix: " + str(element_mean) + '\n' 51 | f.write(sentence8) 52 | sentence9 = "Standard deviation of all elements in confusion matrix: " + str(element_std) + '\n' + '\n' 53 | f.write(sentence9) 54 | element_mean = list(element_mean) 55 | element_mean.extend([np.mean(oa_ae),np.mean(aa_ae),np.mean(kappa_ae)]) 56 | element_std = list(element_std) 57 | element_std.extend([np.std(oa_ae),np.std(aa_ae),np.std(kappa_ae)]) 58 | sentence10 = "All values without std: " + str(element_mean) + '\n' + '\n' 59 | f.write(sentence10) 60 | sentence11 = "All values with std: " 61 | for i,x in enumerate(element_mean): 62 | sentence11 += str(element_mean[i]) + " ± " + str(element_std[i]) + ", " 63 | sentence11 += "\n" 64 | f.write(sentence11) 65 | f.close() 66 | --------------------------------------------------------------------------------