├── MCNN-CP-Source-code.ipynb ├── Oct-MCNN-PS-Source-Code.ipynb ├── README.md ├── data └── readme.md ├── models └── readme.md ├── pretrained_models └── readme.md ├── requirements.txt ├── subpixel_conv2d.py └── validate.py /MCNN-CP-Source-code.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "scrolled": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import tensorflow as tf\n", 12 | "import keras\n", 13 | "from keras.layers import Conv2D, Conv3D, Flatten, Dense, Reshape, BatchNormalization, Lambda\n", 14 | "from keras.layers import Dropout, Input\n", 15 | "from keras.models import Model\n", 16 | "from keras.optimizers import Adam\n", 17 | "from keras.callbacks import ModelCheckpoint\n", 18 | "from keras.utils import np_utils\n", 19 | "\n", 20 | "from sklearn.decomposition import PCA\n", 21 | "from sklearn.model_selection import train_test_split\n", 22 | "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n", 23 | "\n", 24 | "import time\n", 25 | "\n", 26 | "from plotly.offline import init_notebook_mode\n", 27 | "from subpixel_conv2d import SubpixelConv2D\n", 28 | "import numpy as np\n", 29 | "import matplotlib.pyplot as plt\n", 30 | "import scipy.io as sio\n", 31 | "import os\n", 32 | "import spectral\n", 33 | "\n", 34 | "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n", 35 | "\n", 36 | "init_notebook_mode(connected=True)\n", 37 | "%matplotlib inline" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "# Data" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "## GLOBAL VARIABLES\n", 54 | "dataset = 'IP'\n", 55 | "test_ratio = 0.99\n", 56 | "train_val_ratio = 1\n", 57 | "train_ratio = 1-test_ratio\n", 58 | "windowSize = 11\n", 59 | "if dataset == 'UP':\n", 60 | " componentsNum = 15\n", 61 | "elif dataset == 'UH':\n", 62 | " componentsNum = 50 if test_ratio >= 0.99 else 25\n", 63 | "elif dataset == 'IP':\n", 64 | " componentsNum = 110\n", 65 | "else:\n", 66 | " componentsNum = 30\n", 67 | "drop = 0.4" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "def loadData(name):\n", 77 | " data_path = os.path.join(os.getcwd(),'data')\n", 78 | " if name == 'IP':\n", 79 | " data = sio.loadmat(os.path.join(data_path, 'Indian_pines_corrected.mat'))['indian_pines_corrected']\n", 80 | " labels = sio.loadmat(os.path.join(data_path, 'Indian_pines_gt.mat'))['indian_pines_gt']\n", 81 | " elif name == 'SA':\n", 82 | " data = sio.loadmat(os.path.join(data_path, 'Salinas_corrected.mat'))['salinas_corrected']\n", 83 | " labels = sio.loadmat(os.path.join(data_path, 'Salinas_gt.mat'))['salinas_gt']\n", 84 | " elif name == 'UP':\n", 85 | " data = sio.loadmat(os.path.join(data_path, 'PaviaU.mat'))['paviaU']\n", 86 | " labels = sio.loadmat(os.path.join(data_path, 'PaviaU_gt.mat'))['paviaU_gt']\n", 87 | " elif name == 'UH':\n", 88 | " data = sio.loadmat(os.path.join(data_path, 'HoustonU.mat'))['houstonU'] # 601*2384*50\n", 89 | " labels = sio.loadmat(os.path.join(data_path, 'HoustonU_gt.mat'))['houstonU_gt']\n", 90 | " return data, labels" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "def splitTrainTestSet(X, y, testRatio, randomState=345):\n", 100 | " X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testRatio, random_state=randomState,stratify=y)\n", 101 | " return X_train, X_test, y_train, y_test" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "def applyPCA(X, numComponents=64):\n", 111 | " newX = np.reshape(X, (-1, X.shape[2]))\n", 112 | " print(newX.shape)\n", 113 | " pca = PCA(n_components=numComponents, whiten=True)\n", 114 | " newX = pca.fit_transform(newX)\n", 115 | " newX = np.reshape(newX, (X.shape[0],X.shape[1], numComponents))\n", 116 | " return newX, pca, pca.explained_variance_ratio_" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "def padWithZeros(X, margin=2):\n", 126 | " newX = np.zeros((X.shape[0] + 2 * margin, X.shape[1] + 2* margin, X.shape[2]),dtype=\"float16\")\n", 127 | " x_offset = margin\n", 128 | " y_offset = margin\n", 129 | " newX[x_offset:X.shape[0] + x_offset, y_offset:X.shape[1] + y_offset, :] = X\n", 130 | " return newX" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "def createPatches(X, y, windowSize=25, removeZeroLabels = True):\n", 140 | " margin = int((windowSize - 1) / 2)\n", 141 | " zeroPaddedX = padWithZeros(X, margin=margin)\n", 142 | " # split patches\n", 143 | " patchesData = np.zeros((X.shape[0] * X.shape[1], windowSize, windowSize, X.shape[2]),dtype=\"float16\")\n", 144 | " patchesLabels = np.zeros((X.shape[0] * X.shape[1]),dtype=\"float16\")\n", 145 | " patchIndex = 0\n", 146 | " for r in range(margin, zeroPaddedX.shape[0] - margin):\n", 147 | " for c in range(margin, zeroPaddedX.shape[1] - margin):\n", 148 | " patch = zeroPaddedX[r - margin:r + margin + 1, c - margin:c + margin + 1] \n", 149 | " patchesData[patchIndex, :, :, :] = patch\n", 150 | " patchesLabels[patchIndex] = y[r-margin, c-margin]\n", 151 | " patchIndex = patchIndex + 1\n", 152 | " if removeZeroLabels:\n", 153 | " patchesData = patchesData[patchesLabels>0,:,:,:]\n", 154 | " patchesLabels = patchesLabels[patchesLabels>0]\n", 155 | " patchesLabels -= 1\n", 156 | " return patchesData, patchesLabels" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "def infoChange(X,numComponents):\n", 166 | " X_copy = np.zeros((X.shape[0] , X.shape[1], X.shape[2]))\n", 167 | " half = int(numComponents/2)\n", 168 | " for i in range(0,half-1):\n", 169 | " X_copy[:,:,i] = X[:,:,(half-i)*2-1]\n", 170 | " for i in range(half,numComponents):\n", 171 | " X_copy[:,:,i] = X[:,:,(i-half)*2]\n", 172 | " X = X_copy\n", 173 | " return X" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": null, 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "X, y = loadData(dataset)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "X,pca,ratio = applyPCA(X,numComponents=componentsNum)\n", 192 | "X = infoChange(X,componentsNum) # channel-wise shift" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "X, y = createPatches(X, y, windowSize=windowSize)" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "Xtrain, Xtest, ytrain, ytest = splitTrainTestSet(X, y, test_ratio)" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "# Train" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "Xtrain = Xtrain.reshape(-1, windowSize, windowSize, componentsNum, 1)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "ytrain = np_utils.to_categorical(ytrain)" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "metadata": {}, 242 | "outputs": [], 243 | "source": [ 244 | "Xvalid, Xtest, yvalid, ytest = splitTrainTestSet(Xtest, ytest, (test_ratio-train_ratio/train_val_ratio)/test_ratio)\n", 245 | "Xvalid = Xvalid.reshape(-1, windowSize, windowSize, componentsNum, 1)\n", 246 | "yvalid = np_utils.to_categorical(yvalid)" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "metadata": {}, 253 | "outputs": [], 254 | "source": [ 255 | "if dataset == 'UP':\n", 256 | " output_units = 9\n", 257 | "elif dataset == 'UH':\n", 258 | " output_units = 20\n", 259 | "else:\n", 260 | " output_units = 16" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "## implementation of covariance pooling layers\n", 270 | "def cov_pooling(features):\n", 271 | " shape_f = features.shape.as_list()\n", 272 | " centers_batch = tf.reduce_mean(tf.transpose(features, [0, 2, 1]),2) # 均值\n", 273 | " centers_batch = tf.reshape(centers_batch, [-1, 1, shape_f[2]])\n", 274 | " centers_batch = tf.tile(centers_batch, [1, shape_f[1], 1]) # 张量扩展 \n", 275 | " tmp = tf.subtract(features, centers_batch)\n", 276 | " tmp_t = tf.transpose(tmp, [0, 2, 1]) \n", 277 | " features_t = 1/tf.cast((shape_f[1]-1),tf.float32)*tf.matmul(tmp_t, tmp) \n", 278 | " trace_t = tf.trace(features_t)\n", 279 | " trace_t = tf.reshape(trace_t, [-1, 1])\n", 280 | " trace_t = tf.tile(trace_t, [1, shape_f[2]])\n", 281 | " trace_t = 0.0001*tf.matrix_diag(trace_t)\n", 282 | " return tf.add(features_t,trace_t)" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "def feature_vector(features):\n", 292 | " # features，是对称的，由于张量无法像矩阵一样直接取上三角数据拉成一维向量\n", 293 | " shape_f = features.shape.as_list()\n", 294 | " feature_upper = tf.linalg.band_part(features,0,shape_f[2])\n", 295 | " return feature_upper" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": null, 301 | "metadata": { 302 | "scrolled": false 303 | }, 304 | "outputs": [], 305 | "source": [ 306 | "## input layer\n", 307 | "input_layer = Input((windowSize, windowSize, componentsNum, 1))\n", 308 | "\n", 309 | "## convolutional layers\n", 310 | "conv_layer1 = Conv3D(filters=8, kernel_size=(3, 3, 3), activation='relu', padding='same')(input_layer)\n", 311 | "conv_layer2 = Conv3D(filters=16, kernel_size=(3, 3, 3), activation='relu', padding='same')(conv_layer1)\n", 312 | "conv_layer3 = Conv3D(filters=32, kernel_size=(3, 3, 3), activation='relu', padding='same')(conv_layer2)\n", 313 | "# print(conv_layer3._keras_shape)\n", 314 | "conv3d_shape = conv_layer3._keras_shape\n", 315 | "conv_layer3 = Reshape((conv3d_shape[1], conv3d_shape[2], conv3d_shape[3]*conv3d_shape[4]))(conv_layer3)\n", 316 | "conv_layer4 = Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same')(conv_layer3)\n", 317 | "\n", 318 | "# conv2d_shape = conv_layer4._keras_shape\n", 319 | "# conv_layer4 = SubpixelConv2D(upsampling_factor=8)(conv_layer4)\n", 320 | "\n", 321 | "conv2d_shape = conv_layer4._keras_shape\n", 322 | "conv_layer4 = Reshape((conv2d_shape[1] * conv2d_shape[2], conv2d_shape[3]))(conv_layer4)\n", 323 | "conv2d_shape = conv_layer4._keras_shape\n", 324 | "cov_pooling_layer1 = Lambda(cov_pooling,output_shape=(conv2d_shape[2],conv2d_shape[2]),mask=None,arguments=None)(conv_layer4)\n", 325 | "cov_pooling_layer2 = Lambda(feature_vector,output_shape=(conv2d_shape[2],conv2d_shape[2]),mask=None,arguments=None)(cov_pooling_layer1)\n", 326 | "# (int)(cov_pooling_shape[1]*(cov_pooling_shape[2]+1)/2)为有效参数数量\n", 327 | "\n", 328 | "flatten_layer = Flatten()(cov_pooling_layer2)\n", 329 | "\n", 330 | "## fully connected layers\n", 331 | "dense_layer1 = Dense(units=256, activation='relu')(flatten_layer)\n", 332 | "dense_layer1 = Dropout(0.4)(dense_layer1)\n", 333 | "dense_layer2 = Dense(units=128, activation='relu')(dense_layer1)\n", 334 | "dense_layer2 = Dropout(0.4)(dense_layer2)\n", 335 | "output_layer = Dense(units=output_units, activation='softmax')(dense_layer2)" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [ 344 | "# define the model with input layer and output layer\n", 345 | "model = Model(inputs=input_layer, outputs=output_layer)" 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "execution_count": null, 351 | "metadata": { 352 | "scrolled": false 353 | }, 354 | "outputs": [], 355 | "source": [ 356 | "model.summary()" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "# compiling the model\n", 366 | "adam = Adam(lr=0.001, decay=1e-06)\n", 367 | "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": null, 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [ 376 | "# checkpoint\n", 377 | "filepath = \"best-model.hdf5\"\n", 378 | "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n", 379 | "callbacks_list = [checkpoint]" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": null, 385 | "metadata": { 386 | "scrolled": true 387 | }, 388 | "outputs": [], 389 | "source": [ 390 | "start = time.time()\n", 391 | "history = model.fit(x=Xtrain, y=ytrain, batch_size=256, epochs=100, validation_data=(Xvalid,yvalid), callbacks=callbacks_list) #,validation_split=(1/3)\n", 392 | "end = time.time()\n", 393 | "print((end - start)/60)" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": null, 399 | "metadata": {}, 400 | "outputs": [], 401 | "source": [ 402 | "plt.figure(figsize=(7,7))\n", 403 | "plt.grid()\n", 404 | "plt.plot(history.history['loss'])\n", 405 | "plt.ylabel('Loss')\n", 406 | "plt.xlabel('Epochs')\n", 407 | "plt.legend(['Training','Validation'], loc='upper right')\n", 408 | "plt.savefig(\"loss_curve.pdf\")\n", 409 | "plt.show()" 410 | ] 411 | }, 412 | { 413 | "cell_type": "code", 414 | "execution_count": null, 415 | "metadata": {}, 416 | "outputs": [], 417 | "source": [ 418 | "plt.figure(figsize=(5,5))\n", 419 | "plt.ylim(0,1.1)\n", 420 | "plt.grid()\n", 421 | "plt.plot(history.history['acc'])\n", 422 | "plt.ylabel('Accuracy')\n", 423 | "plt.xlabel('Epochs')\n", 424 | "plt.legend(['Training','Validation'])\n", 425 | "plt.savefig(\"acc_curve.pdf\")\n", 426 | "plt.show()" 427 | ] 428 | }, 429 | { 430 | "cell_type": "markdown", 431 | "metadata": {}, 432 | "source": [ 433 | "# Test" 434 | ] 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": null, 439 | "metadata": {}, 440 | "outputs": [], 441 | "source": [ 442 | "# load best weights\n", 443 | "model.load_weights(\"best-model.hdf5\")\n", 444 | "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" 445 | ] 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": null, 450 | "metadata": {}, 451 | "outputs": [], 452 | "source": [ 453 | "Xtest = Xtest.reshape(-1, windowSize, windowSize, componentsNum, 1)\n", 454 | "# Xtest.shape" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": null, 460 | "metadata": {}, 461 | "outputs": [], 462 | "source": [ 463 | "ytest = np_utils.to_categorical(ytest)\n", 464 | "# ytest.shape" 465 | ] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": null, 470 | "metadata": {}, 471 | "outputs": [], 472 | "source": [ 473 | "Y_pred_test = model.predict(Xtest)\n", 474 | "y_pred_test = np.argmax(Y_pred_test, axis=1)\n", 475 | "\n", 476 | "classification = classification_report(np.argmax(ytest, axis=1), y_pred_test)\n", 477 | "print(classification)" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [ 486 | "def AA_andEachClassAccuracy(confusion_matrix):\n", 487 | " counter = confusion_matrix.shape[0]\n", 488 | " list_diag = np.diag(confusion_matrix)\n", 489 | " list_raw_sum = np.sum(confusion_matrix, axis=1)\n", 490 | " each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n", 491 | " average_acc = np.mean(each_acc)\n", 492 | " return each_acc, average_acc" 493 | ] 494 | }, 495 | { 496 | "cell_type": "code", 497 | "execution_count": null, 498 | "metadata": {}, 499 | "outputs": [], 500 | "source": [ 501 | "def reports (X_test,y_test,name):\n", 502 | " start = time.time()\n", 503 | " Y_pred = model.predict(X_test)\n", 504 | " y_pred = np.argmax(Y_pred, axis=1)\n", 505 | " end = time.time()\n", 506 | " print(end - start)\n", 507 | " if name == 'IP':\n", 508 | " target_names = ['Alfalfa', 'Corn-notill', 'Corn-mintill', 'Corn',\n", 509 | " 'Grass-pasture', 'Grass-trees', 'Grass-pasture-mowed', \n", 510 | " 'Hay-windrowed', 'Oats', 'Soybean-notill', 'Soybean-mintill',\n", 511 | " 'Soybean-clean', 'Wheat', 'Woods', 'Buildings-Grass-Trees-Drives',\n", 512 | " 'Stone-Steel-Towers']\n", 513 | " elif name == 'SA':\n", 514 | " target_names = ['Brocoli_green_weeds_1','Brocoli_green_weeds_2','Fallow','Fallow_rough_plow','Fallow_smooth',\n", 515 | " 'Stubble','Celery','Grapes_untrained','Soil_vinyard_develop','Corn_senesced_green_weeds',\n", 516 | " 'Lettuce_romaine_4wk','Lettuce_romaine_5wk','Lettuce_romaine_6wk','Lettuce_romaine_7wk',\n", 517 | " 'Vinyard_untrained','Vinyard_vertical_trellis']\n", 518 | " elif name == 'UP':\n", 519 | " target_names = ['Asphalt','Meadows','Gravel','Trees', 'Painted metal sheets','Bare Soil','Bitumen',\n", 520 | " 'Self-Blocking Bricks','Shadows']\n", 521 | " elif name == 'UH':\n", 522 | " target_names = ['Healthy grass','Stressed grass','Artificial turf','Evergreen trees', 'Deciduous trees','Bare earth','Water',\n", 523 | " 'Residential buildings','Non-residential buildings','Roads','Sidewalks','Crosswalks','Major thoroughfares','Highways',\n", 524 | " 'Railways','Paved parking lots','Unpaved parking lots','Cars','Trains','Stadium seats']\n", 525 | " \n", 526 | " classification = classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names)\n", 527 | " oa = accuracy_score(np.argmax(y_test, axis=1), y_pred)\n", 528 | " confusion = confusion_matrix(np.argmax(y_test, axis=1), y_pred)\n", 529 | " each_acc, aa = AA_andEachClassAccuracy(confusion)\n", 530 | " kappa = cohen_kappa_score(np.argmax(y_test, axis=1), y_pred)\n", 531 | " score = model.evaluate(X_test, y_test, batch_size=32) \n", 532 | " Test_Loss = score[0]*100\n", 533 | " Test_accuracy = score[1]*100\n", 534 | " \n", 535 | " return classification, confusion, Test_Loss, Test_accuracy, oa*100, each_acc*100, aa*100, kappa*100" 536 | ] 537 | }, 538 | { 539 | "cell_type": "code", 540 | "execution_count": null, 541 | "metadata": {}, 542 | "outputs": [], 543 | "source": [ 544 | "classification, confusion, Test_loss, Test_accuracy, oa, each_acc, aa, kappa = reports(Xtest,ytest,dataset)\n", 545 | "classification = str(classification)\n", 546 | "confusion1 = str(confusion)\n", 547 | "file_name = \"classification_report.txt\"\n", 548 | "\n", 549 | "with open(file_name, 'w') as x_file:\n", 550 | " x_file.write('{} Test loss (%)'.format(Test_loss))\n", 551 | " x_file.write('\\n')\n", 552 | " x_file.write('{} Test accuracy (%)'.format(Test_accuracy))\n", 553 | " x_file.write('\\n')\n", 554 | " x_file.write('\\n')\n", 555 | " x_file.write('{} Kappa accuracy (%)'.format(kappa))\n", 556 | " x_file.write('\\n')\n", 557 | " x_file.write('{} Overall accuracy (%)'.format(oa))\n", 558 | " x_file.write('\\n')\n", 559 | " x_file.write('{} Average accuracy (%)'.format(aa))\n", 560 | " x_file.write('\\n')\n", 561 | " x_file.write('\\n')\n", 562 | " x_file.write('{}'.format(classification))\n", 563 | " x_file.write('\\n')\n", 564 | " x_file.write('{}'.format(confusion1))" 565 | ] 566 | }, 567 | { 568 | "cell_type": "markdown", 569 | "metadata": {}, 570 | "source": [ 571 | "# Run Data" 572 | ] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": null, 577 | "metadata": {}, 578 | "outputs": [], 579 | "source": [ 580 | "def Patch(data,height_index,width_index):\n", 581 | " height_slice = slice(height_index, height_index+PATCH_SIZE)\n", 582 | " width_slice = slice(width_index, width_index+PATCH_SIZE)\n", 583 | " patch = data[height_slice, width_slice, :]\n", 584 | " return patch" 585 | ] 586 | }, 587 | { 588 | "cell_type": "code", 589 | "execution_count": null, 590 | "metadata": {}, 591 | "outputs": [], 592 | "source": [ 593 | "X, y = loadData(dataset)" 594 | ] 595 | }, 596 | { 597 | "cell_type": "code", 598 | "execution_count": null, 599 | "metadata": {}, 600 | "outputs": [], 601 | "source": [ 602 | "height = y.shape[0]\n", 603 | "width = y.shape[1]\n", 604 | "PATCH_SIZE = windowSize" 605 | ] 606 | }, 607 | { 608 | "cell_type": "code", 609 | "execution_count": null, 610 | "metadata": {}, 611 | "outputs": [], 612 | "source": [ 613 | "X,pca,ratio = applyPCA(X,numComponents=componentsNum)\n", 614 | "X = infoChange(X,componentsNum)" 615 | ] 616 | }, 617 | { 618 | "cell_type": "code", 619 | "execution_count": null, 620 | "metadata": {}, 621 | "outputs": [], 622 | "source": [ 623 | "X = padWithZeros(X, PATCH_SIZE//2)" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": null, 629 | "metadata": {}, 630 | "outputs": [], 631 | "source": [ 632 | "# calculate the predicted image\n", 633 | "outputs = np.zeros((height,width),dtype=\"float16\")\n", 634 | "outputs2 = np.zeros((height,width),dtype=\"float16\")\n", 635 | "for i in range(height):\n", 636 | " for j in range(width): \n", 637 | " target = int(y[i,j])\n", 638 | " if target == 0 :\n", 639 | " image_patch=Patch(X,i,j)\n", 640 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 641 | " prediction2 = (model.predict(X_test_image))\n", 642 | " prediction2 = np.argmax(prediction2, axis=1)\n", 643 | " outputs2[i][j] = prediction2+1\n", 644 | " else :\n", 645 | " image_patch=Patch(X,i,j)\n", 646 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 647 | " prediction = (model.predict(X_test_image))\n", 648 | " prediction = np.argmax(prediction, axis=1)\n", 649 | " outputs[i][j] = prediction+1\n", 650 | " outputs2[i][j] = prediction+1" 651 | ] 652 | }, 653 | { 654 | "cell_type": "code", 655 | "execution_count": null, 656 | "metadata": {}, 657 | "outputs": [], 658 | "source": [ 659 | "import spectral\n", 660 | "ground_truth = spectral.imshow(classes = y,figsize =(7,7))\n", 661 | "predict_image = spectral.imshow(classes = outputs.astype(int),figsize =(7,7))\n", 662 | "predict_image2 = spectral.imshow(classes = outputs2.astype(int),figsize =(7,7))\n", 663 | "spectral.save_rgb(\"predictions.png\", outputs.astype(int), colors=spectral.spy_colors)\n", 664 | "spectral.save_rgb(\"predictions2.png\", outputs2.astype(int), colors=spectral.spy_colors)" 665 | ] 666 | }, 667 | { 668 | "cell_type": "code", 669 | "execution_count": null, 670 | "metadata": {}, 671 | "outputs": [], 672 | "source": [] 673 | } 674 | ], 675 | "metadata": { 676 | "kernelspec": { 677 | "display_name": "Python 3", 678 | "language": "python", 679 | "name": "python3" 680 | }, 681 | "language_info": { 682 | "codemirror_mode": { 683 | "name": "ipython", 684 | "version": 3 685 | }, 686 | "file_extension": ".py", 687 | "mimetype": "text/x-python", 688 | "name": "python", 689 | "nbconvert_exporter": "python", 690 | "pygments_lexer": "ipython3", 691 | "version": "3.6.9" 692 | } 693 | }, 694 | "nbformat": 4, 695 | "nbformat_minor": 2 696 | } 697 | -------------------------------------------------------------------------------- /Oct-MCNN-PS-Source-Code.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "scrolled": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import tensorflow as tf\n", 12 | "import keras\n", 13 | "from keras.layers import Conv2D, Conv3D,ReLU, Flatten, Dense, Reshape, BatchNormalization, Lambda, AveragePooling2D, AveragePooling3D, Add, Concatenate\n", 14 | "from keras.layers import add\n", 15 | "import keras.backend as K\n", 16 | "from keras.layers import Dropout, Input\n", 17 | "from keras.models import Model, load_model\n", 18 | "from keras.optimizers import Adam, Adagrad, RMSprop\n", 19 | "from keras.callbacks import ModelCheckpoint\n", 20 | "from keras.utils import np_utils\n", 21 | "\n", 22 | "from sklearn.decomposition import PCA\n", 23 | "from sklearn.model_selection import train_test_split\n", 24 | "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n", 25 | "\n", 26 | "from adabound import AdaBound\n", 27 | "import time\n", 28 | "from operator import truediv\n", 29 | "from plotly.offline import init_notebook_mode\n", 30 | "from subpixel_conv2d import SubpixelConv2D\n", 31 | "\n", 32 | "import numpy as np\n", 33 | "import matplotlib.pyplot as plt\n", 34 | "import scipy.io as sio\n", 35 | "import os\n", 36 | "import spectral\n", 37 | "\n", 38 | "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n", 39 | "\n", 40 | "init_notebook_mode(connected=True)\n", 41 | "%matplotlib inline" 42 | ] 43 | }, 44 | { 45 | "cell_type": "raw", 46 | "metadata": {}, 47 | "source": [ 48 | "print(spectral.spy_colors)" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "# Data" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "## GLOBAL VARIABLES\n", 65 | "dataset = 'IP' # IP和HU在下面控制训练样本量\n", 66 | "test_ratio = 0.995\n", 67 | "train_val_ratio = 1\n", 68 | "train_ratio = 1-test_ratio\n", 69 | "windowSize = 11\n", 70 | "if dataset == 'UP':\n", 71 | " componentsNum = 20\n", 72 | "elif dataset == 'BW':\n", 73 | " componentsNum = 50\n", 74 | "elif dataset == 'HU':\n", 75 | " componentsNum = 50 # if test_ratio >= 0.99 else 25\n", 76 | "elif dataset == 'XZ':\n", 77 | " componentsNum = 20\n", 78 | "elif dataset == 'IP':\n", 79 | " componentsNum = 110\n", 80 | "else:\n", 81 | " componentsNum = 30\n", 82 | "drop = 0.4" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "data_path = os.path.join(os.getcwd(),'data')\n", 90 | "data = sio.loadmat(os.path.join(data_path, 'xuzhou_gt.mat'))# 假设文件名为1.mat\n", 91 | "print(data.keys())\n", 92 | "print(data)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "def loadData(name):\n", 102 | " data_path = os.path.join(os.getcwd(),'data')\n", 103 | " if name == 'IP':\n", 104 | " data = sio.loadmat(os.path.join(data_path, 'Indian_pines_corrected.mat'))['indian_pines_corrected']\n", 105 | " labels = sio.loadmat(os.path.join(data_path, 'Indian_pines_gt.mat'))['indian_pines_gt']\n", 106 | " elif name == 'KSC':\n", 107 | " data = sio.loadmat(os.path.join(data_path, 'KSC.mat'))['KSC']\n", 108 | " labels = sio.loadmat(os.path.join(data_path, 'KSC_gt.mat'))['KSC_gt']\n", 109 | " elif name == 'SA':\n", 110 | " data = sio.loadmat(os.path.join(data_path, 'Salinas_corrected.mat'))['salinas_corrected']\n", 111 | " labels = sio.loadmat(os.path.join(data_path, 'Salinas_gt.mat'))['salinas_gt']\n", 112 | " elif name == 'UP':\n", 113 | " data = sio.loadmat(os.path.join(data_path, 'PaviaU.mat'))['paviaU']\n", 114 | " labels = sio.loadmat(os.path.join(data_path, 'PaviaU_gt.mat'))['paviaU_gt']\n", 115 | " # 349*1905*144\n", 116 | " elif name == 'HU':\n", 117 | " data = sio.loadmat(os.path.join(data_path, 'HoustonU.mat'))['houstonU'] # 601*2384*50\n", 118 | " labels = sio.loadmat(os.path.join(data_path, 'HoustonU_gt.mat'))['houstonU_gt']\n", 119 | " return data, labels" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "def splitTrainTestSet(X, y, testRatio, randomState=345):\n", 129 | " X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testRatio, random_state=randomState, stratify=y)\n", 130 | " return X_train, X_test, y_train, y_test" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "def applyPCA(X, numComponents=64):\n", 140 | " newX = np.reshape(X, (-1, X.shape[2]))\n", 141 | " print(newX.shape)\n", 142 | " pca = PCA(n_components=numComponents, whiten=True)\n", 143 | " newX = pca.fit_transform(newX)\n", 144 | " print(newX.shape)\n", 145 | " # print(pca.explained_variance_ratio_)\n", 146 | " # print(pca.explained_variance_)\n", 147 | " newX = np.reshape(newX, (X.shape[0],X.shape[1], numComponents))\n", 148 | " return newX, pca, pca.explained_variance_ratio_" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": null, 154 | "metadata": {}, 155 | "outputs": [], 156 | "source": [ 157 | "def padWithZeros(X, margin=2):\n", 158 | " newX = np.zeros((X.shape[0] + 2 * margin, X.shape[1] + 2* margin, X.shape[2]),dtype=\"float16\")\n", 159 | " x_offset = margin\n", 160 | " y_offset = margin\n", 161 | " newX[x_offset:X.shape[0] + x_offset, y_offset:X.shape[1] + y_offset, :] = X\n", 162 | " return newX" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [ 171 | "def createPatches(X, y, windowSize=25, removeZeroLabels = True):\n", 172 | " margin = int((windowSize - 1) / 2)\n", 173 | " zeroPaddedX = padWithZeros(X, margin=margin)\n", 174 | " # split patches\n", 175 | " patchesData = np.zeros((X.shape[0] * X.shape[1], windowSize, windowSize, X.shape[2]),dtype=\"float16\")\n", 176 | " patchesLabels = np.zeros((X.shape[0] * X.shape[1]),dtype=\"float16\")\n", 177 | " patchIndex = 0\n", 178 | " for r in range(margin, zeroPaddedX.shape[0] - margin):\n", 179 | " for c in range(margin, zeroPaddedX.shape[1] - margin):\n", 180 | " patch = zeroPaddedX[r - margin:r + margin + 1, c - margin:c + margin + 1] \n", 181 | " patchesData[patchIndex, :, :, :] = patch\n", 182 | " patchesLabels[patchIndex] = y[r-margin, c-margin]\n", 183 | " patchIndex = patchIndex + 1\n", 184 | " if removeZeroLabels:\n", 185 | " patchesData = patchesData[patchesLabels>0,:,:,:]\n", 186 | " patchesLabels = patchesLabels[patchesLabels>0]\n", 187 | " patchesLabels -= 1\n", 188 | " return patchesData, patchesLabels" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "X, y = loadData(dataset)\n", 198 | "# X.shape, y.shape" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "X,pca,ratio = applyPCA(X,numComponents=componentsNum)\n", 208 | "X.shape" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "X, y = createPatches(X, y, windowSize=windowSize)\n", 218 | "X.shape, y.shape" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [ 227 | "if dataset != 'IP':\n", 228 | " Xtrain, Xtest, ytrain, ytest = splitTrainTestSet(X, y, test_ratio)\n", 229 | " Xvalid, Xtest, yvalid, ytest = splitTrainTestSet(Xtest, ytest, (test_ratio-train_ratio/train_val_ratio)/test_ratio)\n", 230 | "# Xtrain = tf.Session().run(randn(Xtrain))\n", 231 | "# Xtrain = add_gaussian_noise(Xtrain, noise_sigma)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "import random\n", 241 | "import math\n", 242 | "from itertools import chain\n", 243 | "def train_test_random_new(y,n,number):\n", 244 | " K = max(y)+1\n", 245 | " train_index = []\n", 246 | " val_indexes = []\n", 247 | " test_indexes = []\n", 248 | " print(\"********\")\n", 249 | " print(n)\n", 250 | " for i in range(int(K)):\n", 251 | " index1 = [j for j in range(len(y)) if y[j]==int(i)]\n", 252 | " random.shuffle(index1)\n", 253 | " if len(index1)/2>n:\n", 254 | " train_index.append(index1[0:n])\n", 255 | " val_indexes.append(index1[n:2*n])\n", 256 | " test_indexes.append(index1[2*n:])\n", 257 | " else:\n", 258 | " train_index.append(index1[0:math.floor(len(index1)/2)])\n", 259 | " val_indexes.append(index1[math.floor(len(index1)/2):2*math.floor(len(index1)/2)])\n", 260 | " test_indexes.append(index1[2*math.floor(len(index1)/2):])\n", 261 | " train_index = list(chain.from_iterable(train_index))\n", 262 | " val_indexes = list(chain.from_iterable(val_indexes))\n", 263 | " test_indexes = list(chain.from_iterable(test_indexes))\n", 264 | " return train_index,val_indexes,test_indexes" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "if dataset == 'IP':\n", 274 | " no_classes = 16\n", 275 | " no_train = no_classes * 10 # 10表示每个样本类取10个\n", 276 | " no_val = no_train\n", 277 | " no_test = 10249-no_train*2\n", 278 | " train_index,val_indexes,test_indexes = train_test_random_new(y,math.floor(int(no_train/no_classes)),no_train)\n", 279 | " print(train_index)" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": null, 285 | "metadata": {}, 286 | "outputs": [], 287 | "source": [ 288 | "if dataset == 'IP':\n", 289 | " Xtrain = X[train_index,:,:,:]\n", 290 | " ytrain = y[train_index,]" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "class_sum = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n", 300 | "for i in ytrain:\n", 301 | " class_sum[int(i)] += 1\n", 302 | "print(class_sum)" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "metadata": {}, 309 | "outputs": [], 310 | "source": [ 311 | "Xtrain = Xtrain.reshape(-1, windowSize, windowSize, componentsNum, 1)\n", 312 | "ytrain = np_utils.to_categorical(ytrain)" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "metadata": {}, 319 | "outputs": [], 320 | "source": [ 321 | "print(Xtrain.shape)" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": null, 327 | "metadata": {}, 328 | "outputs": [], 329 | "source": [ 330 | "if dataset == 'IP':\n", 331 | " Xvalid = X[val_indexes,:,:,:]\n", 332 | " yvalid = y[val_indexes,]" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "metadata": {}, 339 | "outputs": [], 340 | "source": [ 341 | "print(Xvalid.shape)" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": null, 347 | "metadata": {}, 348 | "outputs": [], 349 | "source": [ 350 | "Xvalid = Xvalid.reshape(-1, windowSize, windowSize, componentsNum, 1)\n", 351 | "yvalid = np_utils.to_categorical(yvalid)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": null, 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "if dataset == 'IP':\n", 361 | " Xtest = X[test_indexes,:,:,:]\n", 362 | " ytest = y[test_indexes,]" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": null, 368 | "metadata": {}, 369 | "outputs": [], 370 | "source": [ 371 | "class_sum = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n", 372 | "for i in ytest:\n", 373 | " class_sum[int(i)] += 1\n", 374 | "print(class_sum)" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": null, 380 | "metadata": {}, 381 | "outputs": [], 382 | "source": [ 383 | "print(Xtest.shape)" 384 | ] 385 | }, 386 | { 387 | "cell_type": "markdown", 388 | "metadata": {}, 389 | "source": [ 390 | "# Train" 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": null, 396 | "metadata": {}, 397 | "outputs": [], 398 | "source": [ 399 | "if dataset == 'UP':\n", 400 | " output_units = 9\n", 401 | "elif dataset == 'KSC':\n", 402 | " output_units = 13\n", 403 | "elif dataset == 'BW':\n", 404 | " output_units = 14\n", 405 | "elif dataset == 'HU':\n", 406 | " output_units = 20\n", 407 | "elif dataset == 'XZ':\n", 408 | " output_units = 9\n", 409 | "else:\n", 410 | " output_units = 16" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "metadata": {}, 417 | "outputs": [], 418 | "source": [ 419 | "## implementation of covariance pooling layers\n", 420 | "def cov_pooling(features):\n", 421 | " shape_f = features.shape.as_list()\n", 422 | " centers_batch = tf.reduce_mean(tf.transpose(features, [0, 2, 1]),2) # 均值\n", 423 | " centers_batch = tf.reshape(centers_batch, [-1, 1, shape_f[2]])\n", 424 | " centers_batch = tf.tile(centers_batch, [1, shape_f[1], 1]) # 张量扩展 \n", 425 | " tmp = tf.subtract(features, centers_batch)\n", 426 | " tmp_t = tf.transpose(tmp, [0, 2, 1]) \n", 427 | " features_t = 1/tf.cast((shape_f[1]-1),tf.float32)*tf.matmul(tmp_t, tmp) \n", 428 | " trace_t = tf.trace(features_t)\n", 429 | " trace_t = tf.reshape(trace_t, [-1, 1])\n", 430 | " trace_t = tf.tile(trace_t, [1, shape_f[2]])\n", 431 | " trace_t = 0.0001*tf.matrix_diag(trace_t)\n", 432 | " return tf.add(features_t,trace_t)" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": null, 438 | "metadata": {}, 439 | "outputs": [], 440 | "source": [ 441 | "def feature_vector(features):\n", 442 | " # features，是对称的，由于张量无法像矩阵一样直接取上三角数据拉成一维向量\n", 443 | " shape_f = features.shape.as_list()\n", 444 | " feature_upper = tf.linalg.band_part(features,0,shape_f[2])\n", 445 | " return feature_upper" 446 | ] 447 | }, 448 | { 449 | "cell_type": "markdown", 450 | "metadata": {}, 451 | "source": [ 452 | "def upsample(data):\n", 453 | " # (None,25,25,30,1)\n", 454 | " shape_f = data.shape.as_list()\n", 455 | " data = tf.reshape(data, [-1, shape_f[1], shape_f[2], shape_f[3]*shape_f[4]])\n", 456 | " data = tf.image.resize_images(images=data, size=[shape_f[1]*2+1, shape_f[2]*2+1], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n", 457 | " data = tf.reshape(data, [-1, shape_f[1]*2+1, shape_f[2]*2+1, shape_f[3], shape_f[4]])\n", 458 | " return data" 459 | ] 460 | }, 461 | { 462 | "cell_type": "markdown", 463 | "metadata": {}, 464 | "source": [ 465 | "def upsample2(data):\n", 466 | " shape_f = data.shape.as_list()\n", 467 | " data = tf.reshape(data, [-1, shape_f[1], shape_f[2], shape_f[3]*shape_f[4]])\n", 468 | " data = tf.image.resize_images(images=data, size=[shape_f[1]*2, shape_f[2]*2], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n", 469 | " data = tf.reshape(data, [-1, shape_f[1]*2, shape_f[2]*2, shape_f[3], shape_f[4]])\n", 470 | " return data" 471 | ] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": null, 476 | "metadata": {}, 477 | "outputs": [], 478 | "source": [ 479 | "def UpSampling(data):\n", 480 | " shape_f = data.shape.as_list()#(None, 12, 12, 14, 8)\n", 481 | " data = tf.transpose(data, [0, 1, 2, 3, 4])\n", 482 | " bb = tf.reshape(data, [-1, shape_f[1], shape_f[2], shape_f[3] * shape_f[4]])\n", 483 | " # NEAREST_NEIGHBOR BILINEAR\n", 484 | " data_lh = tf.image.resize_images(images=bb, size=[shape_f[1]*2+1, shape_f[2]*2+1], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n", 485 | " data = tf.reshape(data_lh, [-1, shape_f[1]*2+1, shape_f[2]*2+1, shape_f[3], shape_f[4]])\n", 486 | " data = tf.transpose(data, [0, 1, 2, 3, 4])\n", 487 | " return data" 488 | ] 489 | }, 490 | { 491 | "cell_type": "code", 492 | "execution_count": null, 493 | "metadata": {}, 494 | "outputs": [], 495 | "source": [ 496 | "def UpSampling2(data):\n", 497 | " shape_f = data.shape.as_list()#(None, 12, 12, 14, 8)\n", 498 | " data = tf.transpose(data, [0, 1, 2, 3, 4])\n", 499 | " bb = tf.reshape(data, [-1, shape_f[1], shape_f[2], shape_f[3] * shape_f[4]])\n", 500 | " data_lh = tf.image.resize_images(images=bb, size=[shape_f[1]*2, shape_f[2]*2], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n", 501 | " data = tf.reshape(data_lh, [-1, shape_f[1]*2, shape_f[2]*2, shape_f[3], shape_f[4]])\n", 502 | " data = tf.transpose(data, [0, 1, 2, 3, 4])\n", 503 | " return data" 504 | ] 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": null, 509 | "metadata": { 510 | "scrolled": true 511 | }, 512 | "outputs": [], 513 | "source": [ 514 | "## input layer\n", 515 | "input_layer = Input((windowSize, windowSize, componentsNum, 1))\n", 516 | "\n", 517 | "## convolutional layers\n", 518 | "# 3D Octave Conv1\n", 519 | "data_hh_1 = Conv3D(filters=8, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(input_layer)\n", 520 | "high_to_low_1 = AveragePooling3D(pool_size=(2,2,1))(input_layer)\n", 521 | "data_hl_1 = Conv3D(filters=8, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(high_to_low_1)\n", 522 | "\n", 523 | "# 3D Octave Conv2\n", 524 | "data_hh_2 = Conv3D(filters=16, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(data_hh_1)\n", 525 | "data_down_2 = AveragePooling3D(pool_size=(2,2,1))(data_hh_1)\n", 526 | "data_hl_2 = Conv3D(filters=16, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(data_down_2)\n", 527 | "\n", 528 | "data_ll_2 = Conv3D(filters=16, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(data_hl_1)\n", 529 | "data_lh_2 = Conv3D(filters=16, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(data_hl_1)\n", 530 | "h_1_shape = data_hh_2._keras_shape\n", 531 | "data_up_2 = Lambda(UpSampling,output_shape=(h_1_shape[1], h_1_shape[2], h_1_shape[3], h_1_shape[4]),mask=None,arguments=None)(data_lh_2)\n", 532 | "\n", 533 | "data_h_2 = add([data_hh_2, data_up_2])\n", 534 | "data_l_2 = add([data_hl_2, data_ll_2])\n", 535 | "\n", 536 | "# pool\n", 537 | "data_down_3 = AveragePooling3D(pool_size=(2,2,1))(data_h_2)\n", 538 | "data_hl_3 = Conv3D(filters=32, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(data_down_3)\n", 539 | "data_ll_3 = Conv3D(filters=32, kernel_size=(3, 3, 3), activation='relu', padding='SAME')(data_l_2)\n", 540 | "conv_layer3 = add([data_hl_3, data_ll_3])\n", 541 | "\n", 542 | "conv3d_shape = conv_layer3._keras_shape\n", 543 | "conv_layer3 = Reshape((conv3d_shape[1], conv3d_shape[2], conv3d_shape[3]*conv3d_shape[4]))(conv_layer3)\n", 544 | "conv3d_shape = conv_layer3._keras_shape\n", 545 | "\n", 546 | "conv_layer4 = Conv2D(filters=512, kernel_size=(1,1), activation='relu')(conv_layer3)\n", 547 | "conv2d_shape = conv_layer4._keras_shape\n", 548 | "conv_layer4 = SubpixelConv2D(upsampling_factor=16)(conv_layer4)\n", 549 | "\n", 550 | "flatten_layer = Flatten()(conv_layer4)\n", 551 | "\n", 552 | "## fully connected layers\n", 553 | "dense_layer1 = Dense(units=256, activation='relu')(flatten_layer)\n", 554 | "dense_layer1 = Dropout(drop)(dense_layer1)\n", 555 | "dense_layer2 = Dense(units=128, activation='relu')(dense_layer1)\n", 556 | "dense_layer2 = Dropout(drop)(dense_layer2)\n", 557 | "output_layer = Dense(units=output_units, activation='softmax')(dense_layer2)" 558 | ] 559 | }, 560 | { 561 | "cell_type": "code", 562 | "execution_count": null, 563 | "metadata": {}, 564 | "outputs": [], 565 | "source": [ 566 | "# define the model with input layer and output layer\n", 567 | "model = Model(inputs=input_layer, outputs=output_layer)" 568 | ] 569 | }, 570 | { 571 | "cell_type": "raw", 572 | "metadata": {}, 573 | "source": [ 574 | "model = load_model('Oct-MCNN-CP.h5', custom_objects={'SubpixelConv2D': SubpixelConv2D,'tf': tf})\n", 575 | "adam = Adam(lr=0.001, decay=1e-06)\n", 576 | "model.load_weights(\"best-model.hdf5\")" 577 | ] 578 | }, 579 | { 580 | "cell_type": "code", 581 | "execution_count": null, 582 | "metadata": { 583 | "scrolled": false 584 | }, 585 | "outputs": [], 586 | "source": [ 587 | "model.summary()\n", 588 | "model.save('Oct-MCNN-PS.h5')" 589 | ] 590 | }, 591 | { 592 | "cell_type": "code", 593 | "execution_count": null, 594 | "metadata": {}, 595 | "outputs": [], 596 | "source": [ 597 | "# compiling the model\n", 598 | "adam = Adam(lr=0.001, decay=1e-06)\n", 599 | "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" 600 | ] 601 | }, 602 | { 603 | "cell_type": "code", 604 | "execution_count": null, 605 | "metadata": {}, 606 | "outputs": [], 607 | "source": [ 608 | "# checkpoint\n", 609 | "filepath = \"best-model.hdf5\"\n", 610 | "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='auto')\n", 611 | "callbacks_list = [checkpoint]" 612 | ] 613 | }, 614 | { 615 | "cell_type": "code", 616 | "execution_count": null, 617 | "metadata": { 618 | "scrolled": true 619 | }, 620 | "outputs": [], 621 | "source": [ 622 | "start = time.time()\n", 623 | "history = model.fit(x=Xtrain, y=ytrain, batch_size=256, epochs=100, validation_data=(Xvalid,yvalid), callbacks=callbacks_list, class_weight='auto') #,validation_split=(1/3) \n", 624 | "end = time.time()\n", 625 | "print((end - start)/60)" 626 | ] 627 | }, 628 | { 629 | "cell_type": "code", 630 | "execution_count": null, 631 | "metadata": {}, 632 | "outputs": [], 633 | "source": [ 634 | "plt.figure(figsize=(7,7))\n", 635 | "plt.grid()\n", 636 | "plt.plot(history.history['loss'])\n", 637 | "plt.ylabel('Loss')\n", 638 | "plt.xlabel('Epochs')\n", 639 | "plt.legend(['Training','Validation'], loc='upper right')\n", 640 | "plt.savefig(\"loss_curve.pdf\")\n", 641 | "plt.show()" 642 | ] 643 | }, 644 | { 645 | "cell_type": "code", 646 | "execution_count": null, 647 | "metadata": {}, 648 | "outputs": [], 649 | "source": [ 650 | "plt.figure(figsize=(5,5))\n", 651 | "plt.ylim(0,1.1)\n", 652 | "plt.grid()\n", 653 | "plt.plot(history.history['acc'])\n", 654 | "plt.ylabel('Accuracy')\n", 655 | "plt.xlabel('Epochs')\n", 656 | "plt.legend(['Training','Validation'])\n", 657 | "plt.savefig(\"acc_curve.pdf\")\n", 658 | "plt.show()" 659 | ] 660 | }, 661 | { 662 | "cell_type": "markdown", 663 | "metadata": {}, 664 | "source": [ 665 | "# Test" 666 | ] 667 | }, 668 | { 669 | "cell_type": "code", 670 | "execution_count": null, 671 | "metadata": {}, 672 | "outputs": [], 673 | "source": [ 674 | "# load best weights\n", 675 | "model.load_weights(\"best-model.hdf5\")\n", 676 | "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "execution_count": null, 682 | "metadata": {}, 683 | "outputs": [], 684 | "source": [ 685 | "Xtest = Xtest.reshape(-1, windowSize, windowSize, componentsNum, 1)\n", 686 | "# Xtest.shape" 687 | ] 688 | }, 689 | { 690 | "cell_type": "code", 691 | "execution_count": null, 692 | "metadata": {}, 693 | "outputs": [], 694 | "source": [ 695 | "ytest = np_utils.to_categorical(ytest)\n", 696 | "# ytest.shape" 697 | ] 698 | }, 699 | { 700 | "cell_type": "code", 701 | "execution_count": null, 702 | "metadata": {}, 703 | "outputs": [], 704 | "source": [ 705 | "Y_pred_test = model.predict(Xtest)\n", 706 | "y_pred_test = np.argmax(Y_pred_test, axis=1)\n", 707 | "\n", 708 | "classification = classification_report(np.argmax(ytest, axis=1), y_pred_test, digits=4)\n", 709 | "print(classification)" 710 | ] 711 | }, 712 | { 713 | "cell_type": "code", 714 | "execution_count": null, 715 | "metadata": {}, 716 | "outputs": [], 717 | "source": [ 718 | "def AA_andEachClassAccuracy(confusion_matrix):\n", 719 | " counter = confusion_matrix.shape[0]\n", 720 | " list_diag = np.diag(confusion_matrix)\n", 721 | " list_raw_sum = np.sum(confusion_matrix, axis=1)\n", 722 | " each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n", 723 | " average_acc = np.mean(each_acc)\n", 724 | " return each_acc, average_acc" 725 | ] 726 | }, 727 | { 728 | "cell_type": "code", 729 | "execution_count": null, 730 | "metadata": {}, 731 | "outputs": [], 732 | "source": [ 733 | "def reports (X_test,y_test,name):\n", 734 | " start = time.time()\n", 735 | " Y_pred = model.predict(X_test)\n", 736 | " y_pred = np.argmax(Y_pred, axis=1)\n", 737 | " end = time.time()\n", 738 | " print(end - start)\n", 739 | " if name == 'IP':\n", 740 | " target_names = ['Alfalfa', 'Corn-notill', 'Corn-mintill', 'Corn',\n", 741 | " 'Grass-pasture', 'Grass-trees', 'Grass-pasture-mowed', \n", 742 | " 'Hay-windrowed', 'Oats', 'Soybean-notill', 'Soybean-mintill',\n", 743 | " 'Soybean-clean', 'Wheat', 'Woods', 'Buildings-Grass-Trees-Drives',\n", 744 | " 'Stone-Steel-Towers']\n", 745 | " elif name == 'KSC':\n", 746 | " target_names = ['Scrub','Willow-swamp','CP-hammock','Slash-pine','Oak/Broadleaf','Hardwood','Swamp','Graminoid-marsh'\n", 747 | " ,'Spartina-marsh','Cattail-marsh','Salt-marsh','Mud-flats','Water']\n", 748 | " elif name == 'SA':\n", 749 | " target_names = ['Brocoli_green_weeds_1','Brocoli_green_weeds_2','Fallow','Fallow_rough_plow','Fallow_smooth',\n", 750 | " 'Stubble','Celery','Grapes_untrained','Soil_vinyard_develop','Corn_senesced_green_weeds',\n", 751 | " 'Lettuce_romaine_4wk','Lettuce_romaine_5wk','Lettuce_romaine_6wk','Lettuce_romaine_7wk',\n", 752 | " 'Vinyard_untrained','Vinyard_vertical_trellis']\n", 753 | " elif name == 'UP':\n", 754 | " target_names = ['Asphalt','Meadows','Gravel','Trees', 'Painted metal sheets','Bare Soil','Bitumen',\n", 755 | " 'Self-Blocking Bricks','Shadows']\n", 756 | " elif name == 'BW':\n", 757 | " target_names = ['Water','Hippo grass','Floodplain grasses1','Floodplain grasses2', 'Reeds1','Riparian','Firescar2',\n", 758 | " 'Island interior','Acacia woodlands','Acacia shrublands','Acacia grasslands','Short mopane','Mixed mopane','Exposed soils']\n", 759 | " elif name == 'HU':\n", 760 | " target_names = ['Healthy grass','Stressed grass','Artificial turf','Evergreen trees', 'Deciduous trees','Bare earth','Water',\n", 761 | " 'Residential buildings','Non-residential buildings','Roads','Sidewalks','Crosswalks','Major thoroughfares','Highways',\n", 762 | " 'Railways','Paved parking lots','Unpaved parking lots','Cars','Trains','Stadium seats']\n", 763 | " elif name == 'XZ':\n", 764 | " target_names = ['class1','class2','class3','class4', 'class5','class6','class7','class8','class9']\n", 765 | " \n", 766 | " classification = classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names, digits=4)\n", 767 | " oa = accuracy_score(np.argmax(y_test, axis=1), y_pred)\n", 768 | " confusion = confusion_matrix(np.argmax(y_test, axis=1), y_pred)\n", 769 | " each_acc, aa = AA_andEachClassAccuracy(confusion)\n", 770 | " kappa = cohen_kappa_score(np.argmax(y_test, axis=1), y_pred)\n", 771 | " score = model.evaluate(X_test, y_test, batch_size=32) \n", 772 | " Test_Loss = score[0]*100\n", 773 | " Test_accuracy = score[1]*100\n", 774 | " \n", 775 | " return classification, confusion, Test_Loss, Test_accuracy, oa*100, each_acc*100, aa*100, kappa*100" 776 | ] 777 | }, 778 | { 779 | "cell_type": "code", 780 | "execution_count": null, 781 | "metadata": {}, 782 | "outputs": [], 783 | "source": [ 784 | "classification, confusion, Test_loss, Test_accuracy, oa, each_acc, aa, kappa = reports(Xtest,ytest,dataset)\n", 785 | "classification = str(classification)\n", 786 | "confusion1 = str(confusion)\n", 787 | "file_name = \"classification_report.txt\"\n", 788 | "\n", 789 | "with open(file_name, 'w') as x_file:\n", 790 | " x_file.write('{} Test loss (%)'.format(Test_loss))\n", 791 | " x_file.write('\\n')\n", 792 | " x_file.write('{} Test accuracy (%)'.format(Test_accuracy))\n", 793 | " x_file.write('\\n')\n", 794 | " x_file.write('\\n')\n", 795 | " x_file.write('{} Kappa accuracy (%)'.format(kappa))\n", 796 | " x_file.write('\\n')\n", 797 | " x_file.write('{} Overall accuracy (%)'.format(oa))\n", 798 | " x_file.write('\\n')\n", 799 | " x_file.write('{} Average accuracy (%)'.format(aa))\n", 800 | " x_file.write('\\n')\n", 801 | " x_file.write('\\n')\n", 802 | " x_file.write('{}'.format(classification))\n", 803 | " x_file.write('\\n')\n", 804 | " x_file.write('{}'.format(confusion1))" 805 | ] 806 | }, 807 | { 808 | "cell_type": "markdown", 809 | "metadata": {}, 810 | "source": [ 811 | "labels = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16']\n", 812 | "tick_marks = np.array(range(len(labels))) + 0.5\n", 813 | "\n", 814 | "def plot_confusion_matrix(cm, title='Confusion Matrix', cmap=plt.cm.binary):\n", 815 | " plt.imshow(cm, interpolation='nearest', cmap=cmap)\n", 816 | " plt.title(title)\n", 817 | " plt.colorbar()\n", 818 | " xlocations = np.array(range(len(labels)))\n", 819 | " plt.xticks(xlocations, labels, rotation=90)\n", 820 | " plt.yticks(xlocations, labels)\n", 821 | " plt.ylabel('True label')\n", 822 | " plt.xlabel('Predicted label')\n", 823 | " \n", 824 | "cm = confusion\n", 825 | "np.set_printoptions(precision=2)\n", 826 | "cm_normalized = confusion.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", 827 | "plt.figure(figsize=(12, 8), dpi=120)\n", 828 | "\n", 829 | "ind_array = np.arange(len(labels))\n", 830 | "x, y = np.meshgrid(ind_array, ind_array)\n", 831 | "\n", 832 | "for x_val, y_val in zip(x.flatten(), y.flatten()):\n", 833 | " c = cm_normalized[y_val][x_val]\n", 834 | " if c > 0.01:\n", 835 | " plt.text(x_val, y_val, \"%0.2f\" % (c,), color='red', fontsize=7, va='center', ha='center')\n", 836 | "# offset the tick\n", 837 | "plt.gca().set_xticks(tick_marks, minor=True)\n", 838 | "plt.gca().set_yticks(tick_marks, minor=True)\n", 839 | "plt.gca().xaxis.set_ticks_position('none')\n", 840 | "plt.gca().yaxis.set_ticks_position('none')\n", 841 | "plt.grid(True, which='minor', linestyle='-')\n", 842 | "plt.gcf().subplots_adjust(bottom=0.15)\n", 843 | "\n", 844 | "plot_confusion_matrix(cm_normalized, title='Normalized confusion matrix')\n", 845 | "plt.show()" 846 | ] 847 | }, 848 | { 849 | "cell_type": "markdown", 850 | "metadata": {}, 851 | "source": [ 852 | "# Run Data" 853 | ] 854 | }, 855 | { 856 | "cell_type": "code", 857 | "execution_count": null, 858 | "metadata": {}, 859 | "outputs": [], 860 | "source": [ 861 | "def Patch(data,height_index,width_index):\n", 862 | " height_slice = slice(height_index, height_index+PATCH_SIZE)\n", 863 | " width_slice = slice(width_index, width_index+PATCH_SIZE)\n", 864 | " patch = data[height_slice, width_slice, :]\n", 865 | " return patch" 866 | ] 867 | }, 868 | { 869 | "cell_type": "code", 870 | "execution_count": null, 871 | "metadata": {}, 872 | "outputs": [], 873 | "source": [ 874 | "X, y = loadData(dataset)" 875 | ] 876 | }, 877 | { 878 | "cell_type": "code", 879 | "execution_count": null, 880 | "metadata": {}, 881 | "outputs": [], 882 | "source": [ 883 | "height = y.shape[0]\n", 884 | "width = y.shape[1]\n", 885 | "PATCH_SIZE = windowSize" 886 | ] 887 | }, 888 | { 889 | "cell_type": "code", 890 | "execution_count": null, 891 | "metadata": {}, 892 | "outputs": [], 893 | "source": [ 894 | "X,pca,ratio = applyPCA(X,numComponents=componentsNum)\n", 895 | "# X = wightRatio(X,componentsNum,ratio)\n", 896 | "# X = infoChange(X,componentsNum)" 897 | ] 898 | }, 899 | { 900 | "cell_type": "code", 901 | "execution_count": null, 902 | "metadata": {}, 903 | "outputs": [], 904 | "source": [ 905 | "X = padWithZeros(X, PATCH_SIZE//2)" 906 | ] 907 | }, 908 | { 909 | "cell_type": "markdown", 910 | "metadata": {}, 911 | "source": [ 912 | "# calculate the predicted image\n", 913 | "outputs = np.zeros((height,width))\n", 914 | "for i in range(height):\n", 915 | " for j in range(width):\n", 916 | " target = int(y[i,j])\n", 917 | " if target == 0 :\n", 918 | " continue\n", 919 | " else :\n", 920 | " image_patch=Patch(X,i,j)\n", 921 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 922 | " prediction = (model.predict(X_test_image))\n", 923 | " prediction = np.argmax(prediction, axis=1)\n", 924 | " outputs[i][j] = prediction+1" 925 | ] 926 | }, 927 | { 928 | "cell_type": "code", 929 | "execution_count": null, 930 | "metadata": {}, 931 | "outputs": [], 932 | "source": [ 933 | "# calculate the predicted image\n", 934 | "outputs = np.zeros((height,width),dtype=\"float16\")\n", 935 | "outputs2 = np.zeros((height,width),dtype=\"float16\")\n", 936 | "for i in range(height):\n", 937 | " for j in range(width): \n", 938 | " target = int(y[i,j])\n", 939 | " if target == 0 :\n", 940 | " image_patch=Patch(X,i,j)\n", 941 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 942 | " prediction2 = (model.predict(X_test_image))\n", 943 | " prediction2 = np.argmax(prediction2, axis=1)\n", 944 | " outputs2[i][j] = prediction2+1\n", 945 | " else :\n", 946 | " image_patch=Patch(X,i,j)\n", 947 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 948 | " prediction = (model.predict(X_test_image))\n", 949 | " prediction = np.argmax(prediction, axis=1)\n", 950 | " outputs[i][j] = prediction+1\n", 951 | " outputs2[i][j] = prediction+1" 952 | ] 953 | }, 954 | { 955 | "cell_type": "markdown", 956 | "metadata": {}, 957 | "source": [ 958 | "# calculate the predicted image\n", 959 | "outputs = np.zeros((height,width),dtype=\"float16\")\n", 960 | "for i in range(height):\n", 961 | " for j in range(width): \n", 962 | " target = int(y[i,j])\n", 963 | "\n", 964 | " image_patch=Patch(X,i,j)\n", 965 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 966 | " prediction = (model.predict(X_test_image))\n", 967 | " prediction = np.argmax(prediction, axis=1)\n", 968 | " outputs[i][j] = prediction+1" 969 | ] 970 | }, 971 | { 972 | "cell_type": "markdown", 973 | "metadata": {}, 974 | "source": [ 975 | "outputs2 = np.zeros((height,width),dtype=\"float16\")\n", 976 | "for i in range(height):\n", 977 | " for j in range(width):\n", 978 | " target = int(y[i,j])\n", 979 | " if target == 0 :\n", 980 | " continue\n", 981 | " else :\n", 982 | " image_patch=Patch(X,i,j)\n", 983 | " X_test_image = image_patch.reshape(1,image_patch.shape[0],image_patch.shape[1], image_patch.shape[2], 1).astype('float32') \n", 984 | " prediction2 = (model.predict(X_test_image))\n", 985 | " prediction2 = np.argmax(prediction2, axis=1)\n", 986 | " outputs2[i][j] = prediction2+1" 987 | ] 988 | }, 989 | { 990 | "cell_type": "code", 991 | "execution_count": null, 992 | "metadata": {}, 993 | "outputs": [], 994 | "source": [ 995 | "import spectral\n", 996 | "ground_truth = spectral.imshow(classes = y,figsize =(7,7))\n", 997 | "predict_image = spectral.imshow(classes = outputs.astype(int),figsize =(7,7))\n", 998 | "predict_image2 = spectral.imshow(classes = outputs2.astype(int),figsize =(7,7))\n", 999 | "spectral.save_rgb(\"predictions.png\", outputs.astype(int), colors=spectral.spy_colors)\n", 1000 | "spectral.save_rgb(\"predictions2.png\", outputs2.astype(int), colors=spectral.spy_colors)" 1001 | ] 1002 | }, 1003 | { 1004 | "cell_type": "code", 1005 | "execution_count": null, 1006 | "metadata": {}, 1007 | "outputs": [], 1008 | "source": [ 1009 | "import spectral\n", 1010 | "ground_truth = spectral.imshow(classes = y,figsize =(7,7))\n", 1011 | "spectral.save_rgb(\"UH_GT.png\", y, colors=spectral.spy_colors)" 1012 | ] 1013 | }, 1014 | { 1015 | "cell_type": "markdown", 1016 | "metadata": { 1017 | "collapsed": true 1018 | }, 1019 | "source": [ 1020 | "### " 1021 | ] 1022 | }, 1023 | { 1024 | "cell_type": "code", 1025 | "execution_count": null, 1026 | "metadata": {}, 1027 | "outputs": [], 1028 | "source": [] 1029 | } 1030 | ], 1031 | "metadata": { 1032 | "kernelspec": { 1033 | "display_name": "Python 3", 1034 | "language": "python", 1035 | "name": "python3" 1036 | }, 1037 | "language_info": { 1038 | "codemirror_mode": { 1039 | "name": "ipython", 1040 | "version": 3 1041 | }, 1042 | "file_extension": ".py", 1043 | "mimetype": "text/x-python", 1044 | "name": "python", 1045 | "nbconvert_exporter": "python", 1046 | "pygments_lexer": "ipython3", 1047 | "version": "3.7.4" 1048 | } 1049 | }, 1050 | "nbformat": 4, 1051 | "nbformat_minor": 2 1052 | } 1053 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MCNN-based_HSI_Classification 2 | ## Papers 3 | * MCNN-CP: Hyperspectral Image Classification Using Mixed Convolutions and Covariance Pooling (TGARS 2021) [paper](https://ieeexplore.ieee.org/document/9103280/) and [source_code](https://github.com/ZhengJianwei2/MCNN-based_HSI_Classification/blob/master/MCNN-CP-Source-code.ipynb) 4 | * Oct-MCNN-HS: 3D Octave and 2D Vanilla Mixed Convolutional Neural Network for Hyperspectral Image Classification With Limited Samples (Remote Sensing,2021) [paper](https://www.mdpi.com/2072-4292/13/21/4407/htm) 5 | 6 | ## 1. Environment setup 7 | This code has been tested on on a personal laptop with Intel i7-9750H 2.6-GHz processor, 32-GB RAM, and an NVIDIA GTX1650 graphic card, Python 3.6, tensorflow_gpu-1.14.0, Keras-2.2.4, CUDA 10.0, cuDNN 7.6. Please install related libraries before running this code: 8 | 9 | pip install -r requirements.txt 10 | 11 | ## 2. Download the datesets: 12 | * IP: 13 | [Indian Pines corrected](http://www.ehu.eus/ccwintco/uploads/6/67/Indian_pines_corrected.mat) and 14 | [Indian Pines gt](http://www.ehu.eus/ccwintco/uploads/c/c4/Indian_pines_gt.mat) 15 | * UH: 16 | [University of Houston 2018](https://hyperspectral.ee.uh.edu/?page_id=1075) 17 | * UP: 18 | [University of Pavia corrected](http://www.ehu.eus/ccwintco/uploads/e/ee/PaviaU.mat) and 19 | [University of Pavia gt](http://www.ehu.eus/ccwintco/uploads/5/50/PaviaU_gt.mat) 20 | * SA: 21 | [Salinas Scene corrected](http://www.ehu.eus/ccwintco/uploads/a/a3/Salinas_corrected.mat) and 22 | [Salinas Scene gt](http://www.ehu.eus/ccwintco/uploads/f/fa/Salinas_gt.mat) 23 | 24 | and put them into data directory. 25 | 26 | ## 3. Download the models (loading models): 27 | 28 | * [models]() code: 29 | 30 | and put them into models directory. 31 | 32 | ## 4. Download the pretrained_models (loading model parameters): 33 | * IP: 34 | [Indian Pines]() code: 35 | * UH: 36 | [University of Houston]() code: 37 | * UP: 38 | [University of Pavia]() code: 39 | * SA: 40 | [Salinas Scene]() code: 41 | 42 | and put them into pretrained_models directory. 43 | 44 | ## 5. Test 45 | 46 | python validate.py 47 | --dataset IP # dataset_name 48 | --model MCNN-CP # model_name 49 | --ratio 0.99 # test_ratio 50 | 51 | The testing result will be saved in the classification_report.txt. 52 | 53 | ## 6. Cite 54 | If you use MCNN-CP in your work please cite our paper: 55 | * BibTex： 56 | 57 | 58 | @ARTICLE{9103280, 59 | author={J. {Zheng} and Y. {Feng} and C. {Bai} and J. {Zhang}}, 60 | journal={IEEE Transactions on Geoscience and Remote Sensing}, 61 | title={Hyperspectral Image Classification Using Mixed Convolutions and Covariance Pooling}, 62 | year={2021}, 63 | volume={59}, 64 | number={1}, 65 | pages={522-534}, 66 | doi={10.1109/TGRS.2020.2995575}}, 67 | } 68 | 69 | * Plane Text： 70 | 71 | J. Zheng, Y. Feng, C. Bai and J. Zhang, "Hyperspectral Image Classification Using Mixed Convolutions and Covariance Pooling," in IEEE Transactions on Geoscience and Remote Sensing, vol. 59, no. 1, pp. 522-534, Jan. 2021, doi: 10.1109/TGRS.2020.2995575. 72 | 73 | 74 | If you use Oct-MCNN-PS in your work please cite our paper: 75 | * BibTex： 76 | 77 | 78 | @Article{rs13214407, 79 | AUTHOR = {Feng, Yuchao and Zheng, Jianwei and Qin, Mengjie and Bai, Cong and Zhang, Jinglin}, 80 | TITLE = {3D Octave and 2D Vanilla Mixed Convolutional Neural Network for Hyperspectral Image Classification with Limited Samples}, 81 | JOURNAL = {Remote Sensing}, 82 | VOLUME = {13}, 83 | YEAR = {2021}, 84 | NUMBER = {21}, 85 | ARTICLE-NUMBER = {4407}, 86 | URL = {https://www.mdpi.com/2072-4292/13/21/4407}, 87 | ISSN = {2072-4292}, 88 | DOI = {10.3390/rs13214407} 89 | } 90 | -------------------------------------------------------------------------------- /data/readme.md: -------------------------------------------------------------------------------- 1 | Download data sets: 2 | ## IP: 3 | [Indian Pines corrected](http://www.ehu.eus/ccwintco/uploads/6/67/Indian_pines_corrected.mat) 4 | 5 | [Indian Pines gt](http://www.ehu.eus/ccwintco/uploads/c/c4/Indian_pines_gt.mat) 6 | 7 | ## UH: 8 | [University of Houston 2018](https://hyperspectral.ee.uh.edu/?page_id=1075) 9 | 10 | ## UP: 11 | [University of Pavia corrected](http://www.ehu.eus/ccwintco/uploads/e/ee/PaviaU.mat) 12 | 13 | [University of Pavia gt](http://www.ehu.eus/ccwintco/uploads/5/50/PaviaU_gt.mat) 14 | 15 | ## SA: 16 | [Salinas Scene corrected](http://www.ehu.eus/ccwintco/uploads/a/a3/Salinas_corrected.mat) 17 | 18 | [Salinas Scene gt](http://www.ehu.eus/ccwintco/uploads/f/fa/Salinas_gt.mat) 19 | 20 | and put them here! 21 | -------------------------------------------------------------------------------- /models/readme.md: -------------------------------------------------------------------------------- 1 | ## Download the models: 2 | 3 | [models](https://pan.baidu.com/s/1fnthMICtci2lfuWEihkDJA) code: caor 4 | 5 | and put them here！ 6 | -------------------------------------------------------------------------------- /pretrained_models/readme.md: -------------------------------------------------------------------------------- 1 | ## Download the pretrained models: 2 | 3 | 4 | and put them here！ 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | python==3.6 2 | numpy 3 | tensorflow_gpu==1.14.0 4 | Keras==2.2.4 5 | sklearn 6 | matplotlib 7 | operator 8 | time 9 | scipy 10 | os 11 | argparse 12 | -------------------------------------------------------------------------------- /subpixel_conv2d.py: -------------------------------------------------------------------------------- 1 | from keras.engine import Layer 2 | import tensorflow as tf 3 | 4 | from keras.utils.generic_utils import get_custom_objects 5 | 6 | class SubpixelConv2D(Layer): 7 | """ Subpixel Conv2D Layer 8 | 9 | upsampling a layer from (h, w, c) to (h*r, w*r, c/(r*r)), 10 | where r is the scaling factor, default to 4 11 | 12 | # Arguments 13 | upsampling_factor: the scaling factor 14 | 15 | # Input shape 16 | Arbitrary. Use the keyword argument `input_shape` 17 | (tuple of integers, does not include the samples axis) 18 | when using this layer as the first layer in a model. 19 | 20 | # Output shape 21 | the second and the third dimension increased by a factor of 22 | `upsampling_factor`; the last layer decreased by a factor of 23 | `upsampling_factor^2`. 24 | 25 | # References 26 | Real-Time Single Image and Video Super-Resolution Using an Efficient 27 | Sub-Pixel Convolutional Neural Network Shi et Al. https://arxiv.org/abs/1609.05158 28 | """ 29 | 30 | def __init__(self, upsampling_factor=4, **kwargs): 31 | super(SubpixelConv2D, self).__init__(**kwargs) 32 | self.upsampling_factor = upsampling_factor 33 | 34 | def build(self, input_shape): 35 | last_dim = input_shape[-1] 36 | factor = self.upsampling_factor * self.upsampling_factor 37 | if last_dim % (factor) != 0: 38 | raise ValueError('Channel ' + str(last_dim) + ' should be of ' 39 | 'integer times of upsampling_factor^2: ' + 40 | str(factor) + '.') 41 | 42 | def call(self, inputs, **kwargs): 43 | return tf.depth_to_space( inputs, self.upsampling_factor ) 44 | 45 | def get_config(self): 46 | config = { 'upsampling_factor': self.upsampling_factor, } 47 | base_config = super(SubpixelConv2D, self).get_config() 48 | return dict(list(base_config.items()) + list(config.items())) 49 | 50 | def compute_output_shape(self, input_shape): 51 | factor = self.upsampling_factor * self.upsampling_factor 52 | input_shape_1 = None 53 | if input_shape[1] is not None: 54 | input_shape_1 = input_shape[1] * self.upsampling_factor 55 | input_shape_2 = None 56 | if input_shape[2] is not None: 57 | input_shape_2 = input_shape[2] * self.upsampling_factor 58 | dims = [ input_shape[0], 59 | input_shape_1, 60 | input_shape_2, 61 | int(input_shape[3]/factor) 62 | ] 63 | return tuple( dims ) 64 | 65 | get_custom_objects().update({'SubpixelConv2D': SubpixelConv2D}) 66 | 67 | if __name__ == '__main__': 68 | from keras.layers import Input 69 | from keras.models import Model, load_model 70 | ip = Input(shape=(32, 32, 16)) 71 | x = SubpixelConv2D(upsampling_factor=4)(ip) 72 | model = Model(ip, x) 73 | model.summary() 74 | model.save( 'model.h5' ) 75 | 76 | print( '*'*80 ) 77 | nm = load_model( 'model.h5' ) 78 | print( 'new model loaded successfully' ) -------------------------------------------------------------------------------- /validate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import keras 3 | from keras.layers import Conv2D, Conv3D, Flatten, Dense, Reshape, BatchNormalization, Lambda, AveragePooling2D, Add 4 | from keras.layers import Dropout, Input 5 | from keras.models import Model, load_model 6 | from keras.optimizers import Adam 7 | from keras.callbacks import ModelCheckpoint 8 | from keras.utils import np_utils 9 | 10 | from sklearn.decomposition import PCA 11 | from sklearn.model_selection import train_test_split 12 | from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score 13 | 14 | from operator import truediv 15 | import time 16 | from subpixel_conv2d import SubpixelConv2D 17 | import numpy as np 18 | import matplotlib.pyplot as plt 19 | import scipy.io as sio 20 | import os 21 | import argparse 22 | 23 | # MODELS = ['MCNN-CP','Oct-MCNN-PS'] # 可选模型 24 | # DATASETS = ['IP','UH','UP','SA'] # 可选数据集 25 | # RATIOS = [0.99, 0.995] # 可选测试样本量(测试样本+验证样本)，剩余为训练样本量 IP:0.99 UH/UP/SA:0.995 26 | parser = argparse.ArgumentParser(description='evaluation') 27 | parser.add_argument('--dataset', '-d', type=str, default='IP', 28 | help='dataset name') 29 | parser.add_argument('--model', '-m', type=str, default='Oct-MCNN-PS', 30 | help='model name') 31 | parser.add_argument('--ratio', '-r', default=0.99, type=float, 32 | help='test ratio') 33 | args = parser.parse_args() 34 | 35 | def loadData(name): 36 | data_path = os.path.join(os.getcwd(),'data') 37 | if name == 'IP': 38 | data = sio.loadmat(os.path.join(data_path, 'Indian_pines_corrected.mat'))['indian_pines_corrected'] 39 | labels = sio.loadmat(os.path.join(data_path, 'Indian_pines_gt.mat'))['indian_pines_gt'] 40 | elif name == 'UH': 41 | data = sio.loadmat(os.path.join(data_path, 'HoustonU.mat'))['houstonU'] # 601*2384*50 42 | labels = sio.loadmat(os.path.join(data_path, 'HoustonU_gt.mat'))['houstonU_gt'] 43 | elif name == 'SA': 44 | data = sio.loadmat(os.path.join(data_path, 'Salinas_corrected.mat'))['salinas_corrected'] 45 | labels = sio.loadmat(os.path.join(data_path, 'Salinas_gt.mat'))['salinas_gt'] 46 | elif name == 'UP': 47 | data = sio.loadmat(os.path.join(data_path, 'PaviaU.mat'))['paviaU'] 48 | labels = sio.loadmat(os.path.join(data_path, 'PaviaU_gt.mat'))['paviaU_gt'] 49 | return data, labels 50 | 51 | def splitTrainTestSet(X, y, testRatio, randomState=345): 52 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testRatio, random_state=randomState,stratify=y) 53 | return X_train, X_test, y_train, y_test 54 | 55 | def applyPCA(X, numComponents=64): 56 | newX = np.reshape(X, (-1, X.shape[2])) 57 | print(newX.shape) 58 | pca = PCA(n_components=numComponents, whiten=True) 59 | newX = pca.fit_transform(newX) 60 | newX = np.reshape(newX, (X.shape[0],X.shape[1], numComponents)) 61 | return newX, pca, pca.explained_variance_ratio_ 62 | 63 | def padWithZeros(X, margin=2): 64 | newX = np.zeros((X.shape[0] + 2 * margin, X.shape[1] + 2* margin, X.shape[2])) 65 | x_offset = margin 66 | y_offset = margin 67 | newX[x_offset:X.shape[0] + x_offset, y_offset:X.shape[1] + y_offset, :] = X 68 | return newX 69 | 70 | def createPatches(X, y, windowSize=25, removeZeroLabels = True): 71 | margin = int((windowSize - 1) / 2) 72 | zeroPaddedX = padWithZeros(X, margin=margin) 73 | # split patches 74 | patchesData = np.zeros((X.shape[0] * X.shape[1], windowSize, windowSize, X.shape[2])) 75 | patchesLabels = np.zeros((X.shape[0] * X.shape[1])) 76 | patchIndex = 0 77 | for r in range(margin, zeroPaddedX.shape[0] - margin): 78 | for c in range(margin, zeroPaddedX.shape[1] - margin): 79 | patch = zeroPaddedX[r - margin:r + margin + 1, c - margin:c + margin + 1] 80 | patchesData[patchIndex, :, :, :] = patch 81 | patchesLabels[patchIndex] = y[r-margin, c-margin] 82 | patchIndex = patchIndex + 1 83 | if removeZeroLabels: 84 | patchesData = patchesData[patchesLabels>0,:,:,:] 85 | patchesLabels = patchesLabels[patchesLabels>0] 86 | patchesLabels -= 1 87 | return patchesData, patchesLabels 88 | 89 | def infoChange(X,numComponents): 90 | X_copy = np.zeros((X.shape[0] , X.shape[1], X.shape[2])) 91 | half = int(numComponents/2) 92 | for i in range(0,half-1): 93 | X_copy[:,:,i] = X[:,:,(half-i)*2-1] 94 | for i in range(half,numComponents): 95 | X_copy[:,:,i] = X[:,:,(i-half)*2] 96 | X = X_copy 97 | return X 98 | 99 | def AA_andEachClassAccuracy(confusion_matrix): 100 | counter = confusion_matrix.shape[0] 101 | list_diag = np.diag(confusion_matrix) 102 | list_raw_sum = np.sum(confusion_matrix, axis=1) 103 | each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum)) 104 | average_acc = np.mean(each_acc) 105 | return each_acc, average_acc 106 | 107 | def reports (X_test,y_test,name): 108 | start = time.time() 109 | Y_pred = model.predict(X_test) 110 | y_pred = np.argmax(Y_pred, axis=1) 111 | end = time.time() 112 | print(end - start) 113 | if name == 'IP': 114 | target_names = ['Alfalfa', 'Corn-notill', 'Corn-mintill', 'Corn', 115 | 'Grass-pasture', 'Grass-trees', 'Grass-pasture-mowed', 116 | 'Hay-windrowed', 'Oats', 'Soybean-notill', 'Soybean-mintill', 117 | 'Soybean-clean', 'Wheat', 'Woods', 'Buildings-Grass-Trees-Drives', 118 | 'Stone-Steel-Towers'] 119 | elif name == 'UH': 120 | target_names = ['Healthy grass','Stressed grass','Artificial turf','Evergreen trees', 'Deciduous trees','Bare earth','Water', 121 | 'Residential buildings','Non-residential buildings','Roads','Sidewalks','Crosswalks','Major thoroughfares','Highways', 122 | 'Railways','Paved parking lots','Unpaved parking lots','Cars','Trains','Stadium seats'] 123 | elif name == 'SA': 124 | target_names = ['Brocoli_green_weeds_1','Brocoli_green_weeds_2','Fallow','Fallow_rough_plow','Fallow_smooth', 125 | 'Stubble','Celery','Grapes_untrained','Soil_vinyard_develop','Corn_senesced_green_weeds', 126 | 'Lettuce_romaine_4wk','Lettuce_romaine_5wk','Lettuce_romaine_6wk','Lettuce_romaine_7wk', 127 | 'Vinyard_untrained','Vinyard_vertical_trellis'] 128 | elif name == 'UP': 129 | target_names = ['Asphalt','Meadows','Gravel','Trees', 'Painted metal sheets','Bare Soil','Bitumen', 130 | 'Self-Blocking Bricks','Shadows'] 131 | 132 | classification = classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names) 133 | oa = accuracy_score(np.argmax(y_test, axis=1), y_pred) 134 | confusion = confusion_matrix(np.argmax(y_test, axis=1), y_pred) 135 | each_acc, aa = AA_andEachClassAccuracy(confusion) 136 | kappa = cohen_kappa_score(np.argmax(y_test, axis=1), y_pred) 137 | score = model.evaluate(X_test, y_test, batch_size=32) 138 | Test_Loss = score[0]*100 139 | Test_accuracy = score[1]*100 140 | 141 | return classification, confusion, Test_Loss, Test_accuracy, oa*100, each_acc*100, aa*100, kappa*100 142 | 143 | 144 | def main(): 145 | ## GLOBAL VARIABLES 146 | dataset = args.dataset 147 | modelname = args.model 148 | test_ratio = args.ratio 149 | train_ratio = int((1-test_ratio)*100) 150 | 151 | windowSize = 11 152 | componentsNum = 20 if dataset == 'UP' else 28 153 | if dataset == 'UP': 154 | componentsNum = 15 155 | elif dataset == 'UH': 156 | componentsNum = 50 if test_ratio >= 0.99 else 25 157 | elif dataset == 'IP': 158 | componentsNum = 110 159 | else: 160 | componentsNum = 30 161 | drop = 0.4 162 | 163 | X, y = loadData(dataset) 164 | X,pca,ratio = applyPCA(X,numComponents=componentsNum) 165 | X = infoChange(X,componentsNum) 166 | X, y = createPatches(X, y, windowSize=windowSize) 167 | 168 | Xtrain, Xtest, ytrain, ytest = splitTrainTestSet(X, y, test_ratio) 169 | Xvalid, Xtest, yvalid, ytest = splitTrainTestSet(Xtest, ytest, (test_ratio-train_ratio/train_val_ratio)/test_ratio) 170 | 171 | Xtest = Xtest.reshape(-1, windowSize, windowSize, componentsNum, 1) 172 | ytest = np_utils.to_categorical(ytest) 173 | 174 | model = load_model('models/'+str(modelname)+'.h5', custom_objects={'SubpixelConv2D': SubpixelConv2D,'tf': tf}) 175 | adam = Adam(lr=0.001, decay=1e-06) 176 | 177 | model.load_weights('pretrained_models/'+str(modelname)+"_"+str(dataset)+".hdf5") 178 | model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) 179 | 180 | Y_pred_test = model.predict(Xtest) 181 | y_pred_test = np.argmax(Y_pred_test, axis=1) 182 | classification = classification_report(np.argmax(ytest, axis=1), y_pred_test) 183 | print(classification) 184 | 185 | classification, confusion, Test_loss, Test_accuracy, oa, each_acc, aa, kappa = reports(Xtest,ytest,dataset) 186 | classification = str(classification) 187 | confusion1 = str(confusion) 188 | file_name = "classification_report.txt" 189 | 190 | with open(file_name, 'w') as x_file: 191 | x_file.write('{} Test loss (%)'.format(Test_loss)) 192 | x_file.write('\n') 193 | x_file.write('{} Test accuracy (%)'.format(Test_accuracy)) 194 | x_file.write('\n') 195 | x_file.write('\n') 196 | x_file.write('{} Kappa accuracy (%)'.format(kappa)) 197 | x_file.write('\n') 198 | x_file.write('{} Overall accuracy (%)'.format(oa)) 199 | x_file.write('\n') 200 | x_file.write('{} Average accuracy (%)'.format(aa)) 201 | x_file.write('\n') 202 | x_file.write('\n') 203 | x_file.write('{}'.format(classification)) 204 | x_file.write('\n') 205 | x_file.write('{}'.format(confusion1)) 206 | 207 | if __name__ == '__main__': 208 | main() 209 | --------------------------------------------------------------------------------