├── Codes ├── Attention_Unet_v1.ipynb ├── helper.py ├── losses.py └── readme.md ├── Data └── readme.md ├── LICENSE ├── Pictures ├── Attention U-Net-4.png └── readme.md └── README.md /Codes/Attention_Unet_v1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Landslide mapping on SAR data by Attention U-Net\n", 8 | "*Authors: Lorenzo Nava, Kushanav Bhuyan, and Sansar Raj Meena*" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import tensorflow as tf\n", 18 | "import numpy as np\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "from tensorflow.keras import layers\n", 21 | "from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate, Conv2DTranspose, BatchNormalization, Dropout, Lambda\n", 22 | "from losses import dice_loss\n", 23 | "from tensorflow import keras\n", 24 | "from tensorflow.keras.optimizers import Adam\n", 25 | "from helper import *\n", 26 | "import pandas as pd\n", 27 | "\n", 28 | "# Library with segmentation metrics\n", 29 | "import segmentation_models as sm" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "# Load data\n", 39 | "X_train = np.load(f'data/Arrays/Arrays_BAA_S_Ascending/trainX.npy')\n", 40 | "Y_train = np.load(f'data/Arrays/Arrays_BAA_S_Ascending/trainY.npy')\n", 41 | "X_test = np.load(f'data/Arrays/Arrays_BAA_S_Ascending/testX.npy')\n", 42 | "Y_test = np.load(f'data/Arrays/Arrays_BAA_S_Ascending/testY.npy')" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": { 49 | "scrolled": true 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "# Define the evaluation metrics Precision, Recall, F1-Score, IoU\n", 54 | "metrics = [sm.metrics.Precision(threshold=0.5),sm.metrics.Recall(threshold=0.5),sm.metrics.FScore(threshold=0.5,beta=1),sm.metrics.IOUScore(threshold=0.5)]" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": { 61 | "scrolled": true 62 | }, 63 | "outputs": [], 64 | "source": [ 65 | "# Model training - Results are saved in a .csv file\n", 66 | "\n", 67 | "# Size of the tiles\n", 68 | "size = 64\n", 69 | "# Image bands\n", 70 | "img_bands = X_train.shape[3]\n", 71 | "# Sampling method\n", 72 | "sampling = \"no_overlap\"\n", 73 | "# Number of filters \n", 74 | "filters = [4, 8, 16, 32]\n", 75 | "# Learning rates\n", 76 | "lr = [10e-3, 5e-4, 10e-4, 5e-5, 10e-5]\n", 77 | "# Batch sizes \n", 78 | "batch_size = [4, 8, 16, 32]\n", 79 | "# Epochs\n", 80 | "epochs = 200\n", 81 | "\n", 82 | "# Dictionary that will save the results\n", 83 | "dic = {}\n", 84 | "\n", 85 | "# Hyperparameters\n", 86 | "dic[\"model\"] = []\n", 87 | "dic[\"batch_size\"] = []\n", 88 | "dic[\"learning_rate\"] = []\n", 89 | "dic[\"filters\"] = []\n", 90 | "\n", 91 | "# test area 1\n", 92 | "dic[\"precision_area\"] = []\n", 93 | "dic[\"recall_area\"] = []\n", 94 | "dic[\"f1_score_area\"] = []\n", 95 | "dic[\"iou_score_area\"] = []\n", 96 | "\n", 97 | "# loop over all the filters in the filter list\n", 98 | "for fiilter in filters:\n", 99 | " # loop over the learning rates\n", 100 | " for learning_rate in lr:\n", 101 | " # loop over all batch sizes in batch_size list\n", 102 | " for batch in batch_size:\n", 103 | " def attn_unet(lr,filtersFirstLayer, pretrained_weights = None,input_size = (size,size,img_bands)): \n", 104 | " inputs = Input(shape=input_size)\n", 105 | " conv1 = UnetConv2D(inputs, filtersFirstLayer, is_batchnorm=True, name='conv1')\n", 106 | " pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n", 107 | "\n", 108 | " conv2 = UnetConv2D(pool1, filtersFirstLayer, is_batchnorm=True, name='conv2')\n", 109 | " pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n", 110 | "\n", 111 | " conv3 = UnetConv2D(pool2, filtersFirstLayer*2, is_batchnorm=True, name='conv3')\n", 112 | " #conv3 = Dropout(0.2,name='drop_conv3')(conv3)\n", 113 | " pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)\n", 114 | "\n", 115 | " conv4 = UnetConv2D(pool3, filtersFirstLayer*2, is_batchnorm=True, name='conv4')\n", 116 | " #conv4 = Dropout(0.2, name='drop_conv4')(conv4)\n", 117 | " pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)\n", 118 | "\n", 119 | " center = UnetConv2D(pool4, filtersFirstLayer*4, is_batchnorm=True, name='center')\n", 120 | "\n", 121 | " g1 = UnetGatingSignal(center, is_batchnorm=True, name='g1')\n", 122 | " attn1 = AttnGatingBlock(conv4, g1, filtersFirstLayer*4, '_1')\n", 123 | " up1 = concatenate([Conv2DTranspose(filtersFirstLayer, (3,3), strides=(2,2), padding='same', activation='relu', kernel_initializer=kinit)(center), attn1], name='up1')\n", 124 | "\n", 125 | " g2 = UnetGatingSignal(up1, is_batchnorm=True, name='g2')\n", 126 | " attn2 = AttnGatingBlock(conv3, g2, filtersFirstLayer*2, '_2')\n", 127 | " up2 = concatenate([Conv2DTranspose(filtersFirstLayer*2, (3,3), strides=(2,2), padding='same', activation='relu', kernel_initializer=kinit)(up1), attn2], name='up2')\n", 128 | "\n", 129 | " g3 = UnetGatingSignal(up1, is_batchnorm=True, name='g3')\n", 130 | " attn3 = AttnGatingBlock(conv2, g3, filtersFirstLayer, '_3')\n", 131 | " up3 = concatenate([Conv2DTranspose(filtersFirstLayer, (3,3), strides=(2,2), padding='same', activation='relu', kernel_initializer=kinit)(up2), attn3], name='up3')\n", 132 | "\n", 133 | " up4 = concatenate([Conv2DTranspose(filtersFirstLayer, (3,3), strides=(2,2), padding='same', activation='relu', kernel_initializer=kinit)(up3), conv1], name='up4')\n", 134 | " conv10 = Conv2D(1, (1, 1), activation='sigmoid', kernel_initializer=kinit, name='final')(up4)\n", 135 | "\n", 136 | " model = Model(inputs, conv10)\n", 137 | "\n", 138 | " model.compile(optimizer = Adam(lr = lr), loss = dice_loss, metrics = metrics)\n", 139 | " model.summary()\n", 140 | "\n", 141 | " if(pretrained_weights):\n", 142 | " model.load_weights(pretrained_weights)\n", 143 | "\n", 144 | " return model\n", 145 | "\n", 146 | "\n", 147 | "\n", 148 | " # Load the model\n", 149 | " model = attn_unet(filtersFirstLayer= fiilter, lr = learning_rate, input_size = (size,size,img_bands))\n", 150 | " # Stop the training if the validation loss does not decrease after 30 epochs\n", 151 | " early_stop = keras.callbacks.EarlyStopping(monitor = 'val_loss', # what is the metric to measure\n", 152 | " patience = 30, # how many epochs to continue running the model after seeing an increase in val_loss\n", 153 | " restore_best_weights = True) # update the mod\n", 154 | " # Save the models only when validation loss decrease\n", 155 | " model_checkpoint = tf.keras.callbacks.ModelCheckpoint(f'data/model/att_unet/weights/unet_{sampling}_size_{size}_filters_{fiilter}_batch_size_{batch}_lr_{learning_rate}.hdf5', monitor='val_loss', mode='min',verbose=1, save_best_only=True,save_weights_only = True)\n", 156 | " print(fiilter, learning_rate,batch)\n", 157 | " # fit the model 20% of the dataset was used as validation\n", 158 | " history = model.fit(X_train,Y_train,batch_size = batch,epochs=epochs,validation_split=0.2,callbacks = [model_checkpoint, early_stop])\n", 159 | "\n", 160 | " # summarize history for iou score\n", 161 | " plt.plot(history.history['f1-score'])\n", 162 | " plt.plot(history.history['val_f1-score'])\n", 163 | " plt.title('model f1-score')\n", 164 | " plt.ylabel('f1-score')\n", 165 | " plt.xlabel('epoch')\n", 166 | " plt.legend(['train', 'validation'], loc='upper left')\n", 167 | " # save plots \n", 168 | " plt.savefig(f\"data/model/att_unet/plots/unet_{sampling}_size_{size}_filters_{fiilter}_batch_size_{batch}_lr_{learning_rate}_iou_score.png\")\n", 169 | " plt.show()\n", 170 | " # summarize history for loss\n", 171 | " plt.plot(history.history['loss'])\n", 172 | " plt.plot(history.history['val_loss'])\n", 173 | " plt.title('model loss')\n", 174 | " plt.ylabel('loss')\n", 175 | " plt.xlabel('epoch')\n", 176 | " plt.legend(['train', 'validation'], loc='upper left')\n", 177 | " plt.savefig(f\"data/model/att_unet/plots/unet_{sampling}_size_{size}_filters_{fiilter}_batch_size_{batch}_lr_{learning_rate}_val_loss.png\")\n", 178 | " plt.show()\n", 179 | " \n", 180 | " # load unet to evaluate the test data\n", 181 | " attn_unet = attn_unet(filtersFirstLayer= fiilter, lr = learning_rate,input_size=(size,size,img_bands))\n", 182 | " # load the last saved weight from the training\n", 183 | " attn_unet.load_weights(f\"data/model/att_unet/weights/unet_{sampling}_size_{size}_filters_{fiilter}_batch_size_{batch}_lr_{learning_rate}.hdf5\")\n", 184 | " \n", 185 | " # Evaluate the model\n", 186 | " res_1= attn_unet.evaluate(X_test,Y_test)\n", 187 | "\n", 188 | "\n", 189 | " # save results on the dictionary\n", 190 | " dic[\"model\"].append(\"Attention_Unet\")\n", 191 | " dic[\"batch_size\"].append(batch)\n", 192 | " dic[\"learning_rate\"].append(learning_rate)\n", 193 | " dic[\"filters\"].append(fiilter)\n", 194 | " dic[\"precision_area\"].append(res_1[1])\n", 195 | " dic[\"recall_area\"].append(res_1[2])\n", 196 | " dic[\"f1_score_area\"].append(res_1[3])\n", 197 | " dic[\"iou_score_area\"].append(res_1[4])\n", 198 | " \n", 199 | "\n", 200 | "\n", 201 | " # Convert results to a dataframe\n", 202 | " results = pd.DataFrame(dic)\n", 203 | " # Export as csv\n", 204 | " results.to_csv(f'data/model/att_unet/results/results_Att_Unet.csv', index = False)" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": { 211 | "scrolled": true 212 | }, 213 | "outputs": [], 214 | "source": [ 215 | "# Load the best model based on the best performances on the test set (check CSV file)\n", 216 | "# Loading the model weights\n", 217 | "attn_unet_best = attn_unet(filtersFirstLayer= 4,lr = 0.001,input_size=(size,size,img_bands))\n", 218 | "attn_unet_best.load_weights(\"data/model/att_unet/weights/unet_no_overlap_size_64_filters_4_batch_size_4_lr_0.001.hdf5\")\n", 219 | "\n", 220 | "# Plot predictions on test set\n", 221 | "for i in range(X_test.shape[0]):\n", 222 | " preds_train_1 = attn_unet_best.predict(np.expand_dims(X_test[i],axis = 0), verbose=1)\n", 223 | " # It's possible to change the 0.5 threshold to improve the results;\n", 224 | " preds_train_t1 = (preds_train_1 > 0.9).astype(np.uint8)\n", 225 | " f, axarr = plt.subplots(1,3,figsize=(10,10))\n", 226 | " axarr[0].imshow(X_test[i][:,:,0])\n", 227 | " axarr[1].imshow(np.squeeze(preds_train_t1))\n", 228 | " axarr[2].imshow(np.squeeze(Y_test[i]))\n", 229 | " " 230 | ] 231 | } 232 | ], 233 | "metadata": { 234 | "kernelspec": { 235 | "display_name": "Python 3.7 (tensorflow)", 236 | "language": "python", 237 | "name": "tensorflow" 238 | }, 239 | "language_info": { 240 | "codemirror_mode": { 241 | "name": "ipython", 242 | "version": 3 243 | }, 244 | "file_extension": ".py", 245 | "mimetype": "text/x-python", 246 | "name": "python", 247 | "nbconvert_exporter": "python", 248 | "pygments_lexer": "ipython3", 249 | "version": "3.7.9" 250 | } 251 | }, 252 | "nbformat": 4, 253 | "nbformat_minor": 4 254 | } 255 | -------------------------------------------------------------------------------- /Codes/helper.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import time 3 | import os 4 | import h5py 5 | from tensorflow.keras.models import Model 6 | from tensorflow.keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Conv2DTranspose 7 | from tensorflow.keras.layers import Activation, add, multiply, Lambda 8 | from tensorflow.keras.layers import AveragePooling2D, average, UpSampling2D, Dropout 9 | from tensorflow.keras.optimizers import Adam, SGD, RMSprop 10 | from tensorflow.keras.initializers import glorot_normal, random_normal, random_uniform 11 | from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping 12 | 13 | from tensorflow.keras import backend as K 14 | from tensorflow.keras.layers import BatchNormalization 15 | from tensorflow.keras.applications import VGG19, densenet 16 | from tensorflow.keras.models import load_model 17 | import os 18 | import cv2 19 | import tensorflow as tf 20 | import numpy as np 21 | from matplotlib import pyplot as plt 22 | import segmentation_models as sm 23 | from tensorflow.keras.metrics import MeanIoU 24 | import random 25 | import rasterio 26 | from rasterio.plot import show 27 | from sklearn.model_selection import train_test_split 28 | from tensorflow.keras import layers 29 | from tensorflow.keras.models import Model 30 | from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate, Conv2DTranspose, BatchNormalization, Dropout, Lambda 31 | from tensorflow.keras import backend as K 32 | 33 | K.set_image_data_format('channels_last') # TF dimension ordering in this code 34 | kinit = 'glorot_normal' 35 | 36 | 37 | def expend_as(tensor, rep,name): 38 | my_repeat = Lambda(lambda x, repnum: K.repeat_elements(x, repnum, axis=3), arguments={'repnum': rep}, name='psi_up'+name)(tensor) 39 | return my_repeat 40 | 41 | 42 | def AttnGatingBlock(x, g, inter_shape, name): 43 | ''' take g which is the spatially smaller signal, do a conv to get the same 44 | number of feature channels as x (bigger spatially) 45 | do a conv on x to also get same geature channels (theta_x) 46 | then, upsample g to be same size as x 47 | add x and g (concat_xg) 48 | relu, 1x1 conv, then sigmoid then upsample the final - this gives us attn coefficients''' 49 | 50 | shape_x = K.int_shape(x) # 32 51 | shape_g = K.int_shape(g) # 16 52 | 53 | theta_x = Conv2D(inter_shape, (2, 2), strides=(2, 2), padding='same', name='xl'+name)(x) # 16 54 | shape_theta_x = K.int_shape(theta_x) 55 | 56 | phi_g = Conv2D(inter_shape, (1, 1), padding='same')(g) 57 | upsample_g = Conv2DTranspose(inter_shape, (3, 3),strides=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]),padding='same', name='g_up'+name)(phi_g) # 16 58 | 59 | concat_xg = add([upsample_g, theta_x]) 60 | act_xg = Activation('relu')(concat_xg) 61 | psi = Conv2D(1, (1, 1), padding='same', name='psi'+name)(act_xg) 62 | sigmoid_xg = Activation('sigmoid')(psi) 63 | shape_sigmoid = K.int_shape(sigmoid_xg) 64 | upsample_psi = UpSampling2D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2]))(sigmoid_xg) # 32 65 | 66 | upsample_psi = expend_as(upsample_psi, shape_x[3], name) 67 | y = multiply([upsample_psi, x], name='q_attn'+name) 68 | 69 | result = Conv2D(shape_x[3], (1, 1), padding='same',name='q_attn_conv'+name)(y) 70 | result_bn = BatchNormalization(name='q_attn_bn'+name)(result) 71 | return result_bn 72 | 73 | def UnetConv2D(input, outdim, is_batchnorm, name): 74 | x = Conv2D(outdim, (3, 3), strides=(1, 1), kernel_initializer=kinit, padding="same", name=name+'_1')(input) 75 | if is_batchnorm: 76 | x =BatchNormalization(name=name + '_1_bn')(x) 77 | x = Activation('relu',name=name + '_1_act')(x) 78 | 79 | x = Conv2D(outdim, (3, 3), strides=(1, 1), kernel_initializer=kinit, padding="same", name=name+'_2')(x) 80 | if is_batchnorm: 81 | x = BatchNormalization(name=name + '_2_bn')(x) 82 | x = Activation('relu', name=name + '_2_act')(x) 83 | return x 84 | 85 | 86 | def UnetGatingSignal(input, is_batchnorm, name): 87 | ''' this is simply 1x1 convolution, bn, activation ''' 88 | shape = K.int_shape(input) 89 | x = Conv2D(shape[3] * 1, (1, 1), strides=(1, 1), padding="same", kernel_initializer=kinit, name=name + '_conv')(input) 90 | if is_batchnorm: 91 | x = BatchNormalization(name=name + '_bn')(x) 92 | x = Activation('relu', name = name + '_act')(x) 93 | return x 94 | -------------------------------------------------------------------------------- /Codes/losses.py: -------------------------------------------------------------------------------- 1 | import tensorflow.keras.backend as K 2 | 3 | smooth = 1 4 | 5 | def dsc(y_true, y_pred): 6 | smooth = 1. 7 | y_true_f = K.flatten(y_true) 8 | y_pred_f = K.flatten(y_pred) 9 | intersection = K.sum(y_true_f * y_pred_f) 10 | score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth) 11 | return score 12 | 13 | def dice_loss(y_true, y_pred): 14 | loss = 1 - dsc(y_true, y_pred) 15 | return loss -------------------------------------------------------------------------------- /Codes/readme.md: -------------------------------------------------------------------------------- 1 | ## Training and hyperparameter tuning of the Attention U-Net model. 2 | 3 | #### The notebook will save three outputs for which the user has to generate the following folder structure: 4 | 5 | data/ 6 | 7 | model/ 8 | 9 | att_unet/ 10 | 11 | weights/ 12 | "the notebook will save .hdf5 file with the best weights 13 | of each parameters combinations" 14 | 15 | plots/ 16 | "the notebook will save training and validation learning and loss curves" 17 | 18 | results/ 19 | "the notebook will save a .csv file containing all details of each 20 | models with respectives evaluation scores on the test data" 21 | 22 | Credits for the Attention U-Net architecture: [Abraham et al., (2019)](https://ieeexplore.ieee.org/abstract/document/8759329). 23 | -------------------------------------------------------------------------------- /Data/readme.md: -------------------------------------------------------------------------------- 1 | ## Sample data in the numpy array format (.npy) at the [link](https://drive.google.com/drive/folders/1kSyilyQMK_IXww8rkhmPm6oDklyLgNfo?usp=sharing). 2 | 3 | Satellite: Sentinel-1 4 | 5 | Orbit: Ascending 6 | 7 | Patch size: 64 x 64 x 4 8 | 9 | Train: 752 patches 10 | 11 | Test: 189 patches 12 | 13 | Image bands: VV amplitude pre-event + VV amplitude post-event + VV amplitude post-event + Slope angle. 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 lorenzonava96 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pictures/Attention U-Net-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lorenzonava96/Landslide-mapping-on-SAR-data-by-Attention-U-Net/7e36477c9f35282a12af710decc6d576eb21c33d/Pictures/Attention U-Net-4.png -------------------------------------------------------------------------------- /Pictures/readme.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Landslide mapping on SAR data by Attention-U-Net 2 | 3 | This repository contains codes and sample data from the journal paper "Rapid Mapping of landslide on SAR data by Attention U-net" by [Nava et al (2022)](https://www.mdpi.com/2072-4292/14/6/1449/htm). 4 | With this repository files you can train the Attention U-Net segmentation model on Sentinel-1 SAR amplitude landslide data sampled for the Hokkaido multiple landslide event of September 2018. 5 | 6 | Contact emails: lorenava996@gmail.com & kushanavb@gmail.com 7 | 8 | ***Attention U-Net architecture*** 9 | ![Attention U-Net architecture](https://github.com/lorenzonava96/Landslide-mapping-on-SAR-data-by-Attention-U-Net/blob/main/Pictures/Attention%20U-Net-4.png) 10 | --------------------------------------------------------------------------------