├── README.md └── UNet++_MSOF_model.py /README.md: -------------------------------------------------------------------------------- 1 | # End-to-end-CD-for-VHR-satellite-image 2 | The project aims to contribute to geoscience community. 3 |
4 | ## Paper 5 | 6 | End-to-End Change Detection for High Resolution Satellite Images Using Improved UNet++ [https://www.mdpi.com/2072-4292/11/11/1382] 7 | ## Introduction 8 | 9 | Change detection (CD) is essential to the accurate understanding of land surface changes using available Earth observation data. Due to the great advantages in deep feature representation and nonlinear problem modeling, deep learning is becoming increasingly popular to solve CD tasks in remote-sensing community. However, most existing deep learning-based CD methods are implemented by either generating difference images using deep features or learning change relations between pixel patches, which leads to error accumulation problems since many intermediate processing steps are needed to obtain final change maps. To address the above-mentioned issues, a novel end-to-end CD method is proposed based on an effective encoder-decoder architecture for semantic segmentation named UNet++, where change maps could be learned from scratch using available annotated datasets. Firstly, co-registered image pairs are concatenated as an input for the improved UNet++ network, where both global and fine-grained information can be utilized to generate feature maps with high spatial accuracy. Then, the fusion strategy of multiple side outputs is adopted to combine change maps from different semantic levels, thereby generating a final change map with high accuracy. The effectiveness and reliability of our proposed CD method are verified on very-high-resolution (VHR) satellite image datasets. Extensive experimental results have shown that our proposed approach outperforms the other state-of-the-art CD methods 10 | 11 | 12 | ## Citation 13 | Please cite our paper if you find it useful for your research. 14 | ``` 15 | @article{peng2019end, 16 | title={End-to-End Change Detection for High Resolution Satellite Images Using Improved UNet++}, 17 | author={Peng, Daifeng and Zhang, Yongjun and Guan, Haiyan}, 18 | journal={Remote Sensing}, 19 | volume={11}, 20 | number={11}, 21 | pages={1382}, 22 | year={2019}, 23 | publisher={Multidisciplinary Digital Publishing Institute} 24 | } 25 | ``` 26 | -------------------------------------------------------------------------------- /UNet++_MSOF_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from keras import Input, Model 4 | from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten, merge, Concatenate,Conv2DTranspose,add,Concatenate,Add,Subtract 5 | from keras.optimizers import Adam,SGD 6 | from keras.utils import plot_model 7 | from keras import backend as K 8 | from keras.layers import add,BatchNormalization,UpSampling2D 9 | from keras.layers import Embedding,Input,Conv2D,Conv3D,Lambda,concatenate,Flatten,Dense,Dropout,MaxPooling2D,Activation,GlobalAveragePooling2D,GlobalAveragePooling3D,BatchNormalization 10 | from keras import regularizers 11 | import tensorflow as tf 12 | from keras import objectives 13 | from keras.regularizers import l2 14 | from keras.callbacks import Callback 15 | 16 | SEED = 1998 17 | np.random.seed(SEED) 18 | session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, 19 | inter_op_parallelism_threads=1) 20 | tf.set_random_seed(SEED) 21 | sess = tf.Session(graph=tf.get_default_graph(), config=session_conf) 22 | K.set_session(sess) 23 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 24 | 25 | 26 | def dice_coef(y_true, y_pred, smooth=1, weight=0.5): 27 | """ 28 | 加权后的dice coefficient 29 | """ 30 | y_true = y_true[:, :, :, -1] # y_true[:, :, :, :-1]=y_true[:, :, :, -1] if dim(3)=1 等效于[8,256,256,1]==>[8,256,256] 31 | y_pred = y_pred[:, :, :, -1] 32 | intersection = K.sum(y_true * y_pred) 33 | union = K.sum(y_true) + weight * K.sum(y_pred) 34 | # K.mean((2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)) 35 | return ((2. * intersection + smooth) / (union + smooth)) # not working better using mean 36 | def dice_coef_loss(y_true, y_pred): 37 | """ 38 | 目标函数 39 | """ 40 | return 1 - dice_coef(y_true, y_pred) 41 | def weighted_bce_dice_loss(y_true,y_pred): 42 | class_loglosses = K.mean(K.binary_crossentropy(y_true, y_pred), axis=[0, 1, 2]) 43 | 44 | class_weights = [0.1, 0.9]#note that the weights can be computed automatically using the training smaples 45 | weighted_bce = K.sum(class_loglosses * K.constant(class_weights)) 46 | 47 | # return K.weighted_binary_crossentropy(y_true, y_pred,pos_weight) + 0.35 * (self.dice_coef_loss(y_true, y_pred)) #not work 48 | return weighted_bce + 0.5 * (dice_coef_loss(y_true, y_pred)) 49 | def standard_unit(input_tensor, stage, nb_filter, kernel_size=3, mode='None'): 50 | x = Conv2D(nb_filter, (kernel_size, kernel_size), activation='selu', name='conv' + stage + '_1', 51 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(input_tensor) 52 | x0 = x 53 | # x = Dropout(0.2, name='dp' + stage + '_1')(x) 54 | x = BatchNormalization(name='bn' + stage + '_1')(x) # much better than dropout 55 | x = Conv2D(nb_filter, (kernel_size, kernel_size), activation='selu', name='conv' + stage + '_2', 56 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(x) 57 | # x = Dropout(0.2, name='dp' + stage + '_2')(x) 58 | x = BatchNormalization(name='bn' + stage + '_2')(x) 59 | if mode == 'residual': 60 | # x=Add(name='resi'+stage)([x,input_tensor])# 维度不相同！ 61 | x = Add(name='resi' + stage)([x, x0]) 62 | 63 | return x 64 | 65 | def Nest_Net2(input_shape, num_class=1, deep_supervision=False): 66 | nb_filter = [32, 64, 128, 256, 512] 67 | # nb_filter = [16, 32, 64, 128, 256] 68 | mode = 'residual' # mode='residual' seems to improve better than DS 69 | # Handle Dimension Ordering for different backends 70 | bn_axis = 3 71 | inputs = Input(shape=input_shape) 72 | conv1_1 = standard_unit(inputs, stage='11', nb_filter=nb_filter[0]) # add 要求输入输出维度相同 73 | pool1 = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(conv1_1) # (?,128,128,32) 74 | 75 | conv2_1 = standard_unit(pool1, stage='21', nb_filter=nb_filter[1], mode=mode) 76 | pool2 = MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(conv2_1) # (?,64,64,64) 77 | 78 | up1_2 = Conv2DTranspose(nb_filter[0], (2, 2), strides=(2, 2), name='up12', padding='same')(conv2_1) 79 | conv1_2 = concatenate([up1_2, conv1_1], name='merge12', axis=bn_axis) # (?,256,256,64) 80 | conv1_2 = standard_unit(conv1_2, stage='12', nb_filter=nb_filter[0], mode=mode) # (?,256,256,32) 81 | 82 | conv3_1 = standard_unit(pool2, stage='31', nb_filter=nb_filter[2], mode=mode) 83 | pool3 = MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(conv3_1) 84 | 85 | up2_2 = Conv2DTranspose(nb_filter[1], (2, 2), strides=(2, 2), name='up22', padding='same')(conv3_1) 86 | conv2_2 = concatenate([up2_2, conv2_1], name='merge22', axis=bn_axis) 87 | conv2_2 = standard_unit(conv2_2, stage='22', nb_filter=nb_filter[1], mode=mode) 88 | 89 | up1_3 = Conv2DTranspose(nb_filter[0], (2, 2), strides=(2, 2), name='up13', padding='same')(conv2_2) 90 | conv1_3 = concatenate([up1_3, conv1_1, conv1_2], name='merge13', axis=bn_axis) 91 | conv1_3 = standard_unit(conv1_3, stage='13', nb_filter=nb_filter[0], mode=mode) # (?,256,256,32) 92 | 93 | conv4_1 = standard_unit(pool3, stage='41', nb_filter=nb_filter[3], mode=mode) 94 | pool4 = MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(conv4_1) 95 | 96 | up3_2 = Conv2DTranspose(nb_filter[2], (2, 2), strides=(2, 2), name='up32', padding='same')(conv4_1) 97 | conv3_2 = concatenate([up3_2, conv3_1], name='merge32', axis=bn_axis) 98 | conv3_2 =standard_unit(conv3_2, stage='32', nb_filter=nb_filter[2], mode=mode) 99 | 100 | up2_3 = Conv2DTranspose(nb_filter[1], (2, 2), strides=(2, 2), name='up23', padding='same')(conv3_2) 101 | conv2_3 = concatenate([up2_3, conv2_1, conv2_2], name='merge23', axis=bn_axis) 102 | conv2_3 = standard_unit(conv2_3, stage='23', nb_filter=nb_filter[1], mode=mode) 103 | 104 | up1_4 = Conv2DTranspose(nb_filter[0], (2, 2), strides=(2, 2), name='up14', padding='same')(conv2_3) 105 | conv1_4 = concatenate([up1_4, conv1_1, conv1_2, conv1_3], name='merge14', axis=bn_axis) 106 | conv1_4 =standard_unit(conv1_4, stage='14', nb_filter=nb_filter[0], mode=mode) 107 | 108 | conv5_1 = standard_unit(pool4, stage='51', nb_filter=nb_filter[4], mode=mode) 109 | 110 | up4_2 = Conv2DTranspose(nb_filter[3], (2, 2), strides=(2, 2), name='up42', padding='same')(conv5_1) 111 | conv4_2 = concatenate([up4_2, conv4_1], name='merge42', axis=bn_axis) 112 | conv4_2 = standard_unit(conv4_2, stage='42', nb_filter=nb_filter[3], mode=mode) 113 | 114 | up3_3 = Conv2DTranspose(nb_filter[2], (2, 2), strides=(2, 2), name='up33', padding='same')(conv4_2) 115 | conv3_3 = concatenate([up3_3, conv3_1, conv3_2], name='merge33', axis=bn_axis) 116 | conv3_3 = standard_unit(conv3_3, stage='33', nb_filter=nb_filter[2], mode=mode) 117 | 118 | up2_4 = Conv2DTranspose(nb_filter[1], (2, 2), strides=(2, 2), name='up24', padding='same')(conv3_3) 119 | conv2_4 = concatenate([up2_4, conv2_1, conv2_2, conv2_3], name='merge24', axis=bn_axis) 120 | conv2_4 = standard_unit(conv2_4, stage='24', nb_filter=nb_filter[1], mode=mode) 121 | 122 | up1_5 = Conv2DTranspose(nb_filter[0], (2, 2), strides=(2, 2), name='up15', padding='same')(conv2_4) 123 | conv1_5 = concatenate([up1_5, conv1_1, conv1_2, conv1_3, conv1_4], name='merge15', axis=bn_axis) 124 | conv1_5 =standard_unit(conv1_5, stage='15', nb_filter=nb_filter[0], mode=mode) 125 | 126 | nestnet_output_1 = Conv2D(num_class, (1, 1), activation='sigmoid', name='output_1', 127 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(conv1_2) 128 | nestnet_output_2 = Conv2D(num_class, (1, 1), activation='sigmoid', name='output_2', 129 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(conv1_3) 130 | nestnet_output_3 = Conv2D(num_class, (1, 1), activation='sigmoid', name='output_3', 131 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(conv1_4) 132 | nestnet_output_4 = Conv2D(num_class, (1, 1), activation='sigmoid', name='output_4', 133 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(conv1_5) 134 | # using combined loss 135 | conv_fuse = concatenate([conv1_2, conv1_3, conv1_4, conv1_5], name='merge_fuse', axis=bn_axis) 136 | nestnet_output_5 = Conv2D(num_class, (1, 1), activation='sigmoid', name='output_5', 137 | kernel_initializer='he_normal', padding='same', kernel_regularizer=l2(1e-4))(conv_fuse) 138 | 139 | 140 | if deep_supervision: 141 | model = Model(input=inputs, output=[nestnet_output_1, 142 | nestnet_output_2, 143 | nestnet_output_3, 144 | nestnet_output_4, nestnet_output_5]) 145 | model.compile(optimizer=Adam(lr=1e-4), 146 | #loss=['binary_crossentropy','binary_crossentropy','binary_crossentropy','binary_crossentropy'], 147 | loss=[weighted_bce_dice_loss, weighted_bce_dice_loss, weighted_bce_dice_loss, 148 | weighted_bce_dice_loss, weighted_bce_dice_loss], 149 | loss_weights=[0.5, 0.5, 0.75, 0.5, 1.0], 150 | metrics=['accuracy'] 151 | ) 152 | else: 153 | model = Model(input=inputs, output=[nestnet_output_4]) 154 | model.compile(optimizer=Adam(lr=1e-4), loss=weighted_bce_dice_loss, 155 | metrics=['accuracy']) 156 | model.summary() 157 | return model 158 | 159 | if __name__ == '__main__': 160 | input_shape = [256, 256, 6] 161 | model=Nest_Net2(input_shape,deep_supervision=True) 162 | output_layer=model.get_layer('output_5') 163 | print("the output shape is:") 164 | print(output_layer.output_shape) 165 | --------------------------------------------------------------------------------