├── src ├── training │ ├── modeller.py │ ├── plots.py │ ├── data_loader.py │ ├── keras_history.py │ ├── keras_callbacks.py │ ├── augmentation.py │ ├── metrics.py │ ├── training_modes.py │ └── seg_data_generator.py ├── utils │ ├── buzz_utility.py │ ├── create_mask.py │ ├── gdal_utils.py │ ├── train_test_splits.py │ ├── post_process.py │ ├── load_model.py │ ├── image_utils.py │ ├── parallel_model2.py │ ├── parallel_model.py │ ├── infer.py │ └── infer2.py └── networks │ ├── tiramisu.py │ ├── linknet.py │ └── pspnet.py ├── .gitignore ├── LICENSE ├── config.py ├── README.md ├── evaluate.py ├── train.py ├── envs └── bfss.yml └── notebook └── training.ipynb /src/training/modeller.py: -------------------------------------------------------------------------------- 1 | import config 2 | 3 | def finetune_model(base_model): 4 | 5 | for i, layer in enumerate(base_model.layers): 6 | if i in config.trainable_layers: 7 | layer.trainable=True 8 | else: 9 | layer.trainable=False 10 | 11 | print("Layer freezing complete!!") 12 | 13 | return base_model -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | **/.DS_Store 6 | **/data 7 | 8 | # Distribution / packaging 9 | .Python 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | *.egg-info/ 14 | .installed.cfg 15 | *.egg 16 | MANIFEST 17 | 18 | # Installer logs 19 | pip-log.txt 20 | pip-delete-this-directory.txt 21 | 22 | 23 | # Flask stuff: 24 | instance/ 25 | .webassets-cache 26 | 27 | # Jupyter Notebook 28 | .ipynb_checkpoints 29 | 30 | # pyenv 31 | .python-version 32 | -------------------------------------------------------------------------------- /src/utils/buzz_utility.py: -------------------------------------------------------------------------------- 1 | 2 | import buzzard as buzz 3 | 4 | def raster_explorer(raster_path, stats=True): 5 | 6 | ds = buzz.DataSource(allow_interpolation=True) 7 | ds.open_raster('rgb', raster_path) 8 | 9 | fp= buzz.Footprint( 10 | tl=ds.rgb.fp.tl, 11 | size=ds.rgb.fp.size, 12 | rsize=ds.rgb.fp.rsize,) 13 | 14 | tlx, dx, rx, tly, ry, dy = fp.gt 15 | 16 | if stats: 17 | # print("No of channels in raster image:", len(ds.rgb)) 18 | print("Raster size:", ds.rgb.fp.rsize) 19 | print("Dtype of Raster:", ds.rgb.dtype) 20 | # print("Projection System:", ds.rgb.proj4_virtual) 21 | # print("Top left spatial coordinates:", (tlx, tly)) 22 | print("Resolution in x and y:", (dx, dy)) 23 | 24 | return ds -------------------------------------------------------------------------------- /src/training/plots.py: -------------------------------------------------------------------------------- 1 | 2 | import matplotlib.pyplot as plt 3 | 4 | def save_plots(history, exp_name): 5 | # Loss Curves 6 | plt.figure(figsize=[8,6]) 7 | plt.plot(history.history['loss'],'r',linewidth=3.0) 8 | plt.plot(history.history['val_loss'],'b',linewidth=3.0) 9 | plt.legend(['Training loss', 'Validation Loss'],fontsize=18) 10 | plt.xlabel('Epochs ',fontsize=16) 11 | plt.ylabel('Loss',fontsize=16) 12 | plt.title('Loss Curves',fontsize=16) 13 | plt.savefig("./data/"+exp_name+"/"+"loss_curve.jpg") 14 | 15 | # Accuracy Curves 16 | plt.figure(figsize=[8,6]) 17 | plt.plot(history.history['acc'],'r',linewidth=3.0) 18 | plt.plot(history.history['val_acc'],'b',linewidth=3.0) 19 | plt.legend(['Training Accuracy', 'Validation Accuracy'],fontsize=18) 20 | plt.xlabel('Epochs ',fontsize=16) 21 | plt.ylabel('Accuracy',fontsize=16) 22 | plt.title('Accuracy Curves',fontsize=16) 23 | plt.savefig("./data/"+exp_name+"/"+"accuracy_curve.jpg") -------------------------------------------------------------------------------- /src/utils/create_mask.py: -------------------------------------------------------------------------------- 1 | 2 | import buzzard as buzz 3 | 4 | aoi_list = ["aoi1", "aoi2"] 5 | 6 | def create_mask(rgb_path, shp_path, mask_path): 7 | 8 | ds = buzz.DataSource(allow_interpolation=True) 9 | ds.open_raster('rgb', rgb_path) 10 | ds.open_vector('shp', shp_path) 11 | 12 | fp= buzz.Footprint( 13 | tl=ds.rgb.fp.tl, 14 | size=ds.rgb.fp.size, 15 | rsize=ds.rgb.fp.rsize,) 16 | 17 | polygons = ds.shp.iter_data(None) 18 | 19 | mask = fp.burn_polygons(polygons) 20 | mask_tr = mask *255 21 | 22 | with ds.create_raster('mask', mask_path, ds.rgb.fp, 'uint8', 1, 23 | band_schema=None, sr=ds.rgb.proj4_virtual).close: 24 | ds.mask.set_data(mask_tr, band = 1) 25 | 26 | return True 27 | 28 | if __name__ == "__main__": 29 | 30 | for aoi in aoi_list: 31 | 32 | images_dir = "data/datasets/" 33 | rgb_path = images_dir + "images/" + aoi+".tif" 34 | shp_path = images_dir + "shapes/" + aoi+".shp" 35 | mask_path = images_dir + "masks/" + aoi+".tif" 36 | 37 | create_mask(rgb_path, shp_path, mask_path) 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Pradip Gupta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/training/data_loader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | from src.utils.buzz_utility import raster_explorer 4 | import config 5 | 6 | def get_samples(dataset_path): 7 | 8 | train = pd.read_csv(os.path.join(dataset_path, "train.txt"), header=None) 9 | val = pd.read_csv(os.path.join(dataset_path, "validation.txt"), header=None) 10 | test = pd.read_csv(os.path.join(dataset_path, "test.txt"), header=None) 11 | 12 | if not config.perce_aoi_touse == 1.0: 13 | train = train[:int(len(train)*config.perce_aoi_touse)] 14 | val = val[:int(len(val)*config.perce_aoi_touse)] 15 | 16 | print("No of aois:") 17 | print("for train:", len(train)) 18 | print("for validation:", len(val)) 19 | print("for test:", len(test)) 20 | 21 | return train, val, test 22 | 23 | 24 | def build_source(df, dataset_path): 25 | 26 | X = [] 27 | y = [] 28 | 29 | for file in df[0].values: 30 | if not file.startswith("."): 31 | rgb_path = dataset_path + "/images/" + file+".tif" 32 | binary_path = dataset_path + "/masks/" + file+".tif" 33 | 34 | print("\nAdding {} to AOI list".format(file)) 35 | 36 | ds_rgb = raster_explorer(rgb_path, stats=False) 37 | ds_binary = raster_explorer(binary_path, stats=False) 38 | 39 | X.append(ds_rgb) 40 | y.append(ds_binary) 41 | 42 | return X, y -------------------------------------------------------------------------------- /src/training/keras_history.py: -------------------------------------------------------------------------------- 1 | def generate_stats(history, config): 2 | 3 | history_to_save = history.history 4 | 5 | history_to_save['train acc'] = max(history.history['acc']) 6 | history_to_save['train dice_coeff'] = max(history.history['dice_coeff']) 7 | history_to_save['train loss'] = min(history.history['loss']) 8 | history_to_save['last train accuracy'] = history.history['dice_coeff'][-1] 9 | history_to_save['last train loss'] = history.history['loss'][-1] 10 | 11 | history_to_save['val acc'] = max(history.history['val_acc']) 12 | history_to_save['val dice_coeff'] = max(history.history['val_dice_coeff']) 13 | history_to_save['val loss'] = min(history.history['val_loss']) 14 | history_to_save['last val loss'] = history.history['val_loss'][-1] 15 | history_to_save['last val acc'] = history.history['val_dice_coeff'][-1] 16 | 17 | history_to_save['final lr'] = history.history['lr'][-1] 18 | history_to_save['total epochs'] = len(history.history['lr']) 19 | 20 | 21 | history_to_save['downsampling_factor'] = config.down_sampling 22 | history_to_save['initial lr'] = config.learning_rate 23 | history_to_save['optimiser'] = config.optimiser 24 | history_to_save['loss'] = config.loss 25 | history_to_save['metric'] = config.metric 26 | history_to_save['dice_weight'] = config.dice_weight 27 | history_to_save['cross_entropy_weight'] = config.cross_entropy_weight 28 | 29 | 30 | return history_to_save 31 | 32 | -------------------------------------------------------------------------------- /src/utils/gdal_utils.py: -------------------------------------------------------------------------------- 1 | 2 | from osgeo import gdal, ogr, osr 3 | 4 | def raster_to_shp(vector, filename): 5 | """ 6 | This function is a work-in-progress. Not to be used. 7 | """ 8 | 9 | src_ds = gdal.Open(vector) 10 | srs = osr.SpatialReference() 11 | srs.ImportFromWkt(src_ds.GetProjection()) 12 | srcband = src_ds.GetRasterBand(1) 13 | 14 | dst_layername = filename[:-4] 15 | drv = ogr.GetDriverByName("ESRI Shapefile") 16 | dst_ds = drv.CreateDataSource(filename) 17 | 18 | dst_layer = dst_ds.CreateLayer(dst_layername, srs = srs) 19 | 20 | newField = ogr.FieldDefn('polygon', ogr.OFTInteger) 21 | dst_layer.CreateField(newField) 22 | 23 | gdal.Polygonize(srcband, None, dst_layer, 0, [], callback=None ) 24 | 25 | return True 26 | 27 | 28 | def array2D_to_geoJson(geoJsonFileName, array2d, 29 | layerName="BuildingID",fieldName="BuildingID"): 30 | """ 31 | This function is a work-in-progress. Not to be used. 32 | """ 33 | 34 | memdrv = gdal.GetDriverByName('MEM') 35 | src_ds = memdrv.Create('', array2d.shape[1], array2d.shape[0], 1) 36 | band = src_ds.GetRasterBand(1) 37 | band.WriteArray(array2d) 38 | 39 | drv = ogr.GetDriverByName("geojson") 40 | dst_ds = drv.CreateDataSource(geoJsonFileName) 41 | dst_layer = dst_ds.CreateLayer(layerName, srs=None) 42 | 43 | fd = ogr.FieldDefn(fieldName, ogr.OFTInteger) 44 | dst_layer.CreateField(fd) 45 | dst_field = 0 46 | 47 | gdal.Polygonize(band, None, dst_layer, dst_field, [], callback=None) 48 | 49 | return True -------------------------------------------------------------------------------- /src/utils/train_test_splits.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import numpy as np 4 | 5 | base = "/Users/pradip.gupta/jiogis/data/datasets/AerialImageDataset/train/" 6 | config = {"train":0.80, 7 | "validation":0.19, 8 | "test":0.01} 9 | 10 | 11 | def create_splits(base, config): 12 | 13 | phases = ["train", "test", "validation"] 14 | 15 | if os.path.exists(base + "train.txt"): 16 | os.remove(base + "train.txt") 17 | 18 | if os.path.exists(base + "validation.txt"): 19 | os.remove(base + "validation.txt") 20 | 21 | if os.path.exists(base + "test.txt"): 22 | os.remove(base + "test.txt") 23 | 24 | images_list = glob.glob(base + "images/" + "**/*.tif", recursive=True) 25 | aois = [os.path.basename(fname)[:-4] for fname in images_list] 26 | nos = len(aois) 27 | 28 | np.random.shuffle(aois) 29 | 30 | train_size = int(np.ceil(nos*config["train"])) 31 | val_size = int(np.ceil(nos*config["validation"])) 32 | 33 | images = {} 34 | images["train"] = aois[:train_size] 35 | images["validation"] = aois[train_size:(train_size+val_size)] 36 | images["test"] = aois[(train_size+val_size):] 37 | 38 | 39 | for phase in phases: 40 | with open('{}{}.txt'.format(base,phase), 'a') as f: 41 | for image in images[phase]: 42 | f.write('{}\n'.format(image)) 43 | 44 | 45 | if __name__ == "__main__": 46 | 47 | create_splits(base, config) 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /src/utils/post_process.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from skimage import img_as_ubyte 4 | from skimage.morphology import reconstruction as morph_reconstruction 5 | 6 | def morphological_rescontruction(prob_mask, mask_ths, marker_ths): 7 | """Removes noise from a mask. 8 | 9 | Args: 10 | prob_mask: the probability mask to remove pixels having less confidence from. 11 | mask_ths: threshold for creating a binary mask from the probability mask. 12 | marker_ths: threshold for creating a binary mask to be used as a marker in morphological recontruction 13 | 14 | Returns: 15 | The mask after applying morphological reconstruction. 16 | """ 17 | mask = (prob_mask > mask_ths)*1.0 18 | marker = (prob_mask > marker_ths)*1.0 19 | 20 | return morph_reconstruction(marker, mask) 21 | 22 | def denoise(mask, eps=3): 23 | """Removes noise from a mask. 24 | 25 | Args: 26 | mask: the mask to remove noise from. 27 | eps: the morphological operation's kernel size for noise removal, in pixel. 28 | 29 | Returns: 30 | The mask after applying denoising. 31 | """ 32 | 33 | struct = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (eps, eps)) 34 | return cv2.morphologyEx(mask, cv2.MORPH_OPEN, struct) 35 | 36 | 37 | def grow(mask, eps=3): 38 | """Grows a mask to fill in small holes, e.g. to establish connectivity. 39 | 40 | Args: 41 | mask: the mask to grow. 42 | eps: the morphological operation's kernel size for growing, in pixel. 43 | 44 | Returns: 45 | The mask after filling in small holes. 46 | """ 47 | 48 | struct = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (eps, eps)) 49 | return cv2.morphologyEx(mask, cv2.MORPH_CLOSE, struct) 50 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | 2 | dataset_path = "data/datasets/AerialImageDataset/train" 3 | exp_name = "inria_linknet" 4 | 5 | tile_size = 512 6 | down_sampling = 2.0 7 | no_of_gpu = 1 8 | batch_size = 12*no_of_gpu #has to be even no. 9 | 10 | no_of_samples = 700 #no_of_samples_perunit for inria 11 | perce_aoi_touse = 0.1 #decides the no of aois to use for training and validation. 12 | #class_weight = {0: 1.0, 1: 1.0} #0 :Background, 1 : Building 13 | 14 | epochs = 30 15 | patience_lr = 5 16 | factor_lr = 0.75 17 | min_delta = 0.01 18 | patience_es = 10 19 | 20 | dice_weight=0.5 21 | cross_entropy_weight=0.5 22 | 23 | training_frm_scratch = True 24 | training_frm_chkpt = False 25 | transfer_lr = False 26 | trial = False 27 | 28 | if training_frm_scratch: 29 | model = 'linknet' #used for importing from src networks 30 | initial_epoch = 0 31 | optimiser = 'sgd' #enter everything in small letters 32 | loss = 'bce_dice' 33 | metric = 'dice' 34 | learning_rate = 0.001 35 | 36 | if training_frm_chkpt: 37 | initial_epoch = 10 #training starts from 'initial epoch + 1' 38 | model_path = "path/to/checkpoint-10-0.90.h5" 39 | 40 | if fine_tuning: 41 | model = "newmodel" 42 | weights_path="data/pretrained/newmodel/checkpoint-newmodel.h5" 43 | 44 | initial_epoch = 0 45 | 46 | optimiser = 'sgd' #enter everything in small letters 47 | loss = 'wbce_dice' 48 | metric = 'dice' 49 | learning_rate = 0.0001 50 | 51 | if transfer_lr: 52 | model_path = "path/to/checkpoint-10-0.90.h5" 53 | model = "linknet" 54 | initial_epoch = 0 55 | 56 | trainable_layers = list(range(104)) #complete architecture 57 | 58 | optimiser = 'sgd' #enter everything in small letters 59 | loss = 'bce_dice' 60 | metric = 'dice' 61 | learning_rate = 0.0001 62 | 63 | if trial: 64 | print("Trial Mode Activated") 65 | epochs = 3 66 | no_of_samples = 10 67 | perce_aoi_touse = 0.10 68 | -------------------------------------------------------------------------------- /src/utils/load_model.py: -------------------------------------------------------------------------------- 1 | 2 | import os, sys 3 | from unipath import Path 4 | 5 | absolute_path = Path('src/networks/').absolute() 6 | sys.path.append(str(absolute_path)) 7 | 8 | import importlib 9 | import json 10 | from keras.models import model_from_json, load_model 11 | 12 | class LoadModel: 13 | """ 14 | model_dict = {"weights_file":"", 15 | "arch_file":"", 16 | "model_file":""} 17 | 18 | """ 19 | @classmethod 20 | def load(cls, **kwargs): 21 | 22 | if "model_file" in kwargs: 23 | model = cls.load1(kwargs.get("model_file")) 24 | 25 | return model 26 | 27 | elif "model_name" in kwargs: 28 | model = cls.load3(kwargs["model_name"],kwargs["weight_file"]) 29 | 30 | return model 31 | 32 | if kwargs["weight_file"] and kwargs["json_file"]: 33 | model = cls.load2(kwargs["weight_file"], kwargs["json_file"]) 34 | 35 | return model 36 | 37 | @staticmethod 38 | def load1(cls, model_path): 39 | model = load_model(model_path) 40 | 41 | return model 42 | 43 | @classmethod 44 | def load2(cls, weight_file, json_file): 45 | 46 | jsonfile = open(json_file,'r') 47 | loaded_model_json = jsonfile.read() 48 | jsonfile.close() 49 | 50 | model = model_from_json(loaded_model_json) 51 | model.load_weights(weight_file) 52 | 53 | return model 54 | 55 | @classmethod 56 | def load3(cls, model_name, weights_path): 57 | 58 | build = getattr(importlib.import_module("higal_unet_resnet50"),"build") 59 | model = build(256, 3) 60 | model.load_weights(weights_path) 61 | 62 | @classmethod 63 | def load4(cls, custom): 64 | # kwargs.custom = {'bce_dice_loss':bce_dice_loss,'dice_coeff':dice_coeff} 65 | model = load_model("data/pretrained/savera/SAVERA_86_trained_weights.best.hdf5.index", 66 | custom_objects=custom) 67 | 68 | return model 69 | -------------------------------------------------------------------------------- /src/training/keras_callbacks.py: -------------------------------------------------------------------------------- 1 | from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard 2 | import config 3 | 4 | class GPUModelCheckpoint(ModelCheckpoint): 5 | def __init__(self, filepath, org_model, **kwargs): 6 | """ 7 | :param filepath: 8 | :param org_model: Keras model to save instead of the default. 9 | This is used especially when training multi-gpu models built with Keras multi_gpu_model(). 10 | In that case, you would pass the original "template model" to be saved each checkpoint. 11 | :param kwargs: Passed to ModelCheckpoint. 12 | """ 13 | 14 | self.org_model = org_model 15 | super().__init__(filepath, **kwargs) 16 | 17 | def on_epoch_end(self, epoch, logs=None): 18 | model_before = self.model 19 | self.model = self.org_model 20 | super().on_epoch_end(epoch, logs) 21 | self.model = model_before 22 | 23 | 24 | def get_callbacks(model): 25 | 26 | # ============================================================================= 27 | # #Early Stopping 28 | # early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.001, 29 | # patience=config.patience_es, verbose=1, 30 | # mode = "min") 31 | # ============================================================================= 32 | #LR manage 33 | reduce_lr = ReduceLROnPlateau(monitor='val_acc', mode='max', factor=config.factor_lr, 34 | patience=config.patience_lr, min_lr=1e-8, 35 | min_delta=config.min_delta, verbose=1) 36 | 37 | #TB visualzation 38 | tensorb = TensorBoard(log_dir="./data/"+config.exp_name+"/"+"Graph", histogram_freq=0, 39 | write_graph=True, write_images=True) 40 | 41 | #Model Checkpoint 42 | if config.no_of_gpu > 1: 43 | filepath= "./data/"+config.exp_name+"/"+"gpu_checkpoint-{epoch:02d}-{val_acc:.2f}.h5" 44 | checkpoint = GPUModelCheckpoint(filepath, model, monitor='val_acc', mode="max", 45 | save_best_only=False, save_weights_only=False, 46 | verbose=1) 47 | 48 | else: 49 | filepath= "./data/"+config.exp_name+"/"+"checkpoint-{epoch:02d}-{val_acc:.2f}.h5" 50 | checkpoint = ModelCheckpoint(filepath, monitor='val_acc', mode="max", 51 | save_best_only=False, save_weights_only=False, 52 | verbose=1) 53 | 54 | callbacks_list = [checkpoint, reduce_lr, tensorb] 55 | 56 | 57 | return callbacks_list -------------------------------------------------------------------------------- /src/networks/tiramisu.py: -------------------------------------------------------------------------------- 1 | """ 2 | https://github.com/okotaku/kaggle_dsbowl/blob/master/model/tiramisu56.py 3 | """ 4 | 5 | from keras.layers import Input 6 | from keras.layers.core import Dropout, Activation 7 | from keras.layers.convolutional import Conv2D, MaxPooling2D, Conv2DTranspose 8 | from keras.layers.normalization import BatchNormalization 9 | from keras.regularizers import l2 10 | from keras.models import Model 11 | 12 | def _denseBlock(x, layers, filters): 13 | for i in range(layers): 14 | x = BatchNormalization(gamma_regularizer=l2(0.0001), 15 | beta_regularizer=l2(0.0001))(x) 16 | x = Activation('relu')(x) 17 | x = Conv2D(filters, (3, 3), padding='same', 18 | kernel_initializer="he_uniform")(x) 19 | x = Dropout(0.2)(x) 20 | 21 | return x 22 | 23 | 24 | def _transitionDown(x, filters): 25 | x = BatchNormalization(gamma_regularizer=l2(0.0001), 26 | beta_regularizer=l2(0.0001))(x) 27 | x = Activation('relu')(x) 28 | x = Conv2D(filters, (1, 1), padding='same', 29 | kernel_initializer="he_uniform")(x) 30 | x = Dropout(0.2)(x) 31 | x = MaxPooling2D(pool_size=(2, 2))(x) 32 | 33 | return x 34 | 35 | 36 | def _transitionUp(x, filters): 37 | x = Conv2DTranspose(filters, (3, 3), strides=(2, 2), padding='same', 38 | kernel_initializer="he_uniform")(x) 39 | 40 | return x 41 | 42 | 43 | def build(size=256, chs=3, summary=False): 44 | 45 | #Input 46 | inp = Input(shape=(size, size, chs)) 47 | 48 | # Encoder 49 | x = Conv2D(48, kernel_size=(3, 3), padding='same', 50 | input_shape=(size, size, chs), 51 | kernel_initializer="he_uniform", 52 | kernel_regularizer=l2(0.0001))(inp) 53 | 54 | x = _denseBlock(x, 4, 96) # 4*12 = 48 + 48 = 96 55 | x = _transitionDown(x, 96) 56 | x = _denseBlock(x, 4, 144) # 4*12 = 48 + 96 = 144 57 | x = _transitionDown(x, 144) 58 | x = _denseBlock(x, 4, 192) # 4*12 = 48 + 144 = 192 59 | x = _transitionDown(x, 192) 60 | x = _denseBlock(x, 4, 240)# 4*12 = 48 + 192 = 240 61 | x = _transitionDown(x, 240) 62 | x = _denseBlock(x, 4, 288) # 4*12 = 48 + 288 = 336 63 | x = _transitionDown(x, 288) 64 | 65 | #Center 66 | x = _denseBlock(x, 15, 336) # 4 * 12 = 48 + 288 = 336 67 | 68 | #Decoder 69 | x = _transitionUp(x, 384) # m = 288 + 4x12 + 4x12 = 384. 70 | x = _denseBlock(x, 4, 384) 71 | 72 | x = _transitionUp(x, 336) #m = 240 + 4x12 + 4x12 = 336 73 | x = _denseBlock(x, 4, 336) 74 | 75 | x = _transitionUp(x, 288) # m = 192 + 4x12 + 4x12 = 288 76 | x = _denseBlock(x, 4, 288) 77 | 78 | x = _transitionUp(x, 240) # m = 144 + 4x12 + 4x12 = 240 79 | x = _denseBlock(x, 4, 240) 80 | 81 | x = _transitionUp(x, 192) # m = 96 + 4x12 + 4x12 = 192 82 | x = _denseBlock(x, 4, 192) 83 | 84 | #Output 85 | x = Conv2D(1, kernel_size=(1, 1), padding='same', 86 | kernel_initializer="he_uniform", 87 | kernel_regularizer=l2(0.0001))(x) 88 | x = Activation('sigmoid')(x) 89 | 90 | model = Model(inputs=inp, outputs=x) #Trainable params: 42,392,113 91 | 92 | if summary: 93 | model.summary() 94 | 95 | return model 96 | 97 | 98 | -------------------------------------------------------------------------------- /src/utils/image_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import itertools 3 | import PIL 4 | from PIL import Image 5 | import matplotlib.pyplot as plt 6 | import cv2 7 | 8 | def img_to_slices(img, img_rows, img_cols): 9 | """ 10 | Convert image into slices 11 | args: 12 | .img: input image 13 | .img_rows: img_rows of slice 14 | .img_cols: img_rows of slice 15 | return slices, shapes[nb_rows, nb_cols] 16 | """ 17 | nb_rows = img.shape[0] // img_rows 18 | nb_cols = img.shape[1] // img_cols 19 | slices = [] 20 | 21 | # generate img slices 22 | for i, j in itertools.product(range(nb_rows), range(nb_cols)): 23 | slice = img[i * img_rows: i * img_rows + img_rows, 24 | j * img_cols:j * img_cols + img_cols] 25 | slices.append(slice) 26 | 27 | return slices, [nb_rows, nb_cols] 28 | 29 | 30 | def slices_to_img(slices, shapes): 31 | """ 32 | Restore slice into image 33 | args: 34 | slices: image slices 35 | shapes: [nb_rows, nb_cols] of original image 36 | return img 37 | """ 38 | # set img placeholder 39 | if len(slices[0].shape) == 3: 40 | img_rows, img_cols, in_ch = slices[0].shape 41 | img = np.zeros( 42 | (img_rows * shapes[0], img_cols * shapes[1], in_ch), np.uint8) 43 | else: 44 | img_rows, img_cols = slices[0].shape 45 | img = np.zeros((img_rows * shapes[0], img_cols * shapes[1]), np.uint8) 46 | 47 | # merge 48 | for i, j in itertools.product(range(shapes[0]), range(shapes[1])): 49 | img[i * img_rows:i * img_rows + img_rows, 50 | j * img_cols:j * img_cols + img_cols] = slices[i * shapes[1] + j] 51 | 52 | return img 53 | 54 | 55 | def save_np(np_array, filepath): 56 | 57 | im = Image.fromarray(np_array.astype('uint8')) 58 | im.save(filepath) 59 | # matplotlib.image.imsave(filename, np_array) 60 | 61 | return True 62 | 63 | 64 | def show_image(images, labels, preds = None, n=3, figx=10, figy =8): 65 | 66 | l = 3 67 | if preds is None: 68 | l = 2 69 | 70 | f = plt.figure(figsize=(figx, figy), dpi= 80, facecolor='w', edgecolor='k') 71 | f.add_subplot(1,l, 1) 72 | plt.imshow(images[n,:,:,:]) 73 | 74 | f.add_subplot(1,l, 2) 75 | plt.imshow(labels[n,:,:,0], cmap="gray") 76 | 77 | if l==3: 78 | f.add_subplot(1,3, 3) 79 | plt.imshow(preds[n,:,:,0], cmap="gray") 80 | 81 | 82 | def downsample(): 83 | ds = 2 84 | pilimg = Image.open('data/test/rcp2/rcp2.tiff') 85 | #pilimg.show() 86 | 87 | h, w = pilimg.size 88 | pilimg_resized = pilimg.resize((h//ds, w//ds),PIL.Image.LANCZOS) 89 | pilimg_resized.save('data/test/rcp2/rcp2_75.tiff') 90 | 91 | return True 92 | 93 | def plot_hist(image, mode='rgb'): 94 | 95 | if mode=='rgb': 96 | color = ('r','g','b') 97 | else: 98 | color = ('b','g','r') 99 | 100 | for i,col in enumerate(color): 101 | histr = cv2.calcHist([image], [i], None, [256], [0,256]) 102 | plt.plot(histr,color = col) 103 | plt.xlim([0,256]) 104 | plt.show() 105 | 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bfss 2 | Building Footprint Segmentation from Satellite images 3 | 4 | - [Problem Statement](#problem-statement) 5 | - [Project Structure](#project-structure) 6 | - [Project Setup](#project-setup) 7 | - [Project description](#proj-des) 8 | - [Salient features](#salient-fea) 9 | - [To-Do](#to-do) 10 | 11 | 12 | ## 1. Problem Statement 13 | Building extraction from satellite imagery has been a labor-intensive task for many organisations. This is specially true in developing nations (like India) where high resolution satellite images are still far from reach. 14 | 15 | This project was conducted for extracting building footprints for tire-2 cities in India where the resolution for satellite imagery varies from _50cm to 65cm_. As the data is private, the same project flow has been implemented for a public [database](https://project.inria.fr/aerialimagelabeling/) 16 | 17 | 18 | ## 2. Project Structure 19 | 20 | ``` 21 | bfss 22 | ├── train.py 23 | ├── config.py 24 | ├── evaluate.py 25 | ├── src 26 | | ├── training/ 27 | | ├── evaluation/ 28 | | ├── networks/ 29 | | └── utils/ 30 | ├── data 31 | | ├── datasets/AerialImageDataset/ 32 | | └── test/ 33 | ``` 34 | 35 | _Data_:
36 | the `data` folder is not a part of this git project as it was heavy. The same can be downloaded from below link: 37 | 38 | ```sh 39 | https://project.inria.fr/aerialimagelabeling/ 40 | ``` 41 | 42 | 43 | ## 3. Project Setup 44 | To setup the virtual environment for this project do the following steps: 45 | 46 | **Step 1:** ```cd bfss``` #Enter the project folder!
47 | **Step 2:** ```conda env create -f envs/bfss.yml``` #create the virutal environment from the yml file provided.
48 | **Step 3:** ```conda activate bfss``` #activate the virtual env. 49 | 50 | 51 | ## 4. Project description 52 | The training script is `train.py`
53 | 54 | The entire training configuration including the dataset path, hyper-parameters and other arguments are specified in `config.py`, which you can modify and experiment with. It gives a one-shot view of the entire training process.
55 | 56 | The training can be conducted in 3 modes: 57 | - Training from scratch 58 | - Training from checkpoint 59 | - Fine tuning 60 | - Transfer Learning 61 | 62 | Explore `config.py` to learn more about the parameters you can tweak. 63 | 64 | A Notebook has been created for explaining the different training steps:
65 | Jupyter-notebook [LINK](./notebook/training.ipynb) 66 | 67 | 68 | ## 5. Salient Features 69 | 70 | _**Infinity Generator**_ 71 | Random tile generation at run time. This enables us to explode our dataset as random tiling can (at least theoretically) generate infinite unique tiles. 72 | 73 | _**Weighted Loss map**_ 74 | My take on weighted loss map using the concept of signed distance function. Refer to [code](./src/training/metrics.py) for implementation. 75 | 76 | 77 | 78 | ## 6. To-Do 79 | 80 | - [x] Data download and EDA. 81 | - [x] Basic framework for evaluating existing models 82 | - [x] Complete framework with mulitple models tested 83 | - [ ] Pre and Post Processing techniques (on-going) 84 | - [x] Transfer Learning framework 85 | - [ ] Model training (On-Going) 86 | - [ ] Conditional Random Field (CRF) for post-processing 87 | -------------------------------------------------------------------------------- /src/training/augmentation.py: -------------------------------------------------------------------------------- 1 | import imgaug as ia 2 | from imgaug import augmenters as iaa 3 | import numpy as np 4 | 5 | 6 | class SegmentationAugmentation(): 7 | def __init__(self, seed=None): 8 | self.random_seed = seed 9 | 10 | sometimes = lambda aug: iaa.Sometimes(0.6, aug) 11 | 12 | self.sequence = iaa.Sequential( 13 | [ 14 | # Apply the following augmenters to most images. 15 | iaa.Fliplr(0.5), 16 | iaa.Flipud(0.5), # vertically flip 50% of all images 17 | 18 | # Apply affine transformations to some of the images 19 | sometimes(iaa.Affine(rotate=(-10, 10), 20 | scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis 21 | mode='symmetric', cval=(0))), 22 | 23 | iaa.SomeOf((0, 2), 24 | [ 25 | iaa.Multiply((0.75, 1.25)), 26 | 27 | iaa.AddToHueAndSaturation((-10, 10)), 28 | 29 | iaa.ContrastNormalization((0.75, 1.25)), 30 | 31 | # Sharpen each image, overlay the result with the original 32 | # image using an alpha between 0 (no sharpening) and 1 33 | # (full sharpening effect). 34 | iaa.Sharpen(alpha=(0.2, 0.8), lightness=(0.75, 1.5)), 35 | 36 | ], random_order=True) 37 | 38 | ], 39 | random_order=True) 40 | 41 | @staticmethod 42 | def np2img(np_array): 43 | return np.clip(np_array, 0, 255)[None,:,:,:] 44 | 45 | @staticmethod 46 | def np2segmap(np_array, n_classes=1): 47 | return ia.SegmentationMapOnImage(np_array, shape=np_array.shape, nb_classes=n_classes + 1) 48 | 49 | @staticmethod 50 | def segmap2np(segmap): 51 | return segmap.get_arr_int()[:,:,None] 52 | 53 | @staticmethod 54 | def img2np(img): 55 | return img[0] 56 | 57 | def augment_img_and_segmap(self, img, segmap): 58 | 59 | sequence = self.sequence.to_deterministic() 60 | 61 | aug_img = sequence.augment_images(img) 62 | aug_segmap = sequence.augment_segmentation_maps([segmap])[0] 63 | 64 | return aug_img, aug_segmap 65 | 66 | 67 | def run(self, images, segmaps, n_classes=1): 68 | 69 | aug_images = [] 70 | aug_segmaps = [] 71 | 72 | for i in range(len(images)): 73 | 74 | img = self.np2img(images[i]) 75 | segmap = self.np2segmap(segmaps[i], n_classes) 76 | 77 | aug_img, aug_segmap = self.augment_img_and_segmap(img, segmap) 78 | 79 | aug_images.append(self.img2np(aug_img)) 80 | aug_segmaps.append(self.segmap2np(aug_segmap)) 81 | 82 | return aug_images, aug_segmaps 83 | 84 | 85 | def run_single(self, image, mask): 86 | aug_images, aug_masks = self.run(np.array([image]), 87 | np.array([mask])) 88 | return aug_images[0], aug_masks[0] -------------------------------------------------------------------------------- /src/networks/linknet.py: -------------------------------------------------------------------------------- 1 | """ 2 | https://www.kaggle.com/kmader/keras-linknet 3 | https://github.com/okotaku/kaggle_dsbowl/blob/master/model/linknet.py 4 | """ 5 | 6 | from keras.models import Model 7 | from keras.layers import Input, Conv2D, Deconv2D, MaxPool2D, concatenate, AvgPool2D 8 | from keras.layers import BatchNormalization, Activation 9 | from keras.layers.core import Dropout 10 | from keras import backend as K 11 | from keras.regularizers import l2 12 | from keras.layers import add 13 | 14 | 15 | s_c2 = lambda fc, k, s = 1, activation='elu', **kwargs: Conv2D(fc, kernel_size = (k,k), strides= (s,s), 16 | padding = 'same', activation = activation, 17 | **kwargs) 18 | 19 | 20 | s_d2 = lambda fc, k, s = 1, activation='elu', **kwargs: Deconv2D(fc, kernel_size=(k,k), strides=(s,s), 21 | padding = 'same', activation=activation, 22 | **kwargs) 23 | 24 | 25 | c2 = lambda fc, k, s = 1, **kwargs: lambda x: Activation('elu')(BatchNormalization()( 26 | Conv2D(fc, kernel_size = (k,k), strides= (s,s), 27 | padding = 'same', activation = 'linear', **kwargs)(x))) 28 | 29 | 30 | d2 = lambda fc, k, s = 1, **kwargs: lambda x: Activation('elu')(BatchNormalization()( 31 | Deconv2D(fc, kernel_size=(k,k), strides=(s,s), 32 | padding = 'same', activation='linear', **kwargs)(x))) 33 | 34 | def _shortcut(input, residual): 35 | """Adds a shortcut between input and residual block and merges them with "sum" 36 | """ 37 | # Expand channels of shortcut to match residual. 38 | # Stride appropriately to match residual (width, height) 39 | # Should be int if network architecture is correctly configured. 40 | input_shape = K.int_shape(input) 41 | residual_shape = K.int_shape(residual) 42 | stride_width = int(round(input_shape[1] / residual_shape[1])) 43 | stride_height = int(round(input_shape[2] / residual_shape[2])) 44 | equal_channels = input_shape[3] == residual_shape[3] 45 | 46 | shortcut = input 47 | # 1 X 1 conv if shape is different. Else identity. 48 | if stride_width > 1 or stride_height > 1 or not equal_channels: 49 | shortcut = Conv2D(filters=residual_shape[3], 50 | kernel_size=(1, 1), 51 | strides=(stride_width, stride_height), 52 | padding="valid", 53 | kernel_initializer="he_normal", 54 | kernel_regularizer=l2(0.0001))(input) 55 | 56 | return add([shortcut, residual]) 57 | 58 | 59 | def enc_block(m, n): 60 | def block_func(x): 61 | cx = c2(n, 3)(c2(n, 3, 2)(x)) 62 | cs1 = concatenate([AvgPool2D((2,2))(x), 63 | cx]) 64 | cs2 = c2(n, 3)(c2(n, 3)(cs1)) 65 | return concatenate([cs2, cs1]) 66 | return block_func 67 | 68 | 69 | def dec_block(m, n): 70 | def block_func(x): 71 | cx1 = c2(m//4, 1)(x) 72 | cx2 = d2(m//4, 3, 2)(cx1) 73 | return Dropout(0.1)(c2(n, 1)(cx2)) 74 | return block_func 75 | 76 | 77 | def build(size=256, chs=3, summary=False): 78 | 79 | start_in = Input((size, size, chs), name = 'Input') 80 | in_filt = c2(64, 7, 2)(start_in) 81 | in_mp = MaxPool2D((3,3), strides = (2,2), padding = 'same')(in_filt) 82 | 83 | enc1 = enc_block(64, 64)(in_mp) 84 | enc2 = enc_block(64, 128)(enc1) 85 | 86 | dec2 = dec_block(64, 128)(enc2) 87 | dec2_cat = _shortcut(enc1, dec2) 88 | dec1 = dec_block(64, 64)(dec2_cat) 89 | 90 | last_out = _shortcut(dec1, in_mp) 91 | 92 | out_upconv = d2(32, 3, 2)(last_out) 93 | out_conv = c2(32, 3)(out_upconv) 94 | out = s_d2(1, 2, 2, activation = 'sigmoid')(out_conv) 95 | 96 | model = Model(inputs = [start_in], outputs = [out]) #Trainable params: 1,151,297 97 | 98 | return model -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | from tqdm import tqdm 4 | 5 | ## Standard imports 6 | import buzzard as buzz 7 | 8 | ## Custom imports 9 | from src.utils.load_model import LoadModel 10 | from src.utils.infer import predict_from_file 11 | from src.utils.post_process import morphological_rescontruction, grow 12 | 13 | def parse_args(): 14 | """Evaluation options for INRIA building segmentation model""" 15 | parser = argparse.ArgumentParser(description='JioGIS \ 16 | Segmentation') 17 | # model and dataset 18 | parser.add_argument('--weights', type=str, default='data/best_inria_weights.h5', 19 | help='weights file (default: data/best_inria_weights.h5)') 20 | parser.add_argument('--data_dir', type=str, default='../../gis/data/datasets/annotated_data/images/', 21 | help='data directory (default: data/datasets/annotated_data/images/)') 22 | parser.add_argument('--aoi_file', type=str, default='dimapur.tif', 23 | help='AoI filename (default: dimapur.tif)') 24 | parser.add_argument('--results_dir', type=str, default='data/test/geoJSON/', 25 | help='result directory (default: data/test/geoJSON/)') 26 | # test hyper params 27 | parser.add_argument('--downsample', type=int, default=1, 28 | help='downsampling factor (default:1)') 29 | parser.add_argument('--tile_size', type=int, default=128, 30 | help='tile size for cropping the file (default: 128)') 31 | parser.add_argument('--threshold', type=float, default=0.5, metavar='ths', 32 | help='threshold for converting probs into mask (default: 0.5)') 33 | #Post processing boolean 34 | parser.add_argument('--post_process', type=int, default=1, 35 | help='post process boolean: change to 1 for post-processing the output (default:1)') 36 | parser.add_argument('--overlap_factor', type=int, default=1, 37 | help='overlap factor for tiles (default:1)') 38 | # the parser 39 | args = parser.parse_args() 40 | print(args) 41 | return args 42 | 43 | class Evaluator(object): 44 | def __init__(self, args): 45 | self.args = args 46 | self.model = LoadModel.load(model_file = self.args.weights) 47 | 48 | self.rgb_path = self.args.data_dir + args.aoi_file 49 | self.ds_rgb = buzz.DataSource(allow_interpolation=True) 50 | self.ds_rgb.open_raster('rgb', self.rgb_path) 51 | 52 | self.downs = self.args.downsample 53 | self.tile_size = self.args.tile_size 54 | self.threshold = self.args.threshold 55 | 56 | def pre_process(self, image): 57 | image = image/127.5-1.0 58 | return image 59 | 60 | def generate_preds(self, post_process=0): 61 | predicted_binary, fp = predict_from_file(self.rgb_path, self.model, self.pre_process, downsampling_factor=self.downs, tile_size=self.tile_size, overlap_factor=self.args.overlap_factor) 62 | if post_process==1: 63 | print("Post-processing started...") 64 | morph_pred = morphological_rescontruction(predicted_binary, 0.5, 0.7) 65 | dilated_pred = grow(morph_pred, 3) 66 | predicted_binary = dilated_pred 67 | print("Post-processing complete...") 68 | return predicted_binary, fp 69 | 70 | def convert_to_polygons(self, predicted_binary, fp): 71 | predicted_mask = (predicted_binary > self.threshold)*255 72 | poly = fp.find_polygons(predicted_mask) 73 | ds = buzz.DataSource(allow_interpolation=True) 74 | self.geojson_path = args.results_dir + self.args.aoi_file[:-4] +"_ds"+str(self.downs)+str("_ths_")+str(self.threshold)+str("_inria_post_process")+'.geojson' 75 | ds.create_vector('dst', self.geojson_path, 'polygon', driver='GeoJSON') 76 | for i in tqdm(range(len(poly))): 77 | ds.dst.insert_data(poly[i]) 78 | ds.dst.close() 79 | 80 | if __name__ == "__main__": 81 | args = parse_args() 82 | print('Evaluating model on ', args.aoi_file) 83 | evaluator = Evaluator(args) 84 | predicted_binary, fp = evaluator.generate_preds(args.post_process) 85 | print("Converting prediction mask into geoJSON polygons...") 86 | evaluator.convert_to_polygons(predicted_binary, fp) 87 | print("File saved at ", evaluator.geojson_path) 88 | -------------------------------------------------------------------------------- /src/training/metrics.py: -------------------------------------------------------------------------------- 1 | """ 2 | To understand mode on jaquard loss and metrics, follow the link below: 3 | https://www.jeremyjordan.me/semantic-segmentation/ 4 | 5 | To know more on distance_transform: 6 | 1. https://github.com/gunpowder78/PReMVOS/blob/a2d8160560509cac7e6d270ad63b2e1d6a1072c6/code/refinement_net/datasets/util/DistanceTransform.py 7 | 8 | To know more on weighted_bce_loss: 9 | 1. https://www.tensorflow.org/api_docs/python/tf/nn/weighted_cross_entropy_with_logits 10 | 11 | """ 12 | 13 | import numpy as np 14 | from keras import backend as K 15 | import tensorflow as tf 16 | from scipy.ndimage.morphology import distance_transform_edt 17 | import config 18 | from keras.losses import binary_crossentropy 19 | 20 | def dice_coeff(y_true, y_pred, eps=K.epsilon()): 21 | 22 | if np.max(y_true) == 0.0: 23 | return dice_coeff(1-y_true, 1-y_pred) ## empty image; calc IoU of zeros 24 | 25 | y_true_f = K.flatten(y_true) 26 | y_pred_f = K.flatten(y_pred) 27 | 28 | intersection = K.sum(y_true_f * y_pred_f) 29 | union = K.sum(y_true_f) + K.sum(y_pred_f) - intersection 30 | 31 | dice = (intersection + eps) / (union + eps) 32 | 33 | return dice 34 | 35 | 36 | def dice_loss(y_true, y_pred, eps=1e-6): 37 | 38 | dloss = 1 - dice_coeff(y_true, y_pred) 39 | 40 | return dloss 41 | 42 | 43 | def bce_loss(y_true, y_pred): 44 | return binary_crossentropy(y_true, y_pred) 45 | 46 | 47 | def bce_dice_loss(y_true, y_pred): 48 | 49 | if not tf.contrib.framework.is_tensor(y_true): 50 | y_true = tf.convert_to_tensor(y_true, dtype=tf.float32) 51 | 52 | if not tf.contrib.framework.is_tensor(y_pred): 53 | y_pred = tf.convert_to_tensor(y_pred, dtype=tf.float32) 54 | 55 | loss = bce_loss(y_true, y_pred)*config.cross_entropy_weight + \ 56 | dice_loss(y_true, y_pred)*config.dice_weight 57 | 58 | return loss 59 | 60 | 61 | def weight_map(masks): 62 | weight_maps = tf.map_fn(lambda mask: weight_map_tf(mask), masks,dtype="float32") 63 | return weight_maps 64 | 65 | 66 | def weight_map_tf(mask): 67 | weight_map = tf.py_func(weight_map_np, [mask], tf.float32) 68 | weight_map.set_shape(mask.get_shape()) 69 | return weight_map 70 | 71 | 72 | def weight_map_np(mask): 73 | 74 | mask = np.squeeze(mask) 75 | 76 | if np.max(mask) == 0.0: 77 | weight_map = np.ones_like(mask) 78 | weight_map = np.expand_dims(weight_map,axis=-1) 79 | else: 80 | distances_bg = distance_transform_edt(1-mask) 81 | distances_bg = np.clip(distances_bg,0,30) 82 | distances_bg_norm = (distances_bg - np.min(distances_bg))/(np.max(distances_bg) - np.min(distances_bg)) 83 | inv_distances_bg = 1. - distances_bg_norm 84 | 85 | weight_map = np.ones_like(mask) 86 | w0 = np.sum(weight_map) 87 | weight_map = 5.*inv_distances_bg 88 | weight_map = np.clip(weight_map, 0.1,5) 89 | w1 = np.sum(weight_map) 90 | weight_map *= (w0 / w1) 91 | weight_map = np.expand_dims(weight_map,axis=-1) 92 | 93 | return weight_map.astype("float32") 94 | 95 | 96 | def weighted_bce_loss(y_true, y_pred, weight): 97 | 98 | # avoiding overflow 99 | epsilon = 1e-7 100 | y_pred = K.clip(y_pred, epsilon, 1. - epsilon) 101 | 102 | logit_y_pred = K.log(y_pred / (1. - y_pred)) 103 | 104 | # https://www.tensorflow.org/api_docs/python/tf/nn/weighted_cross_entropy_with_logits 105 | loss = (1. - y_true) * logit_y_pred + \ 106 | (1. + (weight - 1.) * y_true) * \ 107 | (K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.)) 108 | 109 | return K.sum(loss) / K.sum(weight) 110 | 111 | 112 | def wbce_dice_loss(y_true, y_pred): 113 | 114 | if not tf.contrib.framework.is_tensor(y_true): 115 | y_true = tf.convert_to_tensor(y_true, dtype=tf.float32) 116 | 117 | if not tf.contrib.framework.is_tensor(y_pred): 118 | y_pred = tf.convert_to_tensor(y_pred, dtype=tf.float32) 119 | 120 | weights = weight_map(y_true) 121 | 122 | loss = weighted_bce_loss(y_true, y_pred, weights)*config.cross_entropy_weight + \ 123 | dice_loss(y_true, y_pred)*config.dice_weight 124 | 125 | return loss 126 | 127 | 128 | -------------------------------------------------------------------------------- /src/training/training_modes.py: -------------------------------------------------------------------------------- 1 | 2 | #Standard imports 3 | from keras.models import load_model 4 | import importlib 5 | from keras.utils import multi_gpu_model 6 | import tensorflow as tf 7 | 8 | #Custom imports 9 | import config 10 | from src.training.metrics import bce_dice_loss, dice_coeff 11 | from src.training.modeller import finetune_model 12 | 13 | def training_scratch(optimiser_class, loss_class, metric_class): 14 | print("Training from scratch") 15 | 16 | optimizer = optimiser_class[config.optimiser][0](lr=config.learning_rate, 17 | **optimiser_class[config.optimiser][1]) 18 | loss = loss_class[config.loss] 19 | metric = metric_class[config.metric] 20 | 21 | if config.no_of_gpu > 1: 22 | print("Running in multi-gpu mode") 23 | with tf.device('/cpu:0'): 24 | build = getattr(importlib.import_module(config.model),"build") 25 | model = build(size = config.tile_size, chs = 3) 26 | 27 | gpu_model = multi_gpu_model(model, gpus = config.no_of_gpu) 28 | gpu_model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy']) 29 | model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy']) 30 | else: 31 | build = getattr(importlib.import_module(config.model),"build") 32 | model = build(size = config.tile_size, chs = 3) 33 | model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy']) 34 | gpu_model = None 35 | 36 | return model, gpu_model 37 | 38 | 39 | def training_checkpoint(): 40 | print("Training from prv checkpoint") 41 | 42 | #build the model 43 | model_path = config.model_path 44 | 45 | if config.no_of_gpu > 1: 46 | gpu_model = load_model(model_path, 47 | custom_objects={'bce_dice_loss': bce_dice_loss, 48 | 'dice_coeff':dice_coeff}) 49 | else: 50 | build = getattr(importlib.import_module(config.model),"build") 51 | model = build(size = config.tile_size, chs = 3) 52 | model.set_weights(gpu_model.layers[-2].get_weights()) 53 | 54 | return model, gpu_model 55 | 56 | 57 | def fine_tune(optimiser_class, loss_class, metric_class): 58 | print("Fine tuning mode") 59 | 60 | optimizer = optimiser_class[config.optimiser][0](lr=config.learning_rate, 61 | **optimiser_class[config.optimiser][1]) 62 | loss = loss_class[config.loss] 63 | metric = metric_class[config.metric] 64 | 65 | if config.no_of_gpu > 1: 66 | print("Running in multi-gpu mode") 67 | with tf.device('/cpu:0'): 68 | build = getattr(importlib.import_module(config.model),"build") 69 | model = build(input_shape=(config.tile_size, config.tile_size, 3)) 70 | model.load_weights(config.weights_path, by_name=True) 71 | 72 | gpu_model = multi_gpu_model(model, gpus = config.no_of_gpu) 73 | gpu_model.layers[-2].set_weights(model.get_weights()) 74 | gpu_model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy']) 75 | model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy']) 76 | 77 | else: 78 | build = getattr(importlib.import_module(config.model),"build") 79 | model = build(input_shape=(config.tile_size, config.tile_size, 3)) 80 | model.load_weights(config.weights_path, by_name=True) 81 | model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy']) 82 | gpu_model = None 83 | 84 | return model, gpu_model 85 | 86 | 87 | def transfer_learning(optimiser_class, loss_class, metric_class): 88 | print("Transfer Learning mode") 89 | 90 | #build the model 91 | model_path = config.model_path 92 | gpu_model = load_model(model_path, 93 | custom_objects={'bce_dice_loss': bce_dice_loss, 94 | 'dice_coeff':dice_coeff}) 95 | 96 | build = getattr(importlib.import_module(config.model),"build") 97 | model = build(size = config.tile_size, chs = 3) 98 | model.set_weights(gpu_model.layers[-2].get_weights()) 99 | 100 | # #freeze layers for transfer learning & load weights 101 | model = finetune_model(model) 102 | 103 | if config.no_of_gpu > 1: 104 | gpu_model = multi_gpu_model(model, gpus = config.no_of_gpu, cpu_relocation=True) 105 | print("Running in multi-gpu mode") 106 | else: 107 | gpu_model = None 108 | 109 | # ============================================================================= 110 | # #compile the model 111 | # gpu_model = compile_model(gpu_model, lr = config.learning_rate, 112 | # optimiser = optimiser_class[config.optimiser], 113 | # loss = loss_class[config.loss] , 114 | # metric = metric_class[config.metric]) 115 | # ============================================================================= 116 | 117 | return model, gpu_model 118 | -------------------------------------------------------------------------------- /src/training/seg_data_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a utility class for Data Generation in Keras for the specific problem of semantic segmentaion. 3 | The api is heavily inspired from the following links: 4 | https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly 5 | https://github.com/BaptisteLevasseur/Semantic-segmentation-on-buildings/blob/master/generator.py 6 | 7 | """ 8 | 9 | import numpy as np 10 | import keras 11 | import buzzard as buzz 12 | 13 | from src.training.augmentation import SegmentationAugmentation 14 | 15 | #TODO: Steps per epoch to discuss. It is getting defined at 2 places. 16 | class SegDataGenerator(keras.utils.Sequence): 17 | 'Generates data for Keras' 18 | def __init__(self, dataset_path, img_source, mask_source, batch_size=8, no_of_samples = 500, tile_size=128, 19 | downsampling_factor = 1.0, aug=True): 20 | 'Initialization' 21 | self.dataset_path = dataset_path 22 | self.ts = tile_size 23 | self.ds = downsampling_factor 24 | self.batch_size = batch_size 25 | self.no_of_samples = no_of_samples 26 | self.img_source = img_source 27 | self.mask_source = mask_source 28 | self.to_augment = aug 29 | self.augmentation = SegmentationAugmentation() 30 | 31 | def pre_process(self, image_list, isbinary=False): 32 | 33 | new_list = [] 34 | 35 | for i in range(self.batch_size): 36 | 37 | #Step1: normalise 38 | image = image_list[i] 39 | 40 | if isbinary: 41 | image = (image > 0)*abs(image) #binary mask with value 0 or 1 42 | else: 43 | # image = image/127.5-1.0 #normalising the image for values between -1 to +1 44 | image = (image - np.min(image) + 1.0) / (np.max(image) - np.min(image) + 1.0) #normalising the image for values between 0 to +1 45 | new_list.append(image) 46 | 47 | return new_list 48 | 49 | def random_crop(self,ds, tile_size=128, factor=1): 50 | 51 | # Cropping parameters 52 | crop_factor = ds.rgb.fp.rsize/tile_size 53 | crop_size = ds.rgb.fp.size/crop_factor 54 | 55 | # Original footprint 56 | fp = buzz.Footprint( 57 | tl=ds.rgb.fp.tl, 58 | size=ds.rgb.fp.size, 59 | rsize=ds.rgb.fp.rsize , 60 | ) 61 | 62 | min = np.random.randint(fp.tl[0],fp.tl[0] + fp.size[0] - factor*crop_size[0]) #x 63 | max = np.random.randint(fp.tl[1] - fp.size[1] + factor*crop_size[1], fp.tl[1]) #y 64 | 65 | # print(min > fp.tlx, max < fp.tly) 66 | # print("fp.tlx", fp.tlx, "fp.tly", fp.tly, "min , max", min, max) 67 | # New random footprint 68 | tl = np.array([min, max]) 69 | 70 | fp = buzz.Footprint( 71 | tl=tl, 72 | size=crop_size*factor, 73 | rsize=[tile_size, tile_size], 74 | ) 75 | 76 | return fp 77 | 78 | 79 | def __len__(self): 80 | 'Denotes the number of batches per epoch' 81 | steps_per_epoch = int(np.floor((len(self.img_source)*self.no_of_samples) / self.batch_size)) 82 | return steps_per_epoch 83 | 84 | def __getitem__(self, index): 85 | 'Generate one batch of data' 86 | 87 | # Generate indexes of the batch 88 | index = np.random.randint(len(self.img_source)) 89 | 90 | # Generate data 91 | X, y = self.__data_generation(index) 92 | 93 | return X, y 94 | 95 | def __data_generation(self, index): 96 | 'Generates data containing batch_size samples' 97 | 98 | # print(index) 99 | ds_rgb = self.img_source[index] 100 | ds_binary = self.mask_source[index] 101 | 102 | # Initialization 103 | X = [] 104 | y = [] 105 | 106 | if self.to_augment: 107 | batch_size = self.batch_size//2 108 | else: 109 | batch_size = self.batch_size 110 | 111 | for i in range(batch_size): 112 | 113 | while True: 114 | while True: 115 | try: 116 | fp = self.random_crop(ds_rgb, tile_size=self.ts, factor=self.ds) 117 | rgb= ds_rgb.rgb.get_data(band=(1, 2, 3), fp=fp).astype('uint8') 118 | 119 | binary= ds_binary.rgb.get_data(band=(1), fp=fp).astype('uint8') 120 | binary = binary.reshape((self.ts, self.ts,1)) 121 | binary = binary // 255 #converting numpy into binary with 0 or 1 122 | 123 | break 124 | 125 | except: 126 | continue 127 | if np.sum(rgb == 0) < self.ts*self.ts*0.7*3: #if the zero region is less than 70% of total image 128 | break 129 | 130 | X.append(rgb) 131 | y.append(binary) 132 | 133 | if self.to_augment: 134 | aug_images, aug_masks = self.augmentation.run(X, y) 135 | X.extend(aug_images) 136 | y.extend(aug_masks) 137 | 138 | X = self.pre_process(X) 139 | 140 | # print(len(X), len(y)) 141 | 142 | return np.array(X), np.array(y) -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | 2 | import os, sys 3 | os.environ['TF_CPP_MIN_LOG_LEVEL']='3' 4 | sys.path.append(os.path.abspath('./src/networks')) 5 | 6 | #To handel OOM errors 7 | import tensorflow as tf 8 | from keras import backend as K 9 | import keras.backend.tensorflow_backend as ktf 10 | def get_session(): 11 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction= 0.9, 12 | allow_growth=True) 13 | return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 14 | ktf.set_session(get_session()) 15 | 16 | #Standard imports 17 | import pandas as pd 18 | import numpy as np 19 | from keras.optimizers import Adam, RMSprop, Nadam, SGD 20 | 21 | #Custom imports 22 | import config 23 | from src.training import data_loader 24 | from src.training.metrics import bce_dice_loss, dice_coeff 25 | from src.training.seg_data_generator import SegDataGenerator 26 | from src.training.keras_callbacks import get_callbacks 27 | from src.training.training_modes import training_scratch, training_checkpoint, transfer_learning 28 | from src.training.keras_history import generate_stats 29 | from src.training.plots import save_plots 30 | 31 | if __name__ == "__main__": 32 | 33 | dataset_path = config.dataset_path 34 | exp_name = config.exp_name 35 | 36 | train, val, test = data_loader.get_samples(dataset_path) 37 | 38 | print("\nPreparing dataset for Training") 39 | X_train, y_train = data_loader.build_source(train, dataset_path) 40 | 41 | print("\nPreparing dataset for Validation") 42 | X_val, y_val = data_loader.build_source(val, dataset_path) 43 | 44 | #Params 45 | tile_size = config.tile_size 46 | no_of_samples = config.no_of_samples 47 | downs = config.down_sampling 48 | 49 | batch_size = config.batch_size 50 | epochs = config.epochs 51 | initial_epoch = config.initial_epoch 52 | 53 | loss_class = {'bin_cross': 'binary_crossentropy', 54 | 'bce_dice': bce_dice_loss, 55 | 'wbce_dice': wbce_dice_loss} 56 | 57 | metric_class = {'dice':dice_coeff} 58 | 59 | optimiser_class = {'adam': (Adam, {}), 60 | 'nadam': (Nadam, {}), 61 | 'rmsprop': (RMSprop, {}), 62 | 'sgd':(SGD, {'decay':1e-6, 'momentum':0.90, 'nesterov':True})} 63 | 64 | training_frm_scratch = config.training_frm_scratch 65 | training_frm_chkpt = config.training_frm_chkpt 66 | transfer_lr = config.transfer_lr 67 | 68 | if sum((training_frm_scratch, training_frm_chkpt, fine_tuning, transfer_lr)) != 1: 69 | raise Exception("Conflicting training modes") 70 | 71 | #spe = Steps per epoch 72 | train_spe = int(np.floor((len(X_train)*no_of_samples*2) / batch_size)) # factor of 2 bcos of Augmentation 73 | val_spe = int(np.floor((len(X_val)*no_of_samples) / batch_size)) 74 | 75 | 76 | # Initialise generators 77 | train_generator = SegDataGenerator(dataset_path, img_source=X_train, 78 | mask_source=y_train, batch_size= batch_size, 79 | no_of_samples = no_of_samples, tile_size= tile_size, 80 | downsampling_factor = downs, aug=True) 81 | 82 | val_generator = SegDataGenerator(dataset_path, img_source=X_val, 83 | mask_source=y_val, batch_size= batch_size, 84 | no_of_samples = no_of_samples, tile_size= tile_size, 85 | downsampling_factor = downs, aug=False) 86 | 87 | if training_frm_scratch: 88 | model, gpu_model = training_scratch(optimiser_class, loss_class, metric_class) 89 | 90 | elif training_frm_chkpt: 91 | model, gpu_model = training_checkpoint() 92 | 93 | elif fine_tuning: 94 | model, gpu_model = fine_tune(optimiser_class, loss_class, metric_class) 95 | 96 | elif transfer_lr: 97 | model, gpu_model = transfer_learning(optimiser_class, loss_class, metric_class) 98 | 99 | #Print the model params 100 | print("Model training params:") 101 | trainable_count = int(np.sum([K.count_params(p) for p in set(model.trainable_weights)])) 102 | non_trainable_count = int(np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) 103 | params = (trainable_count + non_trainable_count,trainable_count, non_trainable_count) 104 | 105 | print('Total params: {:,}'.format(params[0])) 106 | print('Trainable params: {:,}'.format(params[1])) 107 | print('Non-trainable params: {:,}'.format(params[2])) 108 | 109 | #Set callbacks 110 | callbacks_list = get_callbacks(model) 111 | 112 | # Start/resume training 113 | if config.no_of_gpu > 1: 114 | history = gpu_model.fit_generator(steps_per_epoch= train_spe, 115 | generator=train_generator, 116 | epochs=epochs, 117 | validation_data = val_generator, 118 | validation_steps = val_spe, 119 | initial_epoch = initial_epoch, 120 | callbacks = callbacks_list) 121 | 122 | else: 123 | history = model.fit_generator(steps_per_epoch= train_spe, 124 | generator=train_generator, 125 | epochs=epochs, 126 | validation_data = val_generator, 127 | validation_steps = val_spe, 128 | initial_epoch = initial_epoch, 129 | callbacks = callbacks_list) 130 | 131 | #Save final complete model 132 | filename = "model_ep_"+str(int(epochs))+"_batch_"+str(int(batch_size)) 133 | model.save("./data/"+exp_name+"/"+filename+".h5") 134 | print("Saved complete model file at: ", filename+"_model"+".h5") 135 | 136 | #Save history 137 | history_to_save = generate_stats(history, config) 138 | pd.DataFrame(history_to_save).to_csv("./data/"+exp_name+"/"+filename + "_train_results.csv") 139 | save_plots(history, exp_name) 140 | -------------------------------------------------------------------------------- /envs/bfss.yml: -------------------------------------------------------------------------------- 1 | name: bfss 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - alabaster=0.7.12=py_0 8 | - appnope=0.1.0=py36_1000 9 | - asn1crypto=0.24.0=py36_1003 10 | - astroid=2.1.0=py36_1000 11 | - attrs=18.2.0=py_0 12 | - babel=2.6.0=py_1 13 | - backcall=0.1.0=py_0 14 | - blas=1.0=mkl 15 | - bleach=3.1.0=py_0 16 | - boost-cpp=1.68.0=h6f8c590_1000 17 | - bzip2=1.0.6=h1de35cc_1002 18 | - ca-certificates=2018.11.29=ha4d7672_0 19 | - cairo=1.16.0=h9247486_1000 20 | - certifi=2018.11.29=py36_1000 21 | - cffi=1.12.1=py36h342bebf_0 22 | - chardet=3.0.4=py36_1003 23 | - cloudpickle=0.7.0=py_0 24 | - cryptography=2.5=py36hdbc3d79_1 25 | - curl=7.64.0=heae2a1f_0 26 | - decorator=4.3.2=py_0 27 | - docutils=0.14=py36_1001 28 | - entrypoints=0.3=py36_1000 29 | - expat=2.2.5=h0a44026_1002 30 | - fontconfig=2.13.1=h1e4e890_1000 31 | - freetype=2.9.1=h597ad8a_1005 32 | - freexl=1.0.5=h1de35cc_1002 33 | - gdal=2.4.0=py36h0e3174d_1002 34 | - geos=3.7.1=h0a44026_1000 35 | - geotiff=1.4.3=hce09ea4_1000 36 | - gettext=0.19.8.1=hcca000d_1001 37 | - giflib=5.1.4=h1de35cc_1001 38 | - glib=2.58.2=h2836805_1001 39 | - hdf4=4.2.13=hf3c6af0_1002 40 | - hdf5=1.10.4=nompi_h646315f_1105 41 | - icu=58.2=h0a44026_1000 42 | - idna=2.8=py36_1000 43 | - imagesize=1.1.0=py_0 44 | - intel-openmp=2019.1=144 45 | - ipykernel=5.1.0=py36h24bf2e0_1002 46 | - ipython=7.3.0=py36h24bf2e0_0 47 | - ipython_genutils=0.2.0=py_1 48 | - ipywidgets=7.4.2=py_0 49 | - isort=4.3.4=py36_1000 50 | - jedi=0.13.2=py36_1000 51 | - jinja2=2.10=py_1 52 | - jpeg=9c=h1de35cc_1001 53 | - json-c=0.13.1=h1de35cc_1001 54 | - jsonschema=3.0.0=py36_0 55 | - jupyter=1.0.0=py_1 56 | - jupyter_client=5.2.4=py_1 57 | - jupyter_console=6.0.0=py_0 58 | - jupyter_core=4.4.0=py_0 59 | - kealib=1.4.10=hf5ed860_1002 60 | - keyring=18.0.0=py36_0 61 | - krb5=1.16.3=h24a3359_1000 62 | - lazy-object-proxy=1.3.1=py36h1de35cc_1000 63 | - libcurl=7.64.0=h76de61e_0 64 | - libcxx=7.0.0=h2d50403_2 65 | - libdap4=3.19.1=hae55d67_1000 66 | - libedit=3.1.20170329=hcfe32e1_1001 67 | - libffi=3.2.1=h0a44026_1005 68 | - libgdal=2.4.0=h89caebc_1002 69 | - libgfortran=3.0.1=0 70 | - libiconv=1.15=h1de35cc_1004 71 | - libkml=1.3.0=h71ee1b2_1009 72 | - libnetcdf=4.6.2=h6b88ef6_1001 73 | - libpng=1.6.36=ha441bb4_1000 74 | - libpq=10.6=hbe1e24e_1000 75 | - libsodium=1.0.16=h1de35cc_1001 76 | - libspatialite=4.3.0a=h0cd9627_1026 77 | - libssh2=1.8.0=hf30b1f0_1003 78 | - libtiff=4.0.10=h79f4b77_1001 79 | - libxml2=2.9.8=hf14e9c8_1005 80 | - llvm-meta=7.0.0=0 81 | - markupsafe=1.1.0=py36h1de35cc_1000 82 | - mccabe=0.6.1=py_1 83 | - mistune=0.8.4=py36h1de35cc_1000 84 | - mkl=2019.1=144 85 | - mkl_fft=1.0.10=py36h1de35cc_1 86 | - mkl_random=1.0.2=py36h1702cab_2 87 | - nbconvert=5.3.1=py_1 88 | - nbformat=4.4.0=py_1 89 | - ncurses=6.1=h0a44026_1002 90 | - ninja=1.9.0=h04f5b5a_0 91 | - notebook=5.7.4=py36_1000 92 | - numpy=1.15.4=py36hacdab7b_0 93 | - numpy-base=1.15.4=py36h6575580_0 94 | - numpydoc=0.8.0=py_1 95 | - olefile=0.46=py_0 96 | - openjpeg=2.3.0=h3bf0609_1003 97 | - openssl=1.0.2q=h1de35cc_0 98 | - packaging=19.0=py_0 99 | - pandoc=2.6=1 100 | - pandocfilters=1.4.2=py_1 101 | - parso=0.3.4=py_0 102 | - pcre=8.41=h0a44026_1003 103 | - pexpect=4.6.0=py36_1000 104 | - pickleshare=0.7.5=py36_1000 105 | - pillow=5.4.1=py36hbddbef0_1000 106 | - pip=19.0.3=py36_0 107 | - pixman=0.34.0=h1de35cc_1003 108 | - poppler=0.67.0=hb974355_6 109 | - poppler-data=0.4.9=1 110 | - postgresql=10.6=ha1bbaa7_1000 111 | - proj4=5.2.0=h1de35cc_1001 112 | - prometheus_client=0.6.0=py_0 113 | - prompt_toolkit=2.0.9=py_0 114 | - psutil=5.5.1=py36h1de35cc_0 115 | - ptyprocess=0.6.0=py36_1000 116 | - pycodestyle=2.5.0=py_0 117 | - pycparser=2.19=py_0 118 | - pyflakes=2.1.0=py_0 119 | - pygments=2.3.1=py_0 120 | - pylint=2.2.2=py36_1000 121 | - pyopenssl=19.0.0=py36_0 122 | - pyparsing=2.3.1=py_0 123 | - pyqt=5.6.0=py36hc26a216_1008 124 | - pyrsistent=0.14.11=py36h1de35cc_0 125 | - pyshp=2.1.0=py_0 126 | - pysocks=1.6.8=py36_1002 127 | - python=3.6.7=h4a56312_1002 128 | - python-dateutil=2.8.0=py_0 129 | - python.app=1.2=py36h1de35cc_1200 130 | - pytorch=1.0.1=py3.6_2 131 | - pytz=2018.9=py_0 132 | - pyzmq=18.0.0=py36h4cc6ddd_0 133 | - qt=5.6.2=h822fa55_1013 134 | - qtawesome=0.5.6=pyh8a2030e_0 135 | - qtconsole=4.4.3=py_0 136 | - qtpy=1.6.0=pyh8a2030e_0 137 | - readline=7.0=hcfe32e1_1001 138 | - requests=2.21.0=py36_1000 139 | - rope=0.10.7=py_1 140 | - send2trash=1.5.0=py_0 141 | - setuptools=40.8.0=py36_0 142 | - shapely=1.6.4=py36h2bcc7ef_1002 143 | - sip=4.18.1=py36h0a44026_1000 144 | - six=1.12.0=py36_1000 145 | - snowballstemmer=1.2.1=py_1 146 | - sphinx=1.8.4=py36_0 147 | - sphinxcontrib-websupport=1.1.0=py_1 148 | - spyder=3.3.3=py36_0 149 | - spyder-kernels=0.4.2=py36_0 150 | - sqlite=3.26.0=h1765d9f_1000 151 | - terminado=0.8.1=py36_1001 152 | - testpath=0.4.2=py36_1000 153 | - tifffile=0.15.1=py36h917ab60_1001 154 | - tk=8.6.9=ha441bb4_1000 155 | - torchvision=0.2.1=py_2 156 | - tornado=5.1.1=py36h1de35cc_1000 157 | - traitlets=4.3.2=py36_1000 158 | - typed-ast=1.3.1=py36h1de35cc_0 159 | - tzcode=2018g=h1de35cc_1001 160 | - urllib3=1.24.1=py36_1000 161 | - wcwidth=0.1.7=py_1 162 | - webencodings=0.5.1=py_1 163 | - wheel=0.33.1=py36_0 164 | - widgetsnbextension=3.4.2=py36_1000 165 | - wrapt=1.11.1=py36h1de35cc_0 166 | - wurlitzer=1.0.2=py36_1000 167 | - xerces-c=3.2.2=h44e365a_1001 168 | - xz=5.2.4=h1de35cc_1001 169 | - zeromq=4.2.5=h0a44026_1006 170 | - zlib=1.2.11=h1de35cc_1004 171 | - pip: 172 | - absl-py==0.7.0 173 | - affine==2.2.2 174 | - astor==0.7.1 175 | - buzzard==0.4.4 176 | - click==7.0 177 | - click-plugins==1.0.4 178 | - cligj==0.5.0 179 | - cycler==0.10.0 180 | - dask==1.1.2 181 | - gast==0.2.2 182 | - grpcio==1.18.0 183 | - h5py==2.9.0 184 | - imageio==2.5.0 185 | - keras==2.2.3 186 | - keras-applications==1.0.7 187 | - keras-preprocessing==1.0.9 188 | - kiwisolver==1.0.1 189 | - markdown==3.0.1 190 | - matplotlib==3.0.2 191 | - networkx==2.2 192 | - opencv-python==4.0.0.21 193 | - pint==0.9 194 | - protobuf==3.6.1 195 | - pywavelets==1.0.1 196 | - pyyaml==3.13 197 | - rasterio==1.0.18 198 | - scikit-image==0.14.2 199 | - scipy==1.2.1 200 | - snuggs==1.4.2 201 | - tensorboard==1.12.2 202 | - tensorflow==1.12.0 203 | - termcolor==1.1.0 204 | - toolz==0.9.0 205 | - tqdm==4.31.1 206 | - unipath==1.1 207 | - werkzeug==0.14.1 208 | prefix: /anaconda3/envs/bfss 209 | -------------------------------------------------------------------------------- /src/utils/parallel_model2.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | import keras.backend as K 4 | import keras.layers as KL 5 | import keras.models as KM 6 | 7 | def make_parallel(keras_model, gpu_count): 8 | """Creates a new wrapper model that consists of multiple replicas of 9 | the original model placed on different GPUs. 10 | Args: 11 | keras_model: the input model to replicate on multiple gpus 12 | gpu_count: the number of replicas to build 13 | Returns: 14 | Multi-gpu model 15 | """ 16 | # Slice inputs. Slice inputs on the CPU to avoid sending a copy 17 | # of the full inputs to all GPUs. Saves on bandwidth and memory. 18 | input_slices = {name: tf.split(x, gpu_count) 19 | for name, x in zip(keras_model.input_names, 20 | keras_model.inputs)} 21 | 22 | output_names = keras_model.output_names 23 | outputs_all = [] 24 | for i in range(len(keras_model.outputs)): 25 | outputs_all.append([]) 26 | 27 | # Run the model call() on each GPU to place the ops there 28 | for i in range(gpu_count): 29 | with tf.device('/gpu:%d' % i): 30 | with tf.name_scope('tower_%d' % i): 31 | # Run a slice of inputs through this replica 32 | zipped_inputs = zip(keras_model.input_names, 33 | keras_model.inputs) 34 | inputs = [ 35 | KL.Lambda(lambda s: input_slices[name][i], 36 | output_shape=lambda s: (None,) + s[1:])(tensor) 37 | for name, tensor in zipped_inputs] 38 | # Create the model replica and get the outputs 39 | outputs = keras_model(inputs) 40 | if not isinstance(outputs, list): 41 | outputs = [outputs] 42 | # Save the outputs for merging back together later 43 | for l, o in enumerate(outputs): 44 | outputs_all[l].append(o) 45 | 46 | # Merge outputs on CPU 47 | with tf.device('/cpu:0'): 48 | merged = [] 49 | for outputs, name in zip(outputs_all, output_names): 50 | # Concatenate or average outputs? 51 | # Outputs usually have a batch dimension and we concatenate 52 | # across it. If they don't, then the output is likely a loss 53 | # or a metric value that gets averaged across the batch. 54 | # Keras expects losses and metrics to be scalars. 55 | if K.int_shape(outputs[0]) == (): 56 | # Average 57 | m = KL.Lambda(lambda o: tf.add_n( 58 | o) / len(outputs), name=name)(outputs) 59 | else: 60 | # Concatenate 61 | m = KL.Concatenate(axis=0, name=name)(outputs) 62 | merged.append(m) 63 | return merged 64 | 65 | 66 | class ParallelModel(KM.Model): 67 | """Subclasses the standard Keras Model and adds multi-GPU support. 68 | It works by creating a copy of the model on each GPU. Then it slices 69 | the inputs and sends a slice to each copy of the model, and then 70 | merges the outputs together and applies the loss on the combined 71 | outputs. 72 | """ 73 | 74 | def __init__(self, keras_model, gpu_count): 75 | """Class constructor. 76 | keras_model: The Keras model to parallelize 77 | gpu_count: Number of GPUs. Must be > 1 78 | """ 79 | merged_outputs = make_parallel( 80 | keras_model=keras_model, gpu_count=gpu_count) 81 | super(ParallelModel, self).__init__(inputs=keras_model.inputs, 82 | outputs=merged_outputs) 83 | self.inner_model = keras_model 84 | 85 | def __getattribute__(self, attrname): 86 | """Redirect loading and saving methods to the inner model. That's where 87 | the weights are stored.""" 88 | if 'load' in attrname or 'save' in attrname: 89 | return getattr(self.inner_model, attrname) 90 | return super(ParallelModel, self).__getattribute__(attrname) 91 | 92 | def summary(self, *args, **kwargs): 93 | """Override summary() to display summaries of both, the wrapper 94 | and inner models.""" 95 | super(ParallelModel, self).summary(*args, **kwargs) 96 | self.inner_model.summary(*args, **kwargs) 97 | 98 | 99 | if __name__ == "__main__": 100 | # Testing code below. It creates a simple model to train on MNIST and 101 | # tries to run it on 2 GPUs. It saves the graph so it can be viewed 102 | # in TensorBoard. Run it as: 103 | # 104 | # python3 parallel_model.py 105 | 106 | import os 107 | import numpy as np 108 | import keras.optimizers 109 | from keras.datasets import mnist 110 | from keras.preprocessing.image import ImageDataGenerator 111 | 112 | GPU_COUNT = 2 113 | 114 | # Root directory of the project 115 | ROOT_DIR = os.path.abspath("../") 116 | 117 | # Directory to save logs and trained model 118 | MODEL_DIR = os.path.join(ROOT_DIR, "logs") 119 | 120 | def build_model(x_train, num_classes): 121 | # Reset default graph. Keras leaves old ops in the graph, 122 | # which are ignored for execution but clutter graph 123 | # visualization in TensorBoard. 124 | tf.reset_default_graph() 125 | 126 | inputs = KL.Input(shape=x_train.shape[1:], name="input_image") 127 | x = KL.Conv2D(32, (3, 3), activation='relu', padding="same", 128 | name="conv1")(inputs) 129 | x = KL.Conv2D(64, (3, 3), activation='relu', padding="same", 130 | name="conv2")(x) 131 | x = KL.MaxPooling2D(pool_size=(2, 2), name="pool1")(x) 132 | x = KL.Flatten(name="flat1")(x) 133 | x = KL.Dense(128, activation='relu', name="dense1")(x) 134 | x = KL.Dense(num_classes, activation='softmax', name="dense2")(x) 135 | 136 | model = KM.Model(inputs=inputs, outputs=x) 137 | 138 | return model 139 | 140 | # Load MNIST Data 141 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 142 | x_train = np.expand_dims(x_train, -1).astype('float32') / 255 143 | x_test = np.expand_dims(x_test, -1).astype('float32') / 255 144 | 145 | print('x_train shape:', x_train.shape) 146 | print('x_test shape:', x_test.shape) 147 | 148 | # Build data generator and model 149 | datagen = ImageDataGenerator() 150 | model = build_model(x_train, 10) 151 | 152 | # Add multi-GPU support. 153 | print("Adding multi-gpu support") 154 | model = ParallelModel(model, GPU_COUNT) 155 | print("Added") 156 | 157 | optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=5.0) 158 | 159 | model.compile(loss='sparse_categorical_crossentropy', 160 | optimizer=optimizer, metrics=['accuracy']) 161 | 162 | model.summary() 163 | 164 | # Train 165 | model.fit_generator( 166 | datagen.flow(x_train, y_train, batch_size=64), 167 | steps_per_epoch=50, epochs=10, verbose=1, 168 | validation_data=(x_test, y_test), 169 | callbacks=[keras.callbacks.TensorBoard(log_dir=MODEL_DIR, 170 | write_graph=True)] 171 | ) -------------------------------------------------------------------------------- /src/utils/parallel_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Multi-GPU Support for Keras. 3 | 4 | Copyright (c) 2017 Matterport, Inc. 5 | Licensed under the MIT License (see LICENSE for details) 6 | Written by Waleed Abdulla 7 | 8 | Ideas and a small code snippets from these sources: 9 | https://github.com/fchollet/keras/issues/2436 10 | https://medium.com/@kuza55/transparent-multi-gpu-training-on-tensorflow-with-keras-8b0016fd9012 11 | https://github.com/avolkov1/keras_experiments/blob/master/keras_exp/multigpu/ 12 | https://github.com/fchollet/keras/blob/master/keras/utils/training_utils.py 13 | """ 14 | 15 | import tensorflow as tf 16 | import keras.backend as K 17 | import keras.layers as KL 18 | import keras.models as KM 19 | 20 | 21 | class ParallelModel(KM.Model): 22 | """Subclasses the standard Keras Model and adds multi-GPU support. 23 | It works by creating a copy of the model on each GPU. Then it slices 24 | the inputs and sends a slice to each copy of the model, and then 25 | merges the outputs together and applies the loss on the combined 26 | outputs. 27 | """ 28 | 29 | def __init__(self, keras_model, gpu_count): 30 | """Class constructor. 31 | keras_model: The Keras model to parallelize 32 | gpu_count: Number of GPUs. Must be > 1 33 | """ 34 | super(ParallelModel, self).__init__() 35 | self.inner_model = keras_model 36 | self.gpu_count = gpu_count 37 | merged_outputs = self.make_parallel() 38 | super(ParallelModel, self).__init__(inputs=self.inner_model.inputs, 39 | outputs=merged_outputs) 40 | 41 | def __getattribute__(self, attrname): 42 | """Redirect loading and saving methods to the inner model. That's where 43 | the weights are stored.""" 44 | if 'load' in attrname or 'save' in attrname: 45 | return getattr(self.inner_model, attrname) 46 | return super(ParallelModel, self).__getattribute__(attrname) 47 | 48 | def summary(self, *args, **kwargs): 49 | """Override summary() to display summaries of both, the wrapper 50 | and inner models.""" 51 | super(ParallelModel, self).summary(*args, **kwargs) 52 | self.inner_model.summary(*args, **kwargs) 53 | 54 | def make_parallel(self): 55 | """Creates a new wrapper model that consists of multiple replicas of 56 | the original model placed on different GPUs. 57 | """ 58 | # Slice inputs. Slice inputs on the CPU to avoid sending a copy 59 | # of the full inputs to all GPUs. Saves on bandwidth and memory. 60 | input_slices = {name: tf.split(x, self.gpu_count) 61 | for name, x in zip(self.inner_model.input_names, 62 | self.inner_model.inputs)} 63 | 64 | output_names = self.inner_model.output_names 65 | outputs_all = [] 66 | for i in range(len(self.inner_model.outputs)): 67 | outputs_all.append([]) 68 | 69 | # Run the model call() on each GPU to place the ops there 70 | for i in range(self.gpu_count): 71 | with tf.device('/gpu:%d' % i): 72 | with tf.name_scope('tower_%d' % i): 73 | # Run a slice of inputs through this replica 74 | zipped_inputs = zip(self.inner_model.input_names, 75 | self.inner_model.inputs) 76 | inputs = [ 77 | KL.Lambda(lambda s: input_slices[name][i], 78 | output_shape=lambda s: (None,) + s[1:])(tensor) 79 | for name, tensor in zipped_inputs] 80 | # Create the model replica and get the outputs 81 | outputs = self.inner_model(inputs) 82 | if not isinstance(outputs, list): 83 | outputs = [outputs] 84 | # Save the outputs for merging back together later 85 | for l, o in enumerate(outputs): 86 | outputs_all[l].append(o) 87 | 88 | # Merge outputs on CPU 89 | with tf.device('/cpu:0'): 90 | merged = [] 91 | for outputs, name in zip(outputs_all, output_names): 92 | # Concatenate or average outputs? 93 | # Outputs usually have a batch dimension and we concatenate 94 | # across it. If they don't, then the output is likely a loss 95 | # or a metric value that gets averaged across the batch. 96 | # Keras expects losses and metrics to be scalars. 97 | if K.int_shape(outputs[0]) == (): 98 | # Average 99 | m = KL.Lambda(lambda o: tf.add_n(o) / len(outputs), name=name)(outputs) 100 | else: 101 | # Concatenate 102 | m = KL.Concatenate(axis=0, name=name)(outputs) 103 | merged.append(m) 104 | return merged 105 | 106 | 107 | if __name__ == "__main__": 108 | # Testing code below. It creates a simple model to train on MNIST and 109 | # tries to run it on 2 GPUs. It saves the graph so it can be viewed 110 | # in TensorBoard. Run it as: 111 | # 112 | # python3 parallel_model.py 113 | 114 | import os 115 | import numpy as np 116 | import keras.optimizers 117 | from keras.datasets import mnist 118 | from keras.preprocessing.image import ImageDataGenerator 119 | 120 | GPU_COUNT = 2 121 | 122 | # Root directory of the project 123 | ROOT_DIR = os.path.abspath("../") 124 | 125 | # Directory to save logs and trained model 126 | MODEL_DIR = os.path.join(ROOT_DIR, "logs") 127 | 128 | def build_model(x_train, num_classes): 129 | # Reset default graph. Keras leaves old ops in the graph, 130 | # which are ignored for execution but clutter graph 131 | # visualization in TensorBoard. 132 | tf.reset_default_graph() 133 | 134 | inputs = KL.Input(shape=x_train.shape[1:], name="input_image") 135 | x = KL.Conv2D(32, (3, 3), activation='relu', padding="same", 136 | name="conv1")(inputs) 137 | x = KL.Conv2D(64, (3, 3), activation='relu', padding="same", 138 | name="conv2")(x) 139 | x = KL.MaxPooling2D(pool_size=(2, 2), name="pool1")(x) 140 | x = KL.Flatten(name="flat1")(x) 141 | x = KL.Dense(128, activation='relu', name="dense1")(x) 142 | x = KL.Dense(num_classes, activation='softmax', name="dense2")(x) 143 | 144 | model = KM.Model(inputs=inputs, outputs=x) 145 | 146 | return model 147 | 148 | # Load MNIST Data 149 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 150 | x_train = np.expand_dims(x_train, -1).astype('float32') / 255 151 | x_test = np.expand_dims(x_test, -1).astype('float32') / 255 152 | 153 | print('x_train shape:', x_train.shape) 154 | print('x_test shape:', x_test.shape) 155 | 156 | # Build data generator and model 157 | datagen = ImageDataGenerator() 158 | model = build_model(x_train, 10) 159 | 160 | # Add multi-GPU support. 161 | print("Adding multi-gpu support") 162 | model = ParallelModel(model, GPU_COUNT) 163 | print("Added") 164 | 165 | optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=5.0) 166 | 167 | model.compile(loss='sparse_categorical_crossentropy', 168 | optimizer=optimizer, metrics=['accuracy']) 169 | 170 | model.summary() 171 | 172 | # Train 173 | model.fit_generator( 174 | datagen.flow(x_train, y_train, batch_size=64), 175 | steps_per_epoch=50, epochs=10, verbose=1, 176 | validation_data=(x_test, y_test), 177 | callbacks=[keras.callbacks.TensorBoard(log_dir=MODEL_DIR, 178 | write_graph=True)] 179 | ) -------------------------------------------------------------------------------- /src/utils/infer.py: -------------------------------------------------------------------------------- 1 | 2 | import buzzard as buzz 3 | import numpy as np 4 | 5 | import tqdm 6 | from os.path import isfile, join 7 | from os import listdir 8 | 9 | def tile_image(tile_size, ds, fp): 10 | ''' 11 | Tiles image in several tiles of size (tile_size, tile_size) 12 | Params 13 | ------ 14 | tile_siz : int 15 | size of a tile in pixel (tiles are square) 16 | ds: Datasource 17 | Datasource of the input image (binary) 18 | fp : footprint 19 | global footprint 20 | Returns 21 | ------- 22 | rgb_array : np.ndarray 23 | array of dimension 5 (x number of tiles, y number of tiles, 24 | x size of tile, y size of tile, number of canal) 25 | tiles : np.ndarray of footprint 26 | array of of size (x number of tiles, y number of tiles) 27 | that contains footprint information 28 | 29 | 30 | ''' 31 | tiles = fp.tile((tile_size,tile_size), overlapx=0, overlapy=0, 32 | boundary_effect='extend', boundary_effect_locus='br') 33 | 34 | rgb_array = np.zeros([tiles.shape[0],tiles.shape[1],tile_size,tile_size,3], 35 | dtype='uint8') 36 | 37 | for i in range(tiles.shape[0]): 38 | for j in range(tiles.shape[1]): 39 | rgb_array[i,j] = ds.rgb.get_data(band=(1,2,3), fp=tiles[i,j]) 40 | 41 | return rgb_array, tiles 42 | 43 | def untile_and_predict(rgb_array,tiles,fp,model, pre_process): 44 | ''' 45 | Get the tile binary array and the footprint matrix and returns the reconstructed image 46 | Params 47 | ------ 48 | rgb_array : np.ndarray 49 | array of dimension 5 (x number of tiles, y number of tiles, 50 | x size of tile, y size of tile,3) 51 | tiles : np.ndarray of footprint 52 | array of of size (x number of tiles, y number of tiles) 53 | that contains footprint information 54 | Returns 55 | ------- 56 | rgb_reconstruct : np.ndarray 57 | reconstructed rgb image 58 | ''' 59 | # initialization of the reconstructed rgb array 60 | binary_reconstruct = np.zeros([ 61 | rgb_array.shape[0]*rgb_array.shape[2], #pixels along x axis 62 | rgb_array.shape[1]*rgb_array.shape[3], # pixels along y axis 63 | ], dtype='float32') 64 | 65 | for i in tqdm.tqdm(range(tiles.shape[0])): 66 | for j in range(tiles.shape[1]): 67 | tile_size = tiles[i,j].rsize 68 | 69 | # predict the binaryzed image 70 | image = rgb_array[i,j] 71 | image = pre_process(image) 72 | predicted = predict_image(image,model) 73 | 74 | # add the image in the global image 75 | binary_reconstruct[i*tile_size[0]:(i+1)*tile_size[0], 76 | j*tile_size[1]:(j+1)*tile_size[1] 77 | ] = predicted 78 | 79 | # delete the tilling padding 80 | binary_reconstruct = binary_reconstruct[:fp.rsize[1],:fp.rsize[0]] 81 | 82 | return binary_reconstruct 83 | 84 | def predict_image(image,model): 85 | ''' 86 | Predict one image with the model. Returns a binary array 87 | Parameters 88 | ---------- 89 | image : np.ndarray 90 | rgb input array of the (n,d,3) 91 | model : Model object 92 | trained model for the prediction of image (tile_size * tile_size) 93 | ''' 94 | shape_im = image.shape 95 | 96 | predicted_image = model.predict(image.reshape(1,shape_im[0], shape_im[1],3)) 97 | predicted_image = predicted_image.reshape(shape_im[0],shape_im[1]) 98 | 99 | return predicted_image 100 | 101 | def predict_map(model,tile_size,ds_rgb,fp, pre_process): 102 | ''' 103 | Pipeline from the whole rasper and footprint adapted to binary array 104 | Params 105 | ------ 106 | model : Model object 107 | trained model for the prediction of image (tile_size * tile_size) 108 | tile_size : int 109 | size of tiles (i.e. size of the input array for the model) 110 | ds_rgb : datasource 111 | Datasource object for the rgb image 112 | fp : footprint 113 | footprint of the adapted image (with downsampling factor) 114 | ''' 115 | print("Tiling images...") 116 | rgb_array, tiles = tile_image(tile_size, ds_rgb, fp) 117 | 118 | print("Predicting tiles..") 119 | predicted_binary = untile_and_predict(rgb_array,tiles,fp,model, pre_process) 120 | 121 | return predicted_binary 122 | 123 | def predict_from_file(rgb_path, model, pre_process, downsampling_factor=3,tile_size=128): 124 | ''' 125 | Predict binaryzed array and adapted footprint from a file_name 126 | Parameters 127 | ---------- 128 | rgb_path : string 129 | file name (with extension) 130 | model : Model object 131 | trained model for the prediction of image (tile_size * tile_size) 132 | downsampling_factor : int 133 | downsampling factor (to lower resolution) 134 | tile_size : int 135 | size of a tile (in pixel) i.e. size of the input images 136 | for the neural network 137 | ''' 138 | ds_rgb = buzz.DataSource(allow_interpolation=True) 139 | ds_rgb.open_raster('rgb', rgb_path) 140 | 141 | fp= buzz.Footprint( 142 | tl=ds_rgb.rgb.fp.tl, 143 | size=ds_rgb.rgb.fp.size, 144 | rsize=ds_rgb.rgb.fp.rsize/downsampling_factor, 145 | ) #unsampling 146 | 147 | predicted_binary = predict_map(model, tile_size, ds_rgb, fp, pre_process) 148 | 149 | return predicted_binary, fp 150 | 151 | 152 | def save_polynoms(file,binary,fp): 153 | ''' 154 | Find polynoms in a binary array and save them at the geojson format 155 | Parameters 156 | ---------- 157 | file : string 158 | file name (with extension) 159 | binary : np.ndarray 160 | array that contains the whole binaryzed image 161 | fp : footprint 162 | footprint linked to the binary array 163 | ''' 164 | poly = fp.find_polygons(binary) 165 | 166 | path = 'geoJSON/'+file.split('.')[0]+'.geojson' 167 | ds = buzz.DataSource(allow_interpolation=True) 168 | ds.create_vector('dst', path, 'polygon', driver='GeoJSON') 169 | for p in poly: 170 | ds.dst.insert_data(p) 171 | ds.dst.close() 172 | 173 | def compute_files(model_nn,images_train,downsampling_factor,tile_size): 174 | ''' 175 | Compute polynoms for each files in images_train folder and save them in 176 | geojson format. 177 | Parameters 178 | ---------- 179 | model : Model object 180 | trained model for the prediction of image (tile_size * tile_size) 181 | images_traian : string 182 | folder name for whole input images 183 | downsampling_factor : int 184 | downsampling factor (to lower resolution) 185 | tile_size : int 186 | size of a tile (in pixel) i.e. size of the input images 187 | for the neural network 188 | ''' 189 | files = [f for f in listdir(images_train) if isfile(join(images_train, f))] 190 | for i in range(len(files)): 191 | print("Processing file number "+str(i)+"/"+str(len(files))+"...") 192 | predicted_binary, fp = predict_from_file(files[i],model_nn, 193 | images_train, 194 | downsampling_factor,tile_size) 195 | save_polynoms(files[i],predicted_binary,fp) 196 | 197 | 198 | 199 | def poly_to_binary(gt_path, geojson_path, downsampling_factor): 200 | 201 | ds = buzz.DataSource(allow_interpolation=True) 202 | ds.open_raster('binary', gt_path) 203 | ds.open_vector('polygons',geojson_path, driver='geoJSON') 204 | 205 | fp = buzz.Footprint( 206 | tl=ds.binary.fp.tl, 207 | size=ds.binary.fp.size, 208 | rsize=ds.binary.fp.rsize/downsampling_factor , 209 | ) 210 | 211 | binary = ds.binary.get_data(band=(1), fp=fp).astype('uint8') 212 | binary_predict = np.zeros_like(binary) 213 | 214 | for poly in ds.polygons.iter_data(None): 215 | mark_poly = fp.burn_polygons(poly) 216 | binary_predict[mark_poly] = 1 217 | 218 | return binary,binary_predict 219 | 220 | -------------------------------------------------------------------------------- /src/utils/infer2.py: -------------------------------------------------------------------------------- 1 | 2 | import buzzard as buzz 3 | import numpy as np 4 | 5 | import tqdm 6 | from os.path import isfile, join 7 | from os import listdir 8 | 9 | def tile_image(tile_size, ds, fp, overlap_factor = 1): 10 | ''' 11 | Tiles image in several tiles of size (tile_size, tile_size) with overlap of overlapping factor 12 | Params 13 | ------ 14 | tile_siz : int 15 | size of a tile in pixel (tiles are square) 16 | ds: Datasource 17 | Datasource of the input image (binary) 18 | fp : footprint 19 | global footprint 20 | overlap_factor: overlapping factor 21 | Overlap factor of consecutive tiles 22 | Returns 23 | ------- 24 | rgb_array : np.ndarray 25 | array of dimension 5 (x number of tiles, y number of tiles, 26 | x size of tile, y size of tile, number of canal) 27 | tiles : np.ndarray of footprint 28 | array of of size (x number of tiles, y number of tiles) 29 | that contains footprint information 30 | 31 | 32 | ''' 33 | tiles = fp.tile((tile_size,tile_size), overlapx=tile_size/overlap_factor, overlapy=tile_size/overlap_factor) 34 | 35 | rgb_array = np.zeros([tiles.shape[0],tiles.shape[1],tile_size,tile_size,3], 36 | dtype='uint8') 37 | 38 | # ============================================================================= 39 | # rgb_array = np.zeros([tiles.shape[0],tiles.shape[1],tile_size,tile_size,3], 40 | # dtype='float32') 41 | # ============================================================================= 42 | 43 | for i in range(tiles.shape[0]): 44 | for j in range(tiles.shape[1]): 45 | rgb_array[i,j] = ds.rgb.get_data(band=(1,2,3), fp=tiles[i,j]) 46 | 47 | return rgb_array, tiles 48 | 49 | def untile_and_predict(rgb_array,tiles, actual_tiles, fp,model, pre_process, overlap_factor=1): 50 | ''' 51 | Get the tile binary array and the footprint matrix and returns the reconstructed image 52 | Params 53 | ------ 54 | rgb_array : np.ndarray 55 | array of dimension 5 (x number of tiles, y number of tiles, 56 | x size of tile, y size of tile,3) 57 | tiles : np.ndarray of footprint 58 | array of of size (x number of tiles, y number of tiles) 59 | that contains footprint information 60 | Returns 61 | ------- 62 | rgb_reconstruct : np.ndarray 63 | reconstructed rgb image 64 | ''' 65 | #Defining actual bounds of tile 66 | x_bounds = actual_tiles.shape[0]*rgb_array.shape[2] 67 | y_bounds = actual_tiles.shape[1]*rgb_array.shape[3] 68 | 69 | tiles_in_x = tiles.shape[0] 70 | tiles_in_y = tiles.shape[1] 71 | 72 | # initialization of the reconstructed rgb array 73 | binary_reconstruct = np.zeros([ 74 | x_bounds, #pixels along x axis 75 | y_bounds, # pixels along y axis 76 | ], dtype='float32') 77 | 78 | x=0 79 | (x_stride, y_stride) = (rgb_array.shape[2],rgb_array.shape[3]) 80 | for i in tqdm.tqdm(range(tiles_in_x)): 81 | y = 0 82 | for j in range(tiles_in_y): 83 | 84 | tile_size = tiles[i,j].rsize 85 | 86 | # predict the binaryzed image 87 | image = rgb_array[i,j] 88 | image = pre_process(image) 89 | predicted = predict_image(image,model) 90 | #predicted = np.random.random_sample((128,128)) 91 | 92 | binary_reconstruct[x: x + x_stride, y: y + y_stride] += predicted 93 | #print("Predictions added for (",x,", ", y, ") origin block") 94 | y += y_stride//overlap_factor 95 | x += x_stride//overlap_factor 96 | binary_reconstruct /= overlap_factor**2 97 | 98 | # delete the tilling padding 99 | binary_reconstruct = binary_reconstruct[:fp.rsize[1],:fp.rsize[0]] 100 | 101 | return binary_reconstruct 102 | 103 | def predict_image(image,model): 104 | ''' 105 | Predict one image with the model. Returns a binary array 106 | Parameters 107 | ---------- 108 | image : np.ndarray 109 | rgb input array of the (n,d,3) 110 | model : Model object 111 | trained model for the prediction of image (tile_size * tile_size) 112 | ''' 113 | shape_im = image.shape 114 | 115 | predicted_image = model.predict(image.reshape(1,shape_im[0], shape_im[1],3)) 116 | predicted_image = predicted_image.reshape(shape_im[0],shape_im[1]) 117 | 118 | return predicted_image 119 | 120 | def predict_map(model,tile_size,ds_rgb,fp, pre_process, overlap_factor=1): 121 | ''' 122 | Pipeline from the whole rasper and footprint adapted to binary array 123 | Params 124 | ------ 125 | model : Model object 126 | trained model for the prediction of image (tile_size * tile_size) 127 | tile_size : int 128 | size of tiles (i.e. size of the input array for the model) 129 | ds_rgb : datasource 130 | Datasource object for the rgb image 131 | fp : footprint 132 | footprint of the adapted image (with downsampling factor) 133 | ''' 134 | print("Tiling images...") 135 | rgb_array, tiles = tile_image(tile_size, ds_rgb, fp, overlap_factor) 136 | 137 | actual_tiles = fp.tile((tile_size,tile_size)) 138 | 139 | print("Predicting tiles..") 140 | predicted_binary = untile_and_predict(rgb_array,tiles, actual_tiles,fp,model, pre_process, overlap_factor) 141 | 142 | return predicted_binary 143 | 144 | def predict_from_file(rgb_path, model, pre_process, downsampling_factor=3,tile_size=128, overlap_factor=1): 145 | ''' 146 | Predict binaryzed array and adapted footprint from a file_name 147 | Parameters 148 | ---------- 149 | rgb_path : string 150 | file name (with extension) 151 | model : Model object 152 | trained model for the prediction of image (tile_size * tile_size) 153 | downsampling_factor : int 154 | downsampling factor (to lower resolution) 155 | tile_size : int 156 | size of a tile (in pixel) i.e. size of the input images 157 | for the neural network 158 | ''' 159 | ds_rgb = buzz.DataSource(allow_interpolation=True) 160 | ds_rgb.open_raster('rgb', rgb_path) 161 | 162 | fp= buzz.Footprint( 163 | tl=ds_rgb.rgb.fp.tl, 164 | size=ds_rgb.rgb.fp.size, 165 | rsize=ds_rgb.rgb.fp.rsize/downsampling_factor, 166 | ) #unsampling 167 | 168 | predicted_binary = predict_map(model, tile_size, ds_rgb, fp, pre_process, overlap_factor) 169 | 170 | return predicted_binary, fp 171 | 172 | 173 | def save_polynoms(file,binary,fp): 174 | ''' 175 | Find polynoms in a binary array and save them at the geojson format 176 | Parameters 177 | ---------- 178 | file : string 179 | file name (with extension) 180 | binary : np.ndarray 181 | array that contains the whole binaryzed image 182 | fp : footprint 183 | footprint linked to the binary array 184 | ''' 185 | poly = fp.find_polygons(binary) 186 | 187 | path = 'geoJSON/'+file.split('.')[0]+'.geojson' 188 | ds = buzz.DataSource(allow_interpolation=True) 189 | ds.create_vector('dst', path, 'polygon', driver='GeoJSON') 190 | for p in poly: 191 | ds.dst.insert_data(p) 192 | ds.dst.close() 193 | 194 | def compute_files(model_nn,images_train,downsampling_factor,tile_size): 195 | ''' 196 | Compute polynoms for each files in images_train folder and save them in 197 | geojson format. 198 | Parameters 199 | ---------- 200 | model : Model object 201 | trained model for the prediction of image (tile_size * tile_size) 202 | images_traian : string 203 | folder name for whole input images 204 | downsampling_factor : int 205 | downsampling factor (to lower resolution) 206 | tile_size : int 207 | size of a tile (in pixel) i.e. size of the input images 208 | for the neural network 209 | ''' 210 | files = [f for f in listdir(images_train) if isfile(join(images_train, f))] 211 | for i in range(len(files)): 212 | print("Processing file number "+str(i)+"/"+str(len(files))+"...") 213 | predicted_binary, fp = predict_from_file(files[i],model_nn, 214 | images_train, 215 | downsampling_factor,tile_size) 216 | save_polynoms(files[i],predicted_binary,fp) 217 | 218 | 219 | 220 | 221 | 222 | -------------------------------------------------------------------------------- /src/networks/pspnet.py: -------------------------------------------------------------------------------- 1 | """ 2 | orignal: https://github.com/okotaku/kaggle_dsbowl/blob/master/model/pspnet.py 3 | 4 | similar: https://github.com/ykamikawa/tf-keras-PSPNet/blob/master/model.py 5 | """ 6 | 7 | import keras.backend as K 8 | from keras.models import Model 9 | from keras.layers import Input, Reshape, Permute, Dense, Activation 10 | from keras.layers import AveragePooling2D, GlobalAveragePooling2D 11 | from keras.layers import Lambda 12 | from keras.layers.convolutional import Conv2D, MaxPooling2D 13 | from keras.layers.normalization import BatchNormalization 14 | from keras.layers import multiply, add, concatenate 15 | from keras.engine.topology import Layer 16 | from keras.engine import InputSpec 17 | 18 | 19 | def build(size = 512, chs = 3, summary=False): 20 | 21 | n_labels=1 22 | output_stride=16 23 | num_blocks=4 24 | 25 | levels=[6,3,2,1] 26 | 27 | use_se=True 28 | output_mode='sigmoid' 29 | upsample_type='duc' 30 | 31 | input_shape = (size, size, chs) 32 | img_input = Input(shape=input_shape) 33 | 34 | x = Conv2D(64, (7, 7), strides=(2, 2), padding='same')(img_input) 35 | x = BatchNormalization()(x) 36 | x = Activation('relu')(x) 37 | x = MaxPooling2D((3, 3), strides=(2, 2))(x) 38 | 39 | x = conv_block(x, 3, [64, 64, 256], stage=2, strides=(1, 1), use_se=use_se) 40 | x = identity_block(x, 3, [64, 64, 256], stage=2, use_se=use_se) 41 | x = identity_block(x, 3, [64, 64, 256], stage=2, use_se=use_se) 42 | 43 | x = conv_block(x, 3, [128, 128, 512], stage=3, use_se=use_se) 44 | x = identity_block(x, 3, [128, 128, 512], stage=3, use_se=use_se) 45 | x = identity_block(x, 3, [128, 128, 512], stage=3, use_se=use_se) 46 | x = identity_block(x, 3, [128, 128, 512], stage=3, use_se=use_se) 47 | 48 | if output_stride == 8: 49 | rate_scale = 2 50 | elif output_stride == 16: 51 | rate_scale = 1 52 | 53 | x = conv_block(x, 3, [256, 256, 1024], stage=4, dilation_rate=1*rate_scale, 54 | multigrid=[1,1,1], use_se=use_se) 55 | x = identity_block(x, 3, [256, 256, 1024], stage=4, 56 | dilation_rate=1*rate_scale, multigrid=[1, 1, 1], 57 | use_se=use_se) 58 | x = identity_block(x, 3, [256, 256, 1024], stage=4, 59 | dilation_rate=1*rate_scale, multigrid=[1, 1, 1], 60 | use_se=use_se) 61 | x = identity_block(x, 3, [256, 256, 1024], stage=4, 62 | dilation_rate=1*rate_scale, multigrid=[1, 1, 1], 63 | use_se=use_se) 64 | x = identity_block(x, 3, [256, 256, 1024], stage=4, 65 | dilation_rate=1*rate_scale, multigrid=[1, 1, 1], 66 | use_se=use_se) 67 | x = identity_block(x, 3, [256, 256, 1024], stage=4, 68 | dilation_rate=1*rate_scale, multigrid=[1, 1, 1], 69 | use_se=use_se) 70 | 71 | init_rate = 2 72 | for block in range(4, num_blocks+1): 73 | x = conv_block(x, 3, [512, 512, 2048], stage=5, 74 | dilation_rate=init_rate*rate_scale, 75 | multigrid=[1, 1, 1], use_se=use_se) 76 | x = identity_block(x, 3, [512, 512, 2048], stage=5, 77 | dilation_rate=init_rate*rate_scale, 78 | multigrid=[1, 1, 1], use_se=use_se) 79 | x = identity_block(x, 3, [512, 512, 2048], stage=5, 80 | dilation_rate=init_rate*rate_scale, 81 | multigrid=[1, 1, 1], use_se=use_se) 82 | init_rate *= 2 83 | 84 | x = pyramid_pooling_module(x, 512, input_shape, output_stride, levels) 85 | 86 | if upsample_type == 'duc': 87 | x = duc(x, factor=output_stride, 88 | output_shape=(input_shape[0], input_shape[1], n_labels)) 89 | out = Conv2D(n_labels, (1, 1), padding='same', 90 | kernel_initializer="he_normal")(x) 91 | 92 | elif upsample_type == 'bilinear': 93 | x = Conv2D(n_labels, (1, 1), padding='same', 94 | kernel_initializer="he_normal")(x) 95 | out = BilinearUpSampling2D((n_labels, input_shape[0], input_shape[1]), 96 | factor=output_stride)(x) 97 | 98 | out = Activation(output_mode)(out) 99 | 100 | model = Model(inputs=img_input, outputs=out) 101 | 102 | return model 103 | 104 | 105 | def conv_block(input_tensor, kernel_size, filters, stage, strides=(2, 2), 106 | dilation_rate=1, multigrid=[1,2,1], use_se=True): 107 | 108 | filters1, filters2, filters3 = filters 109 | 110 | if dilation_rate > 1: 111 | strides = (1, 1) 112 | else: 113 | multigrid = [1, 1, 1] 114 | 115 | x = Conv2D(filters1, (1, 1), strides=strides, 116 | dilation_rate=dilation_rate*multigrid[0])(input_tensor) 117 | x = BatchNormalization()(x) 118 | x = Activation('relu')(x) 119 | 120 | x = Conv2D(filters2, kernel_size, padding='same', 121 | dilation_rate=dilation_rate*multigrid[1])(x) 122 | x = BatchNormalization()(x) 123 | x = Activation('relu')(x) 124 | 125 | x = Conv2D(filters3, (1, 1), 126 | dilation_rate=dilation_rate*multigrid[2])(x) 127 | x = BatchNormalization()(x) 128 | 129 | shortcut = Conv2D(filters3, (1, 1), strides=strides)(input_tensor) 130 | shortcut = BatchNormalization()(shortcut) 131 | 132 | if use_se and stage < 5: 133 | se = _squeeze_excite_block(x, filters3, k=1) 134 | x = multiply([x, se]) 135 | x = add([x, shortcut]) 136 | x = Activation('relu')(x) 137 | 138 | return x 139 | 140 | 141 | def _squeeze_excite_block(init, filters, k=1): 142 | se_shape = (1, 1, filters * k) 143 | 144 | se = GlobalAveragePooling2D()(init) 145 | se = Dense((filters * k) // 16, activation='relu', 146 | kernel_initializer='he_normal', use_bias=False)(se) 147 | se = Dense(filters * k, activation='sigmoid', 148 | kernel_initializer='he_normal', use_bias=False)(se) 149 | 150 | return se 151 | 152 | 153 | def identity_block(input_tensor, kernel_size, filters, stage, dilation_rate=1, 154 | multigrid=[1, 2, 1], use_se=True): 155 | filters1, filters2, filters3 = filters 156 | 157 | if dilation_rate < 2: 158 | multigrid = [1, 1, 1] 159 | 160 | x = Conv2D(filters1, (1, 1), 161 | dilation_rate=dilation_rate*multigrid[0])(input_tensor) 162 | x = BatchNormalization()(x) 163 | x = Activation('relu')(x) 164 | 165 | x = Conv2D(filters2, kernel_size, padding='same', 166 | dilation_rate=dilation_rate*multigrid[1])(x) 167 | x = BatchNormalization()(x) 168 | x = Activation('relu')(x) 169 | 170 | x = Conv2D(filters3, (1, 1), 171 | dilation_rate=dilation_rate*multigrid[2])(x) 172 | x = BatchNormalization()(x) 173 | 174 | # stage 5 after 175 | if use_se and stage < 5: 176 | se = _squeeze_excite_block(x, filters3, k=1) 177 | x = multiply([x, se]) 178 | x = add([x, input_tensor]) 179 | x = Activation('relu')(x) 180 | 181 | return x 182 | 183 | def interp_block(x, num_filters, input_shape, output_stride, level): 184 | feature_map_shape = (input_shape[0]/output_stride, 185 | input_shape[1]/output_stride) 186 | 187 | if output_stride == 16: 188 | scale = 5 189 | elif output_stride == 8: 190 | scale = 10 191 | 192 | kernel = (level*scale, level*scale) 193 | strides = (level*scale, level*scale) 194 | global_feat = AveragePooling2D(kernel, strides=strides)(x) 195 | global_feat = Conv2D(num_filters, (1, 1), padding='same', 196 | kernel_initializer="he_normal")(global_feat) 197 | global_feat = BatchNormalization()(global_feat) 198 | global_feat = Lambda(Interp, arguments={'shape': feature_map_shape})(global_feat) 199 | 200 | return global_feat 201 | 202 | 203 | def pyramid_pooling_module(x, num_filters, input_shape, output_stride, levels): 204 | pyramid_pooling_blocks = [x] 205 | for level in levels: 206 | pyramid_pooling_blocks.append(interp_block(x, num_filters, input_shape, 207 | output_stride, level)) 208 | 209 | y = concatenate(pyramid_pooling_blocks) 210 | y = Conv2D(num_filters, (3, 3), padding='same', 211 | kernel_initializer="he_normal")(y) 212 | y = BatchNormalization()(y) 213 | y = Activation('relu')(y) 214 | 215 | return y 216 | 217 | 218 | def duc(x, factor=8, output_shape=(512,512,1)): 219 | H,W,c,r = output_shape[0],output_shape[1],output_shape[2],factor 220 | h = H/r 221 | w = W/r 222 | x = Conv2D(c*r*r, 223 | (3, 3), 224 | padding='same', 225 | name='conv_duc_%s'%factor)(x) 226 | x = BatchNormalization(name='bn_duc_%s'%factor)(x) 227 | x = Activation('relu')(x) 228 | x = Permute((3,1,2))(x) 229 | x = Reshape((c,r,r,h,w))(x) 230 | x = Permute((1,4,2,5,3))(x) 231 | x = Reshape((c,H,W))(x) 232 | x = Permute((2,3,1))(x) 233 | 234 | return x 235 | 236 | 237 | class BilinearUpSampling2D(Layer): 238 | def __init__(self, target_shape=None, factor=None, data_format=None, 239 | **kwargs): 240 | if data_format is None: 241 | data_format = K.image_data_format() 242 | assert data_format in { 243 | 'channels_last', 'channels_first'} 244 | 245 | self.data_format = data_format 246 | self.input_spec = [InputSpec(ndim=4)] 247 | self.target_shape = target_shape 248 | self.factor = factor 249 | if self.data_format == 'channels_first': 250 | self.target_size = (target_shape[2], target_shape[3]) 251 | elif self.data_format == 'channels_last': 252 | self.target_size = (target_shape[1], target_shape[2]) 253 | super(BilinearUpSampling2D, self).__init__(**kwargs) 254 | 255 | def compute_output_shape(self, input_shape): 256 | if self.data_format == 'channels_last': 257 | return (input_shape[0], self.target_size[0], 258 | self.target_size[1], input_shape[3]) 259 | else: 260 | return (input_shape[0], input_shape[1], 261 | self.target_size[0], self.target_size[1]) 262 | 263 | def call(self, inputs): 264 | return K.resize_images(inputs, self.factor, self.factor, self.data_format) 265 | 266 | def get_config(self): 267 | config = {'target_shape': self.target_shape, 268 | 'data_format': self.data_format} 269 | base_config = super(BilinearUpSampling2D, self).get_config() 270 | return dict(list(base_config.items()) + list(config.items())) 271 | 272 | 273 | def Interp(x, shape): 274 | from keras.backend import tf as ktf 275 | new_height, new_width = shape 276 | resized = ktf.image.resize_images(x, [int(new_height), int(new_width)], align_corners=True) 277 | return resized -------------------------------------------------------------------------------- /notebook/training.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Building Footprint Segmentation from Satellite images (bfss)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "### This notebook is an attempt to walk through the entire code step-by-step, explaining the different blocks, to give an overview of the porject. " 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "%load_ext autoreload\n", 24 | "%autoreload 2" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "data": { 34 | "text/plain": [ 35 | "'/Users/pradip.gupta/personal-projects/bfss/notebook'" 36 | ] 37 | }, 38 | "execution_count": 2, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "import os\n", 45 | "os.getcwd()" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "data": { 55 | "text/plain": [ 56 | "'/Users/pradip.gupta/personal-projects/bfss'" 57 | ] 58 | }, 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "output_type": "execute_result" 62 | } 63 | ], 64 | "source": [ 65 | "import sys, glob, shutil\n", 66 | "os.chdir(os.path.dirname(os.getcwd()))\n", 67 | "os.getcwd()" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "#### Adding \"src/networks\" folder in path, to enable in-line imports for the network files using importlib" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 4, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "import os, sys\n", 84 | "sys.path.append(os.path.abspath('./src/networks'))" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 5, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "name": "stderr", 94 | "output_type": "stream", 95 | "text": [ 96 | "Using TensorFlow backend.\n" 97 | ] 98 | } 99 | ], 100 | "source": [ 101 | "#To handel OOM errors\n", 102 | "import tensorflow as tf\n", 103 | "import keras.backend.tensorflow_backend as ktf\n", 104 | "def get_session():\n", 105 | " gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction= 0.9,\n", 106 | " allow_growth=True)\n", 107 | " return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n", 108 | "ktf.set_session(get_session())" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 20, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "#Standard imports\n", 118 | "import pandas as pd\n", 119 | "import importlib\n", 120 | "import numpy as np\n", 121 | "from keras.models import load_model\n", 122 | "from keras.utils import multi_gpu_model\n", 123 | "from keras.optimizers import Adam, RMSprop, Nadam, SGD" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "### Loading all the custom functions that we have written. These make the training script neat and help in debugging in case of errors. " 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 24, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "#Custom imports\n", 140 | "import config\n", 141 | "from src.training import data_loader\n", 142 | "from src.training.metrics import bce_dice_loss, dice_coeff\n", 143 | "from src.training.seg_data_generator import SegDataGenerator\n", 144 | "from src.training.keras_callbacks import get_callbacks\n", 145 | "from src.training.modeller import finetune_model\n", 146 | "from src.training.keras_history import generate_stats\n", 147 | "from src.training.plots import save_plots" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "### Load the data from the dataset path. Here we are loading only the meta data for all the AOIs. The X and Y variables hold a buzzard datasource object. \n", 155 | "\n", 156 | "Get to know more on buzzard here: https://github.com/airware/buzzard" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 25, 162 | "metadata": { 163 | "collapsed": true 164 | }, 165 | "outputs": [ 166 | { 167 | "name": "stdout", 168 | "output_type": "stream", 169 | "text": [ 170 | "No of aois:\n", 171 | "for train: 14\n", 172 | "for validation: 3\n", 173 | "for test: 1\n", 174 | "\n", 175 | "Preparing dataset for Training\n", 176 | "\n", 177 | "Adding austin4 to AOI list\n", 178 | "\n", 179 | "Adding tyrol-w25 to AOI list\n", 180 | "\n", 181 | "Adding chicago5 to AOI list\n", 182 | "\n", 183 | "Adding vienna10 to AOI list\n", 184 | "\n", 185 | "Adding chicago28 to AOI list\n", 186 | "\n", 187 | "Adding vienna16 to AOI list\n", 188 | "\n", 189 | "Adding chicago33 to AOI list\n", 190 | "\n", 191 | "Adding kitsap18 to AOI list\n", 192 | "\n", 193 | "Adding kitsap12 to AOI list\n", 194 | "\n", 195 | "Adding chicago11 to AOI list\n", 196 | "\n", 197 | "Adding kitsap17 to AOI list\n", 198 | "\n", 199 | "Adding austin14 to AOI list\n", 200 | "\n", 201 | "Adding tyrol-w21 to AOI list\n", 202 | "\n", 203 | "Adding kitsap1 to AOI list\n", 204 | "\n", 205 | "Preparing dataset for Validation\n", 206 | "\n", 207 | "Adding austin29 to AOI list\n", 208 | "\n", 209 | "Adding kitsap31 to AOI list\n", 210 | "\n", 211 | "Adding tyrol-w5 to AOI list\n" 212 | ] 213 | } 214 | ], 215 | "source": [ 216 | "dataset_path = config.dataset_path \n", 217 | "exp_name = config.exp_name\n", 218 | "\n", 219 | "train, val, test = data_loader.get_samples(dataset_path)\n", 220 | "\n", 221 | "print(\"\\nPreparing dataset for Training\")\n", 222 | "X_train, y_train = data_loader.build_source(train, dataset_path)\n", 223 | "\n", 224 | "print(\"\\nPreparing dataset for Validation\")\n", 225 | "X_val, y_val = data_loader.build_source(val, dataset_path)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": {}, 231 | "source": [ 232 | "### Reading the config.py file" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": 26, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "#Params\n", 242 | "tile_size = config.tile_size\n", 243 | "no_of_samples = config.no_of_samples\n", 244 | "downs = config.down_sampling\n", 245 | "\n", 246 | "batch_size = config.batch_size\n", 247 | "epochs = config.epochs \n", 248 | "initial_epoch = config.initial_epoch\n", 249 | "\n", 250 | "training_frm_scratch = config.training_frm_scratch \n", 251 | "training_frm_chkpt = config.training_frm_chkpt \n", 252 | "transfer_lr = config.transfer_lr\n", 253 | "\n", 254 | "if sum((training_frm_scratch, training_frm_chkpt, transfer_lr)) != 1:\n", 255 | " raise Exception(\"Conflicting training modes\")\n" 256 | ] 257 | }, 258 | { 259 | "cell_type": "markdown", 260 | "metadata": {}, 261 | "source": [ 262 | "### Defining a super set for loss, optimiser and metric functions. The user can select any from there options using the config file" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 27, 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [ 271 | "loss_class = {'bin_cross': 'binary_crossentropy',\n", 272 | " 'bce_dice': bce_dice_loss}\n", 273 | "\n", 274 | "metric_class = {'dice':dice_coeff}\n", 275 | "\n", 276 | "optimiser_class = {'adam': (Adam, {}), \n", 277 | " 'nadam': (Nadam, {}), \n", 278 | " 'rmsprop': (RMSprop, {}),\n", 279 | " 'sgd':(SGD, {'decay':1e-6, 'momentum':0.99, 'nesterov':True})} " 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": {}, 285 | "source": [ 286 | "### Calculating the no of iterations we will use per epoch. For training, the steps per epoch is multiplied by 2 as we are using augmentation." 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": 28, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [ 295 | "train_spe = int(np.floor((len(X_train)*no_of_samples*2) / batch_size)) #spe = Steps per epoch\n", 296 | "val_spe = int(np.floor((len(X_val)*no_of_samples*2) / batch_size))" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "### Initialising the datagenerators for training and validation. \n", 304 | "\n", 305 | "Get to know more about keras generator from here: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 29, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "# Initialise generators \n", 315 | "train_generator = SegDataGenerator(dataset_path, img_source=X_train, \n", 316 | " mask_source=y_train, batch_size= batch_size, \n", 317 | " no_of_samples = no_of_samples, tile_size= tile_size, \n", 318 | " downsampling_factor = downs)\n", 319 | "\n", 320 | "val_generator = SegDataGenerator(dataset_path, img_source=X_val, \n", 321 | " mask_source=y_val, batch_size= batch_size, \n", 322 | " no_of_samples = no_of_samples, tile_size= tile_size, \n", 323 | " downsampling_factor = downs)" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 31, 329 | "metadata": { 330 | "scrolled": false 331 | }, 332 | "outputs": [ 333 | { 334 | "name": "stdout", 335 | "output_type": "stream", 336 | "text": [ 337 | "Training from scratch\n" 338 | ] 339 | } 340 | ], 341 | "source": [ 342 | "if training_frm_scratch:\n", 343 | " print(\"Training from scratch\")\n", 344 | " optimizer = optimiser_class[config.optimiser][0](lr=config.learning_rate, \n", 345 | " **optimiser_class[config.optimiser][1])\n", 346 | " loss = loss_class[config.loss]\n", 347 | " metric = metric_class[config.metric]\n", 348 | " \n", 349 | " if config.no_of_gpu > 1:\n", 350 | " print(\"Running in multi-gpu mode\")\n", 351 | " with tf.device('/cpu:0'):\n", 352 | " build = getattr(importlib.import_module(config.model),\"build\")\n", 353 | " model = build(size = config.tile_size, chs = 3)\n", 354 | " \n", 355 | " gpu_model = multi_gpu_model(model, gpus = config.no_of_gpu)\n", 356 | " gpu_model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy'])\n", 357 | " model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy'])\n", 358 | " \n", 359 | " else:\n", 360 | " build = getattr(importlib.import_module(config.model),\"build\")\n", 361 | " model = build(size = config.tile_size, chs = 3)\n", 362 | " model.compile(loss= loss, optimizer=optimizer, metrics=[metric, 'accuracy'])\n", 363 | " gpu_model = None" 364 | ] 365 | }, 366 | { 367 | "cell_type": "markdown", 368 | "metadata": {}, 369 | "source": [ 370 | "### Resume training\n", 371 | "\n", 372 | "When we load a keras model using `load_model`, we do not need to compile it as it _returns_ a **compiled model**.
\n", 373 | "**\"load_model\"** in keras does 4 things: \n", 374 | " - loads architecure, \n", 375 | " - loads weights, \n", 376 | " - loads optimisers and loss, \n", 377 | " - loads state of optimiser and loss\n", 378 | " \n", 379 | "\n", 380 | "To know more about keras models api: https://keras.io/models/about-keras-models/#about-keras-models" 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 32, 386 | "metadata": {}, 387 | "outputs": [], 388 | "source": [ 389 | "if training_frm_chkpt:\n", 390 | " print(\"Training from prv checkpoint\")\n", 391 | " model_path = config.model_path\n", 392 | " model = load_model(model_path, \n", 393 | " custom_objects={'bce_dice_loss': bce_dice_loss, 'dice_coeff':dice_coeff}) " 394 | ] 395 | }, 396 | { 397 | "cell_type": "markdown", 398 | "metadata": {}, 399 | "source": [ 400 | "## Transfer Learning: \n", 401 | "for Transfer Learning we follow the sequence:
\n", 402 | "build --> load_weights --> finetune --> compile \n", 403 | "\n", 404 | "Note: Compiling a model only defines the loss function, the optimizer and the metrics. That's all. \n", 405 | "Weights after compilation are the same as before compilation.\n", 406 | "\n", 407 | "#### To check the weights:\n", 408 | ">for layer in model.layers:
\n", 409 | ">> weights = layer.get_weights()
\n", 410 | ">> print(weights)
" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 33, 416 | "metadata": { 417 | "scrolled": true 418 | }, 419 | "outputs": [], 420 | "source": [ 421 | "if transfer_lr:\n", 422 | " print(\"Transfer Learning mode\")\n", 423 | " \n", 424 | " #build the model\n", 425 | " model_path = config.model_path\n", 426 | " gpu_model = load_model(model_path, \n", 427 | " custom_objects={'bce_dice_loss': bce_dice_loss,\n", 428 | " 'dice_coeff':dice_coeff}) \n", 429 | " \n", 430 | " build = getattr(importlib.import_module(config.model),\"build\")\n", 431 | " model = build(size = config.tile_size, chs = 3)\n", 432 | " model.set_weights(gpu_model.layers[-2].get_weights())\n", 433 | " \n", 434 | " #freeze layers for transfer learning & load weights\n", 435 | " model = finetune_model(model)\n", 436 | " \n", 437 | " if config.no_of_gpu > 1:\n", 438 | " gpu_model = multi_gpu_model(model, gpus = config.no_of_gpu, cpu_relocation=True)\n", 439 | " print(\"Running in multi-gpu mode\")\n", 440 | " else:\n", 441 | " gpu_model = model\n", 442 | " \n", 443 | " #compile the model\n", 444 | " gpu_model = compile_model(gpu_model, lr = config.learning_rate,\n", 445 | " optimiser = optimiser_class[config.optimiser],\n", 446 | " loss = loss_class[config.loss] , \n", 447 | " metric = metric_class[config.metric]) " 448 | ] 449 | }, 450 | { 451 | "cell_type": "markdown", 452 | "metadata": {}, 453 | "source": [ 454 | "### Set the callbacks to be used for training" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": 34, 460 | "metadata": {}, 461 | "outputs": [], 462 | "source": [ 463 | "#Set callbacks \n", 464 | "callbacks_list = get_callbacks(model)" 465 | ] 466 | }, 467 | { 468 | "cell_type": "markdown", 469 | "metadata": {}, 470 | "source": [ 471 | "## Start/Resume training" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": null, 477 | "metadata": {}, 478 | "outputs": [], 479 | "source": [ 480 | "history = gpu_model.fit_generator(steps_per_epoch= train_spe,\n", 481 | " generator=train_generator,\n", 482 | " epochs=epochs,\n", 483 | " validation_data = val_generator,\n", 484 | " validation_steps = val_spe,\n", 485 | " initial_epoch = initial_epoch,\n", 486 | " callbacks = callbacks_list)" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": null, 492 | "metadata": {}, 493 | "outputs": [], 494 | "source": [ 495 | "#Save final complete model \n", 496 | "filename = \"model_ep_\"+str(int(epochs))+\"_batch_\"+str(int(batch_size))\n", 497 | "model.save(\"./data/\"+exp_name+\"/\"+filename+\".h5\")\n", 498 | "print(\"Saved complete model file at: \", filename+\"_model\"+\".h5\")" 499 | ] 500 | }, 501 | { 502 | "cell_type": "code", 503 | "execution_count": null, 504 | "metadata": {}, 505 | "outputs": [], 506 | "source": [ 507 | "#Save history\n", 508 | "history_to_save = generate_stats(history, config)\n", 509 | "pd.DataFrame(history_to_save).to_csv(\"./data/\"+exp_name+\"/\"+filename + \"_train_results.csv\")\n", 510 | "save_plots(history, exp_name)" 511 | ] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": null, 516 | "metadata": {}, 517 | "outputs": [], 518 | "source": [] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": null, 523 | "metadata": {}, 524 | "outputs": [], 525 | "source": [] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "execution_count": null, 530 | "metadata": {}, 531 | "outputs": [], 532 | "source": [] 533 | } 534 | ], 535 | "metadata": { 536 | "kernelspec": { 537 | "display_name": "Python 3", 538 | "language": "python", 539 | "name": "python3" 540 | }, 541 | "language_info": { 542 | "codemirror_mode": { 543 | "name": "ipython", 544 | "version": 3 545 | }, 546 | "file_extension": ".py", 547 | "mimetype": "text/x-python", 548 | "name": "python", 549 | "nbconvert_exporter": "python", 550 | "pygments_lexer": "ipython3", 551 | "version": "3.6.7" 552 | } 553 | }, 554 | "nbformat": 4, 555 | "nbformat_minor": 2 556 | } 557 | --------------------------------------------------------------------------------