├── .gitignore ├── DenseBlock.jpg ├── DenseNet.jpg ├── FC-DenseNet.py ├── README.md ├── config └── FC-DenseNet103.py ├── data_loader.py ├── layers.py ├── metrics.py ├── test.py ├── train.py └── weights └── FC-DenseNet103_weights.npz /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.pyc 3 | *.ipynb 4 | -------------------------------------------------------------------------------- /DenseBlock.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimJeg/FC-DenseNet/cf2375bf9f6ed20ba029a5ee540261aad89732d5/DenseBlock.jpg -------------------------------------------------------------------------------- /DenseNet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimJeg/FC-DenseNet/cf2375bf9f6ed20ba029a5ee540261aad89732d5/DenseNet.jpg -------------------------------------------------------------------------------- /FC-DenseNet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano.tensor as T 3 | from lasagne.init import HeUniform 4 | from lasagne.layers import (InputLayer, ConcatLayer, Conv2DLayer, Pool2DLayer, Deconv2DLayer, 5 | get_all_param_values, set_all_param_values, get_output_shape, get_all_layers) 6 | from lasagne.nonlinearities import linear 7 | 8 | from layers import BN_ReLU_Conv, TransitionDown, TransitionUp, SoftmaxLayer 9 | 10 | 11 | class Network(): 12 | def __init__(self, 13 | input_shape=(None, 3, None, None), 14 | n_classes=11, 15 | n_filters_first_conv=48, 16 | n_pool=4, 17 | growth_rate=12, 18 | n_layers_per_block=5, 19 | dropout_p=0.2): 20 | """ 21 | This code implements the Fully Convolutional DenseNet described in https://arxiv.org/abs/1611.09326 22 | The network consist of a downsampling path, where dense blocks and transition down are applied, followed 23 | by an upsampling path where transition up and dense blocks are applied. 24 | Skip connections are used between the downsampling path and the upsampling path 25 | Each layer is a composite function of BN - ReLU - Conv and the last layer is a softmax layer. 26 | 27 | :param input_shape: shape of the input batch. Only the first dimension (n_channels) is needed 28 | :param n_classes: number of classes 29 | :param n_filters_first_conv: number of filters for the first convolution applied 30 | :param n_pool: number of pooling layers = number of transition down = number of transition up 31 | :param growth_rate: number of new feature maps created by each layer in a dense block 32 | :param n_layers_per_block: number of layers per block. Can be an int or a list of size 2 * n_pool + 1 33 | :param dropout_p: dropout rate applied after each convolution (0. for not using) 34 | """ 35 | 36 | if type(n_layers_per_block) == list: 37 | assert (len(n_layers_per_block) == 2 * n_pool + 1) 38 | elif type(n_layers_per_block) == int: 39 | n_layers_per_block = [n_layers_per_block] * (2 * n_pool + 1) 40 | else: 41 | raise ValueError 42 | 43 | # Theano variables 44 | self.input_var = T.tensor4('input_var', dtype='float32') # input image 45 | self.target_var = T.tensor4('target_var', dtype='int32') # target 46 | 47 | ##################### 48 | # First Convolution # 49 | ##################### 50 | 51 | inputs = InputLayer(input_shape, self.input_var) 52 | 53 | # We perform a first convolution. All the features maps will be stored in the tensor called stack (the Tiramisu) 54 | stack = Conv2DLayer(inputs, n_filters_first_conv, filter_size=3, pad='same', W=HeUniform(gain='relu'), 55 | nonlinearity=linear, flip_filters=False) 56 | # The number of feature maps in the stack is stored in the variable n_filters 57 | n_filters = n_filters_first_conv 58 | 59 | ##################### 60 | # Downsampling path # 61 | ##################### 62 | 63 | skip_connection_list = [] 64 | 65 | for i in range(n_pool): 66 | # Dense Block 67 | for j in range(n_layers_per_block[i]): 68 | # Compute new feature maps 69 | l = BN_ReLU_Conv(stack, growth_rate, dropout_p=dropout_p) 70 | # And stack it : the Tiramisu is growing 71 | stack = ConcatLayer([stack, l]) 72 | n_filters += growth_rate 73 | # At the end of the dense block, the current stack is stored in the skip_connections list 74 | skip_connection_list.append(stack) 75 | 76 | # Transition Down 77 | stack = TransitionDown(stack, n_filters, dropout_p) 78 | 79 | skip_connection_list = skip_connection_list[::-1] 80 | 81 | ##################### 82 | # Bottleneck # 83 | ##################### 84 | 85 | # We store now the output of the next dense block in a list. We will only upsample these new feature maps 86 | block_to_upsample = [] 87 | 88 | # Dense Block 89 | for j in range(n_layers_per_block[n_pool]): 90 | l = BN_ReLU_Conv(stack, growth_rate, dropout_p=dropout_p) 91 | block_to_upsample.append(l) 92 | stack = ConcatLayer([stack, l]) 93 | 94 | ####################### 95 | # Upsampling path # 96 | ####################### 97 | 98 | for i in range(n_pool): 99 | # Transition Up ( Upsampling + concatenation with the skip connection) 100 | n_filters_keep = growth_rate * n_layers_per_block[n_pool + i] 101 | stack = TransitionUp(skip_connection_list[i], block_to_upsample, n_filters_keep) 102 | 103 | # Dense Block 104 | block_to_upsample = [] 105 | for j in range(n_layers_per_block[n_pool + i + 1]): 106 | l = BN_ReLU_Conv(stack, growth_rate, dropout_p=dropout_p) 107 | block_to_upsample.append(l) 108 | stack = ConcatLayer([stack, l]) 109 | 110 | ##################### 111 | # Softmax # 112 | ##################### 113 | 114 | self.output_layer = SoftmaxLayer(stack, n_classes) 115 | 116 | ################################################################################################################ 117 | 118 | def save(self, path): 119 | """ Save the weights """ 120 | np.savez(path, *get_all_param_values(self.output_layer)) 121 | 122 | def restore(self, path): 123 | """ Load the weights """ 124 | 125 | with np.load(path) as f: 126 | saved_params_values = [f['arr_%d' % i] for i in range(len(f.files))] 127 | set_all_param_values(self.output_layer, saved_params_values) 128 | 129 | def summary(self, light=False): 130 | """ Print a summary of the network architecture """ 131 | 132 | layer_list = get_all_layers(self.output_layer) 133 | 134 | def filter_function(layer): 135 | """ We only display the layers in the list below""" 136 | return np.any([isinstance(layer, layer_type) for layer_type in 137 | [InputLayer, Conv2DLayer, Pool2DLayer, Deconv2DLayer, ConcatLayer]]) 138 | 139 | layer_list = filter(filter_function, layer_list) 140 | output_shape_list = map(get_output_shape, layer_list) 141 | layer_name_function = lambda s: str(s).split('.')[3].split('Layer')[0] 142 | 143 | if not light: 144 | print('-' * 75) 145 | print 'Warning : all the layers are not displayed \n' 146 | print ' {:<15} {:<20} {:<20}'.format('Layer', 'Output shape', 'W shape') 147 | 148 | for i, (layer, output_shape) in enumerate(zip(layer_list, output_shape_list)): 149 | if hasattr(layer, 'W'): 150 | input_shape = layer.W.get_value().shape 151 | else: 152 | input_shape = '' 153 | 154 | print '{:<3} {:<15} {:<20} {:<20}'.format(i + 1, layer_name_function(layer), output_shape, input_shape) 155 | if isinstance(layer, Pool2DLayer) | isinstance(layer, Deconv2DLayer): 156 | print('') 157 | 158 | print '\nNumber of Convolutional layers : {}'.format( 159 | len(filter(lambda x: isinstance(x, Conv2DLayer) | isinstance(x, Deconv2DLayer), layer_list))) 160 | print 'Number of parameters : {}'.format(np.sum(map(np.size, get_all_param_values(self.output_layer)))) 161 | print('-' * 75) 162 | 163 | 164 | if __name__ == '__main__': 165 | Network(input_shape=(5, 3, 224, 224)).summary() 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ⚠️ This repository is not maintained anymore ⚠️ 2 | 3 | ## Introduction 4 | 5 | This repo contains code to train and evaluate FC-DenseNets as described in [The One Hundred Layers Tiramisu: 6 | Fully Convolutional DenseNets for Semantic Segmentation](https://arxiv.org/abs/1611.09326). We investigate the use of [Densely Connected Convolutional Networks](https://arxiv.org/abs/1608.06993) for semantic segmentation, and report state of the art results on datasets such as CamVid. 7 | 8 | ## Installation 9 | 10 | 11 | You need to install : 12 | - [Theano](https://github.com/Theano/Theano). Preferably the latest version 13 | - [Lasagne](https://github.com/Lasagne/Lasagne) 14 | - [The dataset loader](https://github.com/fvisin/dataset_loaders). Thanks a lot to Francesco Visin for its data loader, please cite it if you use it. 15 | - (Recommended) [The new Theano GPU backend](https://github.com/Theano/libgpuarray). Compilation will be much faster. 16 | 17 | 18 | ## Data 19 | 20 | [deprecated] If you do want to train models now, you need to create a function load_data which returns 3 iterators (for training, validation and test). When applying next(), the iterator returns two values X, Y where X is the batch of input images (shape= (batch_size, 3, n_rows, n_cols), dtype=float32) and Y the batch of target segmentation maps (shape=(batch_size, n_rows, n_cols), dtype=int32) where each pixel in Y is an int indicating the class of the pixel. 21 | 22 | The iterator must also have the following methods (so they are not python iterators) : get_n_classes (returns the number of classes), get_n_samples (returns the number of examples in the set), get_n_batches (returns the number of batches necessary to see the entire set) and get_void_labels (returns a list containing the classes associated to void). It might be easier to change directly the files train.py and test.py. 23 | 24 | 25 | ## Run experiments 26 | 27 | The architecture of the model is defined in FC-DenseNet.py. To train a model, you need to prepare a configuration file (folder config) where all the parameters needed for creating and training your model are precised. DenseNets contain lot of connections making graph optimization difficult for Theano. We strongly recommend to use the flags described further. 28 | 29 | To train the FC-DenseNet103 model, use the command : `THEANO_FLAGS='device=cuda,optimizer=fast_compile,optimizer_including=fusion' python train.py -c config/FC-DenseNet103.py -e experiment_name`. All the logs of the experiments are stored in the folder experiment_name. 30 | 31 | On a Titan X 12GB, for the model FC-DenseNet103 (see folder config), compilation takes around 400 sec and 1 epoch 120 sec for training and 40 sec for validation. 32 | 33 | ## Use a pretrained model 34 | 35 | We published the weights of our model FC-DenseNet103. Metrics claimed in the paper (jaccard and accuracy) can be verified running 36 | `THEANO_FLAGS='device=cuda,optimizer=fast_compile,optimizer_including=fusion' python test.py` 37 | 38 | ## About the "m" number in the paper 39 | 40 | There is a small error with the "m" number in the Table 2 of the paper (that you may understand when running the code!). All values from the bottleneck to the last block (880, 1072, 800 and 368) should be incremented by 16 (896, 1088, 816 and 384). 41 | 42 | Here how we compute this value representing the number of feature maps concatenated into the "stack" : 43 | - First convolution : m=48 44 | - In the downsampling part + bottleneck, m[B] = m[B-1] + n_layers[B] * growth_rate [linear growth]. First block : m = 48 + 4x16 = 112. Second block m = 112 + 5x16 = 192. Until the bottleneck : m = 656 + 15x16 = 896. 45 | - In the upsampling part, m[B] is the sum of 3 terms : the m value corresponding to same resolution in the downsampling part (skip connection), the number of feature maps from the upsampled block (n_layers[B-1] * growth_rate) and the number of feature maps in the new block (n_layers[B] * growth_rate). First upsampling, m = 656 + 15x16 + 12x16 = 1088. Second upsampling, m = 464 + 12x16 + 10x16 = 816. Third upsampling, m = 304 + 10x16 + 7x16 = 576, Fourth upsampling, m = 192 + 7x16 + 5x16 = 384 and fifth upsampling, m = 112 + 5x16 + 4x16 = 256 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /config/FC-DenseNet103.py: -------------------------------------------------------------------------------- 1 | from metrics import crossentropy 2 | from lasagne.updates import rmsprop 3 | import imp 4 | import os 5 | 6 | # Dataset 7 | dataset = 'camvid' 8 | train_crop_size = (224, 224) # None for full size 9 | 10 | # Training 11 | seed = 0 12 | learning_rate = 1e-3 13 | lr_sched_decay = 0.995 # Applied each epocjh 14 | weight_decay = 0.0001 15 | num_epochs = 750 16 | max_patience = 150 17 | loss_function = crossentropy 18 | optimizer = rmsprop # Consider adam for training on other dataset, or decrease epsilon to 1e-12 19 | batch_size = 3 20 | 21 | # Architecture 22 | # pretrained_model= None # path of the weights of a pretrained network 23 | 24 | model_path = os.path.join(os.getcwd().split('/config')[0], 'FC-DenseNet.py') 25 | net = imp.load_source('Net', model_path).Network( 26 | input_shape=(None, 3, None, None), 27 | n_classes=11, 28 | n_filters_first_conv=48, 29 | n_pool=5, 30 | growth_rate=16, 31 | n_layers_per_block=[4, 5, 7, 10, 12, 15, 12, 10, 7, 5, 4], 32 | dropout_p=0.2) 33 | 34 | ############################################################################## 35 | 36 | if __name__ == '__main__': 37 | # Display a summary with a given shape for the input image 38 | net2 = imp.load_source('Net', model_path).Network( 39 | input_shape=(None, 3, 360, 480), 40 | n_classes=11, 41 | n_filters_first_conv=48, 42 | n_pool=5, 43 | growth_rate=16, 44 | n_layers_per_block=[4, 5, 7, 10, 12, 15, 12, 10, 7, 5, 4], 45 | dropout_p=0.2) 46 | 47 | net2.summary() 48 | -------------------------------------------------------------------------------- /data_loader.py: -------------------------------------------------------------------------------- 1 | from dataset_loaders.images.camvid import CamvidDataset 2 | from numpy.random import RandomState 3 | 4 | def load_data(dataset, train_crop_size=(224, 224), one_hot=False, 5 | batch_size=10, 6 | horizontal_flip=False, 7 | rng=RandomState(0)): 8 | 9 | if isinstance(batch_size, int): 10 | batch_size = [batch_size] * 3 11 | 12 | train_iter = CamvidDataset(which_set='train', 13 | batch_size=batch_size[0], 14 | seq_per_video=0, 15 | seq_length=0, 16 | crop_size=train_crop_size, 17 | horizontal_flip=horizontal_flip, 18 | get_one_hot=False, 19 | get_01c=False, 20 | overlap=0, 21 | use_threads=True, 22 | rng=rng) 23 | 24 | val_iter = CamvidDataset(which_set='val', 25 | batch_size=batch_size[1], 26 | seq_per_video=0, 27 | seq_length=0, 28 | crop_size=None, 29 | get_one_hot=False, 30 | get_01c=False, 31 | shuffle_at_each_epoch=False, 32 | overlap=0, 33 | use_threads=True, 34 | save_to_dir=False) 35 | 36 | test_iter = CamvidDataset(which_set='test', 37 | batch_size=batch_size[2], 38 | seq_per_video=0, 39 | seq_length=0, 40 | crop_size=None, 41 | get_one_hot=False, 42 | get_01c=False, 43 | shuffle_at_each_epoch=False, 44 | overlap=0, 45 | use_threads=True, 46 | save_to_dir=False) 47 | 48 | return train_iter, val_iter, test_iter 49 | -------------------------------------------------------------------------------- /layers.py: -------------------------------------------------------------------------------- 1 | from lasagne.layers import ( 2 | NonlinearityLayer, Conv2DLayer, DropoutLayer, Pool2DLayer, ConcatLayer, Deconv2DLayer, 3 | DimshuffleLayer, ReshapeLayer, get_output, BatchNormLayer) 4 | 5 | from lasagne.nonlinearities import linear, softmax 6 | from lasagne.init import HeUniform 7 | 8 | def BN_ReLU_Conv(inputs, n_filters, filter_size=3, dropout_p=0.2): 9 | """ 10 | Apply successivly BatchNormalization, ReLu nonlinearity, Convolution and Dropout (if dropout_p > 0) on the inputs 11 | """ 12 | 13 | l = NonlinearityLayer(BatchNormLayer(inputs)) 14 | l = Conv2DLayer(l, n_filters, filter_size, pad='same', W=HeUniform(gain='relu'), nonlinearity=linear, 15 | flip_filters=False) 16 | if dropout_p != 0.0: 17 | l = DropoutLayer(l, dropout_p) 18 | return l 19 | 20 | 21 | def TransitionDown(inputs, n_filters, dropout_p=0.2): 22 | """ Apply first a BN_ReLu_conv layer with filter size = 1, and a max pooling with a factor 2 """ 23 | 24 | l = BN_ReLU_Conv(inputs, n_filters, filter_size=1, dropout_p=dropout_p) 25 | l = Pool2DLayer(l, 2, mode='max') 26 | 27 | return l 28 | # Note : network accuracy is quite similar with average pooling or without BN - ReLU. 29 | # We can also reduce the number of parameters reducing n_filters in the 1x1 convolution 30 | 31 | 32 | def TransitionUp(skip_connection, block_to_upsample, n_filters_keep): 33 | """ 34 | Performs upsampling on block_to_upsample by a factor 2 and concatenates it with the skip_connection """ 35 | 36 | # Upsample 37 | l = ConcatLayer(block_to_upsample) 38 | l = Deconv2DLayer(l, n_filters_keep, filter_size=3, stride=2, 39 | crop='valid', W=HeUniform(gain='relu'), nonlinearity=linear) 40 | # Concatenate with skip connection 41 | l = ConcatLayer([l, skip_connection], cropping=[None, None, 'center', 'center']) 42 | 43 | return l 44 | # Note : we also tried Subpixel Deconvolution without seeing any improvements. 45 | # We can reduce the number of parameters reducing n_filters_keep in the Deconvolution 46 | 47 | 48 | def SoftmaxLayer(inputs, n_classes): 49 | """ 50 | Performs 1x1 convolution followed by softmax nonlinearity 51 | The output will have the shape (batch_size * n_rows * n_cols, n_classes) 52 | """ 53 | 54 | l = Conv2DLayer(inputs, n_classes, filter_size=1, nonlinearity=linear, W=HeUniform(gain='relu'), pad='same', 55 | flip_filters=False, stride=1) 56 | 57 | # We perform the softmax nonlinearity in 2 steps : 58 | # 1. Reshape from (batch_size, n_classes, n_rows, n_cols) to (batch_size * n_rows * n_cols, n_classes) 59 | # 2. Apply softmax 60 | 61 | l = DimshuffleLayer(l, (0, 2, 3, 1)) 62 | batch_size, n_rows, n_cols, _ = get_output(l).shape 63 | l = ReshapeLayer(l, (batch_size * n_rows * n_cols, n_classes)) 64 | l = NonlinearityLayer(l, softmax) 65 | 66 | return l 67 | 68 | # Note : we also tried to apply deep supervision using intermediate outputs at lower resolutions but didn't see 69 | # any improvements. Our guess is that FC-DenseNet naturally permits this multiscale approach -------------------------------------------------------------------------------- /metrics.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import numpy as np 3 | 4 | def theano_metrics(y_pred, y_true, n_classes, void_labels): 5 | """ 6 | Returns the intersection I and union U (to compute the jaccard I/U) and the accuracy. 7 | 8 | :param y_pred: tensor of predictions. shape (b*0*1, c) with c = n_classes 9 | :param y_true: groundtruth, shape (b,0,1) or (b,c,0,1) with c=1 10 | :param n_classes: int 11 | :param void_labels: list of indexes of void labels 12 | :return: return tensors I and U of size (n_classes), and scalar acc 13 | """ 14 | 15 | # Put y_pred and y_true under the same shape 16 | y_true = T.flatten(y_true) 17 | y_pred = T.argmax(y_pred, axis=1) 18 | 19 | # We use not_void in case the prediction falls in the void class of the groundtruth 20 | for i in range(len(void_labels)): 21 | if i == 0: 22 | not_void = T.neq(y_true, void_labels[i]) 23 | else: 24 | not_void = not_void * T.neq(y_true, void_labels[i]) 25 | 26 | I = T.zeros(n_classes) 27 | U = T.zeros(n_classes) 28 | 29 | for i in range(n_classes): 30 | y_true_i = T.eq(y_true, i) 31 | y_pred_i = T.eq(y_pred, i) 32 | I = T.set_subtensor(I[i], T.sum(y_true_i * y_pred_i)) 33 | U = T.set_subtensor(U[i], T.sum(T.or_(y_true_i, y_pred_i) * not_void)) 34 | 35 | accuracy = T.sum(I) / T.sum(not_void) 36 | 37 | return I, U, accuracy 38 | 39 | 40 | def numpy_metrics(y_pred, y_true, n_classes, void_labels): 41 | """ 42 | Similar to theano_metrics to metrics but instead y_pred and y_true are now numpy arrays 43 | """ 44 | 45 | # Put y_pred and y_true under the same shape 46 | y_pred = np.argmax(y_pred, axis=1) 47 | y_true = y_true.flatten() 48 | 49 | # We use not_void in case the prediction falls in the void class of the groundtruth 50 | not_void = ~ np.any([y_true == label for label in void_labels], axis=0) 51 | 52 | I = np.zeros(n_classes) 53 | U = np.zeros(n_classes) 54 | 55 | for i in range(n_classes): 56 | y_true_i = y_true == i 57 | y_pred_i = y_pred == i 58 | 59 | I[i] = np.sum(y_true_i & y_pred_i) 60 | U[i] = np.sum((y_true_i | y_pred_i) & not_void) 61 | 62 | accuracy = np.sum(I) / np.sum(not_void) 63 | return I, U, accuracy 64 | 65 | 66 | def crossentropy(y_pred, y_true, void_labels): 67 | # Flatten y_true 68 | y_true = T.flatten(y_true) 69 | 70 | # Clip predictions 71 | 72 | # Create mask 73 | mask = T.ones_like(y_true) 74 | for el in void_labels: 75 | mask = T.switch(T.eq(y_true, el), np.int32(0), mask) 76 | 77 | # Modify y_true temporarily 78 | y_true_tmp = y_true * mask 79 | 80 | # Compute cross-entropy 81 | loss = T.nnet.categorical_crossentropy(y_pred, y_true_tmp) 82 | 83 | # Compute masked mean loss 84 | loss *= mask 85 | loss = T.sum(loss) / T.sum(mask).astype('float32') 86 | 87 | return loss 88 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import imp 4 | import time 5 | import theano 6 | from lasagne.layers import get_output 7 | 8 | from data_loader import load_data 9 | from metrics import numpy_metrics, theano_metrics 10 | 11 | 12 | def test(config_path, weight_path): 13 | """ 14 | This function builds the model defined in config_path and restores the weights defined in weight_path. It then 15 | reports the jaccard and global accuracy metrics on the CamVid test set. 16 | """ 17 | 18 | cf = imp.load_source('cf', config_path) 19 | 20 | ############### 21 | # Load data # 22 | ############### 23 | 24 | print('-' * 75) 25 | # Load config file 26 | 27 | # Load data 28 | print('Loading data') 29 | batch_size = 10 30 | _, _, iterator = load_data(cf.dataset, batch_size=batch_size) 31 | 32 | n_classes = iterator.get_n_classes() 33 | _, n_rows, n_cols = iterator.data_shape 34 | void_labels = iterator.get_void_labels() 35 | 36 | ################### 37 | # Compile model # 38 | ################### 39 | 40 | # Print summary 41 | net = cf.net 42 | net.restore(weight_path) 43 | 44 | # Compile test functions 45 | prediction = get_output(net.output_layer, deterministic=True, batch_norm_use_averages=False) 46 | metrics = theano_metrics(prediction, net.target_var, n_classes, void_labels) 47 | 48 | print('Compiling functions') 49 | start_time_compilation = time.time() 50 | f = theano.function([net.input_var, net.target_var], metrics) 51 | print('Compilation took {:.3f} seconds'.format(time.time() - start_time_compilation)) 52 | 53 | ################### 54 | # Main loop # 55 | ################### 56 | 57 | n_batches = iterator.get_n_batches() 58 | I_tot = np.zeros(n_classes) 59 | U_tot = np.zeros(n_classes) 60 | acc_tot = 0. 61 | n_imgs = 0 62 | for i in range(n_batches): 63 | X, Y = iterator.next() 64 | I, U, acc = f(X, Y[:, None, :, :]) 65 | I_tot += I 66 | U_tot += U 67 | acc_tot += acc * batch_size 68 | n_imgs += batch_size 69 | 70 | # # Progression bar ( < 74 characters) 71 | sys.stdout.write('\r[{}%]'.format(int(100. * (i + 1) / n_batches))) 72 | sys.stdout.flush() 73 | 74 | labels = ['sky', 'building', 'column_pole', 'road', 'sidewalk', 'tree', 'sign', 'fence', 'car', 'pedestrian', 75 | 'byciclist'] 76 | 77 | for label, jacc in zip(labels, I_tot / U_tot): 78 | print('{} :\t{:.4f}'.format(label, jacc)) 79 | print 'Mean Jaccard', np.mean(I_tot / U_tot) 80 | print 'Global accuracy', acc_tot / n_imgs 81 | 82 | # To visualize an image : np.reshape(np.argmax(g(X), axis = 1), (360, 480)) 83 | # with g = theano.function([net.input_var], prediction) 84 | 85 | 86 | if __name__ == '__main__': 87 | config_path = 'config/FC-DenseNet103.py' 88 | weight_path = 'weights/FC-DenseNet103_weights.npz' 89 | test(config_path, weight_path) 90 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import os 4 | import shutil 5 | import sys 6 | import time 7 | from datetime import datetime 8 | import imp 9 | import lasagne 10 | import numpy as np 11 | import theano 12 | import theano.tensor as T 13 | from lasagne.regularization import regularize_network_params 14 | from lasagne.layers import get_output 15 | from theano.tensor.shared_randomstreams import RandomStreams 16 | from data_loader import load_data 17 | 18 | from metrics import theano_metrics, crossentropy 19 | 20 | 21 | def batch_loop(iterator, f, epoch, phase, history): 22 | """ Loop on the batches """ 23 | 24 | n_batches = iterator.get_n_batches() 25 | n_imgs = 0. 26 | 27 | for i in range(n_batches): 28 | X, Y = iterator.next() 29 | batch_size = X.shape[0] 30 | n_imgs += batch_size 31 | 32 | loss, I, U, acc = f(X, Y[:, None, :, :]) 33 | if i == 0: 34 | loss_tot = loss * batch_size 35 | I_tot = I 36 | U_tot = U 37 | acc_tot = acc * batch_size 38 | else: 39 | loss_tot += loss * batch_size 40 | I_tot += I 41 | U_tot += U 42 | acc_tot += acc * batch_size 43 | 44 | # # Progression bar ( < 74 characters) 45 | sys.stdout.write('\rEpoch {} : [{} : {}%]'.format(epoch, phase, int(100. * (i + 1) / n_batches))) 46 | sys.stdout.flush() 47 | 48 | history[phase]['loss'].append(loss_tot / n_imgs) 49 | history[phase]['jaccard'].append(np.mean(I_tot / U_tot)) 50 | history[phase]['accuracy'].append(acc_tot / n_imgs) 51 | 52 | return history 53 | 54 | 55 | def train(cf): 56 | 57 | ############### 58 | # load data # 59 | ############### 60 | 61 | print('-' * 75) 62 | print('Loading data') 63 | #TODO ; prepare a public version of the data loader 64 | train_iter, val_iter, test_iter = load_data(cf.dataset, 65 | train_crop_size=cf.train_crop_size, 66 | batch_size=cf.batch_size, 67 | horizontal_flip=True, 68 | ) 69 | 70 | n_classes = train_iter.get_n_classes() 71 | void_labels = train_iter.get_void_labels() 72 | 73 | print('Number of images : train : {}, val : {}, test : {}'.format( 74 | train_iter.get_n_samples(), val_iter.get_n_samples(), test_iter.get_n_samples())) 75 | 76 | ################### 77 | # Build model # 78 | ################### 79 | 80 | # Build model and display summary 81 | net = cf.net 82 | net.summary() 83 | 84 | # Restore 85 | if hasattr(cf, 'pretrained_model'): 86 | print('Using a pretrained model : {}'.format(cf.pretrained_model)) 87 | net.restore(cf.pretrained_model) 88 | 89 | # Compile functions 90 | print('Compilation starts at ' + str(datetime.now()).split('.')[0]) 91 | params = lasagne.layers.get_all_params(net.output_layer, trainable=True) 92 | lr_shared = theano.shared(np.array(cf.learning_rate, dtype='float32')) 93 | lr_decay = np.array(cf.lr_sched_decay, dtype='float32') 94 | 95 | # Create loss and metrics 96 | for key in ['train', 'valid']: 97 | 98 | # LOSS 99 | pred = get_output(net.output_layer, deterministic=key == 'valid', 100 | batch_norm_update_averages=False, batch_norm_use_averages=False) 101 | loss = crossentropy(pred, net.target_var, void_labels) 102 | 103 | if cf.weight_decay: 104 | weightsl2 = regularize_network_params(net.output_layer, lasagne.regularization.l2) 105 | loss += cf.weight_decay * weightsl2 106 | 107 | # METRICS 108 | I, U, acc = theano_metrics(pred, net.target_var, n_classes, void_labels) 109 | 110 | # COMPILE 111 | start_time_compilation = time.time() 112 | if key == 'train': 113 | updates = cf.optimizer(loss, params, learning_rate=lr_shared) 114 | train_fn = theano.function([net.input_var, net.target_var], [loss, I, U, acc], updates=updates) 115 | else: 116 | val_fn = theano.function([net.input_var, net.target_var], [loss, I, U, acc]) 117 | 118 | print('{} compilation took {:.3f} seconds'.format(key, time.time() - start_time_compilation)) 119 | 120 | ################### 121 | # Main loops # 122 | ################### 123 | 124 | # metric's sauce 125 | init_history = lambda: {'loss': [], 'jaccard': [], 'accuracy': []} 126 | history = {'train': init_history(), 'val': init_history(), 'test': init_history()} 127 | patience = 0 128 | best_jacc_val = 0 129 | best_epoch = 0 130 | 131 | if hasattr(cf, 'pretrained_model'): 132 | print('Validation score before training') 133 | print batch_loop(val_iter, val_fn, 0, 'val', {'val': init_history()}) 134 | 135 | # Training main loop 136 | print('-' * 30) 137 | print('Training starts at ' + str(datetime.now()).split('.')[0]) 138 | print('-' * 30) 139 | 140 | for epoch in range(cf.num_epochs): 141 | 142 | # Train 143 | start_time_train = time.time() 144 | history = batch_loop(train_iter, train_fn, epoch, 'train', history) 145 | # Validation 146 | start_time_valid = time.time() 147 | history = batch_loop(val_iter, val_fn, epoch, 'val', history) 148 | 149 | # Print 150 | out_str = \ 151 | '\r\x1b[2 Epoch {} took {}+{} sec. ' \ 152 | 'loss = {:.5f} | jacc = {:.5f} | acc = {:.5f} || ' \ 153 | 'loss = {:.5f} | jacc = {:.5f} | acc = {:.5f}'.format( 154 | epoch, int(start_time_valid - start_time_train), int(time.time() - start_time_valid), 155 | history['train']['loss'][-1], history['train']['jaccard'][-1], history['train']['accuracy'][-1], 156 | history['val']['loss'][-1], history['val']['jaccard'][-1], history['val']['accuracy'][-1]) 157 | 158 | # Monitoring jaccard 159 | if history['val']['jaccard'][-1] > best_jacc_val: 160 | out_str += ' (BEST)' 161 | best_jacc_val = history['val']['jaccard'][-1] 162 | best_epoch = epoch 163 | patience = 0 164 | net.save(os.path.join(cf.savepath, 'model.npz')) 165 | else: 166 | patience += 1 167 | 168 | print out_str 169 | 170 | np.savez(os.path.join(cf.savepath, 'errors.npz'), metrics=history, best_epoch=best_epoch) 171 | 172 | # Learning rate scheduler 173 | lr_shared.set_value(lr_shared.get_value() * lr_decay) 174 | 175 | # Finish training if patience has expired or max nber of epochs reached 176 | if patience == cf.max_patience or epoch == cf.num_epochs - 1: 177 | # Load best model weights 178 | net.restore(os.path.join(cf.savepath, 'model.npz')) 179 | 180 | # Test 181 | print('Training ends\nTest') 182 | if test_iter.get_n_samples() == 0: 183 | print 'No test set' 184 | else: 185 | history = batch_loop(test_iter, val_fn, epoch, 'test', history) 186 | 187 | print ('Average cost test = {:.5f} | jacc test = {:.5f} | acc_test = {:.5f} '.format( 188 | history['test']['loss'][-1], 189 | history['test']['jaccard'][-1], 190 | history['test']['accuracy'][-1])) 191 | 192 | np.savez(os.path.join(cf.savepath, 'errors.npz'), metrics=history, best_epoch=best_epoch) 193 | 194 | # Exit 195 | return 196 | 197 | 198 | def initiate_training(cf): 199 | 200 | # Seed : to make experiments reproductible, use deterministic convolution in CuDNN with THEANO_FLAGS 201 | np.random.seed(cf.seed) 202 | theano.tensor.shared_randomstreams.RandomStreams(cf.seed) 203 | 204 | if not os.path.exists(cf.savepath): 205 | os.makedirs(cf.savepath) 206 | else: 207 | stop = raw_input('\033[93m The following folder already exists {}. ' 208 | 'Do you want to overwrite it ? ([y]/n) \033[0m'.format(cf.savepath)) 209 | if stop == 'n': 210 | return 211 | 212 | print('-' * 75) 213 | print('Config\n') 214 | print('Local saving directory : ' + cf.savepath) 215 | print('Model path : ' + cf.model_path) 216 | 217 | # We also copy the model and the training scipt to reproduce exactly the experiments 218 | shutil.copy('train.py', os.path.join(cf.savepath, 'train.py')) 219 | shutil.copy(os.path.join('models', cf.model_path), os.path.join(cf.savepath, 'model.py')) 220 | shutil.copy(cf.config_path, os.path.join(cf.savepath, 'cf.py')) 221 | 222 | # Train 223 | train(cf) 224 | 225 | 226 | if __name__ == '__main__': 227 | 228 | # To launch an experiment, use the following command line : 229 | # THEANO_FLAGS='device=cuda,optimizer=fast_compile,optimizer_including=fusion' python train.py -c config_path -e experiment_name 230 | # Logs of the training will be stored in the folder cf.savepath/experiment_name 231 | 232 | parser = argparse.ArgumentParser(description='DenseNet training') 233 | 234 | parser.add_argument('-c', '--config_path', 235 | type=str, 236 | default=None, 237 | help='Configuration file') 238 | 239 | parser.add_argument('-e', '--exp_name', 240 | type=str, 241 | default=None, 242 | help='Name of the experiment') 243 | arguments = parser.parse_args() 244 | 245 | assert arguments.config_path is not None, 'Please provide a configuration path using ' \ 246 | '-c config/pathname in the command line' 247 | assert arguments.exp_name is not None, 'Please provide a name for the experiment using -e name in the command line' 248 | 249 | # Parse the configuration file 250 | cf = imp.load_source('config', arguments.config_path) 251 | cf.savepath = arguments.exp_name 252 | cf.config_path = arguments.config_path 253 | 254 | # You can easily launch different experiments by slightly changing cf and initiate training 255 | initiate_training(cf) 256 | -------------------------------------------------------------------------------- /weights/FC-DenseNet103_weights.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimJeg/FC-DenseNet/cf2375bf9f6ed20ba029a5ee540261aad89732d5/weights/FC-DenseNet103_weights.npz --------------------------------------------------------------------------------