├── data └── .keep ├── weights └── .keep ├── imgs ├── tiramisu-103.png ├── tiramisu-blocks.png ├── original-result-table.png ├── tiramisu-67-model-acc.png └── tiramisu-67-model-loss.png ├── helper.py ├── LICENSE ├── .gitignore ├── requirements.txt ├── camvid_data_loader.py ├── readme.md ├── train-tiramisu.py ├── model-tiramasu-56.py ├── model-dynamic.py ├── model-tiramasu-67.py ├── model-tiramasu-103.py ├── model-tiramasu-67-func-api.py └── fc-densenet-model.py /data/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /weights/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /imgs/tiramisu-103.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0bserver07/One-Hundred-Layers-Tiramisu/HEAD/imgs/tiramisu-103.png -------------------------------------------------------------------------------- /imgs/tiramisu-blocks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0bserver07/One-Hundred-Layers-Tiramisu/HEAD/imgs/tiramisu-blocks.png -------------------------------------------------------------------------------- /imgs/original-result-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0bserver07/One-Hundred-Layers-Tiramisu/HEAD/imgs/original-result-table.png -------------------------------------------------------------------------------- /imgs/tiramisu-67-model-acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0bserver07/One-Hundred-Layers-Tiramisu/HEAD/imgs/tiramisu-67-model-acc.png -------------------------------------------------------------------------------- /imgs/tiramisu-67-model-loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0bserver07/One-Hundred-Layers-Tiramisu/HEAD/imgs/tiramisu-67-model-loss.png -------------------------------------------------------------------------------- /helper.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | 4 | import cv2 5 | import numpy as np 6 | import itertools 7 | 8 | from helper import * 9 | import os 10 | 11 | def normalized(rgb): 12 | #return rgb/255.0 13 | norm=np.zeros((rgb.shape[0], rgb.shape[1], 3),np.float32) 14 | 15 | b=rgb[:,:,0] 16 | g=rgb[:,:,1] 17 | r=rgb[:,:,2] 18 | 19 | norm[:,:,0]=cv2.equalizeHist(b) 20 | norm[:,:,1]=cv2.equalizeHist(g) 21 | norm[:,:,2]=cv2.equalizeHist(r) 22 | 23 | return norm 24 | 25 | def one_hot_it(labels,w,h): 26 | x = np.zeros([w,h,12]) 27 | for i in range(w): 28 | for j in range(h): 29 | x[i,j,labels[i][j]]=1 30 | return x 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [2016] [Yad Faeq] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # dotenv 80 | .env 81 | 82 | # virtualenv 83 | .venv/ 84 | venv/ 85 | ENV/ 86 | 87 | # Spyder project settings 88 | .spyderproject 89 | 90 | # Rope project settings 91 | .ropeproject 92 | 93 | 94 | .DS_Store 95 | 96 | CamVid/ 97 | *.npy 98 | 99 | *.hdf5 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.3 2 | atari-py==0.0.18 3 | autobahn==20.12.3 4 | bleach==3.3.0 5 | box2d-py==2.3.1 6 | click==6.7 7 | constantly==15.1.0 8 | cycler==0.10.0 9 | decorator==4.0.11 10 | docker-py==1.10.3 11 | docker-pycreds==0.2.1 12 | entrypoints==0.2.2 13 | fastzbarlight==0.0.14 14 | Flask==1.0 15 | gevent==1.2.1 16 | go-vncdriver==0.4.19 17 | greenlet==0.4.12 18 | gym==0.7.3 19 | h5py==2.6.0 20 | html5lib==0.9999999 21 | imageio==2.1.1 22 | incremental==16.10.1 23 | ipykernel==4.5.2 24 | ipython==5.2.2 25 | ipython-genutils==0.1.0 26 | itsdangerous==0.24 27 | Jinja2==2.11.3 28 | jsonschema==2.6.0 29 | jupyter-client==4.4.0 30 | jupyter-core==4.2.1 31 | Keras==2.0.2 32 | MarkupSafe==0.23 33 | matplotlib==2.0.0 34 | mistune==0.8.1 35 | mujoco-py==0.5.7 36 | nbconvert==5.1.1 37 | nbformat==4.2.0 38 | notebook==6.4.1 39 | numpy==1.12.1 40 | olefile==0.44 41 | opencv-python==4.2.0.32 42 | pachi-py==0.0.21 43 | packaging==16.8 44 | pandocfilters==1.4.1 45 | pexpect==4.2.1 46 | pickleshare==0.7.4 47 | Pillow==8.3.2 48 | pip-save==0.2.0 49 | pkg-resources==0.0.0 50 | prompt-toolkit==1.0.13 51 | protobuf==3.2.0 52 | ptyprocess==0.5.1 53 | pyglet==1.2.4 54 | Pygments==2.7.4 55 | PyOpenGL==3.1.0 56 | pyparsing==2.2.0 57 | python-dateutil==2.6.0 58 | pytz==2016.10 59 | PyYAML==5.4 60 | pyzmq==16.0.2 61 | ReinforcePy==0.3 62 | requests==2.20.0 63 | scipy==0.19.0 64 | simplegeneric==0.8.1 65 | six==1.9.0 66 | tensorflow==2.5.2 67 | tensorflow-gpu==2.5.2 68 | terminado==0.6 69 | testpath==0.3 70 | tflearn==0.3 71 | Theano==0.9.0 72 | tornado==4.4.2 73 | tqdm==4.11.2 74 | traitlets==4.3.1 75 | Twisted==20.3.0 76 | txaio==2.6.0 77 | ujson==1.35 78 | wcwidth==0.1.7 79 | websocket-client==0.40.0 80 | Werkzeug==0.15.3 81 | zope.interface==4.3.3 82 | -------------------------------------------------------------------------------- /camvid_data_loader.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | 4 | import cv2 5 | import numpy as np 6 | import itertools 7 | 8 | from helper import * 9 | import os 10 | 11 | # Copy the data to this dir here in the SegNet project /CamVid from here: 12 | # https://github.com/alexgkendall/SegNet-Tutorial 13 | DataPath = './CamVid/' 14 | # data_shape = 360*480 15 | data_shape = 224*224 16 | 17 | 18 | def load_data(mode): 19 | data = [] 20 | label = [] 21 | with open(DataPath + mode +'.txt') as f: 22 | txt = f.readlines() 23 | txt = [line.split(' ') for line in txt] 24 | for i in range(len(txt)): 25 | # data.append(np.rollaxis(normalized(cv2.imread(os.getcwd() + txt[i][0][7:])),2)) 26 | # this load cropped images 27 | data.append(np.rollaxis(normalized(cv2.imread(os.getcwd() + txt[i][0][7:])[136:,256:]),2)) 28 | label.append(one_hot_it(cv2.imread(os.getcwd() + txt[i][1][7:][:-1])[136:,256:][:,:,0],224,224)) 29 | print('.',end='') 30 | return np.array(data), np.array(label) 31 | 32 | 33 | 34 | train_data, train_label = load_data("train") 35 | train_label = np.reshape(train_label,(367,data_shape,12)) 36 | 37 | test_data, test_label = load_data("test") 38 | test_label = np.reshape(test_label,(233,data_shape,12)) 39 | 40 | val_data, val_label = load_data("val") 41 | val_label = np.reshape(val_label,(101,data_shape,12)) 42 | 43 | 44 | np.save("data/train_data", train_data) 45 | np.save("data/train_label", train_label) 46 | 47 | np.save("data/test_data", test_data) 48 | np.save("data/test_label", test_label) 49 | 50 | np.save("data/val_data", val_data) 51 | np.save("data/val_label", val_label) 52 | 53 | # FYI they are: 54 | # Sky = [128,128,128] 55 | # Building = [128,0,0] 56 | # Pole = [192,192,128] 57 | # Road_marking = [255,69,0] 58 | # Road = [128,64,128] 59 | # Pavement = [60,40,222] 60 | # Tree = [128,128,0] 61 | # SignSymbol = [192,128,128] 62 | # Fence = [64,64,128] 63 | # Car = [64,0,128] 64 | # Pedestrian = [64,64,0] 65 | # Bicyclist = [0,128,192] 66 | # Unlabelled = [0,0,0] 67 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ### The One Hundred Layers Tiramisu: Fully Convolutional DenseNets for Semantic Segmentation: 2 | --- 3 | 4 | *Work In Progress, Results can't be replicated yet with the models here* 5 | * UPDATE: April 28th: Skip_Connection added thanks to the reviewers, check model `model-tiramasu-67-func-api.py` 6 | 7 | 8 | ` feel free to open issues for suggestions:)` 9 | 10 | * Keras2 + TF used for the recent updates, which might cause with some confilict from previous version I had in here 11 | 12 | 13 | What is The One Hundred Layers Tiramisu? 14 | 15 | * A state of art (as in Jan 2017) Semantic Pixel-wise Image Segmentation model that consists of a fully deep convolutional blocks with downsampling, skip-layer then to Upsampling architecture. 16 | * An extension of DenseNets to deal with the problem of semantic segmentation. 17 | 18 | **Fully Convolutional DensNet** = **(Dense Blocks + Transition Down Blocks)** + **(Bottleneck Blocks)** + **(Dense Blocks + Transition Up Blocks)** + **Pixel-Wise Classification** layer 19 | 20 | ![model](./imgs/tiramisu-103.png) 21 | 22 | 23 | 24 | ##### *[The One Hundred Layers Tiramisu: Fully Convolutional DenseNets for Semantic Segmentation (Simon Jégou, Michal Drozdzal, David Vazquez, Adriana Romero, Yoshua Bengio) arXiv:1611.09326 cs.CV](https://arxiv.org/abs/1611.09326)* 25 | 26 | #### Requirements: 27 | ----- 28 | 29 | * Keras==2.0.2 30 | * tensorflow-gpu==1.0.1 31 | * or just go ahead and do: `pip install -r requirements.txt` 32 | 33 | #### Model Strucure: 34 | ----- 35 | 36 | * DenseBlock: 37 | `BatchNormalization` + `Activation [ Relu ]` + `Convolution2D` + `Dropout` 38 | 39 | * TransitionDown: 40 | `BatchNormalization` + `Activation [ Relu ]` + `Convolution2D` + `Dropout` + `MaxPooling2D` 41 | 42 | * TransitionUp: 43 | `Deconvolution2D` (Convolutions Transposed) 44 | 45 | ![model-blocks](./imgs/tiramisu-blocks.png) 46 | 47 | ----- 48 | 49 | #### Model Params: 50 | ----- 51 | 52 | * RMSprop is used with Learnining Rete of 0.001 and weight decay 0.995 53 | * However, using those got me nowhere, I switched to SGD and started tweaking the LR + Decay myself. 54 | * There are no details given about BatchNorm params, again I have gone with what the Original DenseNet paper had suggested. 55 | * Things to keep in mind perhaps: 56 | * the weight inti: he_uniform (maybe change it around?) 57 | * the regualzrazation too agressive? 58 | 59 | 60 | 61 | ### Repo (explanation): 62 | --- 63 | 64 | * Download the CamVid Dataset as explained below: 65 | * Use the `data_loader.py` to crop images to `224, 224` as in the paper implementation. 66 | * run `model-tiramasu-67-func-api.py` or `python model-tirmasu-56.py` for now to generate each models file. 67 | * run `python train-tirmasu.py` to start training: 68 | * Saves best checkpoints for the model and `data_loader` included for the `CamVidDataset` 69 | * `helper.py` contains two methods `normalized` and `one_hot_it`, currently for the CamVid Task 70 | 71 | ### Dataset: 72 | --- 73 | 74 | 1. In a different directory run this to download the [dataset from original Implementation](https://github.com/alexgkendall/SegNet-Tutorial). 75 | * `git clone git@github.com:alexgkendall/SegNet-Tutorial.git` 76 | * copy the `/CamVid` to here, or change the `DataPath` in `data_loader.py` to the above directory 77 | 2. The run `python data_loader.py` to generate these two files: 78 | 79 | * `/data/train_data.npz/` and `/data/train_label.npz` 80 | * This will make it easy to process the model over and over, rather than waiting the data to be loaded into memory. 81 | 82 | 83 | 84 | ---- 85 | 86 | * Experiments: 87 | 88 | 89 | 90 | | Models | Acc | Loss | Notes | 91 | | ------------- |:-------------:| -----:|-------| 92 | | FC-DenseNet 67| ![model-results](./imgs/tiramisu-67-model-acc.png) | ![model-results](./imgs/tiramisu-67-model-loss.png) | 150 Epochs, RMSPROP 93 | 94 | ### To Do: 95 | ---- 96 | 97 | [x] FC-DenseNet 103 98 | [x] FC-DenseNet 56 99 | [x] FC-DenseNet 67 100 | [ ] Replicate Test Accuracy CamVid Task 101 | [ ] Replicate Test Accuracy GaTech Dataset Task 102 | [ ] Requirements 103 | 104 | 105 | * Original Results Table: 106 | 107 | ![model-results](./imgs/original-result-table.png) 108 | 109 | 110 | -------------------------------------------------------------------------------- /train-tiramisu.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | 5 | 6 | import keras.models as models 7 | from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape, Permute 8 | from keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D, Cropping2D 9 | from keras.layers.normalization import BatchNormalization 10 | 11 | from keras.layers import Conv2D, Conv2DTranspose 12 | 13 | from keras.optimizers import RMSprop, Adam, SGD 14 | from keras.callbacks import ModelCheckpoint 15 | from keras.layers import Input, merge 16 | from keras.regularizers import l2 17 | from keras.models import Model 18 | from keras import regularizers 19 | 20 | from keras.models import Model 21 | from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, UpSampling2D 22 | 23 | from keras import backend as K 24 | 25 | from keras import callbacks 26 | import math 27 | # remote = callbacks.RemoteMonitor(root='http://localhost:9000', path='/publish/epoch/end/', field='data', headers=None) 28 | 29 | # early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=50, verbose=0, mode='auto') 30 | 31 | # tensor_board = callbacks.TensorBoard(log_dir='./logs', histogram_freq=5, write_graph=True, write_images=True) 32 | 33 | 34 | K.set_image_dim_ordering('tf') 35 | 36 | import cv2 37 | import numpy as np 38 | import json 39 | 40 | 41 | np.random.seed(7) # 0bserver07 for reproducibility 42 | 43 | 44 | 45 | 46 | class_weighting = [ 47 | 0.2595, 48 | 0.1826, 49 | 4.5640, 50 | 0.1417, 51 | 0.5051, 52 | 0.3826, 53 | 9.6446, 54 | 1.8418, 55 | 6.6823, 56 | 6.2478, 57 | 3.0, 58 | 7.3614 59 | ] 60 | 61 | 62 | # load the data 63 | train_data = np.load('./data/train_data.npy') 64 | train_data = train_data.reshape((367, 224, 224, 3)) 65 | 66 | train_label = np.load('./data/train_label.npy')#[:,:,:-1] 67 | 68 | 69 | 70 | test_data = np.load('./data/test_data.npy') 71 | test_data = test_data.reshape((233, 224, 224, 3)) 72 | 73 | 74 | test_label = np.load('./data/test_label.npy')#[:,:,:-1] 75 | 76 | # test_label = to_categorical(test_label, num_classes=None) 77 | 78 | # load the model: 79 | with open('tiramisu_fc_dense67_model_12_func.json') as model_file: 80 | tiramisu = models.model_from_json(model_file.read()) 81 | 82 | 83 | # section 4.1 from the paper 84 | 85 | from keras.callbacks import LearningRateScheduler 86 | 87 | # learning rate schedule 88 | def step_decay(epoch): 89 | initial_lrate = 0.001 90 | drop = 0.00001 91 | epochs_drop = 10.0 92 | lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop)) 93 | return lrate 94 | 95 | lrate = LearningRateScheduler(step_decay) 96 | 97 | # tiramisu.load_weights("weights/prop_tiramisu_weights_67_12_func_10-e5_decay.best.hdf5") 98 | 99 | 100 | 101 | 102 | optimizer = RMSprop(lr=0.001, decay=0.0000001) 103 | # optimizer = SGD(lr=0.01) 104 | # optimizer = Adam(lr=1e-3, decay=0.995) 105 | 106 | tiramisu.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"]) 107 | 108 | # learning schedule callback 109 | # lrate = LearningRateScheduler(step_decay) 110 | 111 | # checkpoint 278 112 | TensorBoard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=5, write_graph=True, write_images=True) 113 | # ReduceLROnPlateau = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=0, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) 114 | 115 | filepath="weights/prop_tiramisu_weights_67_12_func_10-e7_decay.best.hdf5" 116 | checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=2, 117 | save_best_only=True, save_weights_only=False, mode='max') 118 | 119 | 120 | callbacks_list = [checkpoint] 121 | 122 | nb_epoch = 150 123 | batch_size = 2 124 | 125 | 126 | # Fit the model 127 | history = tiramisu.fit(train_data, train_label, batch_size=batch_size, epochs=nb_epoch, 128 | callbacks=callbacks_list, class_weight=class_weighting,verbose=1, validation_data=(test_data, test_label), shuffle=True) # validation_split=0.33 129 | 130 | 131 | 132 | 133 | # This save the trained model weights to this file with number of epochs 134 | tiramisu.save_weights('weights/prop_tiramisu_weights_67_12_func_10-e7_decay{}.hdf5'.format(nb_epoch)) 135 | 136 | import matplotlib.pyplot as plt 137 | # list all data in history 138 | print(history.history.keys()) 139 | # summarize history for accuracy 140 | plt.plot(history.history['acc']) 141 | plt.plot(history.history['val_acc']) 142 | plt.title('model accuracy') 143 | plt.ylabel('accuracy') 144 | plt.xlabel('epoch') 145 | plt.legend(['train', 'test'], loc='upper left') 146 | plt.show() 147 | # summarize history for loss 148 | plt.plot(history.history['loss']) 149 | plt.plot(history.history['val_loss']) 150 | plt.title('model loss') 151 | plt.ylabel('loss') 152 | plt.xlabel('epoch') 153 | plt.legend(['train', 'test'], loc='upper left') 154 | plt.show() 155 | -------------------------------------------------------------------------------- /model-tiramasu-56.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | 5 | 6 | import keras.models as models 7 | from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape, Permute 8 | from keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D, Cropping2D 9 | from keras.layers.normalization import BatchNormalization 10 | 11 | from keras.layers import Conv2D, Conv2DTranspose 12 | 13 | from keras import backend as K 14 | 15 | import cv2 16 | import numpy as np 17 | import json 18 | 19 | # weight_decay = 0.0001 20 | from keras.regularizers import l2 21 | 22 | class Tiramisu(): 23 | 24 | 25 | 26 | 27 | def __init__(self): 28 | self.create() 29 | 30 | def DenseBlock(self, layers, filters): 31 | model = self.model 32 | for i in range(layers): 33 | model.add(BatchNormalization(mode=0, axis=1, 34 | gamma_regularizer=l2(0.0001), 35 | beta_regularizer=l2(0.0001))) 36 | model.add(Activation('relu')) 37 | model.add(Conv2D(filters, kernel_size=(3, 3), padding='same', 38 | kernel_initializer="he_uniform")) 39 | model.add(Dropout(0.2)) 40 | 41 | def TransitionDown(self,filters): 42 | model = self.model 43 | model.add(BatchNormalization(mode=0, axis=1, 44 | gamma_regularizer=l2(0.0001), 45 | beta_regularizer=l2(0.0001))) 46 | model.add(Activation('relu')) 47 | model.add(Conv2D(filters, kernel_size=(1, 1), padding='same', 48 | kernel_initializer="he_uniform")) 49 | model.add(Dropout(0.2)) 50 | model.add(MaxPooling2D( pool_size=(2, 2), 51 | strides=(2, 2), 52 | data_format='channels_last')) 53 | 54 | def TransitionUp(self,filters, input_shape,output_shape): 55 | model = self.model 56 | model.add(Conv2DTranspose(filters, kernel_size=(3, 3), strides=(2, 2), 57 | padding='same', 58 | output_shape=output_shape, 59 | input_shape=input_shape, 60 | kernel_initializer="he_uniform", 61 | data_format='channels_last')) 62 | 63 | 64 | def create(self): 65 | model = self.model = models.Sequential() 66 | # cropping 67 | # model.add(Cropping2D(cropping=((68, 68), (128, 128)), input_shape=(3, 360,480))) 68 | 69 | model.add(Conv2D(48, kernel_size=(3, 3), padding='same', 70 | input_shape=(224,224,3), 71 | kernel_initializer="he_uniform", 72 | kernel_regularizer = l2(0.0001), 73 | data_format='channels_last')) 74 | # (5 * 4)* 2 + 5 + 5 + 1 + 1 +1 75 | # growth_m = 4 * 12 76 | # previous_m = 48 77 | 78 | self.DenseBlock(4,96) # 4*12 = 48 + 48 = 96 79 | self.TransitionDown(96) 80 | self.DenseBlock(4,144) # 4*12 = 48 + 96 = 144 81 | self.TransitionDown(144) 82 | self.DenseBlock(4,192) # 4*12 = 48 + 144 = 192 83 | self.TransitionDown(192) 84 | self.DenseBlock(4,240)# 4*12 = 48 + 192 = 240 85 | self.TransitionDown(240) 86 | self.DenseBlock(4,288) # 4*12 = 48 + 288 = 336 87 | self.TransitionDown(288) 88 | 89 | self.DenseBlock(15,336) # 4 * 12 = 48 + 288 = 336 90 | 91 | self.TransitionUp(384, (384, 7, 7), (None, 384, 14, 14)) # m = 288 + 4x12 + 4x12 = 384. 92 | self.DenseBlock(4,384) 93 | 94 | self.TransitionUp(336, (336, 14, 14), (None, 336, 28, 28)) #m = 240 + 4x12 + 4x12 = 336 95 | self.DenseBlock(4,336) 96 | 97 | self.TransitionUp(288, (288, 28, 28), (None, 288, 56, 56)) # m = 192 + 4x12 + 4x12 = 288 98 | self.DenseBlock(4,288) 99 | 100 | self.TransitionUp(240, (240, 56, 56), (None, 240, 112, 112)) # m = 144 + 4x12 + 4x12 = 240 101 | self.DenseBlock(4,240) 102 | 103 | self.TransitionUp(192, (192, 112, 112), (None, 192, 224, 224)) # m = 96 + 4x12 + 4x12 = 192 104 | self.DenseBlock(4,192) 105 | 106 | model.add(Conv2D(12, kernel_size=(1,1), 107 | padding='same', 108 | kernel_initializer="he_uniform", 109 | kernel_regularizer = l2(0.0001), 110 | data_format='channels_last')) 111 | model.add(Reshape((12, 224 * 224))) 112 | model.add(Permute((2, 1))) 113 | model.add(Activation('softmax')) 114 | model.summary() 115 | 116 | with open('tiramisu_fc_dense56_model.json', 'w') as outfile: 117 | outfile.write(json.dumps(json.loads(model.to_json()), indent=3)) 118 | 119 | Tiramisu() -------------------------------------------------------------------------------- /model-dynamic.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | 5 | 6 | import keras.models as models 7 | from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape, Permute 8 | from keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D, Cropping2D 9 | from keras.layers.normalization import BatchNormalization 10 | 11 | from keras.layers import Conv2D, Conv2DTranspose 12 | 13 | from keras import backend as K 14 | 15 | import cv2 16 | import numpy as np 17 | import json 18 | 19 | K.set_image_dim_ordering('th') 20 | 21 | # weight_decay = 0.0001 22 | from keras.regularizers import l2 23 | 24 | class Tiramisu(): 25 | 26 | 27 | 28 | def __init__(self): 29 | self.create() 30 | 31 | def DenseBlock(self, layers, filters): 32 | model = self.model 33 | for i in range(layers): 34 | model.add(BatchNormalization()) 35 | model.add(Activation('relu')) 36 | model.add(Conv2D(filters, kernel_size=(3, 3), padding='same', init="he_uniform", W_regularizer = l2(0.0001))) 37 | model.add(Dropout(0.2)) 38 | 39 | def TransitionDown(self,filters): 40 | model = self.model 41 | model.add(BatchNormalization()) 42 | model.add(Activation('relu')) 43 | model.add(Conv2D(filters, kernel_size=(1, 1), padding='same', init="he_uniform", W_regularizer = l2(0.0001))) 44 | model.add(Dropout(0.2)) 45 | model.add(MaxPooling2D(pool_size=(2, 2))) 46 | 47 | def TransitionUp(self,filters, input_shape,output_shape): 48 | model = self.model 49 | model.add(Conv2DTranspose(filters,kernel_size=(3, 3), strides=(2, 2),data_format='channels_first', output_shape=output_shape, 50 | padding='same', input_shape=input_shape, init="he_uniform", W_regularizer = l2(0.0001))) 51 | 52 | def gfactorCounterDown(self,model_self,growth_factor,block_size,previous_conv_size,block_count=5): 53 | for i in range(block_count): 54 | m = block_size * growth_factor + previous_conv_size 55 | model_self.DenseBlock(growth_factor,m) 56 | model_self.TransitionDown(growth_factor,m) 57 | 58 | def gfactorCounterUp(self,model_self,growth_factor,block_size,previous_block_size,previous_conv_size,block_count=5): 59 | # previous_conv_size = 288, since: 60 | # self.DenseBlock(4,288) # 4*12 = 48 + 288 = 336 61 | # self.TransitionDown(288) 62 | 63 | for i in range(block_count): 64 | m = block_size * growth_factor + previous_block_size * growth_factor + previous_conv_size 65 | model_self.DenseBlock(growth_factor,m) 66 | model_self.TransitionDown(growth_factor,m) 67 | 68 | def create(self): 69 | model = self.model = models.Sequential() 70 | # cropping 71 | # model.add(Cropping2D(cropping=((68, 68), (128, 128)), input_shape=(3, 360,480))) 72 | 73 | model.add(Conv2D(48, kernel_size=(3, 3), padding='same', input_shape=(3,224,224), init="he_uniform", W_regularizer = l2(0.0001))) 74 | 75 | # (5 * 4)* 2 + 5 + 5 + 1 + 1 +1 76 | # growth_m = 4 * 12 77 | # previous_m = 48 78 | self.gfactorCounterDown(self.model,12,4,48,5) 79 | # self.DenseBlock(4,96) # 4*12 = 48 + 48 = 96 80 | # self.TransitionDown(96) 81 | # self.DenseBlock(4,144) # 4*12 = 48 + 96 = 144 82 | # self.TransitionDown(144) 83 | # self.DenseBlock(4,192) # 4*12 = 48 + 144 = 192 84 | # self.TransitionDown(192) 85 | # self.DenseBlock(4,240)# 4*12 = 48 + 192 = 240 86 | # self.TransitionDown(240) 87 | # self.DenseBlock(4,288) # 4*12 = 48 + 288 = 336 88 | # self.TransitionDown(288) 89 | 90 | self.DenseBlock(15,336) # 4 * 12 = 48 + 288 = 336 91 | 92 | 93 | self.gfactorCounterDown(self.model,12,4,4,288,5) 94 | 95 | # self.TransitionUp(384, (384, 7, 7), (None, 384, 14, 14)) # m = 288 + 4x12 + 4x12 = 384. 96 | # self.DenseBlock(4,384) 97 | 98 | # self.TransitionUp(336, (336, 14, 14), (None, 336, 28, 28)) #m = 240 + 4x12 + 4x12 = 336 99 | # self.DenseBlock(4,336) 100 | 101 | # self.TransitionUp(288, (288, 28, 28), (None, 288, 56, 56)) # m = 192 + 4x12 + 4x12 = 288 102 | # self.DenseBlock(4,288) 103 | 104 | # self.TransitionUp(240, (240, 56, 56), (None, 240, 112, 112)) # m = 144 + 4x12 + 4x12 = 240 105 | # self.DenseBlock(4,240) 106 | 107 | # self.TransitionUp(192, (192, 112, 112), (None, 192, 224, 224)) # m = 96 + 4x12 + 4x12 = 192 108 | # self.DenseBlock(4,192) 109 | 110 | model.add(Conv2D(12, kernel_size=(3, 3), padding='same', init="he_uniform", W_regularizer = l2(0.0001))) 111 | model.add(Reshape((12, 224 * 224))) 112 | model.add(Permute((2, 1))) 113 | model.add(Activation('softmax')) 114 | model.summary() 115 | 116 | with open('tiramisu_fc_dense56_model.json', 'w') as outfile: 117 | outfile.write(json.dumps(json.loads(model.to_json()), indent=3)) 118 | 119 | Tiramisu() -------------------------------------------------------------------------------- /model-tiramasu-67.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | 5 | 6 | import keras.models as models 7 | from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape, Permute 8 | from keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D, Cropping2D 9 | from keras.layers.normalization import BatchNormalization 10 | 11 | from keras.layers import add 12 | 13 | from keras.layers import Conv2D, Conv2DTranspose 14 | 15 | from keras import backend as K 16 | 17 | import cv2 18 | import numpy as np 19 | import json 20 | 21 | K.set_image_dim_ordering('tf') 22 | 23 | # weight_decay = 0.0001 24 | from keras.regularizers import l2 25 | 26 | class Tiramisu(): 27 | 28 | 29 | 30 | def __init__(self): 31 | self.create() 32 | 33 | def DenseBlock(self, layers, filters): 34 | model = self.model 35 | for i in range(layers): 36 | model.add(BatchNormalization(mode=0, axis=1, 37 | gamma_regularizer=l2(0.0001), 38 | beta_regularizer=l2(0.0001))) 39 | model.add(Activation('relu')) 40 | model.add(Conv2D(filters, kernel_size=(3, 3), padding='same', 41 | kernel_initializer="he_uniform", 42 | data_format='channels_last')) 43 | model.add(Dropout(0.2)) 44 | 45 | 46 | def TransitionDown(self,filters): 47 | model = self.model 48 | model.add(BatchNormalization(mode=0, axis=1, 49 | gamma_regularizer=l2(0.0001), 50 | beta_regularizer=l2(0.0001))) 51 | model.add(Activation('relu')) 52 | model.add(Conv2D(filters, kernel_size=(1, 1), padding='same', 53 | kernel_initializer="he_uniform")) 54 | model.add(Dropout(0.2)) 55 | model.add(MaxPooling2D( pool_size=(2, 2), 56 | strides=(2, 2), 57 | data_format='channels_last')) 58 | 59 | def TransitionUp(self,filters,input_shape,output_shape): 60 | model = self.model 61 | model.add(Conv2DTranspose(filters, kernel_size=(3, 3), strides=(2, 2), 62 | padding='same', 63 | output_shape=output_shape, 64 | input_shape=input_shape, 65 | kernel_initializer="he_uniform", 66 | data_format='channels_last')) 67 | 68 | 69 | def create(self): 70 | model = self.model = models.Sequential() 71 | # cropping 72 | # model.add(Cropping2D(cropping=((68, 68), (128, 128)), input_shape=(3, 360,480))) 73 | 74 | model.add(Conv2D(48, kernel_size=(3, 3), padding='same', 75 | input_shape=(224,224,3), 76 | kernel_initializer="he_uniform", 77 | kernel_regularizer = l2(0.0001), 78 | data_format='channels_last')) 79 | # (5 * 4)* 2 + 5 + 5 + 1 + 1 +1 80 | # growth_m = 4 * 12 81 | # previous_m = 48 82 | self.DenseBlock(5,108) # 5*12 = 60 + 48 = 108 83 | self.TransitionDown(108) 84 | self.DenseBlock(5,168) # 5*12 = 60 + 108 = 168 85 | self.TransitionDown(168) 86 | self.DenseBlock(5,228) # 5*12 = 60 + 168 = 228 87 | self.TransitionDown(228) 88 | self.DenseBlock(5,288)# 5*12 = 60 + 228 = 288 89 | self.TransitionDown(288) 90 | self.DenseBlock(5,348) # 5*12 = 60 + 288 = 348 91 | self.TransitionDown(348) 92 | 93 | self.DenseBlock(15,408) # m = 348 + 5*12 = 408 94 | 95 | 96 | self.TransitionUp(468, (468, 7, 7), (None, 468, 14, 14)) # m = 348 + 5x12 + 5x12 = 468. 97 | self.DenseBlock(5,468) 98 | 99 | self.TransitionUp(408, (408, 14, 14), (None, 408, 28, 28)) # m = 288 + 5x12 + 5x12 = 408 100 | self.DenseBlock(5,408) 101 | 102 | self.TransitionUp(348, (348, 28, 28), (None, 348, 56, 56)) # m = 228 + 5x12 + 5x12 = 348 103 | self.DenseBlock(5,348) 104 | 105 | self.TransitionUp(288, (288, 56, 56), (None, 288, 112, 112)) # m = 168 + 5x12 + 5x12 = 288 106 | self.DenseBlock(5,288) 107 | 108 | self.TransitionUp(228, (228, 112, 112), (None, 228, 224, 224)) # m = 108 + 5x12 + 5x12 = 228 109 | self.DenseBlock(5,228) 110 | 111 | model.add(Conv2D(12, kernel_size=(1,1), 112 | padding='same', 113 | kernel_initializer="he_uniform", 114 | kernel_regularizer = l2(0.0001), 115 | data_format='channels_last')) 116 | 117 | model.add(Reshape((12, 224 * 224))) 118 | model.add(Permute((2, 1))) 119 | model.add(Activation('softmax')) 120 | model.summary() 121 | 122 | with open('tiramisu_fc_dense67_model_12.json', 'w') as outfile: 123 | outfile.write(json.dumps(json.loads(model.to_json()), indent=3)) 124 | 125 | Tiramisu() -------------------------------------------------------------------------------- /model-tiramasu-103.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | 5 | 6 | import keras.models as models 7 | from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape, Permute 8 | from keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D, Cropping2D 9 | from keras.layers.normalization import BatchNormalization 10 | 11 | from keras.layers import Conv2D, Conv2DTranspose 12 | 13 | from keras import backend as K 14 | 15 | import cv2 16 | import numpy as np 17 | import json 18 | 19 | K.set_image_dim_ordering('tf') 20 | 21 | # weight_decay = 0.0001 22 | from keras.regularizers import l2 23 | 24 | class Tiramisu(): 25 | 26 | 27 | 28 | def __init__(self): 29 | self.create() 30 | 31 | def DenseBlock(self, layers, filters): 32 | model = self.model 33 | for i in range(layers): 34 | model.add(BatchNormalization(mode=0, axis=1, 35 | gamma_regularizer=l2(0.0001), 36 | beta_regularizer=l2(0.0001))) 37 | model.add(Activation('relu')) 38 | model.add(Conv2D(filters, kernel_size=(3, 3), padding='same', 39 | kernel_initializer="he_uniform", 40 | data_format='channels_last')) 41 | model.add(Dropout(0.2)) 42 | 43 | def TransitionDown(self,filters): 44 | model = self.model 45 | model.add(BatchNormalization(mode=0, axis=1, 46 | gamma_regularizer=l2(0.0001), 47 | beta_regularizer=l2(0.0001))) 48 | model.add(Activation('relu')) 49 | model.add(Conv2D(filters, kernel_size=(1, 1), padding='same', 50 | kernel_initializer="he_uniform")) 51 | model.add(Dropout(0.2)) 52 | model.add(MaxPooling2D( pool_size=(2, 2), 53 | strides=(2, 2), 54 | data_format='channels_last')) 55 | 56 | def TransitionUp(self,filters,input_shape,output_shape): 57 | model = self.model 58 | model.add(Conv2DTranspose(filters, kernel_size=(3, 3), strides=(2, 2), 59 | padding='same', 60 | output_shape=output_shape, 61 | input_shape=input_shape, 62 | kernel_initializer="he_uniform", 63 | data_format='channels_last')) 64 | 65 | 66 | def create(self): 67 | model = self.model = models.Sequential() 68 | # cropping 69 | # model.add(Cropping2D(cropping=((68, 68), (128, 128)), input_shape=(3, 360,480))) 70 | 71 | model.add(Conv2D(48, kernel_size=(3, 3), padding='same', 72 | input_shape=(224,224,3), 73 | kernel_initializer="he_uniform", 74 | kernel_regularizer = l2(0.0001), 75 | data_format='channels_last')) 76 | 77 | # (5 * 4)* 2 + 5 + 5 + 1 + 1 +1 78 | # growth_m = 4 * 12 79 | # previous_m = 48 80 | 81 | self.DenseBlock(4,112) # 4*16 = 64 + 48 = 112 82 | self.TransitionDown(112) 83 | self.DenseBlock(5,192) # 5*16 = 80 + 112 = 192 84 | self.TransitionDown(192) 85 | self.DenseBlock(7,304) # 7 * 16 = 112 + 192 = 304 86 | self.TransitionDown(304) 87 | self.DenseBlock(10,464) 88 | self.TransitionDown(464) 89 | self.DenseBlock(12,656) 90 | self.TransitionDown(656) 91 | 92 | self.DenseBlock(15,896) # m = 656 + 15x16 = 896 93 | 94 | # upsampling part, m[B] is the sum of 3 terms 95 | # 1. the m value corresponding to same resolution in the downsampling part (skip connection) 96 | # 2. the number of feature maps from the upsampled block (n_layers[B-1] * growth_rate) 97 | # 3. the number of feature maps in the new block (n_layers[B] * growth_rate) 98 | # 99 | self.TransitionUp(1088, (1088, 7, 7), (None, 1088, 14, 14)) # m = 656 + 15x16 + 12x16 = 1088. 100 | self.DenseBlock(12,1088) 101 | 102 | self.TransitionUp(816, (816, 14, 14), (None, 816, 28, 28)) #m = 464 + 12x16 + 10x16 = 816 103 | self.DenseBlock(10,816) 104 | 105 | self.TransitionUp(576, (576, 28, 28), (None, 576, 56, 56)) # m = 304 + 10x16 + 7x16 = 576 106 | self.DenseBlock(7,576) 107 | 108 | self.TransitionUp(384, (384, 56, 56), (None, 384, 112, 112)) # m = 192 + 7x16 + 5x16 = 384 109 | self.DenseBlock(5,384) 110 | 111 | self.TransitionUp(256, (256, 112, 112), (None, 256, 224, 224)) # m = 112 + 5x16 + 4x16 = 256 112 | self.DenseBlock(4,256) 113 | 114 | model.add(Conv2D(12, kernel_size=(1,1), 115 | padding='same', 116 | kernel_initializer="he_uniform", 117 | kernel_regularizer = l2(0.0001), 118 | data_format='channels_last')) 119 | 120 | model.add(Reshape((12, 224 * 224))) 121 | model.add(Permute((2, 1))) 122 | model.add(Activation('softmax')) 123 | model.summary() 124 | 125 | with open('tiramisu_fc_dense103_model.json', 'w') as outfile: 126 | outfile.write(json.dumps(json.loads(model.to_json()), indent=3)) 127 | 128 | Tiramisu() -------------------------------------------------------------------------------- /model-tiramasu-67-func-api.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | 5 | 6 | import keras.models as models 7 | from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape, Permute 8 | from keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D, Cropping2D 9 | from keras.layers.normalization import BatchNormalization 10 | 11 | from keras.layers import add 12 | 13 | from keras.layers import Conv2D, Conv2DTranspose 14 | 15 | from keras import backend as K 16 | 17 | 18 | 19 | from keras.models import Model 20 | from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, UpSampling2D 21 | 22 | 23 | import cv2 24 | import numpy as np 25 | import json 26 | 27 | K.set_image_dim_ordering('tf') 28 | 29 | # weight_decay = 0.0001 30 | from keras.regularizers import l2 31 | 32 | class Tiramisu(): 33 | 34 | 35 | 36 | def __init__(self): 37 | self.create() 38 | 39 | def DenseBlock(self, layers_count, filters, previous_layer, model_layers, level): 40 | model_layers[level] = {} 41 | for i in range(layers_count): 42 | model_layers[level]['b_norm'+str(i+1)] = BatchNormalization(mode=0, axis=3, 43 | gamma_regularizer=l2(0.0001), 44 | beta_regularizer=l2(0.0001))(previous_layer) 45 | model_layers[level]['act'+str(i+1)] = Activation('relu')(model_layers[level]['b_norm'+str(i+1)]) 46 | model_layers[level]['conv'+str(i+1)] = Conv2D(filters, kernel_size=(3, 3), padding='same', 47 | kernel_initializer="he_uniform", 48 | data_format='channels_last')(model_layers[level]['act'+str(i+1)]) 49 | model_layers[level]['drop_out'+str(i+1)] = Dropout(0.2)(model_layers[level]['conv'+str(i+1)]) 50 | previous_layer = model_layers[level]['drop_out'+str(i+1)] 51 | # print(model_layers) 52 | return model_layers[level]['drop_out'+ str(layers_count)] # return last layer of this level 53 | 54 | 55 | def TransitionDown(self, filters, previous_layer, model_layers, level): 56 | model_layers[level] = {} 57 | model_layers[level]['b_norm'] = BatchNormalization(mode=0, axis=3, 58 | gamma_regularizer=l2(0.0001), 59 | beta_regularizer=l2(0.0001))(previous_layer) 60 | model_layers[level]['act'] = Activation('relu')(model_layers[level]['b_norm']) 61 | model_layers[level]['conv'] = Conv2D(filters, kernel_size=(1, 1), padding='same', 62 | kernel_initializer="he_uniform")(model_layers[level]['act']) 63 | model_layers[level]['drop_out'] = Dropout(0.2)(model_layers[level]['conv']) 64 | model_layers[level]['max_pool'] = MaxPooling2D(pool_size=(2, 2), 65 | strides=(2, 2), 66 | data_format='channels_last')(model_layers[level]['drop_out']) 67 | return model_layers[level]['max_pool'] 68 | 69 | def TransitionUp(self,filters,input_shape,output_shape, previous_layer, model_layers, level): 70 | model_layers[level] = {} 71 | model_layers[level]['conv'] = Conv2DTranspose(filters, kernel_size=(3, 3), strides=(2, 2), 72 | padding='same', 73 | output_shape=output_shape, 74 | input_shape=input_shape, 75 | kernel_initializer="he_uniform", 76 | data_format='channels_last')(previous_layer) 77 | 78 | return model_layers[level]['conv'] 79 | 80 | def create(self): 81 | inputs = Input((224,224,3)) 82 | 83 | first_conv = Conv2D(48, kernel_size=(3, 3), padding='same', 84 | kernel_initializer="he_uniform", 85 | kernel_regularizer = l2(0.0001), 86 | data_format='channels_last')(inputs) 87 | # first 88 | 89 | enc_model_layers = {} 90 | 91 | layer_1_down = self.DenseBlock(5,108, first_conv, enc_model_layers, 'layer_1_down' ) # 5*12 = 60 + 48 = 108 92 | layer_1a_down = self.TransitionDown(108, layer_1_down, enc_model_layers, 'layer_1a_down') 93 | 94 | layer_2_down = self.DenseBlock(5,168, layer_1a_down, enc_model_layers, 'layer_2_down' ) # 5*12 = 60 + 108 = 168 95 | layer_2a_down = self.TransitionDown(168, layer_2_down, enc_model_layers, 'layer_2a_down') 96 | 97 | layer_3_down = self.DenseBlock(5,228, layer_2a_down, enc_model_layers, 'layer_3_down' ) # 5*12 = 60 + 168 = 228 98 | layer_3a_down = self.TransitionDown(228, layer_3_down, enc_model_layers, 'layer_3a_down') 99 | 100 | layer_4_down = self.DenseBlock(5,288, layer_3a_down, enc_model_layers, 'layer_4_down' )# 5*12 = 60 + 228 = 288 101 | layer_4a_down = self.TransitionDown(288, layer_4_down, enc_model_layers, 'layer_4a_down') 102 | 103 | layer_5_down = self.DenseBlock(5,348, layer_4a_down, enc_model_layers, 'layer_5_down' ) # 5*12 = 60 + 288 = 348 104 | layer_5a_down = self.TransitionDown(348, layer_5_down, enc_model_layers, 'layer_5a_down') 105 | 106 | layer_bottleneck = self.DenseBlock(15,408, layer_5a_down, enc_model_layers, 'layer_bottleneck') # m = 348 + 5*12 = 408 107 | 108 | 109 | layer_1_up = self.TransitionUp(468, (468, 7, 7), (None, 468, 14, 14), layer_bottleneck, enc_model_layers, 'layer_1_up') # m = 348 + 5x12 + 5x12 = 468. 110 | skip_up_down_1 = concatenate([layer_1_up, enc_model_layers['layer_5_down']['conv'+ str(5)]], axis=-1) 111 | layer_1a_up = self.DenseBlock(5,468, skip_up_down_1, enc_model_layers, 'layer_1a_up' ) 112 | 113 | layer_2_up = self.TransitionUp(408, (408, 14, 14), (None, 408, 28, 28), layer_1a_up, enc_model_layers, 'layer_2_up') # m = 288 + 5x12 + 5x12 = 408 114 | skip_up_down_2 = concatenate([layer_2_up, enc_model_layers['layer_4_down']['conv'+ str(5)]], axis=-1) 115 | layer_2a_up = self.DenseBlock(5,408, skip_up_down_2, enc_model_layers, 'layer_2a_up' ) 116 | 117 | layer_3_up = self.TransitionUp(348, (348, 28, 28), (None, 348, 56, 56), layer_2a_up, enc_model_layers, 'layer_3_up') # m = 228 + 5x12 + 5x12 = 348 118 | skip_up_down_3 = concatenate([layer_3_up, enc_model_layers['layer_3_down']['conv'+ str(5)]], axis=-1) 119 | layer_3a_up = self.DenseBlock(5,348, skip_up_down_3, enc_model_layers, 'layer_3a_up' ) 120 | 121 | layer_4_up = self.TransitionUp(288, (288, 56, 56), (None, 288, 112, 112), layer_3a_up, enc_model_layers, 'layer_4_up') # m = 168 + 5x12 + 5x12 = 288 122 | skip_up_down_4 = concatenate([layer_4_up, enc_model_layers['layer_2_down']['conv'+ str(5)]], axis=-1) 123 | layer_4a_up = self.DenseBlock(5,288, skip_up_down_4, enc_model_layers, 'layer_4a_up' ) 124 | 125 | layer_5_up = self.TransitionUp(228, (228, 112, 112), (None, 228, 224, 224), layer_4a_up, enc_model_layers, 'layer_5_up') # m = 108 + 5x12 + 5x12 = 228 126 | skip_up_down_5 = concatenate([layer_5_up, enc_model_layers['layer_1_down']['conv'+ str(5)]], axis=-1) 127 | layer_5a_up = self.DenseBlock(5,228, skip_up_down_5, enc_model_layers, 'concatenate' ) 128 | 129 | # last 130 | last_conv = Conv2D(12, activation='linear', 131 | kernel_size=(1,1), 132 | padding='same', 133 | kernel_regularizer = l2(0.0001), 134 | data_format='channels_last')(layer_5a_up) 135 | 136 | reshape = Reshape((12, 224 * 224))(last_conv) 137 | perm = Permute((2, 1))(reshape) 138 | act = Activation('softmax')(perm) 139 | 140 | model = Model(inputs=[inputs], outputs=[act]) 141 | 142 | with open('tiramisu_fc_dense67_model_12_func.json', 'w') as outfile: 143 | outfile.write(json.dumps(json.loads(model.to_json()), indent=3)) 144 | 145 | Tiramisu() -------------------------------------------------------------------------------- /fc-densenet-model.py: -------------------------------------------------------------------------------- 1 | from keras.models import Model 2 | from keras.layers.core import Activation, Dropout, Activation, Reshape 3 | from keras.layers.convolutional import Convolution2D, Deconvolution2D 4 | from keras.layers.pooling import AveragePooling2D 5 | from keras.layers import Input, merge 6 | from keras.layers.normalization import BatchNormalization 7 | from keras.regularizers import l2 8 | import keras.backend as K 9 | 10 | from layers import SubPixelUpscaling 11 | import json 12 | 13 | class Tiramisu(): 14 | 15 | 16 | 17 | def __init__(self, nb_classes, img_dim, nb_dense_block=5, growth_rate=12, nb_filter=16, nb_layers=4, upsampling_conv=128, 18 | bottleneck=False, reduction=0.0, dropout_rate=None, weight_decay=1E-4, upscaling_type='deconv', 19 | verbose=True): 20 | 21 | self.nb_classes=nb_classes 22 | self.img_dim=img_dim 23 | self.nb_dense_block=nb_dense_block 24 | self.growth_rate=growth_rate 25 | self.nb_filter=nb_filter 26 | self.nb_layers=nb_layers 27 | self.upsampling_conv=upsampling_conv 28 | self.bottleneck=bottleneck 29 | self.reduction=reduction 30 | self.dropout_rate=dropout_rate 31 | self.weight_decay=weight_decay 32 | self.upscaling_type=upscaling_type 33 | self.verbose=verbose 34 | self.create() 35 | 36 | def conv_block(self, input_tensor, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1E-4): 37 | ''' Apply BatchNorm, Relu 3x3, Conv2D, optional bottleneck block and dropout 38 | 39 | Args: 40 | input_tensor: Input keras tensor 41 | nb_filter: number of filters 42 | bottleneck: add bottleneck block 43 | dropout_rate: dropout rate 44 | weight_decay: weight decay factor 45 | 46 | Returns: keras tensor with batch_norm, relu and convolution2d added (optional bottleneck) 47 | 48 | ''' 49 | 50 | concat_axis = 1 51 | 52 | x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay), 53 | beta_regularizer=l2(weight_decay))(input_tensor) 54 | x = Activation('relu')(x) 55 | 56 | if bottleneck: 57 | inter_channel = nb_filter * 4 # Obtained from https://github.com/liuzhuang13/DenseNet/blob/master/densenet.lua 58 | 59 | x = Convolution2D(inter_channel, 1, 1, init='he_uniform', border_mode='same', bias=False, 60 | W_regularizer=l2(weight_decay))(x) 61 | 62 | if dropout_rate: 63 | x = Dropout(dropout_rate)(x) 64 | 65 | x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay), 66 | beta_regularizer=l2(weight_decay))(x) 67 | x = Activation('relu')(x) 68 | 69 | x = Convolution2D(nb_filter, 3, 3, init="he_uniform", border_mode="same", bias=False, 70 | W_regularizer=l2(weight_decay))(x) 71 | if dropout_rate: 72 | x = Dropout(dropout_rate)(x) 73 | 74 | return x 75 | 76 | 77 | def transition_down_block(self, input_tensor, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4): 78 | ''' Apply BatchNorm, Relu 1x1, Conv2D, optional compression, dropout and Maxpooling2D 79 | 80 | Args: 81 | input_tensor: keras tensor 82 | nb_filter: number of filters 83 | dropout_rate: dropout rate 84 | weight_decay: weight decay factor 85 | 86 | Returns: keras tensor, after applying batch_norm, relu-conv, dropout, maxpool 87 | 88 | ''' 89 | 90 | concat_axis = 1 91 | 92 | x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay), 93 | beta_regularizer=l2(weight_decay))(input_tensor) 94 | x = Activation('relu')(x) 95 | x = Convolution2D(int(nb_filter * compression), 1, 1, init="he_uniform", border_mode="same", bias=False, 96 | W_regularizer=l2(weight_decay))(x) 97 | if dropout_rate: 98 | x = Dropout(dropout_rate)(x) 99 | x = AveragePooling2D((2, 2), strides=(2, 2))(x) 100 | 101 | return x 102 | 103 | 104 | def transition_up_block(self, input_tensor, nb_filters, type='deconv', output_shape=None, weight_decay=1E-4): 105 | ''' deconv Upscaling (factor = 2) 106 | 107 | Args: 108 | input_tensor: keras tensor 109 | nb_filters: number of layers 110 | type:'deconv'. Determines type of upsampling performed 111 | output_shape: required if type = 'deconv'. Output shape of tensor 112 | weight_decay: weight decay factor 113 | 114 | Returns: keras tensor, after applying batch_norm, relu-conv, dropout, maxpool 115 | 116 | ''' 117 | 118 | 119 | x = Deconvolution2D(nb_filters, 3, 3, output_shape, activation='relu', border_mode='same', 120 | subsample=(2, 2))(input_tensor) 121 | 122 | 123 | 124 | def dense_block(self, x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None, weight_decay=1E-4): 125 | ''' Build a dense_block where the output of each conv_block is fed to subsequent ones 126 | 127 | Args: 128 | x: keras tensor 129 | nb_layers: the number of layers of conv_block to append to the model. 130 | nb_filter: number of filters 131 | growth_rate: growth rate 132 | bottleneck: bottleneck block 133 | dropout_rate: dropout rate 134 | weight_decay: weight decay factor 135 | 136 | Returns: keras tensor with nb_layers of conv_block appended 137 | 138 | ''' 139 | 140 | concat_axis = 1 141 | 142 | feature_list = [x] 143 | 144 | for i in range(nb_layers): 145 | x = self.conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay) 146 | feature_list.append(x) 147 | x = merge(feature_list, mode='concat', concat_axis=concat_axis) 148 | nb_filter += growth_rate 149 | 150 | return x, nb_filter 151 | 152 | 153 | def create(self): 154 | ''' Build the create_dense_net model 155 | 156 | Args: 157 | nb_classes: Number of classes 158 | img_dim: tuple of shape (channels, rows, columns) or (rows, columns, channels) 159 | depth: number or layers 160 | nb_dense_block: number of dense blocks to add to end (generally = 3) 161 | growth_rate: number of filters to add per dense block 162 | nb_filter: initial number of filters. Setting -1 indicates initial number of filters is 2 * growth_rate 163 | nb_layers: number of layers in each dense block. Can be an -1, a positive integer or a list 164 | 165 | If -1, it computes the nb_layer from depth 166 | 167 | If positive integer, a set number of layers per dense block 168 | 169 | If list, nb_layer is used as provided. 170 | Note that list size must be (nb_dense_block + 1) 171 | 172 | upsampling_conv: number of convolutional layers in upsampling via subpixel convolution 173 | bottleneck: add bottleneck blocks 174 | reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression 175 | dropout_rate: dropout rate 176 | weight_decay: weight decay 177 | upscaling_type: method of upscaling. Can be 'subpixel' or 'deconv' 178 | verbose: print the model type 179 | 180 | Returns: keras tensor with nb_layers of conv_block appended 181 | 182 | ''' 183 | 184 | batch_size = None 185 | 186 | model_input = Input(shape=self.img_dim) 187 | 188 | concat_axis = 1 189 | 190 | _, rows, cols = self.img_dim 191 | 192 | 193 | if self.reduction != 0.0: 194 | assert self.reduction <= 1.0 and self.reduction > 0.0, "reduction value must lie between 0.0 and 1.0" 195 | 196 | # check if upsampling_conv has minimum number of filters 197 | # minimum is set to 12, as at least 3 color channels are needed for correct upsampling 198 | assert self.upsampling_conv > 12 and self.upsampling_conv % 4 == 0, "upsampling_conv number of channels must " \ 199 | "be a positive number divisible by 4 and greater " \ 200 | "than 12" 201 | 202 | 203 | 204 | # layers in each dense block 205 | if type(self.nb_layers) is list or type(self.nb_layers) is tuple: 206 | self.nb_layers = list(nb_layers) # Convert tuple to list 207 | 208 | assert len(self.nb_layers) == (self.nb_dense_block + 1), "If list, nb_layer is used as provided. " \ 209 | "Note that list size must be (nb_dense_block + 1)" 210 | 211 | final_nb_layer = self.nb_layers[-1] 212 | self.nb_layers = self.nb_layers[:-1] 213 | 214 | else: 215 | final_nb_layer = self.nb_layers 216 | self.nb_layers = [self.nb_layers] * self.nb_dense_block 217 | 218 | if self.bottleneck: 219 | self.nb_layers = [int(layer // 2) for layer in self.nb_layers] 220 | 221 | # compute initial nb_filter if -1, else accept users initial nb_filter 222 | if self.nb_filter <= 0: 223 | self.nb_filter = 2 * self.growth_rate 224 | 225 | # compute compression factor 226 | compression = 1.0 - self.reduction 227 | 228 | # Initial convolution 229 | x = Convolution2D(48, 3, 3, init="he_uniform", border_mode="same", name="initial_conv2D", bias=False, 230 | W_regularizer=l2(self.weight_decay))(model_input) 231 | 232 | skinput_tensor_connection = x 233 | skinput_tensor_list = [] 234 | 235 | # Add dense blocks and transition down block 236 | for block_idx in range(self.nb_dense_block): 237 | x, nb_filter = self.dense_block(x, self.nb_layers[block_idx], self.nb_filter, self.growth_rate, bottleneck=self.bottleneck, 238 | dropout_rate=self.dropout_rate, weight_decay=self.weight_decay) 239 | 240 | # Skinput_tensor connection 241 | x = merge([x, skinput_tensor_connection], mode='concat', concat_axis=concat_axis) 242 | skinput_tensor_list.append(x) 243 | 244 | # add transition_block 245 | x = self.transition_down_block(x, nb_filter, compression=compression, dropout_rate=self.dropout_rate, 246 | weight_decay=self.weight_decay) 247 | nb_filter = int(nb_filter * compression) 248 | 249 | # Preserve transition for next skinput_tensor connection after dense 250 | skinput_tensor_connection = x 251 | 252 | # The last dense_block does not have a transition_down_block 253 | x, nb_filter = self.dense_block(x, final_nb_layer, nb_filter, self.growth_rate, bottleneck=self.bottleneck, 254 | dropout_rate=self.dropout_rate, weight_decay=self.weight_decay) 255 | 256 | out_shape = [batch_size, nb_filter, rows // 16, cols // 16] 257 | 258 | # Add dense blocks and transition up block 259 | for block_idx in range(self.nb_dense_block): 260 | x = self.transition_up_block(x, nb_filters=self.upsampling_conv, type=self.upscaling_type, output_shape=out_shape) 261 | 262 | out_shape[2] *= 2 263 | out_shape[3] *= 2 264 | 265 | 266 | x = merge([x, skinput_tensor_list.pop()], mode='concat', concat_axis=concat_axis) 267 | 268 | x, nb_filter = self.dense_block(x, self.nb_layers[-block_idx], nb_filter, growth_rate, bottleneck=self.bottleneck, 269 | dropout_rate=self.dropout_rate, weight_decay=self.weight_decay) 270 | 271 | x = Convolution2D(nb_classes, 1, 1, activation='linear', border_mode='same', W_regularizer=l2(self.weight_decay), 272 | bias=False)(x) 273 | 274 | channel, row, col = self.img_dim 275 | 276 | 277 | x = Reshape((row * col, self.nb_classes))(x) 278 | 279 | x = Activation('softmax')(x) 280 | 281 | densenet = Model(input=model_input, output=x, name="create_dense_net") 282 | 283 | # Compute depth 284 | nb_conv_layers = len([layer.name for layer in densenet.layers 285 | if layer.__class__.__name__ == 'Convolution2D']) 286 | 287 | depth = nb_conv_layers - self.nb_dense_block # For 1 extra convolution layers per transition up 288 | 289 | if self.verbose: print('Total number of convolutions', depth) 290 | 291 | if self.verbose: 292 | if self.bottleneck and not self.reduction: 293 | print("Bottleneck DenseNet-B-%d-%d created." % (depth, self.growth_rate)) 294 | elif not self.bottleneck and self.reduction > 0.0: 295 | print("DenseNet-C-%d-%d with %0.1f compression created." % (depth, self.growth_rate, compression)) 296 | elif self.bottleneck and self.reduction > 0.0: 297 | print("Bottleneck DenseNet-BC-%d-%d with %0.1f compression created." % (depth, self.growth_rate, compression)) 298 | else: 299 | print("DenseNet-%d-%d created." % (depth, self.growth_rate)) 300 | 301 | return densenet 302 | 303 | 304 | 305 | nb_layers = [4, 5, 7, 10, 12, 15] 306 | model = Tiramisu(nb_classes=12,img_dim=(3, 224, 224), nb_layers=nb_layers) 307 | model.summary() 308 | 309 | with open('tiramisu_fc_dense103_model.json', 'w') as outfile: 310 | outfile.write(json.dumps(json.loads(model.to_json()), indent=3)) --------------------------------------------------------------------------------