├── BuildNetVgg16.py ├── CheckVGG16Model.py ├── Data_Reader.py ├── Evaluate_Net_IOU.py ├── Figure1.png ├── IOU.py ├── Inference.py ├── OverrlayLabelOnImage.py ├── README.md ├── TRAIN.py └── TensorflowUtils.py /BuildNetVgg16.py: -------------------------------------------------------------------------------- 1 | ################Class which build the fully convolutional neural net########################################################### 2 | 3 | import inspect 4 | import os 5 | import TensorflowUtils as utils 6 | import numpy as np 7 | import tensorflow as tf 8 | 9 | 10 | VGG_MEAN = [103.939, 116.779, 123.68]# Mean value of pixels in R G and B channels 11 | 12 | #========================Class for building the FCN neural network based on VGG16================================================================================== 13 | class BUILD_NET_VGG16: 14 | def __init__(self, vgg16_npy_path=None): 15 | # if vgg16_npy_path is None: 16 | # path = inspect.getfile(BUILD_NET_VGG16) 17 | # path = os.path.abspath(os.path.join(path, os.pardir)) 18 | # path = os.path.join(path, "vgg16.npy") 19 | # vgg16_npy_path = path 20 | # 21 | # print(path) 22 | 23 | self.data_dict = np.load(vgg16_npy_path, encoding='latin1').item() #Load weights of trained VGG16 for encoder 24 | print("npy file loaded") 25 | ########################################Build Net##################################################################################################################### 26 | def build(self, rgb,NUM_CLASSES,keep_prob): #Build the fully convolutional neural network (FCN) and load weight for decoder based on trained VGG16 network 27 | """ 28 | load variable from npy to build the VGG 29 | 30 | :param rgb: rgb image [batch, height, width, 3] values 0-255 31 | """ 32 | self.SumWeights = tf.constant(0.0, name="SumFiltersWeights") #Sum of weights of all filters for weight decay loss 33 | 34 | 35 | print("build model started") 36 | # rgb_scaled = rgb * 255.0 37 | 38 | # Convert RGB to BGR and substract pixels mean 39 | red, green, blue = tf.split(axis=3, num_or_size_splits=3, value=rgb) 40 | 41 | bgr = tf.concat(axis=3, values=[ 42 | blue - VGG_MEAN[0], 43 | green - VGG_MEAN[1], 44 | red - VGG_MEAN[2], 45 | ]) 46 | 47 | #-----------------------------Build network encoder based on VGG16 network and load the trained VGG16 weights----------------------------------------- 48 | #Layer 1 49 | self.conv1_1 = self.conv_layer(bgr, "conv1_1") #Build Convolution layer and load weights 50 | self.conv1_2 = self.conv_layer(self.conv1_1, "conv1_2")#Build Convolution layer +Relu and load weights 51 | self.pool1 = self.max_pool(self.conv1_2, 'pool1') #Max Pooling 52 | # Layer 2 53 | self.conv2_1 = self.conv_layer(self.pool1, "conv2_1") 54 | self.conv2_2 = self.conv_layer(self.conv2_1, "conv2_2") 55 | self.pool2 = self.max_pool(self.conv2_2, 'pool2') 56 | # Layer 3 57 | self.conv3_1 = self.conv_layer(self.pool2, "conv3_1") 58 | self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2") 59 | self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3") 60 | self.pool3 = self.max_pool(self.conv3_3, 'pool3') 61 | # Layer 4 62 | self.conv4_1 = self.conv_layer(self.pool3, "conv4_1") 63 | self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2") 64 | self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3") 65 | self.pool4 = self.max_pool(self.conv4_3, 'pool4') 66 | # Layer 5 67 | self.conv5_1 = self.conv_layer(self.pool4, "conv5_1") 68 | self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2") 69 | self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3") 70 | self.pool5 = self.max_pool(self.conv5_3, 'pool5') 71 | ##-----------------------Build Net Fully connvolutional layers------------------------------------------------------------------------------------ 72 | W6 = utils.weight_variable([7, 7, 512, 4096],name="W6") # Create tf weight for the new layer with initial weights with normal random distrubution mean zero and std 0.02 73 | b6 = utils.bias_variable([4096], name="b6") # Create tf biasefor the new layer with initial weights of 0 74 | self.conv6 = utils.conv2d_basic(self.pool5 , W6, b6) # Check the size of this net input is it same as input or is it 1X1 75 | self.relu6 = tf.nn.relu(self.conv6, name="relu6") 76 | # if FLAGS.debug: utils.add_activation_summary(relu6) 77 | self.relu_dropout6 = tf.nn.dropout(self.relu6,keep_prob=keep_prob) # Apply dropout for traning need to be added only for training 78 | 79 | W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7") # 1X1 Convloution 80 | b7 = utils.bias_variable([4096], name="b7") 81 | self.conv7 = utils.conv2d_basic(self.relu_dropout6, W7, b7) # 1X1 Convloution 82 | self.relu7 = tf.nn.relu(self.conv7, name="relu7") 83 | # if FLAGS.debug: utils.add_activation_summary(relu7) 84 | self.relu_dropout7 = tf.nn.dropout(self.relu7, keep_prob=keep_prob) # Another dropout need to be used only for training 85 | 86 | W8 = utils.weight_variable([1, 1, 4096, NUM_CLASSES],name="W8") # Basically the output num of classes imply the output is already the prediction this is flexible can be change however in multinet class number of 2 give good results 87 | b8 = utils.bias_variable([NUM_CLASSES], name="b8") 88 | self.conv8 = utils.conv2d_basic(self.relu_dropout7, W8, b8) 89 | # annotation_pred1 = tf.argmax(conv8, dimension=3, name="prediction1") 90 | #-------------------------------------Build Decoder -------------------------------------------------------------------------------------------------- 91 | # now to upscale to actual image size 92 | deconv_shape1 = self.pool4.get_shape() # Set the output shape for the the transpose convolution output take only the depth since the transpose convolution will have to have the same depth for output 93 | W_t1 = utils.weight_variable([4, 4, deconv_shape1[3].value, NUM_CLASSES],name="W_t1") # Deconvolution/transpose in size 4X4 note that the output shape is of depth NUM_OF_CLASSES this is not necessary in will need to be fixed if you only have 2 catagories 94 | b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1") 95 | self.conv_t1 = utils.conv2d_transpose_strided(self.conv8, W_t1, b_t1, output_shape=tf.shape(self.pool4)) # Use strided convolution to double layer size (depth is the depth of pool4 for the later element wise addition 96 | self.fuse_1 = tf.add(self.conv_t1, self.pool4, name="fuse_1") # Add element wise the pool layer from the decoder 97 | 98 | deconv_shape2 = self.pool3.get_shape() 99 | W_t2 = utils.weight_variable([4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2") 100 | b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2") 101 | self.conv_t2 = utils.conv2d_transpose_strided(self.fuse_1, W_t2, b_t2, output_shape=tf.shape(self.pool3)) 102 | self.fuse_2 = tf.add(self.conv_t2, self.pool3, name="fuse_2") 103 | 104 | shape = tf.shape(rgb) 105 | W_t3 = utils.weight_variable([16, 16, NUM_CLASSES, deconv_shape2[3].value], name="W_t3") 106 | b_t3 = utils.bias_variable([NUM_CLASSES], name="b_t3") 107 | 108 | self.Prob = utils.conv2d_transpose_strided(self.fuse_2, W_t3, b_t3, output_shape=[shape[0], shape[1], shape[2], NUM_CLASSES], stride=8) 109 | #--------------------Transform probability vectors to label maps----------------------------------------------------------------- 110 | self.Pred = tf.argmax(self.Prob, dimension=3, name="Pred") 111 | 112 | print("FCN model built") 113 | ##################################################################################################################################################### 114 | def max_pool(self, bottom, name): 115 | return tf.nn.max_pool(bottom, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name=name) 116 | ############################################################################################################################################################ 117 | def conv_layer(self, bottom, name): 118 | with tf.variable_scope(name): 119 | filt = self.get_conv_filter(name) 120 | 121 | conv = tf.nn.conv2d(bottom, filt, [1, 1, 1, 1], padding='SAME') 122 | 123 | conv_biases = self.get_bias(name) 124 | bias = tf.nn.bias_add(conv, conv_biases) 125 | 126 | relu = tf.nn.relu(bias) 127 | return relu 128 | 129 | ############################################################################################################################################################ 130 | def conv_layer_NoRelu(self, bottom, name): 131 | with tf.variable_scope(name): 132 | filt = self.get_conv_filter(name) 133 | 134 | conv = tf.nn.conv2d(bottom, filt, [1, 1, 1, 1], padding='SAME') 135 | 136 | conv_biases = self.get_bias(name) 137 | bias = tf.nn.bias_add(conv, conv_biases) 138 | return bias 139 | 140 | #########################################Build fully convolutional layer############################################################################################################## 141 | def fc_layer(self, bottom, name): 142 | with tf.variable_scope(name): 143 | shape = bottom.get_shape().as_list() 144 | dim = 1 145 | for d in shape[1:]: 146 | dim *= d 147 | x = tf.reshape(bottom, [-1, dim]) 148 | 149 | weights = self.get_fc_weight(name) 150 | biases = self.get_bias(name) 151 | 152 | # Fully connected layer. Note that the '+' operation automatically 153 | # broadcasts the biases. 154 | fc = tf.nn.bias_add(tf.matmul(x, weights), biases) 155 | 156 | return fc 157 | ######################################Get VGG filter ############################################################################################################ 158 | def get_conv_filter(self, name): 159 | var=tf.Variable(self.data_dict[name][0], name="filter_" + name) 160 | self.SumWeights+=tf.nn.l2_loss(var) 161 | return var 162 | ################################################################################################################################################## 163 | def get_bias(self, name): 164 | return tf.Variable(self.data_dict[name][1], name="biases_"+name) 165 | ############################################################################################################################################# 166 | def get_fc_weight(self, name): 167 | return tf.Variable(self.data_dict[name][0], name="weights_"+name) 168 | -------------------------------------------------------------------------------- /CheckVGG16Model.py: -------------------------------------------------------------------------------- 1 | import TensorflowUtils 2 | import os 3 | #-----------------------------------------Check if pretrain vgg16 models and data are availale---------------------------------------------------- 4 | def CheckVGG16(model_path): # Check if pretrained vgg16 model avialable and if not try to download it 5 | TensorflowUtils.maybe_download_and_extract(model_path.split('/')[0], "ftp://mi.eng.cam.ac.uk/pub/mttt2/models/vgg16.npy") # If not exist try to download pretrained vgg16 net for network initiation 6 | if not os.path.isfile(model_path): 7 | print("Error: Cant find pretrained vgg16 model for network initiation. Please download model from:") 8 | print("ftp://mi.eng.cam.ac.uk/pub/mttt2/models/vgg16.npy") 9 | print("Or from:") 10 | print("https://drive.google.com/file/d/0B6njwynsu2hXZWcwX0FKTGJKRWs/view?usp=sharing") 11 | print("and place in the path pointed by model_path") 12 | 13 | #This can be download manualy from:"https://drive.google.com/drive/folders/0B6njwynsu2hXcDYwb1hxMW9HMEU" or from ftp://mi.eng.cam.ac.uk/pub/mttt2/models/vgg16.npy 14 | # and placed in the /Model_Zoo folder in the code dir 15 | 16 | -------------------------------------------------------------------------------- /Data_Reader.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import scipy.misc as misc 4 | import random 5 | #------------------------Class for reading training and validation data--------------------------------------------------------------------- 6 | class Data_Reader: 7 | 8 | 9 | ################################Initiate folders were files are and list of train images############################################################################ 10 | def __init__(self, ImageDir,GTLabelDir="", BatchSize=1,Suffle=True): 11 | #ImageDir directory were images are 12 | #GTLabelDir Folder wehere ground truth Labels map are save in png format (same name as corresponnding image in images folder) 13 | self.NumFiles = 0 # Number of files in reader 14 | self.Epoch = 0 # Training epochs passed 15 | self.itr = 0 #Iteration 16 | #Image directory 17 | self.Image_Dir=ImageDir # Image Dir 18 | if GTLabelDir=="":# If no label dir use 19 | self.ReadLabels=False 20 | else: 21 | self.ReadLabels=True 22 | self.Label_Dir = GTLabelDir # Folder with ground truth pixels was annotated (optional for training only) 23 | self.OrderedFiles=[] 24 | # Read list of all files 25 | self.OrderedFiles += [each for each in os.listdir(self.Image_Dir) if each.endswith('.PNG') or each.endswith('.JPG') or each.endswith('.TIF') or each.endswith('.GIF') or each.endswith('.png') or each.endswith('.jpg') or each.endswith('.tif') or each.endswith('.gif') ] # Get list of training images 26 | self.BatchSize=BatchSize #Number of images used in single training operation 27 | self.NumFiles=len(self.OrderedFiles) 28 | self.OrderedFiles.sort() # Sort files by names 29 | self.SuffleBatch() # suffle file list 30 | ####################################### Suffle list of files in group that fit the batch size this is important since we want the batch to contain images of the same size########################################################################################## 31 | def SuffleBatch(self): 32 | self.SFiles = [] 33 | Sf=np.array(range(np.int32(np.ceil(self.NumFiles/self.BatchSize)+1)))*self.BatchSize 34 | random.shuffle(Sf) 35 | self.SFiles=[] 36 | for i in range(len(Sf)): 37 | for k in range(self.BatchSize): 38 | if Sf[i]+k=self.NumFiles: # End of an epoch 43 | self.itr=0 44 | self.SuffleBatch() 45 | self.Epoch+=1 46 | batch_size=np.min([self.BatchSize,self.NumFiles-self.itr]) 47 | Sy =Sx= 0 48 | XF=YF=1 49 | Cry=1 50 | Crx=1 51 | #--------------Resize Factor-------------------------------------------------------- 52 | if np.random.rand() < 1: 53 | YF = XF = 0.3+np.random.rand()*0.7 54 | #------------Stretch image------------------------------------------------------------------- 55 | if np.random.rand()<0.8: 56 | if np.random.rand()<0.5: 57 | XF*=0.5+np.random.rand()*0.5 58 | else: 59 | YF*=0.5+np.random.rand()*0.5 60 | #-----------Crop Image------------------------------------------------------ 61 | if np.random.rand()<0.0: 62 | Cry=0.7+np.random.rand()*0.3 63 | Crx = 0.7 + np.random.rand() * 0.3 64 | 65 | #-----------Augument Images and labeles------------------------------------------------------------------- 66 | 67 | for f in range(batch_size): 68 | #.............Read image and labels from files......................................................... 69 | Img = misc.imread(self.Image_Dir + "/" + self.SFiles[self.itr]) 70 | Img=Img[:,:,0:3] 71 | LabelName=self.SFiles[self.itr][0:-4]+".png"# Assume Label name is same as image only with png ending 72 | if self.ReadLabels: 73 | Label= misc.imread(self.Label_Dir + "/" + LabelName) 74 | self.itr+=1 75 | #............Set Batch image size according to first image in the batch................................................... 76 | if f==0: 77 | Sy, Sx,d = Img.shape 78 | Sy,Sx 79 | Sy*=YF 80 | Sx*=XF 81 | Cry*=Sy 82 | Crx*=Sx 83 | Sy = np.int32(Sy) 84 | Sx = np.int32(Sx) 85 | Cry = np.int32(Cry) 86 | Crx = np.int32(Crx) 87 | Images = np.zeros([batch_size,Cry,Crx,3], dtype=np.float) 88 | if self.ReadLabels: Labels= np.zeros([batch_size,Cry,Crx,1], dtype=np.int) 89 | 90 | 91 | #..........Resize and strecth image and labels.................................................................... 92 | Img = misc.imresize(Img, [Sy,Sx], interp='bilinear') 93 | if self.ReadLabels: Label= misc.imresize(Label, [Sy, Sx], interp='nearest') 94 | 95 | #-------------------------------Crop Image....................................................................... 96 | MinOccupancy=501 97 | if not (Cry==Sy and Crx==Sx): 98 | for u in range(501): 99 | MinOccupancy-=1 100 | Xi=np.int32(np.floor(np.random.rand()*(Sx-Crx))) 101 | Yi=np.int32(np.floor(np.random.rand()*(Sy-Cry))) 102 | if np.sum(Label[Yi:Yi+Cry,Xi:Xi+Crx]>0)>MinOccupancy: 103 | Img=Img[Yi:Yi+Cry,Xi:Xi+Crx,:] 104 | if self.ReadLabels: Label=Label[Yi:Yi+Cry,Xi:Xi+Crx] 105 | break 106 | #------------------------Mirror Image-------------------------------# -------------------------------------------- 107 | if random.random()<0.5: # Agument the image by mirror image 108 | Img=np.fliplr(Img) 109 | if self.ReadLabels: 110 | Label=np.fliplr(Label) 111 | 112 | #-----------------------Agument color of Image----------------------------------------------------------------------- 113 | Img = np.float32(Img) 114 | 115 | 116 | if np.random.rand() < 0.8: # Play with shade 117 | Img *= 0.4 + np.random.rand() * 0.6 118 | if np.random.rand() < 0.4: # Turn to grey 119 | Img[:, :, 2] = Img[:, :, 1]=Img[:, :, 0] = Img[:,:,0]=Img.mean(axis=2) 120 | 121 | if np.random.rand() < 0.0: # Play with color 122 | if np.random.rand() < 0.6: 123 | for i in range(3): 124 | Img[:, :, i] *= 0.1 + np.random.rand() 125 | 126 | 127 | 128 | if np.random.rand() < 0.2: # Add Noise 129 | Img *=np.ones(Img.shape)*0.95 + np.random.rand(Img.shape[0],Img.shape[1],Img.shape[2])*0.1 130 | Img[Img>255]=255 131 | Img[Img<0]=0 132 | #----------------------Add images and labels to to the batch---------------------------------------------------------- 133 | Images[f]=Img 134 | if self.ReadLabels: 135 | Labels[f,:,:,0]=Label 136 | 137 | #.......................Return aumented images and labels........................................................... 138 | if self.ReadLabels: 139 | return Images, Labels# return image and pixelwise labels 140 | else: 141 | return Images# Return image 142 | 143 | 144 | 145 | 146 | ######################################Read next batch of images and labels with no augmentation###################################################################################################### 147 | def ReadNextBatchClean(self): #Read image and labels without agumenting 148 | if self.itr>=self.NumFiles: # End of an epoch 149 | self.itr=0 150 | #self.SuffleBatch() 151 | self.Epoch+=1 152 | batch_size=np.min([self.BatchSize,self.NumFiles-self.itr]) 153 | 154 | for f in range(batch_size): 155 | ##.............Read image and labels from files......................................................... 156 | Img = misc.imread(self.Image_Dir + "/" + self.OrderedFiles[self.itr]) 157 | Img=Img[:,:,0:3] 158 | LabelName=self.OrderedFiles[self.itr][0:-4]+".png"# Assume label name is same as image only with png ending 159 | if self.ReadLabels: 160 | Label= misc.imread(self.Label_Dir + "/" + LabelName) 161 | self.itr+=1 162 | #............Set Batch size according to first image................................................... 163 | if f==0: 164 | Sy,Sx,Depth=Img.shape 165 | Images = np.zeros([batch_size,Sy,Sx,3], dtype=np.float) 166 | if self.ReadLabels: Labels= np.zeros([batch_size,Sy,Sx,1], dtype=np.int) 167 | 168 | #..........Resize image and labels.................................................................... 169 | Img = misc.imresize(Img, [Sy, Sx], interp='bilinear') 170 | if self.ReadLabels: Label = misc.imresize(Label, [Sy, Sx], interp='nearest') 171 | #...................Load image and label to batch.................................................................. 172 | Images[f] = Img 173 | if self.ReadLabels: 174 | Labels[f, :, :, 0] = Label 175 | #...................................Return images and labels........................................ 176 | if self.ReadLabels: 177 | return Images, Labels # return image and and pixelwise labels 178 | else: 179 | return Images # Return image 180 | 181 | -------------------------------------------------------------------------------- /Evaluate_Net_IOU.py: -------------------------------------------------------------------------------- 1 | # Evaluate the perfomance of trained network by evaluating intersection over union (IOU) of the network predcition 2 | # and ground truth of the validation set 3 | # 1) Make sure you you have trained model in logs_dir (See Train.py for creating trained model) 4 | # 2) Set the Image_Dir to the folder where the input images for prediction are located 5 | # 3) Set folder for ground truth labels in Label_DIR 6 | # The Label Maps should be saved as png image with same name as the corresponding image and png ending 7 | # 4) Set number of classes number in NUM_CLASSES 8 | # 5) Run script 9 | ########################################################################################################################################################### 10 | import tensorflow as tf 11 | import numpy as np 12 | import scipy.misc as misc 13 | import os 14 | import IOU 15 | import sys 16 | import Data_Reader 17 | import BuildNetVgg16 18 | import CheckVGG16Model 19 | #...................................................................................................................................... 20 | logs_dir= "logs/"# "path to logs directory where trained model and information will be stored" 21 | Label_Dir="Data_Zoo/Materials_In_Vessels/LiquidSolidLabels/"# Annotetion in png format for train images and validation images (assume the name of the images and annotation images are the same (but annotation is always png format)) 22 | Image_Dir="Data_Zoo/Materials_In_Vessels/Test_Images_All/"# Test image folder 23 | model_path="Model_Zoo/vgg16.npy"# "Path to pretrained vgg16 model for encoder" 24 | 25 | #------------------------------------------------------------------------------------------------------------------------- 26 | 27 | Batch_Size=1 28 | NUM_CLASSES = 4 #Number of classes 29 | Classes = ["BackGround", "Empty Vessel","Liquid","Solid"] #List of classes 30 | #VesseClasses=["Background","Vessel"] #Classes predicted for vessel region prediction 31 | #PhaseClasses=["BackGround","Empty Vessel region","Filled Vessel region"]# 32 | 33 | #ExactPhaseClasses=["BackGround","Vessel","Liquid","Liquid Phase two","Suspension", "Emulsion","Foam","Solid","Gel","Powder","Granular","Bulk","Bulk Liquid","Solid Phase two","Vapor"] 34 | ################################################################################################################################################################################ 35 | def main(argv=None): 36 | tf.reset_default_graph() 37 | keep_prob = tf.placeholder(tf.float32, name="keep_probabilty") # Dropout probability 38 | # .........................Placeholders for input image and labels........................................................................................... 39 | image = tf.placeholder(tf.float32, shape=[None, None, None, 3], name="input_image") # Input image batch first dimension image number second dimension width third dimension height 4 dimension RGB 40 | # .........................Build FCN Net............................................................................................... 41 | Net = BuildNetVgg16.BUILD_NET_VGG16(vgg16_npy_path=model_path) # Create class for the network 42 | Net.build(image, NUM_CLASSES, keep_prob) # Create the net and load intial weights 43 | # # -------------------------Data reader for validation image----------------------------------------------------------------------------------------------------------------------------- 44 | 45 | ValidReader = Data_Reader.Data_Reader(Image_Dir,GTLabelDir=Label_Dir, BatchSize=Batch_Size) # build reader that will be used to load images and labels from validation set 46 | 47 | #........................................................................................................................ 48 | sess = tf.Session() # Start Tensorflow session 49 | #--------Load trained model-------------------------------------------------------------------------------------------------- 50 | print("Setting up Saver...") 51 | saver = tf.train.Saver() 52 | sess.run(tf.global_variables_initializer()) 53 | ckpt = tf.train.get_checkpoint_state(logs_dir) 54 | if ckpt and ckpt.model_checkpoint_path: # if train model exist restore it 55 | saver.restore(sess, ckpt.model_checkpoint_path) 56 | print("Model restored...") 57 | else: # if 58 | print("ERROR NO TRAINED MODEL IN: " + ckpt.model_checkpoint_path+"See TRAIN.py for training") 59 | sys.exit() 60 | #--------------------Sum of intersection from all validation images for all classes and sum of union for all images and all classes---------------------------------------------------------------------------------- 61 | Union = np.float64(np.zeros(len(Classes))) #Sum of union 62 | Intersection = np.float64(np.zeros(len(Classes))) #Sum of Intersection 63 | fim = 0 64 | print("Start Evaluating intersection over union for "+str(ValidReader.NumFiles)+" images") 65 | #===========================GO over all validation images and caclulate IOU============================================================ 66 | while (ValidReader.itr0: print(Classes[i]+"\t"+str(Intersection[i]/Union[i])) 86 | 87 | 88 | ################################################################################################################################################## 89 | main()#Run script 90 | print("Finished") -------------------------------------------------------------------------------- /Figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sagieppel/Fully-convolutional-neural-network-FCN-for-semantic-segmentation-Tensorflow-implementation/ed772ba20a5c6c2e37c6a7bf3db60f74c64df3b5/Figure1.png -------------------------------------------------------------------------------- /IOU.py: -------------------------------------------------------------------------------- 1 | ##Calculate Intersection Over Union Score for predicted layer 2 | import numpy as np 3 | import scipy.misc as misc 4 | 5 | def GetIOU(Pred,GT,NumClasses,ClassNames=[], DisplyResults=False): #Given A ground true and predicted labels return the intersection over union for each class 6 | # and the union for each class 7 | ClassIOU=np.zeros(NumClasses)#Vector that Contain IOU per class 8 | ClassWeight=np.zeros(NumClasses)#Vector that Contain Number of pixel per class Predicted U Ground true (Union for this class) 9 | for i in range(NumClasses): # Go over all classes 10 | Intersection=np.float32(np.sum((Pred==GT)*(GT==i)))# Calculate intersection 11 | Union=np.sum(GT==i)+np.sum(Pred==i)-Intersection # Calculate Union 12 | if Union>0: 13 | ClassIOU[i]=Intersection/Union# Calculate intesection over union 14 | ClassWeight[i]=Union 15 | 16 | #------------Display results------------------------------------------------------------------------------------- 17 | if DisplyResults: 18 | for i in range(len(ClassNames)): 19 | print(ClassNames[i]+") "+str(ClassIOU[i])) 20 | print("Mean Classes IOU) "+str(np.mean(ClassIOU))) 21 | print("Image Predicition Accuracy)" + str(np.float32(np.sum(Pred == GT)) / GT.size)) 22 | #------------------------------------------------------------------------------------------------- 23 | 24 | return ClassIOU, ClassWeight 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /Inference.py: -------------------------------------------------------------------------------- 1 | # Run prediction and genertae pixelwise annotation for every pixels in the image using fully coonvolutional neural net 2 | # Output saved as label images, and label image overlay on the original image 3 | # 1) Make sure you you have trained model in logs_dir (See Train.py for creating trained model) 4 | # 2) Set the Image_Dir to the folder where the input image for prediction are located 5 | # 3) Set number of classes number in NUM_CLASSES 6 | # 4) Set Pred_Dir the folder where you want the output annotated images to be save 7 | # 5) Run script 8 | #-------------------------------------------------------------------------------------------------------------------- 9 | import tensorflow as tf 10 | import numpy as np 11 | import scipy.misc as misc 12 | import sys 13 | import BuildNetVgg16 14 | import TensorflowUtils 15 | import os 16 | import Data_Reader 17 | import OverrlayLabelOnImage as Overlay 18 | import CheckVGG16Model 19 | logs_dir= "logs/"# "path to logs directory where trained model and information will be stored" 20 | Image_Dir="Data_Zoo/Materials_In_Vessels/Test_Images_All/"# Test image folder 21 | w=0.6# weight of overlay on image 22 | Pred_Dir="Output_Prediction/" # Library where the output prediction will be written 23 | model_path="Model_Zoo/vgg16.npy"# "Path to pretrained vgg16 model for encoder" 24 | NameEnd="" # Add this string to the ending of the file name optional 25 | NUM_CLASSES = 2 # Number of classes 26 | #------------------------------------------------------------------------------------------------------------------------- 27 | CheckVGG16Model.CheckVGG16(model_path)# Check if pretrained vgg16 model avialable and if not try to download it 28 | 29 | ################################################################################################################################################################################ 30 | def main(argv=None): 31 | # .........................Placeholders for input image and labels........................................................................ 32 | keep_prob = tf.placeholder(tf.float32, name="keep_probabilty") # Dropout probability 33 | image = tf.placeholder(tf.float32, shape=[None, None, None, 3], name="input_image") # Input image batch first dimension image number second dimension width third dimension height 4 dimension RGB 34 | 35 | # -------------------------Build Net---------------------------------------------------------------------------------------------- 36 | Net = BuildNetVgg16.BUILD_NET_VGG16(vgg16_npy_path=model_path) # Create class instance for the net 37 | Net.build(image, NUM_CLASSES, keep_prob) # Build net and load intial weights (weights before training) 38 | # -------------------------Data reader for validation/testing images----------------------------------------------------------------------------------------------------------------------------- 39 | ValidReader = Data_Reader.Data_Reader(Image_Dir, BatchSize=1) 40 | #-------------------------Load Trained model if you dont have trained model see: Train.py----------------------------------------------------------------------------------------------------------------------------- 41 | 42 | sess = tf.Session() #Start Tensorflow session 43 | 44 | print("Setting up Saver...") 45 | saver = tf.train.Saver() 46 | 47 | sess.run(tf.global_variables_initializer()) 48 | ckpt = tf.train.get_checkpoint_state(logs_dir) 49 | if ckpt and ckpt.model_checkpoint_path: # if train model exist restore it 50 | saver.restore(sess, ckpt.model_checkpoint_path) 51 | print("Model restored...") 52 | else: 53 | print("ERROR NO TRAINED MODEL IN: "+ckpt.model_checkpoint_path+" See Train.py for creating train network ") 54 | sys.exit() 55 | 56 | #--------------------Create output directories for predicted label, one folder for each granulairy of label prediciton--------------------------------------------------------------------------------------------------------------------------------------------- 57 | 58 | if not os.path.exists(Pred_Dir): os.makedirs(Pred_Dir) 59 | if not os.path.exists(Pred_Dir+"/OverLay"): os.makedirs(Pred_Dir+"/OverLay") 60 | if not os.path.exists(Pred_Dir + "/Label"): os.makedirs(Pred_Dir + "/Label") 61 | 62 | 63 | print("Running Predictions:") 64 | print("Saving output to:" + Pred_Dir) 65 | #----------------------Go over all images and predict semantic segmentation in various of classes------------------------------------------------------------- 66 | fim = 0 67 | print("Start Predicting " + str(ValidReader.NumFiles) + " images") 68 | while (ValidReader.itr < ValidReader.NumFiles): 69 | print(str(fim * 100.0 / ValidReader.NumFiles) + "%") 70 | fim += 1 71 | # ..................................Load image....................................................................................... 72 | FileName=ValidReader.OrderedFiles[ValidReader.itr] #Get input image name 73 | Images = ValidReader.ReadNextBatchClean() # load testing image 74 | 75 | # Predict annotation using net 76 | LabelPred = sess.run(Net.Pred, feed_dict={image: Images, keep_prob: 1.0}) 77 | #------------------------Save predicted labels overlay on images--------------------------------------------------------------------------------------------- 78 | misc.imsave(Pred_Dir + "/OverLay/"+ FileName+NameEnd , Overlay.OverLayLabelOnImage(Images[0],LabelPred[0], w)) #Overlay label on image 79 | misc.imsave(Pred_Dir + "/Label/" + FileName[:-4] + ".png" + NameEnd, LabelPred[0].astype(np.uint8)) 80 | ################################################################################################################################################## 81 | main()#Run script 82 | print("Finished") -------------------------------------------------------------------------------- /OverrlayLabelOnImage.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | ###################Overlay Label on image Mark label on the image in transperent form################################################################################### 3 | def OverLayLabelOnImage(ImgIn,Label,W): 4 | #ImageIn is the image 5 | #Label is the label per pixel 6 | # W is the relative weight in which the labels will be marked on the image 7 | # Return image with labels marked over it 8 | Img=ImgIn.copy() 9 | TR = [0,1, 0, 0, 0, 1, 1, 0, 0, 0.5, 0.7, 0.3, 0.5, 1, 0.5] 10 | TB = [0,0, 1, 0, 1, 0, 1, 0, 0.5, 0, 0.2, 0.2, 0.7, 0.5, 0.5] 11 | TG = [0,0, 0, 0.5, 1, 1, 0, 1, 0.7, 0.4, 0.7, 0.2, 0, 0.25, 0.5] 12 | R = Img[:, :, 0].copy() 13 | G = Img[:, :, 1].copy() 14 | B = Img[:, :, 2].copy() 15 | for i in range(Label.max()+1): 16 | if i> Downloading %s %.1f%%' % (filename, float(count * block_size) / float(total_size) * 100.0)) 32 | sys.stdout.flush() 33 | 34 | filepath, _ = urllib.request.urlretrieve(url_name, filepath, reporthook=_progress) 35 | print() 36 | statinfo = os.stat(filepath) 37 | print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.') 38 | if is_tarfile: 39 | tarfile.open(filepath, 'r:gz').extractall(dir_path) 40 | elif is_zipfile: 41 | with zipfile.ZipFile(filepath) as zf: 42 | zip_dir = zf.namelist()[0] 43 | zf.extractall(dir_path) 44 | 45 | 46 | def save_image(image, save_dir, name, mean=None): 47 | """ 48 | Save image by unprocessing if mean given else just save 49 | :param mean: 50 | :param image: 51 | :param save_dir: 52 | :param name: 53 | :return: 54 | """ 55 | if mean: 56 | image = unprocess_image(image, mean) 57 | misc.imsave(os.path.join(save_dir, name + ".png"), image) 58 | 59 | 60 | def get_variable(weights, name): 61 | init = tf.constant_initializer(weights, dtype=tf.float32) 62 | var = tf.get_variable(name=name, initializer=init, shape=weights.shape) 63 | return var 64 | 65 | 66 | def weight_variable(shape, stddev=0.02, name=None): #Create tensorflow matrix with normal random distubotion mean 0 and standart deviation 0.02 67 | # print(shape) 68 | initial = tf.truncated_normal(shape, stddev=stddev) 69 | if name is None: 70 | return tf.Variable(initial) 71 | else: 72 | return tf.get_variable(name, initializer=initial) 73 | 74 | 75 | def bias_variable(shape, name=None): 76 | initial = tf.constant(0.0, shape=shape) 77 | if name is None: 78 | return tf.Variable(initial) 79 | else: 80 | return tf.get_variable(name, initializer=initial) 81 | 82 | 83 | def get_tensor_size(tensor): 84 | from operator import mul 85 | return reduce(mul, (d.value for d in tensor.get_shape()), 1) 86 | 87 | 88 | def conv2d_basic(x, W, bias): #Simple conv and biase addition this function is waste of time replace in the code with the tensorflow command 89 | conv = tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding="SAME") #Padding same mean the output is same size as input? 90 | return tf.nn.bias_add(conv, bias) 91 | 92 | 93 | def conv2d_strided(x, W, b):# Simple strided convolution for transpose convolution waste of time replace in code 94 | conv = tf.nn.conv2d(x, W, strides=[1, 2, 2, 1], padding="SAME") 95 | return tf.nn.bias_add(conv, b) 96 | 97 | 98 | def conv2d_transpose_strided(x, W, b, output_shape=None, stride = 2): # Use traspose convolution with stride of 2 to increase image size to output shape if output shape is none double input image shape 99 | # print x.get_shape() 100 | # print W.get_shape() 101 | if output_shape is None: 102 | output_shape = x.get_shape().as_list() 103 | output_shape[1] *= 2 104 | output_shape[2] *= 2 105 | output_shape[3] = W.get_shape().as_list()[2] 106 | # print output_shape 107 | conv = tf.nn.conv2d_transpose(x, W, output_shape, strides=[1, stride, stride, 1], padding="SAME") 108 | return tf.nn.bias_add(conv, b) 109 | 110 | 111 | def leaky_relu(x, alpha=0.0, name=""): 112 | return tf.maximum(alpha * x, x, name) 113 | 114 | 115 | def max_pool_2x2(x): 116 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME") 117 | 118 | 119 | def avg_pool_2x2(x): #simple tensorflow average pooling (why average?) not really needed 120 | return tf.nn.avg_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME") 121 | 122 | 123 | def local_response_norm(x): 124 | return tf.nn.lrn(x, depth_radius=5, bias=2, alpha=1e-4, beta=0.75) 125 | 126 | 127 | def batch_norm(x, n_out, phase_train, scope='bn', decay=0.9, eps=1e-5): 128 | """ 129 | Code taken from http://stackoverflow.com/a/34634291/2267819 130 | """ 131 | with tf.variable_scope(scope): 132 | beta = tf.get_variable(name='beta', shape=[n_out], initializer=tf.constant_initializer(0.0) 133 | , trainable=True) 134 | gamma = tf.get_variable(name='gamma', shape=[n_out], initializer=tf.random_normal_initializer(1.0, 0.02), 135 | trainable=True) 136 | batch_mean, batch_var = tf.nn.moments(x, [0, 1, 2], name='moments') 137 | ema = tf.train.ExponentialMovingAverage(decay=decay) 138 | 139 | def mean_var_with_update(): 140 | ema_apply_op = ema.apply([batch_mean, batch_var]) 141 | with tf.control_dependencies([ema_apply_op]): 142 | return tf.identity(batch_mean), tf.identity(batch_var) 143 | 144 | mean, var = tf.cond(phase_train, 145 | mean_var_with_update, 146 | lambda: (ema.average(batch_mean), ema.average(batch_var))) 147 | normed = tf.nn.batch_normalization(x, mean, var, beta, gamma, eps) 148 | return normed 149 | 150 | 151 | def process_image(image, mean_pixel): 152 | return image - mean_pixel 153 | 154 | 155 | def unprocess_image(image, mean_pixel): 156 | return image + mean_pixel 157 | 158 | 159 | def bottleneck_unit(x, out_chan1, out_chan2, down_stride=False, up_stride=False, name=None): 160 | """ 161 | Modified implementation from github ry?! 162 | """ 163 | 164 | def conv_transpose(tensor, out_channel, shape, strides, name=None): 165 | out_shape = tensor.get_shape().as_list() 166 | in_channel = out_shape[-1] 167 | kernel = weight_variable([shape, shape, out_channel, in_channel], name=name) 168 | shape[-1] = out_channel 169 | return tf.nn.conv2d_transpose(x, kernel, output_shape=out_shape, strides=[1, strides, strides, 1], 170 | padding='SAME', name='conv_transpose') 171 | 172 | def conv(tensor, out_chans, shape, strides, name=None): 173 | in_channel = tensor.get_shape().as_list()[-1] 174 | kernel = weight_variable([shape, shape, in_channel, out_chans], name=name) 175 | return tf.nn.conv2d(x, kernel, strides=[1, strides, strides, 1], padding='SAME', name='conv') 176 | 177 | def bn(tensor, name=None): 178 | """ 179 | :param tensor: 4D tensor input 180 | :param name: name of the operation 181 | :return: local response normalized tensor - not using batch normalization :( 182 | """ 183 | return tf.nn.lrn(tensor, depth_radius=5, bias=2, alpha=1e-4, beta=0.75, name=name) 184 | 185 | in_chans = x.get_shape().as_list()[3] 186 | 187 | if down_stride or up_stride: 188 | first_stride = 2 189 | else: 190 | first_stride = 1 191 | 192 | with tf.variable_scope('res%s' % name): 193 | if in_chans == out_chan2: 194 | b1 = x 195 | else: 196 | with tf.variable_scope('branch1'): 197 | if up_stride: 198 | b1 = conv_transpose(x, out_chans=out_chan2, shape=1, strides=first_stride, 199 | name='res%s_branch1' % name) 200 | else: 201 | b1 = conv(x, out_chans=out_chan2, shape=1, strides=first_stride, name='res%s_branch1' % name) 202 | b1 = bn(b1, 'bn%s_branch1' % name, 'scale%s_branch1' % name) 203 | 204 | with tf.variable_scope('branch2a'): 205 | if up_stride: 206 | b2 = conv_transpose(x, out_chans=out_chan1, shape=1, strides=first_stride, name='res%s_branch2a' % name) 207 | else: 208 | b2 = conv(x, out_chans=out_chan1, shape=1, strides=first_stride, name='res%s_branch2a' % name) 209 | b2 = bn(b2, 'bn%s_branch2a' % name, 'scale%s_branch2a' % name) 210 | b2 = tf.nn.relu(b2, name='relu') 211 | 212 | with tf.variable_scope('branch2b'): 213 | b2 = conv(b2, out_chans=out_chan1, shape=3, strides=1, name='res%s_branch2b' % name) 214 | b2 = bn(b2, 'bn%s_branch2b' % name, 'scale%s_branch2b' % name) 215 | b2 = tf.nn.relu(b2, name='relu') 216 | 217 | with tf.variable_scope('branch2c'): 218 | b2 = conv(b2, out_chans=out_chan2, shape=1, strides=1, name='res%s_branch2c' % name) 219 | b2 = bn(b2, 'bn%s_branch2c' % name, 'scale%s_branch2c' % name) 220 | 221 | x = b1 + b2 222 | return tf.nn.relu(x, name='relu') 223 | 224 | 225 | def add_to_regularization_and_summary(var): 226 | if var is not None: 227 | tf.summary.histogram(var.op.name, var) 228 | tf.add_to_collection("reg_loss", tf.nn.l2_loss(var)) 229 | 230 | 231 | def add_activation_summary(var): 232 | if var is not None: 233 | tf.summary.histogram(var.op.name + "/activation", var) 234 | tf.summary.scalar(var.op.name + "/sparsity", tf.nn.zero_fraction(var)) 235 | 236 | 237 | def add_gradient_summary(grad, var): 238 | if grad is not None: 239 | tf.summary.histogram(var.op.name + "/gradient", grad) 240 | --------------------------------------------------------------------------------