├── README.md ├── data ├── sample_1 │ ├── label.png │ ├── mask.png │ └── sample_1.png ├── sample_2 │ ├── label.png │ ├── mask.png │ └── sample_2.png └── sample_3 │ ├── label.png │ ├── mask.png │ └── sample_3.png ├── models ├── checkpoint ├── nucles_model_v3.data-00000-of-00001 ├── nucles_model_v3.index └── nucles_model_v3.meta ├── nuclei_DS.py ├── screenshots ├── screenshot_1.png ├── screenshot_2.png └── screenshots_3.png └── util ├── __init__.py ├── __pycache__ ├── __init__.cpython-34.pyc ├── run_restored_model.cpython-34.pyc └── util.cpython-34.pyc ├── run_restored_model.py └── util.py /README.md: -------------------------------------------------------------------------------- 1 | # Cell-Nuclei-Detection-and-Segmentation 2 | Detecting the location and draw boundary of nuclei from tissue microscopic images (H&E stained). 3 | Model is based on U-net with contour enhancement in loss function. Overlap patch based strategy is used to 1) adapt to variant input image size (resize image may stretch features); 2) use random clip and rotation for data augmentation; 3) each region in output mask is determined by combining inference result from multiple patches. More details can be found in [1]. 4 | ![sample_1](screenshots/screenshots_3.png) 5 | ![sample_2](screenshots/screenshot_2.png) 6 | 7 | ### Dependencies 8 | - Tensorflow 9 | - OpenCV 10 | - Scikit-image 11 | - Numpy 12 | - Matplotlib 13 | 14 | #### More 15 | - [x] detection and segmentation model 16 | - [x] consider edge into loss function during training 17 | - [x] morphology operation to calculate center and boundary 18 | - [ ] better color normalization method for preprocess 19 | - [ ] identify overlapping samples with local segmentation model 20 | - [ ] identify tissue types 21 | 22 | #### Reference 23 | [1] K.Chen, N. Zhang, L.S.Powers, J.M.Roveda, Cell Nuclei Detection and Segmentation for Computational Pathology Using Deep Learning, SpringSim 2019 Modeling and Simulation in Medicine, Society for Modeling and Simuation (SCS) International. 24 | -------------------------------------------------------------------------------- /data/sample_1/label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_1/label.png -------------------------------------------------------------------------------- /data/sample_1/mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_1/mask.png -------------------------------------------------------------------------------- /data/sample_1/sample_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_1/sample_1.png -------------------------------------------------------------------------------- /data/sample_2/label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_2/label.png -------------------------------------------------------------------------------- /data/sample_2/mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_2/mask.png -------------------------------------------------------------------------------- /data/sample_2/sample_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_2/sample_2.png -------------------------------------------------------------------------------- /data/sample_3/label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_3/label.png -------------------------------------------------------------------------------- /data/sample_3/mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_3/mask.png -------------------------------------------------------------------------------- /data/sample_3/sample_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/data/sample_3/sample_3.png -------------------------------------------------------------------------------- /models/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "nucles_model_v3" 2 | all_model_checkpoint_paths: "nucles_model_v3" 3 | -------------------------------------------------------------------------------- /models/nucles_model_v3.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/models/nucles_model_v3.data-00000-of-00001 -------------------------------------------------------------------------------- /models/nucles_model_v3.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/models/nucles_model_v3.index -------------------------------------------------------------------------------- /models/nucles_model_v3.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/models/nucles_model_v3.meta -------------------------------------------------------------------------------- /nuclei_DS.py: -------------------------------------------------------------------------------- 1 | ''' 2 | detect and segement potential nuclei in miscropic images (H&E stained) 3 | @author: Kemeng Chen 4 | ''' 5 | import os 6 | import numpy as np 7 | import cv2 8 | from time import time 9 | from util import* 10 | import matplotlib.pyplot as plt 11 | 12 | def process(data_folder, model_name, format): 13 | patch_size=128 14 | stride=16 15 | file_path=os.path.join(os.getcwd(), data_folder) 16 | name_list=os.listdir(file_path) 17 | print(str(len(name_list)), ' files detected') 18 | model_path=os.path.join(os.getcwd(), 'models') 19 | model=restored_model(os.path.join(model_path, model_name), model_path) 20 | print('Start time:') 21 | print_ctime() 22 | 23 | for index, temp_name in enumerate(name_list): 24 | ts=time() 25 | print('process: ', str(index), ' name: ', temp_name) 26 | temp_path=os.path.join(file_path, temp_name) 27 | if not os.path.isdir(temp_path): 28 | continue 29 | # result_path=os.path.join(temp_path, 'mask.png') 30 | temp_image=cv2.imread(os.path.join(temp_path, temp_name+format)) 31 | if temp_image is None: 32 | raise AssertionError(temp_path, ' not found') 33 | batch_group, shape=preprocess(temp_image, patch_size, stride, temp_path) 34 | mask_list=sess_interference(model, batch_group) 35 | c_mask=patch2image(mask_list, patch_size, stride, shape) 36 | c_mask=cv2.medianBlur((255*c_mask).astype(np.uint8), 3) 37 | c_mask=c_mask.astype(np.float)/255 38 | thr=0.5 39 | c_mask[c_mask=thr]=1 41 | center_edge_mask, gray_map=center_edge(c_mask, temp_image) 42 | cv2.imwrite(os.path.join(temp_path, 'mask.png'), gray_map) 43 | cv2.imwrite(os.path.join(temp_path, 'label.png'), center_edge_mask) 44 | te=time() 45 | print('Time cost: ', str(te-ts)) 46 | # fig, ax=plt.subplots(1,2) 47 | # ax[0].imshow(cv2.cvtColor(center_edge_mask, cv2.COLOR_BGR2RGB)) 48 | # ax[0].set_title('label') 49 | # ax[1].imshow(gray_map) 50 | # ax[1].set_title('Center and contour') 51 | 52 | 53 | model.close_sess() 54 | print('mask generation done') 55 | print_ctime() 56 | # plt.show() 57 | 58 | def main(): 59 | data_folder='data' 60 | model_name='nucles_model_v3.meta' 61 | format='.png' 62 | process(data_folder, model_name, format) 63 | 64 | if __name__ == '__main__': 65 | main() 66 | -------------------------------------------------------------------------------- /screenshots/screenshot_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/screenshots/screenshot_1.png -------------------------------------------------------------------------------- /screenshots/screenshot_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/screenshots/screenshot_2.png -------------------------------------------------------------------------------- /screenshots/screenshots_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/screenshots/screenshots_3.png -------------------------------------------------------------------------------- /util/__init__.py: -------------------------------------------------------------------------------- 1 | from .util import* 2 | from .run_restored_model import* -------------------------------------------------------------------------------- /util/__pycache__/__init__.cpython-34.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/util/__pycache__/__init__.cpython-34.pyc -------------------------------------------------------------------------------- /util/__pycache__/run_restored_model.cpython-34.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/util/__pycache__/run_restored_model.cpython-34.pyc -------------------------------------------------------------------------------- /util/__pycache__/util.cpython-34.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KChen89/Cell-Nuclei-Detection-and-Segmentation/698bbe39662dc4301de411268ccd947df25d3d0d/util/__pycache__/util.cpython-34.pyc -------------------------------------------------------------------------------- /util/run_restored_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | restored a model and run session 3 | @author: Kemeng Chen 4 | ''' 5 | import tensorflow as tf 6 | import numpy as np 7 | 8 | class restored_model(object): 9 | 10 | def __init__(self, model_name, model_folder): 11 | self.graph=tf.Graph() 12 | self.sess=tf.Session(graph=self.graph) 13 | print('Read model: ', model_name) 14 | 15 | with self.graph.as_default(): 16 | self.model_saver=tf.train.import_meta_graph(model_name) 17 | self.model_saver.restore(self.sess, tf.train.latest_checkpoint(model_folder+'/.')) 18 | self.graph=self.graph 19 | self.sample_in=self.graph.get_tensor_by_name('sample:0') 20 | self.c_mask_out=self.graph.get_tensor_by_name('c_mask:0') 21 | 22 | def run_sess(self, patches): 23 | feed_dict={self.sample_in: patches} 24 | generated_mask=self.sess.run([self.c_mask_out], feed_dict) 25 | return generated_mask 26 | 27 | def close_sess(self): 28 | self.sess.close() -------------------------------------------------------------------------------- /util/util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | utility functions assisting nuclei detection and segmentation 3 | @author: Kemeng Chen 4 | ''' 5 | import numpy as np 6 | import cv2 7 | import os 8 | import sys 9 | import math 10 | from time import time, ctime 11 | from skimage.morphology import square, erosion, dilation 12 | from skimage.measure import label, regionprops 13 | from .run_restored_model import restored_model 14 | 15 | def print_ctime(): 16 | current_time=ctime(int(time())) 17 | print(str(current_time)) 18 | 19 | def batch2list(batch): 20 | mask_list=list() 21 | for index in range(batch.shape[0]): 22 | mask_list.append(batch[index,:,:]) 23 | return mask_list 24 | 25 | def patch2image(patch_list, patch_size, stride, shape): 26 | if shape[0]2: 63 | full_image=np.pad(in_image, ((0, patch_size+stride*L-shape[0]), (0, patch_size+stride*W-shape[1]), (0,0)), mode='symmetric') 64 | else: 65 | full_image=np.pad(in_image, ((0, patch_size+stride*L-shape[0]), (0, patch_size+stride*W-shape[1])), mode='symmetric') 66 | for i in range(L+1): 67 | for j in range(W+1): 68 | if len(shape)>2: 69 | patch_list.append(full_image[i*stride:i*stride+patch_size, j*stride:j*stride+patch_size, :]) 70 | else: 71 | patch_list.append(full_image[i*stride:i*stride+patch_size, j*stride:j*stride+patch_size]) 72 | if len(patch_list)!=(L+1)*(W+1): 73 | raise ValueError('Patch_list: ', str(len(patch_list), ' L: ', str(L), ' W: ', str(W))) 74 | 75 | return patch_list 76 | 77 | def list2batch(patches): 78 | ''' 79 | covert patch to flat batch 80 | args: 81 | patches: list 82 | return: 83 | batch: numpy array 84 | ''' 85 | patch_shape=list(patches[0].shape) 86 | 87 | batch_size=len(patches) 88 | 89 | if len(patch_shape)>2: 90 | batch=np.zeros([batch_size]+patch_shape) 91 | for index, temp in enumerate(patches): 92 | batch[index,:,:,:]=temp 93 | else: 94 | batch=np.zeros([batch_size]+patch_shape+[1]) 95 | for index, temp in enumerate(patches): 96 | batch[index,:,:,:]=np.expand_dims(temp, axis=-1) 97 | return batch 98 | 99 | def preprocess(input_image, patch_size, stride, file_path): 100 | f_size=5 101 | g_size=10 102 | shape=input_image.shape 103 | patch_list=image2patch(input_image.astype(np.float32)/255.0, patch_size, stride) 104 | num_group=math.ceil(len(patch_list)/g_size) 105 | batch_group=list() 106 | for i in range(num_group): 107 | temp_batch=list2batch(patch_list[i*g_size:(i+1)*g_size]) 108 | batch_group.append(temp_batch) 109 | return batch_group, shape 110 | 111 | def sess_interference(sess, batch_group): 112 | patch_list=list() 113 | for temp_batch in batch_group: 114 | mask_batch=sess.run_sess(temp_batch)[0] 115 | mask_batch=np.squeeze(mask_batch, axis=-1) 116 | mask_list=batch2list(mask_batch) 117 | patch_list+=mask_list 118 | return patch_list 119 | 120 | def center_point(mask): 121 | v,h=mask.shape 122 | center_mask=np.zeros([v,h]) 123 | mask=erosion(mask, square(3)) 124 | individual_mask=label(mask, connectivity=2) 125 | prop=regionprops(individual_mask) 126 | for cordinates in prop: 127 | temp_center=cordinates.centroid 128 | if not math.isnan(temp_center[0]) and not math.isnan(temp_center[1]): 129 | temp_mask=np.zeros([v,h]) 130 | temp_mask[int(temp_center[0]), int(temp_center[1])]=1 131 | center_mask+=dilation(temp_mask, square(2)) 132 | return np.clip(center_mask, a_min=0, a_max=1).astype(np.uint8) 133 | 134 | def draw_individual_edge(mask): 135 | v,h=mask.shape 136 | edge=np.zeros([v,h]) 137 | individual_mask=label(mask, connectivity=2) 138 | for index in np.unique(individual_mask): 139 | if index==0: 140 | continue 141 | temp_mask=np.copy(individual_mask) 142 | temp_mask[temp_mask!=index]=0 143 | temp_mask[temp_mask==index]=1 144 | temp_mask=dilation(temp_mask, square(3)) 145 | temp_edge=cv2.Canny(temp_mask.astype(np.uint8), 2,5)/255 146 | edge+=temp_edge 147 | return np.clip(edge, a_min=0, a_max=1).astype(np.uint8) 148 | 149 | def center_edge(mask, image): 150 | center_map=center_point(mask) 151 | edge_map=draw_individual_edge(mask) 152 | comb_mask=center_map+edge_map 153 | comb_mask=np.clip(comb_mask, a_min=0, a_max=1) 154 | check_image=np.copy(image) 155 | comb_mask*=255 156 | check_image[:,:,1]=np.maximum(check_image[:,:,1], comb_mask) 157 | return check_image.astype(np.uint8), comb_mask.astype(np.uint8) --------------------------------------------------------------------------------