├── utils ├── __init__.py ├── config.py ├── slide.py ├── tf_serving.py └── libs.py ├── main.py ├── LICENSE ├── README.md └── inference.py /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from utils import config 2 | GPU_LIST = config.INFERENCE_GPUS 3 | import os 4 | os.environ["CUDA_VISIBLE_DEVICES"] = ','.join('{0}'.format(n) for n in GPU_LIST) 5 | from inference import Inference 6 | 7 | 8 | if __name__ == '__main__': 9 | pg = Inference(data_dir='/path/to/data/', data_list='/path/to/list', 10 | class_num=2, result_dir='./result', use_level=1) 11 | pg.run() 12 | -------------------------------------------------------------------------------- /utils/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | INFERENCE_GPUS = [8] 4 | CENTER_SIZE = 2000 # The effective patch size. 5 | BORDER_SIZE = 100 # The boarder size containing surrounding information. 6 | PATCH_SIZE = CENTER_SIZE + 2 * BORDER_SIZE 7 | format_mapping = { 8 | 'tif': 'OpenSlide', 9 | 'svs': 'OpenSlide', 10 | 'ndpi': 'OpenSlide', 11 | 'scn': 'OpenSlide', 12 | 'mrxs': 'OpenSlide', 13 | 'bif': 'OpenSlide', 14 | 'vms': 'OpenSlide', 15 | } 16 | MODEL_NAME = 'stomach' 17 | INPUT_KEY = 'output' 18 | PREDICT_KEY = 'output' 19 | TF_SERVING_HOST = os.environ.get('TF_SERVING_HOST', '127.0.0.1') 20 | TF_SERVING_PORT = int(os.environ.get('TF_SERVING_PORT', 9000)) 21 | TEMP_DIR = './temp/' # Where to save the predicted patches. 22 | THUMBNAIL_RATIO = 10 # Down-sample ratio from the predictions to the thumbnail. 23 | KEEP_TEMP = False # Whether to keep the predicted patches after generated the thumbnail. 24 | DO_POST_PROCESSING = False # Whether to post-process the predicted patches. 25 | FILTER_KERNEL = 9 # The kernel size for post-processing. 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 S.Wang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PathologyGo 2 | Core components of PathologyGo, the AI assistance system designed for histopathological inference. 3 | 4 | Dependency 5 | 6 | * Docker 7 | * Python 2.7 and 3.x 8 | * openslide 9 | * tensorflow_serving 10 | * grpc 11 | * pillow 12 | * numpy 13 | * opencv-python 14 | 15 | Dockerized TensorFlow Serving 16 | 17 | * GPU version: [GitHub](https://github.com/physicso/tensorflow_serving_gpu), [Docker Hub](https://hub.docker.com/r/physicso/tf_serving_gpu). 18 | * CPU version: [Docker Hub](https://hub.docker.com/r/tensorflow/serving). 19 | 20 | Quick Start 21 | 22 | This code is easy to implement. Just change the path to your data repo: 23 | 24 | ```python 25 | from utils import config 26 | GPU_LIST = config.INFERENCE_GPUS 27 | import os 28 | os.environ["CUDA_VISIBLE_DEVICES"] = ','.join('{0}'.format(n) for n in GPU_LIST) 29 | from inference import Inference 30 | 31 | 32 | if __name__ == '__main__': 33 | pg = Inference(data_dir='/path/to/data/', data_list='/path/to/list', 34 | class_num=2, result_dir='./result', use_level=1) 35 | pg.run() 36 | 37 | ``` 38 | 39 | You may configure all the model-specific parameters in `utils/config.py`. 40 | 41 | Example 42 | 43 | Use the [CAMELYON16](https://camelyon16.grand-challenge.org/) test dataset as an example, 44 | the data path should be `/data/CAMELYON/`, and the content of the data list is 45 | 46 | ``` 47 | 001.tif 48 | 002.tif 49 | ... 50 | ``` 51 | 52 | The predicted heatmaps will be written to `./result`. 53 | 54 | DIY Notes 55 | 56 | You may use other exported models. You can change the model name for TensorFlow Serving in `utils/config.py`. Just remember to modify `class_num` and `use_level`. 57 | 58 | Note that the default input / output tensor name should be `input` / `output`. 59 | -------------------------------------------------------------------------------- /utils/slide.py: -------------------------------------------------------------------------------- 1 | """ 2 | This class provides a unified interface decoding various slide formats. 3 | Cancheng Liu 4 | Email: liucancheng@thorough.ai 5 | """ 6 | import os.path 7 | from openslide import OpenSlide 8 | from utils import config 9 | 10 | 11 | class Slide(object): 12 | def __init__(self, image_path): 13 | super(Slide, self).__init__() 14 | self.image_path = image_path.strip() 15 | self.image_file = os.path.basename(image_path) 16 | self.image_name, self.suffix = self.image_file.split('.') 17 | if not self.suffix in config.format_mapping.keys(): 18 | raise Exception('Error: File format ' + self.suffix + ' is supported yet.') 19 | self._slide = OpenSlide(image_path) 20 | self._level_downsamples = self._slide.level_downsamples 21 | self._level_dimensions = self._slide.level_dimensions 22 | self.width, self.height = self._level_dimensions[0] 23 | 24 | @property 25 | def dimensions(self): 26 | return self._level_dimensions[0] 27 | 28 | @property 29 | def level_dimensions(self): 30 | return self._level_dimensions 31 | 32 | @property 33 | def level_downsamples(self): 34 | return self._level_downsamples 35 | 36 | def read_region(self, level, location, size): 37 | (x, y), (w, h) = location, size 38 | try: 39 | _ds = self._level_downsamples[level] 40 | patch = self._slide.read_region( 41 | level=level, location=(int(x * _ds), int(y * _ds)), size=size) 42 | except Exception: 43 | raise 44 | else: 45 | return patch 46 | 47 | def get_thumbnail(self): 48 | try: 49 | image = self._slide.read_region( 50 | location=(0, 0), level=0, size=(1000, 1000)) 51 | except Exception: 52 | raise 53 | else: 54 | return image 55 | -------------------------------------------------------------------------------- /utils/tf_serving.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic components for using TF Serving. 3 | """ 4 | import os 5 | os.environ["CUDA_VISIBLE_DEVICES"] = "" 6 | import io 7 | import numpy as np 8 | from PIL import Image 9 | import grpc 10 | import tensorflow as tf 11 | from grpc.beta import implementations 12 | from tensorflow_serving.apis import predict_pb2 13 | from tensorflow_serving.apis import prediction_service_pb2 14 | from grpc._cython import cygrpc 15 | import config 16 | 17 | 18 | def decode_tensor(contents, downsample_factor): 19 | images = [Image.open(io.BytesIO(content)) for content in contents] 20 | dsize = (images[0].size[0] * downsample_factor, images[0].size[1] * downsample_factor) 21 | images = [image.resize(dsize, Image.BILINEAR) for image in images] 22 | mtx = np.array([np.asarray(image, dtype=np.uint8) for image in images], dtype=np.uint8) 23 | mtx = mtx.transpose((1, 2, 0)) 24 | return mtx 25 | 26 | 27 | class TFServing(object): 28 | """ 29 | Tensorflow Serving client, send prediction request. 30 | """ 31 | 32 | def __init__(self, host, port): 33 | super(TFServing, self).__init__() 34 | 35 | channel = self._insecure_channel(host, port) 36 | self._stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) 37 | 38 | def _insecure_channel(self, host, port): 39 | channel = grpc.insecure_channel( 40 | target=host if port is None else '{}:{}'.format(host, port), 41 | options=[(cygrpc.ChannelArgKey.max_send_message_length, -1), 42 | (cygrpc.ChannelArgKey.max_receive_message_length, -1)]) 43 | return grpc.beta.implementations.Channel(channel) 44 | 45 | def predict(self, image_input, model_name): 46 | request = predict_pb2.PredictRequest() 47 | request.model_spec.name = model_name 48 | request.inputs[config.INPUT_KEY].CopyFrom(tf.contrib.util.make_tensor_proto( 49 | image_input.astype(np.uint8, copy=False))) 50 | try: 51 | result = self._stub.Predict(request, 1000.0) 52 | image_prob = np.array(result.outputs[config.PREDICT_KEY].int_val) 53 | except Exception as e: 54 | raise e 55 | else: 56 | return image_prob.astype(np.uint8) 57 | -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- 1 | """ 2 | Inference module supporting whole slide images using TF Serving. 3 | Eric Wang 4 | Email: eric.wang@thorough.ai 5 | """ 6 | import os 7 | import shutil 8 | import numpy as np 9 | from utils.tf_serving import TFServing 10 | from utils.slide import Slide 11 | from utils import config 12 | from utils.libs import write, generate_effective_regions, generate_overlap_tile, \ 13 | post_processing, concat_patches 14 | 15 | 16 | class Inference: 17 | def __init__(self, data_dir, data_list, class_num, result_dir, use_level): 18 | """ 19 | This is the main inference module for the sake of easy to call. 20 | :param data_dir: The directory storing the while image slides. 21 | :param data_list: The text file indicating the slide names. 22 | :param class_num: Number of predicted classes. 23 | :param result_dir: Where to put the predicted results. 24 | :param use_level: Which slide size we want to analyze, 0 for 40x, 1 for 20x, etc. 25 | """ 26 | if data_dir.endswith('/'): 27 | self.data_dir = data_dir 28 | else: 29 | self.data_dir = data_dir + '/' 30 | self.data_list = data_list 31 | self.class_num = class_num 32 | if result_dir.endswith('/'): 33 | self.result_dir = result_dir 34 | else: 35 | self.result_dir = result_dir + '/' 36 | if not os.path.exists(self.result_dir): 37 | os.mkdir(self.result_dir) 38 | self.use_level = use_level 39 | self.config = config 40 | 41 | @staticmethod 42 | def _infer(tfs_client, image): 43 | """ 44 | Inference for an image patch using TF Serving. 45 | :param tfs_client: TF Serving client. 46 | :param image: The image patch. 47 | :return: Predicted heatmap. 48 | """ 49 | try: 50 | prediction = tfs_client.predict(image, config.MODEL_NAME) 51 | except Exception as e: 52 | print('TF_SERVING_HOST: {}'.format(config.TF_SERVING_HOST)) 53 | print(e) 54 | raise 55 | else: 56 | return prediction 57 | 58 | def run(self): 59 | """ 60 | Proceeds the inference procedure. 61 | """ 62 | inference_list = open(self.data_list).readlines() 63 | tfs_client = TFServing(config.TF_SERVING_HOST, config.TF_SERVING_PORT) 64 | for item in inference_list: 65 | image_name, image_suffix = item.split('\n')[0].split('/')[-1].split('.') 66 | print('[INFO] Analyzing: ' + self.data_dir + item.split('\n')[0]) 67 | if not image_suffix in self.config.format_mapping.keys(): 68 | print('[ERROR] File ' + item + ' format not supported yet.') 69 | continue 70 | image_handle = Slide(self.data_dir + item.split('\n')[0]) 71 | image_dimensions = image_handle.level_dimensions[self.use_level] 72 | regions = generate_effective_regions(image_dimensions) 73 | index = 0 74 | region_num = len(regions) 75 | temp_dir = self.config.TEMP_DIR + image_name + '/' 76 | if not os.path.exists(temp_dir): 77 | os.makedirs(temp_dir) 78 | for region in regions: 79 | shifted_region, clip_region = generate_overlap_tile(region, image_dimensions) 80 | index += 1 81 | if index % 1 == 0: 82 | print('[INFO] Progress: ' + str(index) + ' / ' + str(region_num)) 83 | input_image = np.array(image_handle.read_region( 84 | location=(int(shifted_region[0]), 85 | int(shifted_region[1])), 86 | level=self.use_level, size=(self.config.PATCH_SIZE, self.config.PATCH_SIZE)))[:, :, 0: 3] 87 | prediction_result = self._infer(tfs_client, input_image) 88 | prediction_result = prediction_result[clip_region[0]: (self.config.CENTER_SIZE + clip_region[0]), 89 | clip_region[1]: (self.config.CENTER_SIZE + clip_region[1])] 90 | prediction_result = prediction_result[region[2]:(region[4] + 1), region[3]:(region[5] + 1)] 91 | if self.config.DO_POST_PROCESSING: 92 | prediction_result = post_processing(prediction_result) 93 | write(temp_dir + image_name + '_' + str(region[0]) + '_' + str(region[1]) 94 | + '_prediction.png', prediction_result, self.class_num) 95 | print('[INFO] Postprocessing...') 96 | full_prediction = concat_patches(temp_dir, image_name) 97 | write(self.result_dir + 98 | '_'.join([image_name, 'prediction_thumbnail']) + '.png', full_prediction, color_map=False) 99 | if not self.config.KEEP_TEMP: 100 | shutil.rmtree(temp_dir) 101 | print('[INFO] Prediction saved to ' + self.result_dir + '_'.join( 102 | [image_name, 'prediction_thumbnail']) + '.png') 103 | -------------------------------------------------------------------------------- /utils/libs.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import numpy as np 3 | import cv2 4 | from PIL import Image 5 | from utils import config 6 | 7 | 8 | def write(filename, content, class_num=2, color_map=True): 9 | """ 10 | Save image array to a specified path. 11 | The image will be automatically recolored via the class number. 12 | :param filename: The specified path. 13 | :param content: Numpy array containing the image. 14 | :param class_num: Total class number. 15 | :param color_map: Whether change the probability into gray grade. 16 | """ 17 | if class_num <= 1: 18 | raise Exception('ERROR: Class number should be >= 2.') 19 | color_stage = 255. / (class_num - 1) if color_map else 1.0 20 | new_image = Image.fromarray(np.uint8(content * color_stage)) 21 | new_image.save(filename, "PNG") 22 | 23 | 24 | def generate_effective_regions(size): 25 | """ 26 | This function is used to generate effective regions for inference according to the given slide size. 27 | :param size: Given slide size, should be in the form of [w, h]. 28 | """ 29 | width = size[0] 30 | height = size[1] 31 | x_step = int(width / config.CENTER_SIZE) 32 | y_step = int(height / config.CENTER_SIZE) 33 | regions = [] 34 | for x in range(0, x_step): 35 | for y in range(0, y_step): 36 | regions.append([x * config.CENTER_SIZE, y * config.CENTER_SIZE, 0, 0, 37 | config.CENTER_SIZE - 1, config.CENTER_SIZE - 1]) 38 | if not height % config.CENTER_SIZE == 0: 39 | for x in range(0, x_step): 40 | regions.append([x * config.CENTER_SIZE, height - config.CENTER_SIZE, 41 | 0, (y_step + 1) * config.CENTER_SIZE - height, 42 | config.CENTER_SIZE - 1, config.CENTER_SIZE - 1]) 43 | if not width % config.CENTER_SIZE == 0: 44 | for y in range(0, y_step): 45 | regions.append([width - config.CENTER_SIZE, y * config.CENTER_SIZE, 46 | (x_step + 1) * config.CENTER_SIZE - width, 0, 47 | config.CENTER_SIZE - 1, config.CENTER_SIZE - 1]) 48 | if not (height % config.CENTER_SIZE == 0 or width % config.CENTER_SIZE == 0): 49 | regions.append([width - config.CENTER_SIZE, height - config.CENTER_SIZE, 50 | (x_step + 1) * config.CENTER_SIZE - width, (y_step + 1) * config.CENTER_SIZE - height, 51 | config.CENTER_SIZE - 1, config.CENTER_SIZE - 1]) 52 | return regions 53 | 54 | 55 | def generate_overlap_tile(region, dimensions): 56 | """ 57 | This function is used to process border patches. 58 | """ 59 | shifted_region_x = region[0] - config.BORDER_SIZE 60 | shifted_region_y = region[1] - config.BORDER_SIZE 61 | clip_region_x = config.BORDER_SIZE 62 | clip_region_y = config.BORDER_SIZE 63 | if region[0] == 0: 64 | shifted_region_x = shifted_region_x + config.BORDER_SIZE 65 | clip_region_x = 0 66 | if region[1] == 0: 67 | shifted_region_y = shifted_region_y + config.BORDER_SIZE 68 | clip_region_y = 0 69 | if region[0] == dimensions[0] - config.CENTER_SIZE: 70 | shifted_region_x = shifted_region_x - config.BORDER_SIZE 71 | clip_region_x = 2 * config.BORDER_SIZE 72 | if region[1] == dimensions[1] - config.CENTER_SIZE: 73 | shifted_region_y = shifted_region_y - config.BORDER_SIZE 74 | clip_region_y = 2 * config.BORDER_SIZE 75 | return [shifted_region_x, shifted_region_y], [clip_region_x, clip_region_y] 76 | 77 | 78 | def image_to_array(input_image): 79 | """ 80 | Loads image into numpy array. 81 | """ 82 | im_array = np.array(input_image.getdata(), dtype=np.uint8) 83 | im_array = im_array.reshape((input_image.size[0], input_image.size[1])) 84 | return im_array 85 | 86 | 87 | def post_processing(image_patch): 88 | """ 89 | Remove small noisy points. 90 | """ 91 | kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (config.FILTER_KERNEL, config.FILTER_KERNEL)) 92 | open_patch = cv2.morphologyEx(image_patch, cv2.MORPH_OPEN, kernel) 93 | close_patch = cv2.morphologyEx(open_patch, cv2.MORPH_CLOSE, kernel) 94 | return close_patch 95 | 96 | 97 | def concat_patches(temp_dir, image_name): 98 | """ 99 | Concatenate the predicted patches into a thumbnail result. 100 | """ 101 | prediction_list = glob.glob(temp_dir + image_name + '*_prediction.png') 102 | patch_list = [] 103 | for prediction_image in prediction_list: 104 | name_parts = prediction_image.split('/')[-1].split('_') 105 | pos_x, pos_y = int(name_parts[-3]), int(name_parts[-2]) 106 | patch_list.append([pos_x, pos_y]) 107 | image_patches = [] 108 | patch_list.sort() 109 | last_x = -1 110 | row_patch = [] 111 | for position in patch_list: 112 | pos_x = position[0] 113 | pos_y = position[1] 114 | image = Image.open(temp_dir + '_'.join([image_name, str(pos_x), str(pos_y), 'prediction']) + '.png') 115 | original_width, original_height = image.size 116 | if original_width < config.THUMBNAIL_RATIO or original_height < config.THUMBNAIL_RATIO: 117 | continue 118 | image = image.resize( 119 | (int(original_width / config.THUMBNAIL_RATIO), 120 | int(original_height / config.THUMBNAIL_RATIO)), Image.NEAREST) 121 | image_patch = image_to_array(image) 122 | if not pos_x == last_x: 123 | last_x = pos_x 124 | if len(row_patch) == 0: 125 | row_patch = image_patch 126 | else: 127 | if not len(image_patches) == 0: 128 | image_patches = np.column_stack((image_patches, row_patch)) 129 | else: 130 | image_patches = row_patch 131 | row_patch = image_patch 132 | else: 133 | row_patch = np.row_stack((row_patch, image_patch)) 134 | prediction = np.column_stack((image_patches, row_patch)) 135 | return prediction 136 | --------------------------------------------------------------------------------