├── Dockerfile ├── LICENSE ├── README.md ├── docker_readme └── readme.md ├── process_results ├── process_results_patches.py └── process_results_superpatches.py ├── src_incep_vgg_tf ├── NNFramework_TF │ ├── config │ │ ├── config_incep-all.ini │ │ ├── config_incep-manual.ini │ │ ├── config_incep-mix.ini │ │ ├── config_incep-semi.ini │ │ ├── config_vgg-all.ini │ │ ├── config_vgg-manual.ini │ │ ├── config_vgg-mix.ini │ │ ├── config_vgg-semi.ini │ │ ├── test_cancer_types │ │ │ ├── config_incep-all_test.ini │ │ │ ├── config_incep-manual_test.ini │ │ │ ├── config_incep-mix_test.ini │ │ │ ├── config_incep-semi_test.ini │ │ │ ├── config_vgg-all_test.ini │ │ │ ├── config_vgg-manual_test.ini │ │ │ ├── config_vgg-mix_test.ini │ │ │ └── config_vgg-semi_test.ini │ │ ├── test_superpatch_new │ │ │ ├── config_incep-all_test.ini │ │ │ ├── config_incep-manual_test.ini │ │ │ ├── config_incep-mix_test.ini │ │ │ ├── config_incep-semi_test.ini │ │ │ ├── config_vgg-all_test.ini │ │ │ ├── config_vgg-manual_test.ini │ │ │ ├── config_vgg-mix_test.ini │ │ │ └── config_vgg-semi_test.ini │ │ ├── test_val_luad_strat │ │ │ ├── config_incep-all_test.ini │ │ │ ├── config_incep-manual_test.ini │ │ │ ├── config_incep-mix_test.ini │ │ │ ├── config_incep-semi_test.ini │ │ │ ├── config_vgg-all_test.ini │ │ │ ├── config_vgg-manual_test.ini │ │ │ ├── config_vgg-mix_test.ini │ │ │ └── config_vgg-semi_test.ini │ │ └── test_wsi_ext │ │ │ ├── config_incep-all_test_ext.ini │ │ │ ├── config_incep-manual_test_ext.ini │ │ │ ├── config_incep-mix_test_ext.ini │ │ │ ├── config_incep-mix_test_ext_binary.ini │ │ │ ├── config_incep-semi_test_ext.ini │ │ │ ├── config_vgg-all_test_ext.ini │ │ │ ├── config_vgg-manual_test_ext.ini │ │ │ ├── config_vgg-mix_test_ext.ini │ │ │ ├── config_vgg-mix_test_ext_binary.ini │ │ │ └── config_vgg-semi_test_ext.ini │ ├── readme.md │ ├── sa_cost_func │ │ └── cross_entropy_cost_func.py │ ├── sa_data_providers │ │ ├── TCGA_batch_data_provider.py │ │ ├── TCGA_batch_data_provider_test_labeled.py │ │ ├── TCGA_data_provider.py │ │ └── TCGA_superpatch_data_provider.py │ ├── sa_net_arch.py │ ├── sa_net_arch_utilities.py │ ├── sa_net_cost_func.py │ ├── sa_net_data_provider.py │ ├── sa_net_loss_func_helper.py │ ├── sa_net_model_params.py │ ├── sa_net_optimizer.py │ ├── sa_net_output_helper.py │ ├── sa_net_runner.py │ ├── sa_net_test.py │ ├── sa_net_train.py │ ├── sa_networks │ │ ├── inception_utils.py │ │ ├── inception_v4.py │ │ ├── inception_v4_classifier_arch.py │ │ ├── vgg.py │ │ └── vgg_16_classifier_arch.py │ ├── sa_runners │ │ ├── tf_classifier_runner.py │ │ └── tf_classifier_runner_external_input.py │ ├── sa_testers │ │ ├── sa_net_test_classifier.py │ │ ├── sa_net_test_classifier_batch.py │ │ ├── sa_net_test_classifier_external_input.py │ │ ├── sa_net_test_classifier_external_input_binary_output.py │ │ └── sa_net_test_superpatch_classifier_batch.py │ └── sa_trainers │ │ └── sa_net_train_classifier.py └── NNFramework_TF_external_call │ ├── external_model.py │ └── external_train.py └── u24_lymphocyte ├── README.md ├── conf └── variables.sh ├── data └── keep.txt ├── heatmap_gen ├── combine_lym_necrosis.sh ├── combine_lym_necrosis_all.sh ├── combine_lym_no_necrosis.sh ├── cp_heatmaps_all.sh ├── gen_all_json.sh ├── gen_json_multipleheat.py ├── gen_json_multipleheat_eagle.py ├── low_res_all.py ├── start.sh ├── start_manual.sh └── upload_heatmaps.sh ├── patch_extraction ├── save_svs_to_tiles.py ├── save_svs_to_tiles.sh └── start.sh ├── prediction ├── NNFramework_TF │ ├── config │ │ ├── config_incep-all.ini │ │ ├── config_incep-manual.ini │ │ ├── config_incep-mix.ini │ │ ├── config_incep-semi.ini │ │ ├── config_vgg-all.ini │ │ ├── config_vgg-manual.ini │ │ ├── config_vgg-mix.ini │ │ ├── config_vgg-semi.ini │ │ ├── test_cancer_types │ │ │ ├── config_incep-all_test.ini │ │ │ ├── config_incep-manual_test.ini │ │ │ ├── config_incep-mix_test.ini │ │ │ ├── config_incep-semi_test.ini │ │ │ ├── config_vgg-all_test.ini │ │ │ ├── config_vgg-manual_test.ini │ │ │ ├── config_vgg-mix_test.ini │ │ │ └── config_vgg-semi_test.ini │ │ ├── test_superpatch_new │ │ │ ├── config_incep-all_test.ini │ │ │ ├── config_incep-manual_test.ini │ │ │ ├── config_incep-mix_test.ini │ │ │ ├── config_incep-semi_test.ini │ │ │ ├── config_vgg-all_test.ini │ │ │ ├── config_vgg-manual_test.ini │ │ │ ├── config_vgg-mix_test.ini │ │ │ └── config_vgg-semi_test.ini │ │ ├── test_val_luad_strat │ │ │ ├── config_incep-all_test.ini │ │ │ ├── config_incep-manual_test.ini │ │ │ ├── config_incep-mix_test.ini │ │ │ ├── config_incep-semi_test.ini │ │ │ ├── config_vgg-all_test.ini │ │ │ ├── config_vgg-manual_test.ini │ │ │ ├── config_vgg-mix_test.ini │ │ │ └── config_vgg-semi_test.ini │ │ └── test_wsi_ext │ │ │ ├── config_incep-all_test_ext.ini │ │ │ ├── config_incep-manual_test_ext.ini │ │ │ ├── config_incep-mix_test_ext.ini │ │ │ ├── config_incep-mix_test_ext_binary.ini │ │ │ ├── config_incep-semi_test_ext.ini │ │ │ ├── config_vgg-all_test_ext.ini │ │ │ ├── config_vgg-manual_test_ext.ini │ │ │ ├── config_vgg-mix_test_ext.ini │ │ │ ├── config_vgg-mix_test_ext_binary.ini │ │ │ └── config_vgg-semi_test_ext.ini │ ├── readme.md │ ├── sa_cost_func │ │ ├── __pycache__ │ │ │ └── cross_entropy_cost_func.cpython-35.pyc │ │ └── cross_entropy_cost_func.py │ ├── sa_data_providers │ │ ├── TCGA_batch_data_provider.py │ │ ├── TCGA_batch_data_provider_test_labeled.py │ │ ├── TCGA_data_provider.py │ │ └── TCGA_superpatch_data_provider.py │ ├── sa_net_arch.py │ ├── sa_net_arch_utilities.py │ ├── sa_net_cost_func.py │ ├── sa_net_data_provider.py │ ├── sa_net_loss_func_helper.py │ ├── sa_net_model_params.py │ ├── sa_net_optimizer.py │ ├── sa_net_output_helper.py │ ├── sa_net_runner.py │ ├── sa_net_test.py │ ├── sa_net_train.py │ ├── sa_networks │ │ ├── __pycache__ │ │ │ ├── inception_utils.cpython-35.pyc │ │ │ ├── inception_v4.cpython-35.pyc │ │ │ ├── inception_v4_classifier_arch.cpython-35.pyc │ │ │ ├── vgg.cpython-35.pyc │ │ │ └── vgg_16_classifier_arch.cpython-35.pyc │ │ ├── inception_utils.py │ │ ├── inception_v4.py │ │ ├── inception_v4_classifier_arch.py │ │ ├── vgg.py │ │ ├── vgg_0.py │ │ ├── vgg_16_classifier_arch.py │ │ └── vgg_16_classifier_arch_0.py │ ├── sa_runners │ │ ├── __pycache__ │ │ │ ├── tf_classifier_runner_external_input.cpython-35.pyc │ │ │ └── tf_classifier_runner_external_input.cpython-36.pyc │ │ ├── tf_classifier_runner.py │ │ └── tf_classifier_runner_external_input.py │ ├── sa_testers │ │ ├── __pycache__ │ │ │ └── sa_net_test_classifier_external_input.cpython-35.pyc │ │ ├── sa_net_test_classifier.py │ │ ├── sa_net_test_classifier_batch.py │ │ ├── sa_net_test_classifier_external_input.py │ │ ├── sa_net_test_classifier_external_input_binary_output.py │ │ └── sa_net_test_superpatch_classifier_batch.py │ └── sa_trainers │ │ └── sa_net_train_classifier.py ├── NNFramework_TF_external_call │ ├── external_model.py │ └── external_train.py ├── color │ ├── color_stats.py │ └── color_stats.sh ├── common │ ├── __init__.py │ ├── batch_norms.py │ ├── ch_inner_prod.py │ └── shape.py ├── lymphocyte │ ├── data_aug.py │ ├── external_model.py │ ├── pred.py │ ├── pred_by_external_model.py │ └── pred_thread_lym.sh ├── models │ └── update_path.sh ├── necrosis │ ├── data_aug_necrosis.py │ ├── pred_necrosis.py │ ├── pred_necrosis_batch.py │ ├── pred_necrosis_wsi.py │ └── pred_thread_nec.sh ├── pytorch_resnet34_model │ ├── load_til_resnet34_external_input.py │ ├── model_til_resnet34_external_input.py │ └── resnet34_arch.py ├── start.sh ├── start_color_only.sh └── start_lym_only.sh └── scripts ├── cleanup_heatmap.sh ├── patch_2_heatmap.sh ├── svs_2_heatmap.sh ├── threshold_probability_heatmaps.py └── threshold_probability_heatmaps.sh /Dockerfile: -------------------------------------------------------------------------------- 1 | # FROM nvcr.io/nvidia/tensorflow:18.11-py3 2 | FROM tensorflow/tensorflow:1.13.1-gpu-py3 3 | MAINTAINER til_classification 4 | 5 | RUN apt-get -y update && \ 6 | apt-get -y install python3-pip openslide-tools wget git && \ 7 | pip install openslide-python scikit-image pymongo && \ 8 | pip install torch==1.0.1 torchvision==0.2.2 9 | 10 | ENV BASE_DIR="/quip_app/til_classification" 11 | ENV PATH="./":$PATH 12 | 13 | COPY . ${BASE_DIR}/. 14 | 15 | ENV MODEL_VER="v2.0" 16 | ENV MODEL_URL="https://stonybrookmedicine.box.com/shared/static/r0p2h2oh53zw52o6fm6qrqjtcwd3p0ga.zip" 17 | 18 | RUN cd ${BASE_DIR}/u24_lymphocyte/prediction/models && \ 19 | wget -v -O models.zip -L $MODEL_URL >/dev/null 2>&1 && \ 20 | unzip -o models.zip && rm -f models.zip && \ 21 | bash ./update_path.sh && \ 22 | chmod 0755 ${BASE_DIR}/u24_lymphocyte/scripts/* 23 | 24 | WORKDIR ${BASE_DIR}/u24_lymphocyte/scripts 25 | 26 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, ShahiraAbousamra 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning-Based Mapping of Tumor Infiltrating Lymphocytes in Whole Slide Images of 23 Types of Cancer 2 | 3 | #### Shahira Abousamra, Rajarsi Gupta, Le Hou, Rebecca Batiste, Tianhao Zhao, Anand Shankar, Arvind Rao, Chao Chen, Dimitris Samaras, Tahsin Kurc, and Joel Saltz, Frontiers in Oncology, 2022. 4 | 5 | This is the code associated with our journal. 6 | A deep learning workflow to classify 50x50 um tiled image patches (100x100 pixels at 20x magnification) as TIL positive or negative based on the presence of TILs in gigapixel whole slide images (WSIs) from the Cancer Genome Atlas (TCGA). This workflow generates TIL maps to study the abundance and spatial distribution of TILs in 23 different types of cancer. 7 | 8 | **WSI Prediction:** 9 | The code for TIL map prediction on WSIs is in the folder: `u24_lymphocyte`. 10 | To run it through docker, use the docker file: `Dockerfile`. 11 | Instructions for running the docker are availabel in the folder: `docker_readme`. 12 | 13 | **Source codes for the tensor flow models:** 14 | are in the folder `src_incep_vgg_tf`. 15 | 16 | **Source codes for processing predictions on test data:** 17 | are in the folder `process_results`. 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_incep-all.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-all 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 299 31 | input_img_width = 299 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 299 77 | input_img_width = 299 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | 91 | [TRAINER] 92 | class_name = ClassifierTrainer 93 | optimizer_type = ADAM 94 | max_epochs = 200 95 | learning_rate = 0.00005 96 | #batch_size = 1 97 | batch_size = 128 98 | #batch_size = 8 99 | # if epoch_size < 0 then use data size 100 | #epoch_size = 1 101 | epoch_size = -1 102 | display_step = 10 103 | save_best_only=False 104 | subepoch_checkpoint_step = 500 105 | 106 | 107 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_incep-manual.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-manual 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 299 31 | input_img_width = 299 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 299 77 | input_img_width = 299 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | 91 | [TRAINER] 92 | class_name = ClassifierTrainer 93 | optimizer_type = ADAM 94 | max_epochs = 200 95 | learning_rate = 0.00005 96 | #batch_size = 1 97 | batch_size = 128 98 | #batch_size = 8 99 | # if epoch_size < 0 then use data size 100 | #epoch_size = 1 101 | epoch_size = -1 102 | display_step = 10 103 | save_best_only=True 104 | subepoch_checkpoint_step = 500 105 | 106 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_incep-mix.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-mix.ckpt 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | 92 | 93 | [TRAINER] 94 | class_name = ClassifierTrainer 95 | optimizer_type = ADAM 96 | max_epochs = 200 97 | learning_rate = 0.00005 98 | #batch_size = 1 99 | batch_size = 128 100 | #batch_size = 8 101 | # if epoch_size < 0 then use data size 102 | #epoch_size = 1 103 | epoch_size = -1 104 | display_step = 10 105 | save_best_only=False 106 | subepoch_checkpoint_step = 500 107 | 108 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_incep-semi.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-semi.ckpt 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | 92 | 93 | [TRAINER] 94 | class_name = ClassifierTrainer 95 | optimizer_type = ADAM 96 | max_epochs = 200 97 | learning_rate = 0.00005 98 | #batch_size = 1 99 | batch_size = 128 100 | #batch_size = 8 101 | # if epoch_size < 0 then use data size 102 | #epoch_size = 1 103 | epoch_size = -1 104 | display_step = 10 105 | save_best_only=False 106 | subepoch_checkpoint_step = 500 107 | 108 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_vgg-all.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = TCGABatchDataProvider 93 | filepath_data = /pylon5/ac3uump/lhou/patches_test 94 | filepath_label = /pylon5/ac3uump/lhou/patches_test 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTester 131 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 132 | #out_ext = csv 133 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_vgg-manual.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-manual 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 224 31 | input_img_width = 224 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 224 77 | input_img_width = 224 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | [TEST_DATA] 91 | provider_class_name = TCGABatchDataProvider 92 | filepath_data = /pylon5/ac3uump/lhou/patches_test 93 | filepath_label = /pylon5/ac3uump/lhou/patches_test 94 | preprocess = true 95 | augment = false 96 | permute = false 97 | input_img_height = 224 98 | input_img_width = 224 99 | split_name=test 100 | pre_resize=true 101 | pre_center=false 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | #file_name_suffix=__clrnorm1 106 | 107 | [TRAINER] 108 | class_name = ClassifierTrainer 109 | optimizer_type = ADAM 110 | max_epochs = 200 111 | learning_rate = 0.00005 112 | #batch_size = 1 113 | batch_size = 128 114 | #batch_size = 8 115 | # if epoch_size < 0 then use data size 116 | #epoch_size = 1 117 | epoch_size = -1 118 | display_step = 10 119 | save_best_only=True 120 | subepoch_checkpoint_step = 500 121 | 122 | 123 | [TESTER] 124 | class_name = ClassifierTester 125 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 126 | #out_ext = csv 127 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_vgg-mix.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-mix 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 25 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 224 31 | input_img_width = 224 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 224 77 | input_img_width = 224 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | [TEST_DATA] 91 | provider_class_name = TCGABatchDataProvider 92 | filepath_data = /pylon5/ac3uump/lhou/patches_test 93 | filepath_label = /pylon5/ac3uump/lhou/patches_test 94 | preprocess = true 95 | augment = false 96 | permute = false 97 | input_img_height = 224 98 | input_img_width = 224 99 | split_name=test 100 | pre_resize=true 101 | pre_center=false 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | #file_name_suffix=__clrnorm1 106 | 107 | [TRAINER] 108 | class_name = ClassifierTrainer 109 | optimizer_type = ADAM 110 | max_epochs = 200 111 | learning_rate = 0.00005 112 | #batch_size = 1 113 | batch_size = 128 114 | #batch_size = 8 115 | # if epoch_size < 0 then use data size 116 | #epoch_size = 1 117 | epoch_size = -1 118 | display_step = 10 119 | save_best_only=False 120 | subepoch_checkpoint_step = 500 121 | 122 | 123 | [TESTER] 124 | class_name = ClassifierTester 125 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 126 | #out_ext = csv 127 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/config_vgg-semi.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-semi 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 224 31 | input_img_width = 224 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 224 77 | input_img_width = 224 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | [TEST_DATA] 91 | provider_class_name = TCGABatchDataProvider 92 | filepath_data = /pylon5/ac3uump/lhou/patches_test 93 | filepath_label = /pylon5/ac3uump/lhou/patches_test 94 | preprocess = true 95 | augment = false 96 | permute = false 97 | input_img_height = 224 98 | input_img_width = 224 99 | split_name=test 100 | pre_resize=true 101 | pre_center=false 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | #file_name_suffix=__clrnorm1 106 | 107 | [TRAINER] 108 | class_name = ClassifierTrainer 109 | optimizer_type = ADAM 110 | max_epochs = 200 111 | learning_rate = 0.00005 112 | #batch_size = 1 113 | batch_size = 128 114 | #batch_size = 8 115 | # if epoch_size < 0 then use data size 116 | #epoch_size = 1 117 | epoch_size = -1 118 | display_step = 10 119 | save_best_only=False 120 | subepoch_checkpoint_step = 500 121 | 122 | 123 | [TESTER] 124 | class_name = ClassifierTester 125 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 126 | #out_ext = csv 127 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_superpatch_new/config_vgg-all_test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 224 33 | input_img_width = 224 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 224 79 | input_img_width = 224 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = TCGASuperpatchBatchDataProvider 94 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/super-patches-evaluation-merged/super-patches 95 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/super-patches-evaluation-merged/super-patches-label_m.csv 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 224 101 | input_img_width = 224 102 | split_name=val 103 | #pre_crop_center=true 104 | pre_crop_center=false 105 | pre_crop_height=100 106 | pre_crop_width=100 107 | pre_resize=true 108 | pre_center=false 109 | post_crop_center=true 110 | post_crop_height=100 111 | post_crop_width=100 112 | post_resize=true 113 | #file_name_suffix=__clrnorm1 114 | 115 | [TRAINER] 116 | class_name = ClassifierTrainer 117 | optimizer_type = ADAM 118 | max_epochs = 200 119 | learning_rate = 0.00005 120 | #batch_size = 1 121 | batch_size = 128 122 | #batch_size = 8 123 | # if epoch_size < 0 then use data size 124 | #epoch_size = 1 125 | epoch_size = -1 126 | display_step = 10 127 | save_best_only=False 128 | subepoch_checkpoint_step = 500 129 | 130 | 131 | [TESTER] 132 | class_name = ClassifierTesterSuperpatchBatch 133 | out_dir = /pylon5/ac3uump/shahira/tcga/test_out/superpatch_new 134 | batch_size = 1 135 | out_ext = csv 136 | threshold = 0.47 137 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_val_luad_strat/config_vgg-all_test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 224 33 | input_img_width = 224 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 224 79 | input_img_width = 224 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = TCGABatchDataProviderTestLabelled 94 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected/luad_pos_and_neg 95 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected/luad_pos_and_neg 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 224 101 | input_img_width = 224 102 | split_name=val 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | pre_resize=true 107 | pre_center=false 108 | post_crop_center=true 109 | post_crop_height=100 110 | post_crop_width=100 111 | post_resize=true 112 | #file_name_suffix=__clrnorm1 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterSuperpatchBatch 132 | out_dir = /pylon5/ac3uump/shahira/tcga/test_out/val_luad_strat 133 | #batch_size = 64 134 | batch_size = 1 135 | out_ext = csv 136 | threshold = 0.47 137 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_val_luad_strat/config_vgg-semi_test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-semi 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-semi.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = TCGABatchDataProviderTestLabelled 93 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected/luad_pos_and_neg 94 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected/luad_pos_and_neg 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterSuperpatchBatch 132 | out_dir = /pylon5/ac3uump/shahira/tcga/test_out/val_luad_strat 133 | #batch_size = 64 134 | batch_size = 1 135 | out_ext = csv 136 | threshold = 0.33 137 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_incep-all_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/incep-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = InceptionV4Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 299 16 | input_img_width = 299 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 299 100 | input_img_width = 299 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTesterExternalInput 131 | #class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.13 135 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_incep-manual_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-manual 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/incep-manual.ckpt 9 | 10 | [NETWORK] 11 | class_name = InceptionV4Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 299 16 | input_img_width = 299 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/luad_additional+ 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/luad_additional+ 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 299 100 | input_img_width = 299 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=True 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTesterExternalInput 131 | #class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.42 135 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_incep-semi_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-semi 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/incep-semi.ckpt 9 | 10 | [NETWORK] 11 | class_name = InceptionV4Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 299 16 | input_img_width = 299 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 299 33 | input_img_width = 299 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 299 79 | input_img_width = 299 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = None 94 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 299 101 | input_img_width = 299 102 | split_name=val 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | pre_resize=true 107 | pre_center=false 108 | post_crop_center=true 109 | post_crop_height=100 110 | post_crop_width=100 111 | post_resize=true 112 | #file_name_suffix=__clrnorm1 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterExternalInput 132 | #class_name = ClassifierTesterExternalInputBinaryOutput 133 | out_dir = None 134 | out_ext = None 135 | threshold = 0.48 136 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_vgg-all_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 224 33 | input_img_width = 224 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 224 79 | input_img_width = 224 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = None 94 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 224 101 | input_img_width = 224 102 | split_name=val 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | pre_resize=true 107 | pre_center=false 108 | post_crop_center=true 109 | post_crop_height=100 110 | post_crop_width=100 111 | post_resize=true 112 | #file_name_suffix=__clrnorm1 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterExternalInput 132 | #class_name = ClassifierTesterExternalInputBinaryOutput 133 | out_dir = None 134 | out_ext = None 135 | threshold = 0.47 136 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_vgg-manual_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-manual 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-manual.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/luad_additional+ 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/luad_additional+ 95 | preprocess = true 96 | augment = false 97 | permute = false 98 | input_img_height = 224 99 | input_img_width = 224 100 | split_name=test 101 | pre_resize=true 102 | pre_center=false 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | #file_name_suffix=__clrnorm1 107 | 108 | [TRAINER] 109 | class_name = ClassifierTrainer 110 | optimizer_type = ADAM 111 | max_epochs = 200 112 | learning_rate = 0.00005 113 | #batch_size = 1 114 | batch_size = 128 115 | #batch_size = 8 116 | # if epoch_size < 0 then use data size 117 | #epoch_size = 1 118 | epoch_size = -1 119 | display_step = 10 120 | save_best_only=True 121 | subepoch_checkpoint_step = 500 122 | 123 | 124 | [TESTER] 125 | class_name = ClassifierTesterExternalInput 126 | out_dir = None 127 | out_ext = None 128 | threshold = 0.43 129 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_vgg-mix_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-mix 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-mix.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTesterExternalInput 131 | #class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.42 135 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_vgg-mix_test_ext_binary.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-mix 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-mix.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | #class_name = ClassifierTesterExternalInput 131 | class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.42 135 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/config/test_wsi_ext/config_vgg-semi_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-semi 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-semi.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterExternalInput 132 | #class_name = ClassifierTesterExternalInputBinaryOutput 133 | out_dir = None 134 | out_ext = None 135 | threshold = 0.33 136 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_cost_func/cross_entropy_cost_func.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import sys; 5 | import os; 6 | import tensorflow as tf; 7 | 8 | from ..sa_net_cost_func import AbstractCostFunc; 9 | from ..sa_net_loss_func_helper import CNNLossFuncHelper; 10 | 11 | class CrossEntropyCost(AbstractCostFunc): 12 | def __init__(self, n_classes, kwargs): 13 | # predefined list of arguments 14 | args = {'class_weights':None}; 15 | 16 | args.update(kwargs); 17 | class_weights = args['class_weights']; 18 | if(class_weights is not None): 19 | class_weights = [float(x) for x in class_weights.split(',')] 20 | self.n_classes = n_classes; 21 | 22 | print('class_weights = ', class_weights); 23 | if(class_weights == None): 24 | self.class_weights = tf.Variable(tf.ones([self.n_classes])); 25 | else: 26 | self.class_weights = tf.Variable(class_weights); 27 | 28 | 29 | 30 | def calc_cost(self, logits, labels): 31 | return CNNLossFuncHelper.cost_cross_entropy(logits, labels, self.class_weights, self.n_classes); 32 | 33 | 34 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_cost_func.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import sys; 5 | import os; 6 | import tensorflow as tf; 7 | 8 | #from sa_net_loss_func import CNNLossFuncHelper; 9 | 10 | class AbstractCostFunc: 11 | def __init__(self, n_classes, kwargs): 12 | self.n_classes = n_classes; 13 | self.class_weights = tf.Variable(tf.ones([n_classes])); 14 | 15 | self.cost = self.calc_cost(self.logits, self.labels, kwargs); 16 | 17 | def calc_cost(self, logits, labels): 18 | pass; 19 | 20 | 21 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_data_provider.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | 5 | class AbstractDataProvider: 6 | def __init__(self, is_test, filepath_data, filepath_label, n_channels, n_classes, do_preprocess, do_augment, data_var_name=None, label_var_name=None, permute=False, repeat=True): 7 | self.is_test = is_test; 8 | self.filepath_data = ''; 9 | self.filepath_label = ''; 10 | self.n_channels = n_channels; 11 | self.n_classes = n_classes; 12 | self.do_preprocess = do_preprocess; 13 | self.do_augment = do_augment; 14 | self.data_var_name = 'c488gb'; 15 | self.label_var_name = label_var_name; 16 | self.do_permute = permute; 17 | self.do_repeat = repeat; # go over the data multiple times like with training or only once like with test 18 | 19 | self.is_loaded = False; 20 | 21 | def load_data(self): 22 | self.data = None; 23 | self.label = None; 24 | self.last_fetched_indx = -1; 25 | self.permutation = None; 26 | self.data_count = 0; 27 | 28 | def reset(self, repermute=None): 29 | self.last_fetched_indx = -1; 30 | 31 | def get_next_one(self): 32 | pass; 33 | 34 | def get_next_n(self, n:int): 35 | pass; 36 | 37 | def preprocess(self, data_point, label): 38 | return data_point, label; 39 | 40 | def augment(self, data_point, label): 41 | return data_point, label; 42 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_loss_func_helper.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import tensorflow as tf; 5 | from enum import Enum; 6 | import numpy as np; 7 | 8 | class CostFuncTypes(Enum): 9 | CROSS_ENTROPY = 1 10 | 11 | 12 | class CNNLossFuncHelper: 13 | 14 | @staticmethod 15 | def cost_cross_entropy(logits, labels, class_weights, n_classes): 16 | flat_logits = tf.reshape(logits, [-1, n_classes]); 17 | flat_labels = tf.reshape(labels, [-1, n_classes]); 18 | print(class_weights) 19 | print(flat_labels) 20 | print(flat_logits) 21 | return tf.abs(tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits( \ 22 | logits=flat_logits \ 23 | , targets=flat_labels \ 24 | , pos_weight=class_weights \ 25 | ))); 26 | 27 | @staticmethod 28 | def cost_mse(logits, labels, class_weights, n_classes): 29 | flat_logits = tf.reshape(logits, [-1, n_classes]); 30 | flat_labels = tf.reshape(labels, [-1, n_classes]); 31 | print(class_weights) 32 | print(flat_labels) 33 | print(flat_logits) 34 | return tf.losses.mean_squared_error(\ 35 | labels=flat_labels \ 36 | , predictions=flat_logits \ 37 | , weights=class_weights \ 38 | ); 39 | 40 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_model_params.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | 5 | class Modes: 6 | TRAIN = 0; 7 | TEST = 1; 8 | 9 | class ModelParams: 10 | def __init__(self): 11 | self.params = {}; 12 | self.params['arch_name'] = 'basic'; 13 | self.params['mode'] = Modes.TRAIN; 14 | self.params['n_classes'] = 2; 15 | self.params['n_channels'] = 1; 16 | self.params['train_out_path'] = ''; 17 | self.params['test_out_path'] = ''; 18 | self.params['validate_out_path'] = ''; 19 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_optimizer.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import tensorflow as tf; 5 | from enum import Enum; 6 | 7 | 8 | class OptimizerTypes(Enum): 9 | ADAM = 1 10 | SGD = 2 11 | 12 | 13 | class CNNOptimizer: 14 | 15 | @staticmethod 16 | def adam_optimizer(learning_rate, cost, global_step): 17 | return tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-08,) \ 18 | .minimize(cost, global_step=global_step); 19 | 20 | @staticmethod 21 | def sgd_optimizer(learning_rate, cost, global_step): 22 | return tf.train.GradientDescentOptimizer(learning_rate) \ 23 | .minimize(cost, global_step=global_step); 24 | 25 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_output_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/src_incep_vgg_tf/NNFramework_TF/sa_net_output_helper.py -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_net_test.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import tensorflow as tf; 5 | 6 | from sa_net_arch import AbstractCNNArch; 7 | from sa_net_arch_utilities import CNNArchUtils; 8 | from sa_net_optimizer import OptimizerTypes, CNNOptimizer; 9 | from sa_net_data_provider import AbstractDataProvider; 10 | from skimage import io; 11 | import os; 12 | 13 | class CNNTest: 14 | def __init__(self, cnn_arch:AbstractCNNArch, test_data_provider:AbstractDataProvider, session_config, output_dir, output_ext, kwargs): 15 | # predefined list of arguments 16 | 17 | self.cnn_arch = cnn_arch; 18 | #self.cost_func = cost_func; 19 | self.test_data_provider = test_data_provider; 20 | if(session_config == None): 21 | self.session_config = tf.ConfigProto(); 22 | else: 23 | self.session_config = session_config; 24 | self.output_dir = output_dir; 25 | self.output_ext = output_ext; 26 | 27 | self.init = tf.global_variables_initializer(); 28 | 29 | def test(self, do_init, do_restore, do_load_data): 30 | with tf.Session(config=self.session_config) as sess: 31 | #with tf.Session() as sess: 32 | with sess.as_default(): 33 | if(do_init): 34 | sess.run(tf.global_variables_initializer()); 35 | #sess.run(self.init); 36 | if(do_restore): 37 | self.cnn_arch.restore_model(sess); 38 | if(do_load_data): 39 | self.test_data_provider.load_data(); 40 | out_basefilename = self.test_data_provider.data_tag; 41 | 42 | 43 | batch_x, batch_label = self.test_data_provider.get_next_one(); 44 | indx = 0; 45 | while(batch_x is not None): 46 | batch_x, batch_label = self.test_data_provider.get_next_one(); 47 | batch_y, batch_y_softmax, batch_y_class = sess.run([self.cnn_arch.logits, self.cnn_arch.prediction_softmax, self.cnn_arch.prediction_class] \ 48 | , feed_dict={self.cnn_arch.input_x: batch_x \ 49 | , self.cnn_arch.isTest: True \ 50 | }); 51 | io.imsave(os.path.join(self.output_dir, out_basefilename[indx] + self.output_ext), batch_y); 52 | indx = indx + 1; 53 | 54 | 55 | 56 | 57 | 58 | print("Test Finished!") 59 | 60 | 61 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_networks/inception_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Contains common code shared by all inception models. 16 | 17 | Usage of arg scope: 18 | with slim.arg_scope(inception_arg_scope()): 19 | logits, end_points = inception.inception_v3(images, num_classes, 20 | is_training=is_training) 21 | 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | import tensorflow as tf 28 | 29 | slim = tf.contrib.slim 30 | 31 | 32 | def inception_arg_scope(weight_decay=0.00004, 33 | use_batch_norm=True, 34 | batch_norm_decay=0.9997, 35 | batch_norm_epsilon=0.001, 36 | activation_fn=tf.nn.relu, 37 | batch_norm_updates_collections=tf.GraphKeys.UPDATE_OPS, 38 | batch_norm_scale=False): 39 | """Defines the default arg scope for inception models. 40 | 41 | Args: 42 | weight_decay: The weight decay to use for regularizing the model. 43 | use_batch_norm: "If `True`, batch_norm is applied after each convolution. 44 | batch_norm_decay: Decay for batch norm moving average. 45 | batch_norm_epsilon: Small float added to variance to avoid dividing by zero 46 | in batch norm. 47 | activation_fn: Activation function for conv2d. 48 | batch_norm_updates_collections: Collection for the update ops for 49 | batch norm. 50 | batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the 51 | activations in the batch normalization layer. 52 | 53 | Returns: 54 | An `arg_scope` to use for the inception models. 55 | """ 56 | batch_norm_params = { 57 | # Decay for the moving averages. 58 | 'decay': batch_norm_decay, 59 | # epsilon to prevent 0s in variance. 60 | 'epsilon': batch_norm_epsilon, 61 | # collection containing update_ops. 62 | 'updates_collections': batch_norm_updates_collections, 63 | # use fused batch norm if possible. 64 | 'fused': None, 65 | 'scale': batch_norm_scale, 66 | } 67 | if use_batch_norm: 68 | normalizer_fn = slim.batch_norm 69 | normalizer_params = batch_norm_params 70 | else: 71 | normalizer_fn = None 72 | normalizer_params = {} 73 | # Set weight_decay for weights in Conv and FC layers. 74 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 75 | weights_regularizer=slim.l2_regularizer(weight_decay)): 76 | with slim.arg_scope( 77 | [slim.conv2d], 78 | weights_initializer=slim.variance_scaling_initializer(), 79 | activation_fn=activation_fn, 80 | normalizer_fn=normalizer_fn, 81 | normalizer_params=normalizer_params) as sc: 82 | return sc 83 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_testers/sa_net_test_classifier.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf; 2 | 3 | from ..sa_net_arch import AbstractCNNArch; 4 | from ..sa_net_arch_utilities import CNNArchUtils; 5 | from ..sa_net_optimizer import OptimizerTypes, CNNOptimizer; 6 | from ..sa_net_data_provider import AbstractDataProvider; 7 | import os; 8 | import numpy as np; 9 | 10 | class ClassifierTester: 11 | def __init__(self, cnn_arch:AbstractCNNArch, test_data_provider:AbstractDataProvider, session_config, output_dir, output_ext, kwargs): 12 | # predefined list of arguments 13 | 14 | self.cnn_arch = cnn_arch; 15 | #self.cost_func = cost_func; 16 | self.test_data_provider = test_data_provider; 17 | if(session_config == None): 18 | self.session_config = tf.ConfigProto(); 19 | else: 20 | self.session_config = session_config; 21 | self.output_dir = output_dir; 22 | self.output_ext = output_ext; 23 | 24 | self.init = tf.global_variables_initializer(); 25 | 26 | def test(self, do_init, do_restore, do_load_data): 27 | with tf.Session(config=self.session_config) as sess: 28 | #with tf.Session() as sess: 29 | with sess.as_default(): 30 | if(do_init): 31 | sess.run(tf.global_variables_initializer()); 32 | #sess.run(self.init); 33 | if(do_restore): 34 | self.cnn_arch.restore_model(sess); 35 | if(do_load_data): 36 | self.test_data_provider.load_data(); 37 | out_basefilename = self.test_data_provider.data_tag; 38 | 39 | 40 | batch_x, batch_label = self.test_data_provider.get_next_one(); 41 | indx = 0; 42 | total_correct_pred = 0; 43 | while(batch_x is not None): 44 | print(out_basefilename[indx]) 45 | batch_x = batch_x.reshape(1,batch_x.shape[0],batch_x.shape[1],batch_x.shape[2]) 46 | batch_y, correct_pred = sess.run([self.cnn_arch.logits, self.cnn_arch.correct_pred] \ 47 | , feed_dict={self.cnn_arch.input_x: batch_x \ 48 | , self.cnn_arch.labels: batch_label \ 49 | , self.cnn_arch.isTest: True \ 50 | }); 51 | total_correct_pred += correct_pred; 52 | 53 | batch_x = self.test_data_provider.get_next_one(); 54 | indx = indx + 1; 55 | 56 | print(indx); 57 | print(total_correct_pred); 58 | 59 | 60 | 61 | print("Test Finished!") 62 | 63 | 64 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_testers/sa_net_test_classifier_external_input.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf; 2 | 3 | from ..sa_net_arch import AbstractCNNArch; 4 | from ..sa_net_arch_utilities import CNNArchUtils; 5 | from ..sa_net_optimizer import OptimizerTypes, CNNOptimizer; 6 | from ..sa_net_data_provider import AbstractDataProvider; 7 | import os; 8 | import numpy as np; 9 | 10 | class ClassifierTesterExternalInput: 11 | def __init__(self, cnn_arch:AbstractCNNArch, session_config, output_dir, output_ext, kwargs): 12 | # predefined list of arguments 13 | 14 | self.cnn_arch = cnn_arch; 15 | #self.cost_func = cost_func; 16 | #self.test_data_provider = test_data_provider; 17 | if(session_config == None): 18 | self.session_config = tf.ConfigProto(); 19 | self.session_config.gpu_options.allow_growth = True 20 | else: 21 | self.session_config = session_config; 22 | self.output_dir = output_dir; 23 | self.output_ext = output_ext; 24 | 25 | self.init = tf.global_variables_initializer(); 26 | 27 | def init_model(self, do_init, do_restore): 28 | self.sess = tf.Session(config=self.session_config); 29 | with self.sess.as_default(): 30 | if(do_init): 31 | self.sess.run(tf.global_variables_initializer()); 32 | #sess.run(self.init); 33 | if(do_restore): 34 | self.cnn_arch.restore_model(self.sess); 35 | 36 | 37 | def predict(self, inputs): 38 | with self.sess.as_default(): 39 | batch_x = inputs; 40 | if (batch_x is None): 41 | return None; 42 | 43 | batch_x = self.preprocess_input(inputs); 44 | 45 | batch_y = self.sess.run([self.cnn_arch.logits] \ 46 | , feed_dict={self.cnn_arch.input_x: batch_x \ 47 | , self.cnn_arch.isTest: True \ 48 | }); 49 | #print(np.array(batch_y).shape) 50 | #print(np.array(batch_y)) 51 | #print(np.array(batch_y)[...,-1]) 52 | batch_y_sig = self.sigmoid(np.array(batch_y)[...,-1]).reshape((-1,1)); 53 | return batch_y_sig; 54 | 55 | def sigmoid(self, x): 56 | return (1 / (1 + np.exp(-x))) 57 | 58 | def restart_model(self): 59 | self.sess.close(); 60 | #tf.reset_default_graph(); 61 | self.init_model(True, True); 62 | 63 | def preprocess_input(self, inputs): 64 | # normalize (mean 0, std=2) 65 | np.clip(inputs, 0, 255, inputs); 66 | inputs /= 255; 67 | inputs -= 0.5; 68 | inputs *= 2; 69 | inputs = tf.image.resize_images(inputs, (self.cnn_arch.input_img_height, self.cnn_arch.input_img_width)); 70 | inputs = inputs.eval() 71 | return inputs; -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF/sa_testers/sa_net_test_classifier_external_input_binary_output.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf; 2 | 3 | from ..sa_net_arch import AbstractCNNArch; 4 | from ..sa_net_arch_utilities import CNNArchUtils; 5 | from ..sa_net_optimizer import OptimizerTypes, CNNOptimizer; 6 | from ..sa_net_data_provider import AbstractDataProvider; 7 | import os; 8 | import numpy as np; 9 | 10 | class ClassifierTesterExternalInputBinaryOutput: 11 | def __init__(self, cnn_arch:AbstractCNNArch, session_config, output_dir, output_ext, kwargs): 12 | # predefined list of arguments 13 | args = {'threshold':0.5}; 14 | args.update(kwargs); 15 | 16 | self.cnn_arch = cnn_arch; 17 | #self.cost_func = cost_func; 18 | #self.test_data_provider = test_data_provider; 19 | if(session_config == None): 20 | self.session_config = tf.ConfigProto(); 21 | self.session_config.gpu_options.allow_growth = True 22 | else: 23 | self.session_config = session_config; 24 | self.output_dir = output_dir; 25 | self.output_ext = output_ext; 26 | self.threshold = float(args['threshold']); 27 | 28 | self.init = tf.global_variables_initializer(); 29 | 30 | def init_model(self, do_init, do_restore): 31 | self.sess = tf.Session(config=self.session_config); 32 | with self.sess.as_default(): 33 | if(do_init): 34 | self.sess.run(tf.global_variables_initializer()); 35 | #sess.run(self.init); 36 | if(do_restore): 37 | self.cnn_arch.restore_model(self.sess); 38 | 39 | 40 | def predict(self, inputs): 41 | with self.sess.as_default(): 42 | batch_x = inputs; 43 | if (batch_x is None): 44 | return None; 45 | 46 | batch_x = self.preprocess_input(inputs); 47 | 48 | batch_y = self.sess.run([self.cnn_arch.logits] \ 49 | , feed_dict={self.cnn_arch.input_x: batch_x \ 50 | , self.cnn_arch.isTest: True \ 51 | }); 52 | batch_y_sig = self.sigmoid(np.array(batch_y)[...,-1]); 53 | batch_y_binary = np.array(batch_y_sig > self.threshold).astype(np.float).reshape((-1,1)); 54 | 55 | return batch_y_binary; 56 | 57 | 58 | def sigmoid(self, x): 59 | return (1 / (1 + np.exp(-x))) 60 | 61 | def restart_model(self): 62 | self.sess.close(); 63 | #tf.reset_default_graph(); 64 | self.init_model(True, True); 65 | 66 | def preprocess_input(self, inputs): 67 | # normalize (mean 0, std=2) 68 | np.clip(inputs, 0, 255, inputs); 69 | inputs /= 255; 70 | inputs -= 0.5; 71 | inputs *= 2; 72 | inputs = tf.image.resize_images(inputs, (self.cnn_arch.input_img_height, self.cnn_arch.input_img_width)); 73 | inputs = inputs.eval() 74 | return inputs; 75 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF_external_call/external_model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | sys.path.append(".."); 4 | sys.path.append("."); 5 | sys.path.append("../.."); 6 | sys.path.append("..."); 7 | from NNFramework_TF.sa_runners.tf_classifier_runner_external_input import load_model; 8 | 9 | 10 | def load_external_model(model_path): 11 | # Load your model here 12 | model = load_model(model_path) 13 | return model 14 | 15 | def pred_by_external_model(model, inputs): 16 | # Get prediction here 17 | # model: 18 | # A model loaded by load_external_model 19 | # inputs : 20 | # float32 numpy array with shape N x 3 x 100 x 100 21 | # Range of value: 0.0 ~ 255.0 22 | # You may need to rearrange inputs: 23 | # inputs = inputs.transpose((0, 2, 3, 1)) 24 | # Expected output: 25 | # float32 numpy array with shape N x 1 26 | # Each entry is a probability ranges 0.0 ~ 1.0 27 | inputs = inputs.transpose((0, 2, 3, 1)) 28 | pred = model.predict(inputs)[0] 29 | #exp_pred = np.exp(pred - np.max(pred, axis=-1, keepdims=True) + 1) 30 | #return exp_pred[..., -1:] / np.sum(exp_pred, axis=-1, keepdims=True) 31 | 32 | #sig_pred = (1 / (1 + np.exp(-pred))) 33 | #sig_pred = sig_pred[..., -1:] ; 34 | #return sig_pred 35 | 36 | return pred; 37 | 38 | if __name__ == "__main__": 39 | 40 | ## LUAD ############################################################### 41 | ## LUAD - semiauto - InceptionV4 42 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_semiauto.ini"; 43 | ## LUAD - semiauto - VGG16 44 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_semiauto.ini"; 45 | ## LUAD - manual - InceptionV4 46 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_manual.ini"; 47 | ## LUAD - manual - VGG16 48 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_manual.ini"; 49 | 50 | ## SKCM ############################################################### 51 | ## SKCM - semiauto - InceptionV4 52 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_semiauto.ini"; 53 | ## SKCM - semiauto - VGG16 54 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_semiauto.ini"; 55 | # SKCM - manual - InceptionV4 56 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_manual.ini"; 57 | ## SKCM - manual - VGG16 58 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_manual.ini"; 59 | 60 | config_filepath = "/home/shahira/quip_classification/NNFramework_TF/config/test_wsi_ext/config_vgg-semi_test_ext.ini"; 61 | 62 | 63 | print(config_filepath) 64 | model = load_external_model(config_filepath); 65 | print('load_external_model called') 66 | inputs = np.random.rand(10, 3, 100, 100); 67 | print('inputs created') 68 | pred = pred_by_external_model(model, inputs) 69 | print('after predict') 70 | print(pred); 71 | print(pred.shape); 72 | -------------------------------------------------------------------------------- /src_incep_vgg_tf/NNFramework_TF_external_call/external_train.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | sys.path.append(".."); 4 | sys.path.append("."); 5 | sys.path.append("../.."); 6 | sys.path.append("..."); 7 | from NNFramework_TF.sa_runners import tf_classifier_runner; 8 | 9 | 10 | 11 | if __name__ == "__main__": 12 | 13 | ## LUAD ############################################################### 14 | ## LUAD - semiauto - InceptionV4 15 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_semiauto.ini"; 16 | ## LUAD - semiauto - VGG16 17 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_semiauto.ini"; 18 | ## LUAD - manual - InceptionV4 19 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_manual.ini"; 20 | ## LUAD - manual - VGG16 21 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_manual.ini"; 22 | 23 | ## SKCM ############################################################### 24 | ## SKCM - semiauto - InceptionV4 25 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_semiauto.ini"; 26 | ## SKCM - semiauto - VGG16 27 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_semiauto.ini"; 28 | ## SKCM - manual - InceptionV4 29 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_manual.ini"; 30 | ## SKCM - manual - VGG16 31 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_manual.ini"; 32 | 33 | #config_filepath = "/home/shahira/NNFramework_TF/config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_HEaug_luad_semiauto.ini"; 34 | 35 | tf_classifier_runner.main(sys.argv[1:]) -------------------------------------------------------------------------------- /u24_lymphocyte/README.md: -------------------------------------------------------------------------------- 1 | # u24_lymphocyte 2 | 3 | This software implements the pipeline for the lymphocyte classification project. 4 | 5 | List of folders and functionalities are below: 6 | 7 | scripts/: contains scripts that connect several sub-functionalities together for complete functionalities such as patch extraction and prediction. 8 | 9 | conf/: contains configuration. 10 | 11 | data/: a place where should contain all logs, input/output images, and large files. 12 | 13 | patch_extraction/: extracts all patches from svs images. Mainly used in the test phase. 14 | 15 | prediction/: CNN prediction code. 16 | 17 | heatmap_gen/: generate json files that represents heatmaps for quip, using the lymphocyte and necrosis CNNs' raw output txt files. 18 | 19 | -------------------------------------------------------------------------------- /u24_lymphocyte/data/keep.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/combine_lym_necrosis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # merges two heatmaps 3 | # usage: 4 | # bash merge_lym_necrosis.sh ${FILE} 5 | # ${FILE}: something like prediction-TCGA-44-2655-01Z-00-DX1.ee255271-780c-461c-ab23-5cd3504b5e4a 6 | 7 | FN=$1 8 | LYM_FOLDER=./patch-level-lym/ 9 | NEC_FOLDER=./patch-level-nec/ 10 | OUT_FOLDER=./patch-level-merged/ 11 | 12 | awk 'NR==FNR{ 13 | x=$1; 14 | y=$2; 15 | h[x" "y]=$3; 16 | if(x_before!=x){ 17 | lower[x]=x_before; 18 | higher[x_before]=x; 19 | } 20 | if(y_before!=y){ 21 | lower[y]=y_before; 22 | higher[y_before]=y; 23 | } 24 | x_before=x; 25 | y_before=y; 26 | } 27 | 28 | NR!=FNR{ 29 | x=-1; 30 | y=-1; 31 | for (i=$1-2;i<=$1+2;++i) { 32 | for (j=$2-2;j<=$2+2;++j) { 33 | if (i" "j in h) { 34 | x = i; 35 | y = j; 36 | } 37 | } 38 | } 39 | lym = $3; 40 | 41 | if (x" "y in h) { 42 | necrosis = h[x" "y]; 43 | 44 | if (h[lower[x]" "y] > necrosis) 45 | necrosis = h[lower[x]" "y]; 46 | if (h[higher[x]" "y] > necrosis) 47 | necrosis = h[higher[x]" "y]; 48 | if (h[x" "lower[y]] > necrosis) 49 | necrosis = h[x" "lower[y]]; 50 | if (h[x" "higher[y]] > necrosis) 51 | necrosis = h[x" "higher[y]]; 52 | 53 | if (necrosis > 0.4) { 54 | combo = 0.01; 55 | } else { 56 | combo = lym; 57 | } 58 | } else { 59 | combo = lym; 60 | necrosis = 0; 61 | } 62 | 63 | if (length(necrosis) == 0) { 64 | necrosis = 0; 65 | } 66 | print $1, $2, lym, necrosis; 67 | }' ./${NEC_FOLDER}/${FN} ./${LYM_FOLDER}/${1} > ${OUT_FOLDER}/${FN} 68 | 69 | exit 0 70 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/combine_lym_necrosis_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | LYM_FOLDER=./patch-level-lym/ 6 | NEC_FOLDER=./patch-level-nec/ 7 | 8 | for files in ${LYM_FOLDER}/*; do 9 | if [ ! -f ${files} ]; then continue; fi 10 | 11 | fn=`echo ${files} | awk -F'/' '{print $NF}'` 12 | if [ -f ${NEC_FOLDER}/${fn} ]; then 13 | bash combine_lym_necrosis.sh ${fn} &> ${LOG_OUTPUT_FOLDER}/log.combine_lym_necrosis.txt 14 | else 15 | bash combine_lym_no_necrosis.sh ${fn} &> ${LOG_OUTPUT_FOLDER}/log.combine_lym_no_necrosis.txt 16 | fi 17 | done 18 | 19 | python low_res_all.py 20 | 21 | exit 0 22 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/combine_lym_no_necrosis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FN=$1 4 | LYM_FOLDER=./patch-level-lym/ 5 | OUT_FOLDER=./patch-level-merged/ 6 | 7 | awk '{ 8 | print $1, $2, $3, 0.0; 9 | }' ./${LYM_FOLDER}/${FN} > ${OUT_FOLDER}/${FN} 10 | 11 | exit 0 12 | 13 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/cp_heatmaps_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FOLDER=${1} 4 | 5 | PRED_VERSION=patch-level-lym.txt 6 | DIS_FOLDER=./patch-level-lym/ 7 | for files in ${FOLDER}/*/${PRED_VERSION}; do 8 | dis=`echo ${files} | awk -F'/' '{print "prediction-"substr($(NF-1),1,length($(NF-1))-4);}'` 9 | cp ${files} ${DIS_FOLDER}/${dis} 10 | done 11 | 12 | PRED_VERSION=patch-level-necrosis.txt 13 | DIS_FOLDER=./patch-level-nec/ 14 | for files in ${FOLDER}/*/${PRED_VERSION}; do 15 | dis=`echo ${files} | awk -F'/' '{print "prediction-"substr($(NF-1),1,length($(NF-1))-4);}'` 16 | cp ${files} ${DIS_FOLDER}/${dis} 17 | done 18 | 19 | PRED_VERSION=patch-level-color.txt 20 | DIS_FOLDER=./patch-level-color/ 21 | for files in ${FOLDER}/*/${PRED_VERSION}; do 22 | dis=`echo ${files} | awk -F'/' '{print "color-"substr($(NF-1),1,length($(NF-1))-4);}'` 23 | cp ${files} ${DIS_FOLDER}/${dis} 24 | done 25 | 26 | exit 0 27 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/gen_all_json.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | for files in ${HEATMAP_TXT_OUTPUT_FOLDER}/prediction-*; do 6 | if [[ "$files" == *.low_res* ]]; then 7 | python gen_json_multipleheat.py ${files} ${HEATMAP_VERSION}-low_res ${SVS_INPUT_PATH} lym 0.5 necrosis 0.5 &>> ${LOG_OUTPUT_FOLDER}/log.gen_json_multipleheat.low_res.txt 8 | else 9 | python gen_json_multipleheat.py ${files} ${HEATMAP_VERSION}-high_res ${SVS_INPUT_PATH} lym 0.5 necrosis 0.5 &>> ${LOG_OUTPUT_FOLDER}/log.gen_json_multipleheat.txt 10 | fi 11 | done 12 | 13 | exit 0 14 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/low_res_all.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import math 4 | import scipy.stats as ss 5 | 6 | def low_res(fpath): 7 | x = np.zeros((10000000,), np.uint32); 8 | y = np.zeros((10000000,), np.uint32); 9 | p = np.zeros((10000000,), np.float32); 10 | n = np.zeros((10000000,), np.float32); 11 | 12 | nline = 0; 13 | with open(fpath) as f: 14 | for line in f: 15 | fields = line.split(); 16 | if (len(fields) == 4): 17 | x[nline] = int(fields[0]); 18 | y[nline] = int(fields[1]); 19 | p[nline] = float(fields[2]); 20 | n[nline] = float(fields[3]); 21 | nline += 1; 22 | x = x[0:nline]; 23 | y = y[0:nline]; 24 | p = p[0:nline]; 25 | n = n[0:nline]; 26 | 27 | max_x = np.max(x) + np.min(x); 28 | x = (ss.rankdata(x, method='dense') - 1).astype(np.uint32); 29 | y = (ss.rankdata(y, method='dense') - 1).astype(np.uint32); 30 | step = max_x / (np.max(x) + 1); 31 | 32 | imp = np.zeros((np.max(x)+1, np.max(y)+1), np.float32); 33 | for it in range(len(x)): 34 | imp[x[it], y[it]] = p[it]; 35 | 36 | imn = np.zeros((np.max(x)+1, np.max(y)+1), np.float32); 37 | for it in range(len(x)): 38 | imn[x[it], y[it]] = n[it]; 39 | 40 | f = open(fpath + '.low_res', 'w'); 41 | for i in range(int(imp.shape[0]/4)): 42 | for j in range(int(imp.shape[1]/4)): 43 | p_val = np.max(imp[i*4:i*4+4, j*4:j*4+4]); 44 | n_val = np.min(imn[i*4:i*4+4, j*4:j*4+4]); 45 | f.write('{} {} {} {}\n'.format( \ 46 | int(round((i+0.5)*step*4.0)), int(round((j+0.5)*step*4.0)), round(p_val, 6), round(n_val, 6))); 47 | f.close(); 48 | 49 | 50 | directory = './patch-level-merged/'; 51 | for fn in os.listdir(directory): 52 | fpath = directory + fn; 53 | if not os.path.isfile(fpath): 54 | continue; 55 | low_res(fpath); 56 | 57 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | rm -rf json patch-level-lym patch-level-nec patch-level-color patch-level-merged 6 | mkdir json patch-level-lym patch-level-nec patch-level-color patch-level-merged 7 | 8 | # Copy heatmap files from lym and necrosis prediction models 9 | # to patch-level/ and necrosis/ folders respectively. 10 | bash cp_heatmaps_all.sh ${PATCH_PATH} &> ${LOG_OUTPUT_FOLDER}/log.cp_heatmaps_all.txt 11 | 12 | # Combine patch-level and necrosis heatmaps into one heatmap. 13 | # Also generate high-res and low-res version. 14 | bash combine_lym_necrosis_all.sh &> ${LOG_OUTPUT_FOLDER}/log.combine_lym_necrosis_all.txt 15 | cp ./patch-level-merged/* ${HEATMAP_TXT_OUTPUT_FOLDER}/ 16 | cp ./patch-level-color/* ${HEATMAP_TXT_OUTPUT_FOLDER}/ 17 | 18 | # Generate meta and heatmap files for high-res and low-res heatmaps. 19 | bash gen_all_json.sh &> ${LOG_OUTPUT_FOLDER}/log.gen_all_json.txt 20 | cp ./json/* ${JSON_OUTPUT_FOLDER}/ 21 | 22 | # Put all jsons to camicroscope 23 | #bash upload_heatmaps.sh &> ${LOG_OUTPUT_FOLDER}/log.upload_heatmaps.txt 24 | 25 | exit 0 26 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/start_manual.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | # rm -rf json patch-level-lym patch-level-nec patch-level-color patch-level-merged 6 | # mkdir json patch-level-lym patch-level-nec patch-level-color patch-level-merged 7 | 8 | # # Copy heatmap files from lym and necrosis prediction models 9 | # to patch-level/ and necrosis/ folders respectively. 10 | # bash cp_heatmaps_all.sh ${PATCH_PATH} &> ${LOG_OUTPUT_FOLDER}/log.cp_heatmaps_all.txt 11 | 12 | # # Combine patch-level and necrosis heatmaps into one heatmap. 13 | # # Also generate high-res and low-res version. 14 | # bash combine_lym_necrosis_all.sh &> ${LOG_OUTPUT_FOLDER}/log.combine_lym_necrosis_all.txt 15 | # cp ./patch-level-merged/* ${HEATMAP_TXT_OUTPUT_FOLDER}/ 16 | # cp ./patch-level-color/* ${HEATMAP_TXT_OUTPUT_FOLDER}/ 17 | 18 | # # Generate meta and heatmap files for high-res and low-res heatmaps. 19 | # bash gen_all_json.sh &> ${LOG_OUTPUT_FOLDER}/log.gen_all_json.txt 20 | # cp ./json/* ${JSON_OUTPUT_FOLDER}/ 21 | 22 | # # Put all jsons to camicroscope 23 | # #bash upload_heatmaps.sh &> ${LOG_OUTPUT_FOLDER}/log.upload_heatmaps.txt 24 | 25 | exit 0 26 | -------------------------------------------------------------------------------- /u24_lymphocyte/heatmap_gen/upload_heatmaps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | echo Uploading json files to database ${CANCER_TYPE} in ${MONGODB_HOST}:${MONGODB_PORT} 6 | 7 | for file in json/meta_*.json; do 8 | if [ -f ${file} ]; then 9 | echo ${file} 10 | FN=`echo ${file} | awk -F'meta_' '{print substr($2,1,length($2)-5);}'` 11 | mongoimport --port ${MONGODB_PORT} --host ${MONGODB_HOST} -d ${CANCER_TYPE} -c objects json/heatmap_${FN}.json 12 | mongoimport --port ${MONGODB_PORT} --host ${MONGODB_HOST} -d ${CANCER_TYPE} -c metadata json/meta_${FN}.json 13 | fi 14 | done 15 | 16 | exit 0 17 | -------------------------------------------------------------------------------- /u24_lymphocyte/patch_extraction/save_svs_to_tiles.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import openslide 3 | import sys 4 | import os 5 | from PIL import Image 6 | 7 | slide_name = sys.argv[2] + '/' + sys.argv[1]; 8 | output_folder = sys.argv[3] + '/' + sys.argv[1]; 9 | patch_size_20X = 1000; 10 | 11 | fdone = '{}/extraction_done.txt'.format(output_folder); 12 | if os.path.isfile(fdone): 13 | print('fdone {} exist, skipping'.format(fdone)); 14 | exit(0); 15 | 16 | print('extracting {}'.format(output_folder)); 17 | 18 | if not os.path.exists(output_folder): 19 | os.mkdir(output_folder); 20 | 21 | try: 22 | oslide = openslide.OpenSlide(slide_name); 23 | # mag = 10.0 / float(oslide.properties[openslide.PROPERTY_NAME_MPP_X]); 24 | if openslide.PROPERTY_NAME_MPP_X in oslide.properties: 25 | mag = 10.0 / float(oslide.properties[openslide.PROPERTY_NAME_MPP_X]); 26 | elif "XResolution" in oslide.properties: 27 | mag = 10.0 / float(oslide.properties["XResolution"]); 28 | elif 'tiff.XResolution' in oslide.properties: # for Multiplex IHC WSIs, .tiff images 29 | mag = 10.0 / float(oslide.properties["tiff.XResolution"]); 30 | else: 31 | mag = 10.0 / float(0.254); 32 | pw = int(patch_size_20X * mag / 20); 33 | width = oslide.dimensions[0]; 34 | height = oslide.dimensions[1]; 35 | except: 36 | print('{}: exception caught'.format(slide_name)); 37 | exit(1); 38 | 39 | 40 | print(slide_name, width, height); 41 | 42 | for x in range(1, width, pw): 43 | for y in range(1, height, pw): 44 | if x + pw > width: 45 | pw_x = width - x; 46 | else: 47 | pw_x = pw; 48 | if y + pw > height: 49 | pw_y = height - y; 50 | else: 51 | pw_y = pw; 52 | 53 | if (int(patch_size_20X * pw_x / pw) <= 0) or \ 54 | (int(patch_size_20X * pw_y / pw) <= 0) or \ 55 | (pw_x <= 0) or (pw_y <= 0): 56 | continue; 57 | 58 | patch = oslide.read_region((x, y), 0, (pw_x, pw_y)); 59 | #shahira: skip where the alpha channel is zero 60 | patch_arr = np.array(patch); 61 | if(patch_arr[:,:,3].max() == 0): 62 | continue; 63 | 64 | # Resize into 20X. 65 | patch = patch.resize((int(patch_size_20X * pw_x / pw), int(patch_size_20X * pw_y / pw)), Image.ANTIALIAS); 66 | fname = '{}/{}_{}_{}_{}.png'.format(output_folder, x, y, pw, patch_size_20X); 67 | patch.save(fname); 68 | 69 | open(fdone, 'w').close(); 70 | 71 | -------------------------------------------------------------------------------- /u24_lymphocyte/patch_extraction/save_svs_to_tiles.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | COD_PARA=$1 6 | MAX_PARA=$2 7 | IN_FOLDER=${SVS_INPUT_PATH} 8 | OUT_FOLDER=${PATCH_PATH} 9 | 10 | LINE_N=0 11 | for files in ${IN_FOLDER}/*.*; do 12 | LINE_N=$((LINE_N+1)) 13 | if [ $((LINE_N % MAX_PARA)) -ne ${COD_PARA} ]; then continue; fi 14 | 15 | SVS=`echo ${files} | awk -F'/' '{print $NF}'` 16 | python save_svs_to_tiles.py $SVS $IN_FOLDER $OUT_FOLDER 17 | if [ $? -ne 0 ]; then 18 | echo "failed extracting patches for " ${SVS} 19 | rm -rf ${OUT_FOLDER}/${SVS} 20 | fi 21 | done 22 | 23 | exit 0; 24 | 25 | -------------------------------------------------------------------------------- /u24_lymphocyte/patch_extraction/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo ${LOG_OUTPUT_FOLDER} 3 | source ../conf/variables.sh 4 | echo ${LOG_OUTPUT_FOLDER} 5 | nohup bash save_svs_to_tiles.sh 0 4 &> ${LOG_OUTPUT_FOLDER}/log.save_svs_to_tiles.thread_0.txt & 6 | nohup bash save_svs_to_tiles.sh 1 4 &> ${LOG_OUTPUT_FOLDER}/log.save_svs_to_tiles.thread_1.txt & 7 | nohup bash save_svs_to_tiles.sh 2 4 &> ${LOG_OUTPUT_FOLDER}/log.save_svs_to_tiles.thread_2.txt & 8 | nohup bash save_svs_to_tiles.sh 3 4 &> ${LOG_OUTPUT_FOLDER}/log.save_svs_to_tiles.thread_3.txt & 9 | wait 10 | 11 | exit 0 12 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_incep-all.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-all 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 299 31 | input_img_width = 299 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 299 77 | input_img_width = 299 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | 91 | [TRAINER] 92 | class_name = ClassifierTrainer 93 | optimizer_type = ADAM 94 | max_epochs = 200 95 | learning_rate = 0.00005 96 | #batch_size = 1 97 | batch_size = 128 98 | #batch_size = 8 99 | # if epoch_size < 0 then use data size 100 | #epoch_size = 1 101 | epoch_size = -1 102 | display_step = 10 103 | save_best_only=False 104 | subepoch_checkpoint_step = 500 105 | 106 | 107 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_incep-manual.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-manual 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 299 31 | input_img_width = 299 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 299 77 | input_img_width = 299 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | 91 | [TRAINER] 92 | class_name = ClassifierTrainer 93 | optimizer_type = ADAM 94 | max_epochs = 200 95 | learning_rate = 0.00005 96 | #batch_size = 1 97 | batch_size = 128 98 | #batch_size = 8 99 | # if epoch_size < 0 then use data size 100 | #epoch_size = 1 101 | epoch_size = -1 102 | display_step = 10 103 | save_best_only=True 104 | subepoch_checkpoint_step = 500 105 | 106 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_incep-mix.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-mix.ckpt 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | 92 | 93 | [TRAINER] 94 | class_name = ClassifierTrainer 95 | optimizer_type = ADAM 96 | max_epochs = 200 97 | learning_rate = 0.00005 98 | #batch_size = 1 99 | batch_size = 128 100 | #batch_size = 8 101 | # if epoch_size < 0 then use data size 102 | #epoch_size = 1 103 | epoch_size = -1 104 | display_step = 10 105 | save_best_only=False 106 | subepoch_checkpoint_step = 500 107 | 108 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_incep-semi.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-semi.ckpt 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | 9 | [NETWORK] 10 | class_name = InceptionV4Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 299 15 | input_img_width = 299 16 | official_checkpoint = true 17 | 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | 92 | 93 | [TRAINER] 94 | class_name = ClassifierTrainer 95 | optimizer_type = ADAM 96 | max_epochs = 200 97 | learning_rate = 0.00005 98 | #batch_size = 1 99 | batch_size = 128 100 | #batch_size = 8 101 | # if epoch_size < 0 then use data size 102 | #epoch_size = 1 103 | epoch_size = -1 104 | display_step = 10 105 | save_best_only=False 106 | subepoch_checkpoint_step = 500 107 | 108 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_vgg-all.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = TCGABatchDataProvider 93 | filepath_data = /pylon5/ac3uump/lhou/patches_test 94 | filepath_label = /pylon5/ac3uump/lhou/patches_test 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTester 131 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 132 | #out_ext = csv 133 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_vgg-manual.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-manual 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 224 31 | input_img_width = 224 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 224 77 | input_img_width = 224 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | [TEST_DATA] 91 | provider_class_name = TCGABatchDataProvider 92 | filepath_data = /pylon5/ac3uump/lhou/patches_test 93 | filepath_label = /pylon5/ac3uump/lhou/patches_test 94 | preprocess = true 95 | augment = false 96 | permute = false 97 | input_img_height = 224 98 | input_img_width = 224 99 | split_name=test 100 | pre_resize=true 101 | pre_center=false 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | #file_name_suffix=__clrnorm1 106 | 107 | [TRAINER] 108 | class_name = ClassifierTrainer 109 | optimizer_type = ADAM 110 | max_epochs = 200 111 | learning_rate = 0.00005 112 | #batch_size = 1 113 | batch_size = 128 114 | #batch_size = 8 115 | # if epoch_size < 0 then use data size 116 | #epoch_size = 1 117 | epoch_size = -1 118 | display_step = 10 119 | save_best_only=True 120 | subepoch_checkpoint_step = 500 121 | 122 | 123 | [TESTER] 124 | class_name = ClassifierTester 125 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 126 | #out_ext = csv 127 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_vgg-mix.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-mix 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 25 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 224 31 | input_img_width = 224 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 224 77 | input_img_width = 224 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | [TEST_DATA] 91 | provider_class_name = TCGABatchDataProvider 92 | filepath_data = /pylon5/ac3uump/lhou/patches_test 93 | filepath_label = /pylon5/ac3uump/lhou/patches_test 94 | preprocess = true 95 | augment = false 96 | permute = false 97 | input_img_height = 224 98 | input_img_width = 224 99 | split_name=test 100 | pre_resize=true 101 | pre_center=false 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | #file_name_suffix=__clrnorm1 106 | 107 | [TRAINER] 108 | class_name = ClassifierTrainer 109 | optimizer_type = ADAM 110 | max_epochs = 200 111 | learning_rate = 0.00005 112 | #batch_size = 1 113 | batch_size = 128 114 | #batch_size = 8 115 | # if epoch_size < 0 then use data size 116 | #epoch_size = 1 117 | epoch_size = -1 118 | display_step = 10 119 | save_best_only=False 120 | subepoch_checkpoint_step = 500 121 | 122 | 123 | [TESTER] 124 | class_name = ClassifierTester 125 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 126 | #out_ext = csv 127 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/config_vgg-semi.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | mode = train 4 | #mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-semi 7 | model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | 9 | [NETWORK] 10 | class_name = VGG16Classifier 11 | n_channels = 3 12 | n_classes = 2 13 | pretrained=true 14 | input_img_height = 224 15 | input_img_width = 224 16 | official_checkpoint = true 17 | 18 | [COST] 19 | class_name = CrossEntropyCost 20 | class_weights = 1,1 21 | 22 | [TRAIN_DATA] 23 | provider_class_name = TCGABatchDataProvider 24 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 25 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | preprocess = false 27 | augment = true 28 | postprocess = true 29 | permute = true 30 | input_img_height = 224 31 | input_img_width = 224 32 | split_name=train 33 | aug_rot90=true 34 | aug_rot270=true 35 | aug_flip_hv=true 36 | aug_rot180=true 37 | aug_rot_rand=false 38 | aug_rot_min=-180 39 | aug_rot_max=180 40 | aug_brightness=true 41 | aug_brightness_min=-20 42 | aug_brightness_max=20 43 | aug_saturation=false 44 | aug_saturation_min=0.5 45 | aug_saturation_max=1.6 46 | aug_hue=true 47 | aug_hue_min=-10 48 | aug_hue_max=10 49 | aug_translate=true 50 | aug_translate_y_min=-20 51 | aug_translate_y_max=20 52 | aug_translate_x_min=-20 53 | aug_translate_x_max=20 54 | #aug_scale=false 55 | #aug_scale_min=1.0 56 | #aug_scale_max=1.5 57 | pre_resize=false 58 | pre_center=false 59 | pre_crop_center=false 60 | pre_crop_height=100 61 | pre_crop_width=100 62 | post_crop_center=true 63 | post_crop_height=100 64 | post_crop_width=100 65 | post_resize=true 66 | #file_name_suffix=__clrnorm1 67 | 68 | [VALIDATE_DATA] 69 | provider_class_name = TCGABatchDataProvider 70 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 71 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | preprocess = true 73 | augment = false 74 | postprocess = false 75 | permute = false 76 | input_img_height = 224 77 | input_img_width = 224 78 | split_name=val 79 | pre_crop_center=true 80 | pre_crop_height=100 81 | pre_crop_width=100 82 | pre_resize=true 83 | pre_center=false 84 | post_crop_center=true 85 | post_crop_height=100 86 | post_crop_width=100 87 | post_resize=true 88 | #file_name_suffix=__clrnorm1 89 | 90 | [TEST_DATA] 91 | provider_class_name = TCGABatchDataProvider 92 | filepath_data = /pylon5/ac3uump/lhou/patches_test 93 | filepath_label = /pylon5/ac3uump/lhou/patches_test 94 | preprocess = true 95 | augment = false 96 | permute = false 97 | input_img_height = 224 98 | input_img_width = 224 99 | split_name=test 100 | pre_resize=true 101 | pre_center=false 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | #file_name_suffix=__clrnorm1 106 | 107 | [TRAINER] 108 | class_name = ClassifierTrainer 109 | optimizer_type = ADAM 110 | max_epochs = 200 111 | learning_rate = 0.00005 112 | #batch_size = 1 113 | batch_size = 128 114 | #batch_size = 8 115 | # if epoch_size < 0 then use data size 116 | #epoch_size = 1 117 | epoch_size = -1 118 | display_step = 10 119 | save_best_only=False 120 | subepoch_checkpoint_step = 500 121 | 122 | 123 | [TESTER] 124 | class_name = ClassifierTester 125 | #out_dir = /gpfs/projects/KurcGroup/sabousamra/BreaKHis_v1_test 126 | #out_ext = csv 127 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_val_luad_strat/config_vgg-all_test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 224 33 | input_img_width = 224 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 224 79 | input_img_width = 224 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = TCGABatchDataProviderTestLabelled 94 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected/luad_pos_and_neg 95 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected/luad_pos_and_neg 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 224 101 | input_img_width = 224 102 | split_name=val 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | pre_resize=true 107 | pre_center=false 108 | post_crop_center=true 109 | post_crop_height=100 110 | post_crop_width=100 111 | post_resize=true 112 | #file_name_suffix=__clrnorm1 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterSuperpatchBatch 132 | out_dir = /pylon5/ac3uump/shahira/tcga/test_out/val_luad_strat 133 | #batch_size = 64 134 | batch_size = 1 135 | out_ext = csv 136 | threshold = 0.47 137 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_incep-all_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/incep-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = InceptionV4Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 299 16 | input_img_width = 299 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 299 32 | input_img_width = 299 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 299 78 | input_img_width = 299 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 299 100 | input_img_width = 299 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTesterExternalInput 131 | #class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.13 135 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_incep-semi_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-incv4 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = incep-semi 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/inception_v4.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/incep-semi.ckpt 9 | 10 | [NETWORK] 11 | class_name = InceptionV4Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 299 16 | input_img_width = 299 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 299 33 | input_img_width = 299 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label =/pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 299 79 | input_img_width = 299 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = None 94 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 299 101 | input_img_width = 299 102 | split_name=val 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | pre_resize=true 107 | pre_center=false 108 | post_crop_center=true 109 | post_crop_height=100 110 | post_crop_width=100 111 | post_resize=true 112 | #file_name_suffix=__clrnorm1 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterExternalInput 132 | #class_name = ClassifierTesterExternalInputBinaryOutput 133 | out_dir = None 134 | out_ext = None 135 | threshold = 0.48 136 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_vgg-all_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-all 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-all.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | 20 | [COST] 21 | class_name = CrossEntropyCost 22 | class_weights = 1,1 23 | 24 | [TRAIN_DATA] 25 | provider_class_name = TCGABatchDataProvider 26 | filepath_data = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 27 | filepath_label = /pylon5/ac3uump/lhou/patches_train_filtered_by_testset 28 | preprocess = false 29 | augment = true 30 | postprocess = true 31 | permute = true 32 | input_img_height = 224 33 | input_img_width = 224 34 | split_name=train 35 | aug_rot90=true 36 | aug_rot270=true 37 | aug_flip_hv=true 38 | aug_rot180=true 39 | aug_rot_rand=false 40 | aug_rot_min=-180 41 | aug_rot_max=180 42 | aug_brightness=true 43 | aug_brightness_min=-20 44 | aug_brightness_max=20 45 | aug_saturation=false 46 | aug_saturation_min=0.5 47 | aug_saturation_max=1.6 48 | aug_hue=true 49 | aug_hue_min=-10 50 | aug_hue_max=10 51 | aug_translate=true 52 | aug_translate_y_min=-20 53 | aug_translate_y_max=20 54 | aug_translate_x_min=-20 55 | aug_translate_x_max=20 56 | #aug_scale=false 57 | #aug_scale_min=1.0 58 | #aug_scale_max=1.5 59 | pre_resize=false 60 | pre_center=false 61 | pre_crop_center=false 62 | pre_crop_height=100 63 | pre_crop_width=100 64 | post_crop_center=true 65 | post_crop_height=100 66 | post_crop_width=100 67 | post_resize=true 68 | #file_name_suffix=__clrnorm1 69 | 70 | [VALIDATE_DATA] 71 | provider_class_name = TCGABatchDataProvider 72 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 74 | preprocess = true 75 | augment = false 76 | postprocess = false 77 | permute = false 78 | input_img_height = 224 79 | input_img_width = 224 80 | split_name=val 81 | pre_crop_center=true 82 | pre_crop_height=100 83 | pre_crop_width=100 84 | pre_resize=true 85 | pre_center=false 86 | post_crop_center=true 87 | post_crop_height=100 88 | post_crop_width=100 89 | post_resize=true 90 | #file_name_suffix=__clrnorm1 91 | 92 | [TEST_DATA] 93 | provider_class_name = None 94 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 96 | preprocess = true 97 | augment = false 98 | postprocess = false 99 | permute = false 100 | input_img_height = 224 101 | input_img_width = 224 102 | split_name=val 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | pre_resize=true 107 | pre_center=false 108 | post_crop_center=true 109 | post_crop_height=100 110 | post_crop_width=100 111 | post_resize=true 112 | #file_name_suffix=__clrnorm1 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterExternalInput 132 | #class_name = ClassifierTesterExternalInputBinaryOutput 133 | out_dir = None 134 | out_ext = None 135 | threshold = 0.47 136 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_vgg-manual_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-manual 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-manual.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train/patches_manual_labeled 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/luad_additional+ 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/luad_additional+ 95 | preprocess = true 96 | augment = false 97 | permute = false 98 | input_img_height = 224 99 | input_img_width = 224 100 | split_name=test 101 | pre_resize=true 102 | pre_center=false 103 | pre_crop_center=true 104 | pre_crop_height=100 105 | pre_crop_width=100 106 | #file_name_suffix=__clrnorm1 107 | 108 | [TRAINER] 109 | class_name = ClassifierTrainer 110 | optimizer_type = ADAM 111 | max_epochs = 200 112 | learning_rate = 0.00005 113 | #batch_size = 1 114 | batch_size = 128 115 | #batch_size = 8 116 | # if epoch_size < 0 then use data size 117 | #epoch_size = 1 118 | epoch_size = -1 119 | display_step = 10 120 | save_best_only=True 121 | subepoch_checkpoint_step = 500 122 | 123 | 124 | [TESTER] 125 | class_name = ClassifierTesterExternalInput 126 | out_dir = None 127 | out_ext = None 128 | threshold = 0.43 129 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_vgg-mix_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-mix 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-mix.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | class_name = ClassifierTesterExternalInput 131 | #class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.42 135 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_vgg-mix_test_ext_binary.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-mix 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-mix.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/shahira/tcga/patches_manual7c_semi_otherc_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | [TRAINER] 114 | class_name = ClassifierTrainer 115 | optimizer_type = ADAM 116 | max_epochs = 200 117 | learning_rate = 0.00005 118 | #batch_size = 1 119 | batch_size = 128 120 | #batch_size = 8 121 | # if epoch_size < 0 then use data size 122 | #epoch_size = 1 123 | epoch_size = -1 124 | display_step = 10 125 | save_best_only=False 126 | subepoch_checkpoint_step = 500 127 | 128 | 129 | [TESTER] 130 | #class_name = ClassifierTesterExternalInput 131 | class_name = ClassifierTesterExternalInputBinaryOutput 132 | out_dir = None 133 | out_ext = None 134 | threshold = 0.42 135 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/config/test_wsi_ext/config_vgg-semi_test_ext.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | config_name= tcga-vgg16 3 | #mode = train 4 | mode = test 5 | model_path = /pylon5/ac3uump/shahira/tcga/models 6 | model_base_filename = vgg-semi 7 | #model_restore_filename = /pylon5/ac3uump/shahira/tf-slim_models/pretrained-imagenet/vgg_16.ckpt 8 | model_restore_filename = /pylon5/ac3uump/shahira/tcga/models_final/vgg-semi.ckpt 9 | 10 | [NETWORK] 11 | class_name = VGG16Classifier 12 | n_channels = 3 13 | n_classes = 2 14 | pretrained=true 15 | input_img_height = 224 16 | input_img_width = 224 17 | official_checkpoint = false 18 | 19 | [COST] 20 | class_name = CrossEntropyCost 21 | class_weights = 1,1 22 | 23 | [TRAIN_DATA] 24 | provider_class_name = TCGABatchDataProvider 25 | filepath_data = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 26 | filepath_label = /pylon5/ac3uump/lhou/patches_train_semiauto_filtered_by_testset 27 | preprocess = false 28 | augment = true 29 | postprocess = true 30 | permute = true 31 | input_img_height = 224 32 | input_img_width = 224 33 | split_name=train 34 | aug_rot90=true 35 | aug_rot270=true 36 | aug_flip_hv=true 37 | aug_rot180=true 38 | aug_rot_rand=false 39 | aug_rot_min=-180 40 | aug_rot_max=180 41 | aug_brightness=true 42 | aug_brightness_min=-20 43 | aug_brightness_max=20 44 | aug_saturation=false 45 | aug_saturation_min=0.5 46 | aug_saturation_max=1.6 47 | aug_hue=true 48 | aug_hue_min=-10 49 | aug_hue_max=10 50 | aug_translate=true 51 | aug_translate_y_min=-20 52 | aug_translate_y_max=20 53 | aug_translate_x_min=-20 54 | aug_translate_x_max=20 55 | #aug_scale=false 56 | #aug_scale_min=1.0 57 | #aug_scale_max=1.5 58 | pre_resize=false 59 | pre_center=false 60 | pre_crop_center=false 61 | pre_crop_height=100 62 | pre_crop_width=100 63 | post_crop_center=true 64 | post_crop_height=100 65 | post_crop_width=100 66 | post_resize=true 67 | #file_name_suffix=__clrnorm1 68 | 69 | [VALIDATE_DATA] 70 | provider_class_name = TCGABatchDataProvider 71 | filepath_data = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 72 | filepath_label = /pylon5/ac3uump/shahira/patches_val_luad_stratified_corrected 73 | preprocess = true 74 | augment = false 75 | postprocess = false 76 | permute = false 77 | input_img_height = 224 78 | input_img_width = 224 79 | split_name=val 80 | pre_crop_center=true 81 | pre_crop_height=100 82 | pre_crop_width=100 83 | pre_resize=true 84 | pre_center=false 85 | post_crop_center=true 86 | post_crop_height=100 87 | post_crop_width=100 88 | post_resize=true 89 | #file_name_suffix=__clrnorm1 90 | 91 | [TEST_DATA] 92 | provider_class_name = None 93 | filepath_data = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 94 | filepath_label = /pylon5/ac3uump/shahira/tcga/datasets/patches_val_allcancertype_luad_stratified_corrected/brca 95 | preprocess = true 96 | augment = false 97 | postprocess = false 98 | permute = false 99 | input_img_height = 224 100 | input_img_width = 224 101 | split_name=val 102 | pre_crop_center=true 103 | pre_crop_height=100 104 | pre_crop_width=100 105 | pre_resize=true 106 | pre_center=false 107 | post_crop_center=true 108 | post_crop_height=100 109 | post_crop_width=100 110 | post_resize=true 111 | #file_name_suffix=__clrnorm1 112 | 113 | 114 | [TRAINER] 115 | class_name = ClassifierTrainer 116 | optimizer_type = ADAM 117 | max_epochs = 200 118 | learning_rate = 0.00005 119 | #batch_size = 1 120 | batch_size = 128 121 | #batch_size = 8 122 | # if epoch_size < 0 then use data size 123 | #epoch_size = 1 124 | epoch_size = -1 125 | display_step = 10 126 | save_best_only=False 127 | subepoch_checkpoint_step = 500 128 | 129 | 130 | [TESTER] 131 | class_name = ClassifierTesterExternalInput 132 | #class_name = ClassifierTesterExternalInputBinaryOutput 133 | out_dir = None 134 | out_ext = None 135 | threshold = 0.33 136 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_cost_func/__pycache__/cross_entropy_cost_func.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_cost_func/__pycache__/cross_entropy_cost_func.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_cost_func/cross_entropy_cost_func.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import sys; 5 | import os; 6 | import tensorflow as tf; 7 | 8 | from ..sa_net_cost_func import AbstractCostFunc; 9 | from ..sa_net_loss_func_helper import CNNLossFuncHelper; 10 | 11 | class CrossEntropyCost(AbstractCostFunc): 12 | def __init__(self, n_classes, kwargs): 13 | # predefined list of arguments 14 | args = {'class_weights':None}; 15 | 16 | args.update(kwargs); 17 | class_weights = args['class_weights']; 18 | if(class_weights is not None): 19 | class_weights = [float(x) for x in class_weights.split(',')] 20 | self.n_classes = n_classes; 21 | 22 | print('class_weights = ', class_weights); 23 | if(class_weights == None): 24 | self.class_weights = tf.Variable(tf.ones([self.n_classes])); 25 | else: 26 | self.class_weights = tf.Variable(class_weights); 27 | 28 | 29 | 30 | def calc_cost(self, logits, labels): 31 | return CNNLossFuncHelper.cost_cross_entropy(logits, labels, self.class_weights, self.n_classes); 32 | 33 | 34 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_cost_func.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import sys; 5 | import os; 6 | import tensorflow as tf; 7 | 8 | #from sa_net_loss_func import CNNLossFuncHelper; 9 | 10 | class AbstractCostFunc: 11 | def __init__(self, n_classes, kwargs): 12 | self.n_classes = n_classes; 13 | self.class_weights = tf.Variable(tf.ones([n_classes])); 14 | 15 | self.cost = self.calc_cost(self.logits, self.labels, kwargs); 16 | 17 | def calc_cost(self, logits, labels): 18 | pass; 19 | 20 | 21 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_data_provider.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | 5 | class AbstractDataProvider: 6 | def __init__(self, is_test, filepath_data, filepath_label, n_channels, n_classes, do_preprocess, do_augment, data_var_name=None, label_var_name=None, permute=False, repeat=True): 7 | self.is_test = is_test; 8 | self.filepath_data = ''; 9 | self.filepath_label = ''; 10 | self.n_channels = n_channels; 11 | self.n_classes = n_classes; 12 | self.do_preprocess = do_preprocess; 13 | self.do_augment = do_augment; 14 | self.data_var_name = 'c488gb'; 15 | self.label_var_name = label_var_name; 16 | self.do_permute = permute; 17 | self.do_repeat = repeat; # go over the data multiple times like with training or only once like with test 18 | 19 | self.is_loaded = False; 20 | 21 | def load_data(self): 22 | self.data = None; 23 | self.label = None; 24 | self.last_fetched_indx = -1; 25 | self.permutation = None; 26 | self.data_count = 0; 27 | 28 | def reset(self, repermute=None): 29 | self.last_fetched_indx = -1; 30 | 31 | def get_next_one(self): 32 | pass; 33 | 34 | def get_next_n(self, n:int): 35 | pass; 36 | 37 | def preprocess(self, data_point, label): 38 | return data_point, label; 39 | 40 | def augment(self, data_point, label): 41 | return data_point, label; 42 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_loss_func_helper.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import tensorflow as tf; 5 | from enum import Enum; 6 | import numpy as np; 7 | 8 | class CostFuncTypes(Enum): 9 | CROSS_ENTROPY = 1 10 | 11 | 12 | class CNNLossFuncHelper: 13 | 14 | @staticmethod 15 | def cost_cross_entropy(logits, labels, class_weights, n_classes): 16 | flat_logits = tf.reshape(logits, [-1, n_classes]); 17 | flat_labels = tf.reshape(labels, [-1, n_classes]); 18 | print(class_weights) 19 | print(flat_labels) 20 | print(flat_logits) 21 | return tf.abs(tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits( \ 22 | logits=flat_logits \ 23 | , targets=flat_labels \ 24 | , pos_weight=class_weights \ 25 | ))); 26 | 27 | @staticmethod 28 | def cost_mse(logits, labels, class_weights, n_classes): 29 | flat_logits = tf.reshape(logits, [-1, n_classes]); 30 | flat_labels = tf.reshape(labels, [-1, n_classes]); 31 | print(class_weights) 32 | print(flat_labels) 33 | print(flat_logits) 34 | return tf.losses.mean_squared_error(\ 35 | labels=flat_labels \ 36 | , predictions=flat_logits \ 37 | , weights=class_weights \ 38 | ); 39 | 40 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_model_params.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | 5 | class Modes: 6 | TRAIN = 0; 7 | TEST = 1; 8 | 9 | class ModelParams: 10 | def __init__(self): 11 | self.params = {}; 12 | self.params['arch_name'] = 'basic'; 13 | self.params['mode'] = Modes.TRAIN; 14 | self.params['n_classes'] = 2; 15 | self.params['n_channels'] = 1; 16 | self.params['train_out_path'] = ''; 17 | self.params['test_out_path'] = ''; 18 | self.params['validate_out_path'] = ''; 19 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_optimizer.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import tensorflow as tf; 5 | from enum import Enum; 6 | 7 | 8 | class OptimizerTypes(Enum): 9 | ADAM = 1 10 | SGD = 2 11 | 12 | 13 | class CNNOptimizer: 14 | 15 | @staticmethod 16 | def adam_optimizer(learning_rate, cost, global_step): 17 | return tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-08,) \ 18 | .minimize(cost, global_step=global_step); 19 | 20 | @staticmethod 21 | def sgd_optimizer(learning_rate, cost, global_step): 22 | return tf.train.GradientDescentOptimizer(learning_rate) \ 23 | .minimize(cost, global_step=global_step); 24 | 25 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_output_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_net_output_helper.py -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_net_test.py: -------------------------------------------------------------------------------- 1 | # author: Shahira Abousamra 2 | # created: 12.23.2018 3 | # ============================================================================== 4 | import tensorflow as tf; 5 | 6 | from sa_net_arch import AbstractCNNArch; 7 | from sa_net_arch_utilities import CNNArchUtils; 8 | from sa_net_optimizer import OptimizerTypes, CNNOptimizer; 9 | from sa_net_data_provider import AbstractDataProvider; 10 | from skimage import io; 11 | import os; 12 | 13 | class CNNTest: 14 | def __init__(self, cnn_arch:AbstractCNNArch, test_data_provider:AbstractDataProvider, session_config, output_dir, output_ext, kwargs): 15 | # predefined list of arguments 16 | 17 | self.cnn_arch = cnn_arch; 18 | #self.cost_func = cost_func; 19 | self.test_data_provider = test_data_provider; 20 | if(session_config == None): 21 | self.session_config = tf.ConfigProto(); 22 | else: 23 | self.session_config = session_config; 24 | self.output_dir = output_dir; 25 | self.output_ext = output_ext; 26 | 27 | self.init = tf.global_variables_initializer(); 28 | 29 | def test(self, do_init, do_restore, do_load_data): 30 | with tf.Session(config=self.session_config) as sess: 31 | #with tf.Session() as sess: 32 | with sess.as_default(): 33 | if(do_init): 34 | sess.run(tf.global_variables_initializer()); 35 | #sess.run(self.init); 36 | if(do_restore): 37 | self.cnn_arch.restore_model(sess); 38 | if(do_load_data): 39 | self.test_data_provider.load_data(); 40 | out_basefilename = self.test_data_provider.data_tag; 41 | 42 | 43 | batch_x, batch_label = self.test_data_provider.get_next_one(); 44 | indx = 0; 45 | while(batch_x is not None): 46 | batch_x, batch_label = self.test_data_provider.get_next_one(); 47 | batch_y, batch_y_softmax, batch_y_class = sess.run([self.cnn_arch.logits, self.cnn_arch.prediction_softmax, self.cnn_arch.prediction_class] \ 48 | , feed_dict={self.cnn_arch.input_x: batch_x \ 49 | , self.cnn_arch.isTest: True \ 50 | }); 51 | io.imsave(os.path.join(self.output_dir, out_basefilename[indx] + self.output_ext), batch_y); 52 | indx = indx + 1; 53 | 54 | 55 | 56 | 57 | 58 | print("Test Finished!") 59 | 60 | 61 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/inception_utils.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/inception_utils.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/inception_v4.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/inception_v4.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/inception_v4_classifier_arch.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/inception_v4_classifier_arch.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/vgg.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/vgg.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/vgg_16_classifier_arch.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_networks/__pycache__/vgg_16_classifier_arch.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_networks/inception_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Contains common code shared by all inception models. 16 | 17 | Usage of arg scope: 18 | with slim.arg_scope(inception_arg_scope()): 19 | logits, end_points = inception.inception_v3(images, num_classes, 20 | is_training=is_training) 21 | 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | import tensorflow as tf 28 | 29 | slim = tf.contrib.slim 30 | 31 | 32 | def inception_arg_scope(weight_decay=0.00004, 33 | use_batch_norm=True, 34 | batch_norm_decay=0.9997, 35 | batch_norm_epsilon=0.001, 36 | activation_fn=tf.nn.relu, 37 | batch_norm_updates_collections=tf.GraphKeys.UPDATE_OPS, 38 | batch_norm_scale=False): 39 | """Defines the default arg scope for inception models. 40 | 41 | Args: 42 | weight_decay: The weight decay to use for regularizing the model. 43 | use_batch_norm: "If `True`, batch_norm is applied after each convolution. 44 | batch_norm_decay: Decay for batch norm moving average. 45 | batch_norm_epsilon: Small float added to variance to avoid dividing by zero 46 | in batch norm. 47 | activation_fn: Activation function for conv2d. 48 | batch_norm_updates_collections: Collection for the update ops for 49 | batch norm. 50 | batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the 51 | activations in the batch normalization layer. 52 | 53 | Returns: 54 | An `arg_scope` to use for the inception models. 55 | """ 56 | batch_norm_params = { 57 | # Decay for the moving averages. 58 | 'decay': batch_norm_decay, 59 | # epsilon to prevent 0s in variance. 60 | 'epsilon': batch_norm_epsilon, 61 | # collection containing update_ops. 62 | 'updates_collections': batch_norm_updates_collections, 63 | # use fused batch norm if possible. 64 | 'fused': None, 65 | 'scale': batch_norm_scale, 66 | } 67 | if use_batch_norm: 68 | normalizer_fn = slim.batch_norm 69 | normalizer_params = batch_norm_params 70 | else: 71 | normalizer_fn = None 72 | normalizer_params = {} 73 | # Set weight_decay for weights in Conv and FC layers. 74 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 75 | weights_regularizer=slim.l2_regularizer(weight_decay)): 76 | with slim.arg_scope( 77 | [slim.conv2d], 78 | weights_initializer=slim.variance_scaling_initializer(), 79 | activation_fn=activation_fn, 80 | normalizer_fn=normalizer_fn, 81 | normalizer_params=normalizer_params) as sc: 82 | return sc 83 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_runners/__pycache__/tf_classifier_runner_external_input.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_runners/__pycache__/tf_classifier_runner_external_input.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_runners/__pycache__/tf_classifier_runner_external_input.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_runners/__pycache__/tf_classifier_runner_external_input.cpython-36.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_testers/__pycache__/sa_net_test_classifier_external_input.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShahiraAbousamra/til_classification/c98db4c47f94b06de100ca4b13f02ca1ab9c748f/u24_lymphocyte/prediction/NNFramework_TF/sa_testers/__pycache__/sa_net_test_classifier_external_input.cpython-35.pyc -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_testers/sa_net_test_classifier.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf; 2 | 3 | from ..sa_net_arch import AbstractCNNArch; 4 | from ..sa_net_arch_utilities import CNNArchUtils; 5 | from ..sa_net_optimizer import OptimizerTypes, CNNOptimizer; 6 | from ..sa_net_data_provider import AbstractDataProvider; 7 | import os; 8 | import numpy as np; 9 | 10 | class ClassifierTester: 11 | def __init__(self, cnn_arch:AbstractCNNArch, test_data_provider:AbstractDataProvider, session_config, output_dir, output_ext, kwargs): 12 | # predefined list of arguments 13 | 14 | self.cnn_arch = cnn_arch; 15 | #self.cost_func = cost_func; 16 | self.test_data_provider = test_data_provider; 17 | if(session_config == None): 18 | self.session_config = tf.ConfigProto(); 19 | else: 20 | self.session_config = session_config; 21 | self.output_dir = output_dir; 22 | self.output_ext = output_ext; 23 | 24 | self.init = tf.global_variables_initializer(); 25 | 26 | def test(self, do_init, do_restore, do_load_data): 27 | with tf.Session(config=self.session_config) as sess: 28 | #with tf.Session() as sess: 29 | with sess.as_default(): 30 | if(do_init): 31 | sess.run(tf.global_variables_initializer()); 32 | #sess.run(self.init); 33 | if(do_restore): 34 | self.cnn_arch.restore_model(sess); 35 | if(do_load_data): 36 | self.test_data_provider.load_data(); 37 | out_basefilename = self.test_data_provider.data_tag; 38 | 39 | 40 | batch_x, batch_label = self.test_data_provider.get_next_one(); 41 | indx = 0; 42 | total_correct_pred = 0; 43 | while(batch_x is not None): 44 | print(out_basefilename[indx]) 45 | batch_x = batch_x.reshape(1,batch_x.shape[0],batch_x.shape[1],batch_x.shape[2]) 46 | batch_y, correct_pred = sess.run([self.cnn_arch.logits, self.cnn_arch.correct_pred] \ 47 | , feed_dict={self.cnn_arch.input_x: batch_x \ 48 | , self.cnn_arch.labels: batch_label \ 49 | , self.cnn_arch.isTest: True \ 50 | }); 51 | total_correct_pred += correct_pred; 52 | 53 | batch_x = self.test_data_provider.get_next_one(); 54 | indx = indx + 1; 55 | 56 | print(indx); 57 | print(total_correct_pred); 58 | 59 | 60 | 61 | print("Test Finished!") 62 | 63 | 64 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_testers/sa_net_test_classifier_external_input.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf; 2 | 3 | from ..sa_net_arch import AbstractCNNArch; 4 | from ..sa_net_arch_utilities import CNNArchUtils; 5 | from ..sa_net_optimizer import OptimizerTypes, CNNOptimizer; 6 | from ..sa_net_data_provider import AbstractDataProvider; 7 | import os; 8 | import numpy as np; 9 | 10 | class ClassifierTesterExternalInput: 11 | def __init__(self, cnn_arch:AbstractCNNArch, session_config, output_dir, output_ext, kwargs): 12 | # predefined list of arguments 13 | 14 | self.cnn_arch = cnn_arch; 15 | #self.cost_func = cost_func; 16 | #self.test_data_provider = test_data_provider; 17 | if(session_config == None): 18 | self.session_config = tf.ConfigProto(); 19 | self.session_config.gpu_options.allow_growth = True 20 | else: 21 | self.session_config = session_config; 22 | self.output_dir = output_dir; 23 | self.output_ext = output_ext; 24 | 25 | self.init = tf.global_variables_initializer(); 26 | 27 | def init_model(self, do_init, do_restore): 28 | self.sess = tf.Session(config=self.session_config); 29 | with self.sess.as_default(): 30 | if(do_init): 31 | self.sess.run(tf.global_variables_initializer()); 32 | #sess.run(self.init); 33 | if(do_restore): 34 | self.cnn_arch.restore_model(self.sess); 35 | 36 | 37 | def predict(self, inputs): 38 | with self.sess.as_default(): 39 | batch_x = inputs; 40 | if (batch_x is None): 41 | return None; 42 | 43 | batch_x = self.preprocess_input(inputs); 44 | 45 | batch_y = self.sess.run([self.cnn_arch.logits] \ 46 | , feed_dict={self.cnn_arch.input_x: batch_x \ 47 | , self.cnn_arch.isTest: True \ 48 | }); 49 | #print(np.array(batch_y).shape) 50 | #print(np.array(batch_y)) 51 | #print(np.array(batch_y)[...,-1]) 52 | batch_y_sig = self.sigmoid(np.array(batch_y)[...,-1]).reshape((-1,1)); 53 | return batch_y_sig; 54 | 55 | def sigmoid(self, x): 56 | return (1 / (1 + np.exp(-x))) 57 | 58 | def restart_model(self): 59 | self.sess.close(); 60 | #tf.reset_default_graph(); 61 | self.init_model(True, True); 62 | 63 | def preprocess_input(self, inputs): 64 | # normalize (mean 0, std=2) 65 | np.clip(inputs, 0, 255, inputs); 66 | inputs /= 255; 67 | inputs -= 0.5; 68 | inputs *= 2; 69 | inputs = tf.image.resize_images(inputs, (self.cnn_arch.input_img_height, self.cnn_arch.input_img_width)); 70 | inputs = inputs.eval() 71 | return inputs; -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF/sa_testers/sa_net_test_classifier_external_input_binary_output.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf; 2 | 3 | from ..sa_net_arch import AbstractCNNArch; 4 | from ..sa_net_arch_utilities import CNNArchUtils; 5 | from ..sa_net_optimizer import OptimizerTypes, CNNOptimizer; 6 | from ..sa_net_data_provider import AbstractDataProvider; 7 | import os; 8 | import numpy as np; 9 | 10 | class ClassifierTesterExternalInputBinaryOutput: 11 | def __init__(self, cnn_arch:AbstractCNNArch, session_config, output_dir, output_ext, kwargs): 12 | # predefined list of arguments 13 | args = {'threshold':0.5}; 14 | args.update(kwargs); 15 | 16 | self.cnn_arch = cnn_arch; 17 | #self.cost_func = cost_func; 18 | #self.test_data_provider = test_data_provider; 19 | if(session_config == None): 20 | self.session_config = tf.ConfigProto(); 21 | self.session_config.gpu_options.allow_growth = True 22 | else: 23 | self.session_config = session_config; 24 | self.output_dir = output_dir; 25 | self.output_ext = output_ext; 26 | self.threshold = float(args['threshold']); 27 | 28 | self.init = tf.global_variables_initializer(); 29 | 30 | def init_model(self, do_init, do_restore): 31 | self.sess = tf.Session(config=self.session_config); 32 | with self.sess.as_default(): 33 | if(do_init): 34 | self.sess.run(tf.global_variables_initializer()); 35 | #sess.run(self.init); 36 | if(do_restore): 37 | self.cnn_arch.restore_model(self.sess); 38 | 39 | 40 | def predict(self, inputs): 41 | with self.sess.as_default(): 42 | batch_x = inputs; 43 | if (batch_x is None): 44 | return None; 45 | 46 | batch_x = self.preprocess_input(inputs); 47 | 48 | batch_y = self.sess.run([self.cnn_arch.logits] \ 49 | , feed_dict={self.cnn_arch.input_x: batch_x \ 50 | , self.cnn_arch.isTest: True \ 51 | }); 52 | batch_y_sig = self.sigmoid(np.array(batch_y)[...,-1]); 53 | batch_y_binary = np.array(batch_y_sig > self.threshold).astype(np.float).reshape((-1,1)); 54 | 55 | return batch_y_binary; 56 | 57 | 58 | def sigmoid(self, x): 59 | return (1 / (1 + np.exp(-x))) 60 | 61 | def restart_model(self): 62 | self.sess.close(); 63 | #tf.reset_default_graph(); 64 | self.init_model(True, True); 65 | 66 | def preprocess_input(self, inputs): 67 | # normalize (mean 0, std=2) 68 | np.clip(inputs, 0, 255, inputs); 69 | inputs /= 255; 70 | inputs -= 0.5; 71 | inputs *= 2; 72 | inputs = tf.image.resize_images(inputs, (self.cnn_arch.input_img_height, self.cnn_arch.input_img_width)); 73 | inputs = inputs.eval() 74 | return inputs; 75 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF_external_call/external_model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | sys.path.append(".."); 4 | sys.path.append("."); 5 | sys.path.append("../.."); 6 | sys.path.append("..."); 7 | from NNFramework_TF.sa_runners.tf_classifier_runner_external_input import load_model; 8 | 9 | 10 | def load_external_model(model_path): 11 | # Load your model here 12 | model = load_model(model_path) 13 | return model 14 | 15 | def pred_by_external_model(model, inputs): 16 | # Get prediction here 17 | # model: 18 | # A model loaded by load_external_model 19 | # inputs : 20 | # float32 numpy array with shape N x 3 x 100 x 100 21 | # Range of value: 0.0 ~ 255.0 22 | # You may need to rearrange inputs: 23 | # inputs = inputs.transpose((0, 2, 3, 1)) 24 | # Expected output: 25 | # float32 numpy array with shape N x 1 26 | # Each entry is a probability ranges 0.0 ~ 1.0 27 | inputs = inputs.transpose((0, 2, 3, 1)) 28 | pred = model.predict(inputs)[0] 29 | #exp_pred = np.exp(pred - np.max(pred, axis=-1, keepdims=True) + 1) 30 | #return exp_pred[..., -1:] / np.sum(exp_pred, axis=-1, keepdims=True) 31 | 32 | #sig_pred = (1 / (1 + np.exp(-pred))) 33 | #sig_pred = sig_pred[..., -1:] ; 34 | #return sig_pred 35 | 36 | return pred; 37 | 38 | if __name__ == "__main__": 39 | 40 | ## LUAD ############################################################### 41 | ## LUAD - semiauto - InceptionV4 42 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_semiauto.ini"; 43 | ## LUAD - semiauto - VGG16 44 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_semiauto.ini"; 45 | ## LUAD - manual - InceptionV4 46 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_manual.ini"; 47 | ## LUAD - manual - VGG16 48 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_manual.ini"; 49 | 50 | ## SKCM ############################################################### 51 | ## SKCM - semiauto - InceptionV4 52 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_semiauto.ini"; 53 | ## SKCM - semiauto - VGG16 54 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_semiauto.ini"; 55 | # SKCM - manual - InceptionV4 56 | config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_manual.ini"; 57 | ## SKCM - manual - VGG16 58 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_manual.ini"; 59 | 60 | print(config_filepath) 61 | model = load_external_model(config_filepath); 62 | print('load_external_model called') 63 | inputs = np.random.rand(10, 3, 100, 100); 64 | print('inputs created') 65 | pred = pred_by_external_model(model, inputs) 66 | print('after predict') 67 | print(pred); 68 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/NNFramework_TF_external_call/external_train.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | sys.path.append(".."); 4 | sys.path.append("."); 5 | sys.path.append("../.."); 6 | sys.path.append("..."); 7 | from NNFramework_TF.sa_runners import tf_classifier_runner; 8 | 9 | 10 | 11 | if __name__ == "__main__": 12 | 13 | ## LUAD ############################################################### 14 | ## LUAD - semiauto - InceptionV4 15 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_semiauto.ini"; 16 | ## LUAD - semiauto - VGG16 17 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_semiauto.ini"; 18 | ## LUAD - manual - InceptionV4 19 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_luad_manual.ini"; 20 | ## LUAD - manual - VGG16 21 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_luad_manual.ini"; 22 | 23 | ## SKCM ############################################################### 24 | ## SKCM - semiauto - InceptionV4 25 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_semiauto.ini"; 26 | ## SKCM - semiauto - VGG16 27 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_semiauto.ini"; 28 | ## SKCM - manual - InceptionV4 29 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_skcm_manual.ini"; 30 | ## SKCM - manual - VGG16 31 | #config_filepath = "/home/shahira/NNFramework_TF_model_config/config_tcga_vgg16_b128_crop100_skcm_manual.ini"; 32 | 33 | #config_filepath = "/home/shahira/NNFramework_TF/config/config_tcga_incv4_b128_crop100_noBN_wd5e-4_d75_HEaug_luad_semiauto.ini"; 34 | 35 | tf_classifier_runner.main(sys.argv[1:]) -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/color/color_stats.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import numpy as np 4 | import time 5 | from PIL import Image 6 | 7 | APS = 100; 8 | TileFolder = sys.argv[1] + '/'; 9 | heat_map_out = sys.argv[2]; 10 | 11 | def whiteness(png): 12 | wh = (np.std(png[:,:,0].flatten()) + np.std(png[:,:,1].flatten()) + np.std(png[:,:,2].flatten())) / 3.0; 13 | return wh; 14 | 15 | 16 | def blackness(png): 17 | bk = np.mean(png); 18 | return bk; 19 | 20 | 21 | def redness(png): 22 | rd = np.mean((png[:,:,0] >= 190) * (png[:,:,1] <= 100) * (png[:,:,2] <= 100)); 23 | return rd; 24 | 25 | 26 | def load_data(): 27 | X = np.zeros(shape=(1000000, 3), dtype=np.float32); 28 | coor = np.zeros(shape=(1000000, 2), dtype=np.int32); 29 | 30 | ind = 0; 31 | for fn in os.listdir(TileFolder): 32 | full_fn = TileFolder + '/' + fn; 33 | if not os.path.isfile(full_fn): 34 | continue; 35 | if len(fn.split('_')) < 4: 36 | continue; 37 | 38 | x_off = float(fn.split('_')[0]); 39 | y_off = float(fn.split('_')[1]); 40 | svs_pw = float(fn.split('_')[2]); 41 | png_pw = float(fn.split('_')[3].split('.png')[0]); 42 | 43 | png = np.array(Image.open(full_fn).convert('RGB')); 44 | for x in range(0, png.shape[1], APS): 45 | if x + APS > png.shape[1]: 46 | continue; 47 | for y in range(0, png.shape[0], APS): 48 | if y + APS > png.shape[0]: 49 | continue; 50 | X[ind, 0] = whiteness(png[y:y+APS, x:x+APS, :]); 51 | X[ind, 1] = blackness(png[y:y+APS, x:x+APS, :]); 52 | X[ind, 2] = redness(png[y:y+APS, x:x+APS, :]); 53 | coor[ind, 0] = np.int32(x_off + (x + APS/2) * svs_pw / png_pw); 54 | coor[ind, 1] = np.int32(y_off + (y + APS/2) * svs_pw / png_pw); 55 | ind += 1; 56 | 57 | X = X[0:ind]; 58 | coor = coor[0:ind]; 59 | 60 | return X, coor; 61 | 62 | 63 | def split_validation(): 64 | Wh, coor = load_data(); 65 | 66 | fid = open(TileFolder + '/' + heat_map_out, 'w'); 67 | for idx in range(0, Wh.shape[0]): 68 | fid.write('{} {} {} {} {}\n'.format(coor[idx][0], coor[idx][1], Wh[idx][0], Wh[idx][1], Wh[idx][2])); 69 | fid.close(); 70 | 71 | 72 | def main(): 73 | split_validation(); 74 | 75 | 76 | if __name__ == "__main__": 77 | main(); 78 | 79 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/color/color_stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FOLDER=$1 4 | # PARAL = [0, MAX_PARAL-1] 5 | PARAL=$2 6 | MAX_PARAL=$3 7 | 8 | DATA_FILE=patch-level-color.txt 9 | DONE_FILE=extraction_done.txt 10 | EXEC_FILE=color_stats.py 11 | 12 | PRE_FILE_NUM=0 13 | while [ 1 ]; do 14 | LINE_N=0 15 | FILE_NUM=0 16 | EXTRACTING=0 17 | for files in ${FOLDER}/*/; do 18 | FILE_NUM=$((FILE_NUM+1)) 19 | if [ ! -f ${files}/${DONE_FILE} ]; then EXTRACTING=1; fi 20 | 21 | LINE_N=$((LINE_N+1)) 22 | if [ $((LINE_N % MAX_PARAL)) -ne ${PARAL} ]; then continue; fi 23 | 24 | if [ -f ${files}/${DONE_FILE} ]; then 25 | if [ ! -f ${files}/${DATA_FILE} ]; then 26 | echo ${files}/${DATA_FILE} generating 27 | python -u ${EXEC_FILE} ${files} ${DATA_FILE} 28 | fi 29 | fi 30 | done 31 | 32 | if [ ${EXTRACTING} -eq 0 ] && [ ${PRE_FILE_NUM} -eq ${FILE_NUM} ]; then break; fi 33 | PRE_FILE_NUM=${FILE_NUM} 34 | done 35 | 36 | exit 0 37 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/common/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ch_inner_prod', 'batch_norms', 'shape']; 2 | 3 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/common/ch_inner_prod.py: -------------------------------------------------------------------------------- 1 | import lasagne 2 | import theano.tensor as T 3 | import numpy as np 4 | 5 | class ChInnerProd(lasagne.layers.Layer): 6 | def __init__(self, incoming, **kwargs): 7 | super(ChInnerProd, self).__init__(incoming, **kwargs); 8 | 9 | def get_output_shape_for(self, input_shape): 10 | output_shape = list(input_shape); 11 | output_shape[1] -= 1; 12 | return tuple(output_shape); 13 | 14 | def get_output_for(self, input, **kwargs): 15 | output = T.shape_padaxis(input[:, 0], axis=1) * input[:, 1:]; 16 | return output; 17 | 18 | class ChInnerProdMerge(lasagne.layers.MergeLayer): 19 | def __init__(self, feat_map, mask_map, **kwargs): 20 | super(ChInnerProdMerge, self).__init__([feat_map, mask_map], **kwargs); 21 | 22 | def get_output_shape_for(self, input_shapes): 23 | output_shape = list(input_shapes[0]); 24 | return tuple(output_shape); 25 | 26 | def get_output_for(self, inputs, **kwargs): 27 | output = inputs[0] * inputs[1]; 28 | return output; 29 | 30 | class CenterCrop(lasagne.layers.Layer): 31 | def __init__(self, incoming, crop_size, **kwargs): 32 | super(CenterCrop, self).__init__(incoming, **kwargs); 33 | self.crop_size = crop_size; 34 | 35 | def get_output_for(self, input, **kwargs): 36 | return input[:, :, self.crop_size:-self.crop_size, self.crop_size:-self.crop_size]; 37 | 38 | def get_output_shape_for(self, input_shape): 39 | output_shape = list(input_shape); 40 | return (input_shape[0], input_shape[1], input_shape[2]-2*self.crop_size, input_shape[3]-2*self.crop_size); 41 | 42 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/lymphocyte/data_aug.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import misc 3 | from PIL import Image 4 | from skimage.color import hed2rgb, rgb2hed 5 | 6 | APS = 100; 7 | PS = 100; 8 | MARGIN = 90; 9 | 10 | def rotate_img(x, degree): 11 | rotate_angle = (np.random.rand(1)[0]-0.5)*2 * degree; 12 | im = Image.fromarray(x.astype(np.uint8)); 13 | x = np.array(im.rotate(rotate_angle, Image.BICUBIC)).astype(np.float32); 14 | return x; 15 | 16 | def data_aug_img(img, mu, sigma, deterministic=False, idraw=-1, jdraw=-1): 17 | # mirror and flip 18 | if np.random.rand(1)[0] < 0.5: 19 | img = img[:, ::-1, :]; 20 | if np.random.rand(1)[0] < 0.5: 21 | img = img[:, :, ::-1]; 22 | 23 | # transpose 24 | if np.random.rand(1)[0] < 0.5: 25 | img = img.transpose((0, 2, 1)); 26 | 27 | img = (img / 255.0 - mu) / sigma; 28 | 29 | return img; 30 | 31 | def zero_centering(img): 32 | x0 = (img.shape[1] - PS) // 2; 33 | y0 = (img.shape[2] - PS) // 2; 34 | im = Image.fromarray(img.transpose().astype(np.uint8)); 35 | img = np.array(im.crop((x0, y0, x0+PS, y0+PS))).transpose().astype(np.float32); 36 | return img; 37 | 38 | def data_aug(X, mu, sigma, deterministic=False, idraw=-1, jdraw=-1): 39 | Xc = np.zeros(shape=(X.shape[0], X.shape[1], PS, PS), dtype=np.float32); 40 | Xcopy = X.copy(); 41 | for i in range(len(Xcopy)): 42 | Xc[i] = data_aug_img(Xcopy[i], mu, sigma, deterministic=deterministic, idraw=idraw, jdraw=jdraw); 43 | return Xc; 44 | 45 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/lymphocyte/pred_thread_lym.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../../conf/variables.sh 4 | 5 | FOLDER=$1 6 | # PARAL = [0, MAX_PARAL-1] 7 | PARAL=$2 8 | MAX_PARAL=$3 9 | DEVICE=$4 10 | 11 | DATA_FILE=patch-level-lym.txt 12 | DONE_FILE=extraction_done.txt 13 | 14 | if [ ${EXTERNAL_LYM_MODEL} -eq 0 ]; then 15 | EXEC_FILE=pred.py 16 | else 17 | EXEC_FILE=pred_by_external_model.py 18 | fi 19 | 20 | PRE_FILE_NUM=0 21 | while [ 1 ]; do 22 | LINE_N=0 23 | FILE_NUM=0 24 | EXTRACTING=0 25 | for files in ${FOLDER}/*/; do 26 | FILE_NUM=$((FILE_NUM+1)) 27 | if [ ! -f ${files}/${DONE_FILE} ]; then EXTRACTING=1; fi 28 | 29 | LINE_N=$((LINE_N+1)) 30 | if [ $((LINE_N % MAX_PARAL)) -ne ${PARAL} ]; then continue; fi 31 | 32 | if [ -f ${files}/${DONE_FILE} ]; then 33 | if [ ! -f ${files}/${DATA_FILE} ]; then 34 | echo ${files}/${DATA_FILE} generating 35 | python -u ${EXEC_FILE} \ 36 | ${files} ${LYM_NECRO_CNN_MODEL_PATH} ${DATA_FILE} ${LYM_PREDICTION_BATCH_SIZE} 37 | fi 38 | fi 39 | done 40 | 41 | if [ ${EXTRACTING} -eq 0 ] && [ ${PRE_FILE_NUM} -eq ${FILE_NUM} ]; then break; fi 42 | PRE_FILE_NUM=${FILE_NUM} 43 | done 44 | 45 | exit 0 46 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/models/update_path.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -f 2 | 3 | source ../../conf/variables.sh 4 | 5 | # replace /root in the model ini files 6 | for i in `ls *.ini`; do 7 | sed -i 's/\/root/\'"${APP_DIR}"'/g' $i; 8 | done 9 | 10 | exit 0; -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/necrosis/data_aug_necrosis.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | from skimage.color import hed2rgb, rgb2hed 4 | 5 | MARGIN = 10; 6 | 7 | def data_aug_img(img, msk, mu, sigma, deterministic=False, idraw=-1, jdraw=-1, APS=500, PS=200): 8 | # crop 9 | icut = APS - PS; 10 | jcut = APS - PS; 11 | if deterministic: 12 | if idraw < -0.5 or jdraw < -0.5: 13 | ioff = int(icut // 2); 14 | joff = int(jcut // 2); 15 | else: 16 | ioff = idraw; 17 | joff = jdraw; 18 | else: 19 | ioff = np.random.randint(MARGIN, icut + 1 - MARGIN); 20 | joff = np.random.randint(MARGIN, jcut + 1 - MARGIN); 21 | img = img[:, ioff : ioff+PS, joff : joff+PS]; 22 | msk = msk[ioff : ioff+PS, joff : joff+PS]; 23 | 24 | # adjust color 25 | if not deterministic: 26 | adj_add = np.array([[[0.15, 0.15, 0.02]]], dtype=np.float32); 27 | img = np.clip(hed2rgb( \ 28 | rgb2hed(img.transpose((2, 1, 0)) / 255.0) + np.random.uniform(-1.0, 1.0, (1, 1, 3))*adj_add \ 29 | ).transpose((2, 1, 0))*255.0, 0.0, 255.0); 30 | 31 | if not deterministic: 32 | adj_range = 0.1; 33 | adj_add = 5; 34 | rgb_mean = np.mean(img, axis=(1,2), keepdims=True).astype(np.float32); 35 | adj_magn = np.random.uniform(1 - adj_range, 1 + adj_range, (3, 1, 1)).astype(np.float32); 36 | img = np.clip((img-rgb_mean)*adj_magn + rgb_mean + np.random.uniform(-1.0, 1.0, (3, 1, 1))*adj_add, 0.0, 255.0); 37 | 38 | # mirror and flip 39 | if not deterministic: 40 | if np.random.rand(1)[0] < 0.5: 41 | img = img[:, ::-1, :]; 42 | msk = msk[::-1, :]; 43 | if np.random.rand(1)[0] < 0.5: 44 | img = img[:, :, ::-1]; 45 | msk = msk[:, ::-1]; 46 | 47 | # transpose 48 | if not deterministic: 49 | if np.random.rand(1)[0] < 0.5: 50 | img = img.transpose((0, 2, 1)); 51 | msk = msk.transpose((1, 0)); 52 | 53 | img, msk = zero_centering(img, msk, APS=APS, PS=PS); 54 | img = (img - mu) / sigma; 55 | 56 | return img, msk; 57 | 58 | def zero_centering(img, msk, APS=500, PS=224): 59 | x0 = (img.shape[1] - PS) // 2; 60 | y0 = (img.shape[2] - PS) // 2; 61 | im = Image.fromarray(img.transpose().astype(np.uint8)); 62 | mk = Image.fromarray(msk.transpose().astype(np.uint8)); 63 | img = np.array(im.crop((x0, y0, x0+PS, y0+PS))).transpose().astype(np.float32); 64 | msk = np.array(mk.crop((x0, y0, x0+PS, y0+PS))).transpose().astype(np.float32); 65 | return img, msk; 66 | 67 | def data_aug(X, Y, mu, sigma, deterministic=False, idraw=-1, jdraw=-1, APS=500, PS=200): 68 | Xc = np.zeros(shape=(X.shape[0], X.shape[1], PS, PS), dtype=np.float32); 69 | Yc = np.zeros(shape=(Y.shape[0], PS, PS), dtype=np.int32); 70 | Xcopy = X.copy(); 71 | Ycopy = Y.copy(); 72 | for i in range(len(Xcopy)): 73 | Xc[i], Yc[i] = data_aug_img(Xcopy[i], Ycopy[i], mu, sigma, deterministic=deterministic, idraw=idraw, jdraw=jdraw, APS=APS, PS=PS); 74 | return Xc, Yc; 75 | 76 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/necrosis/pred_necrosis.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pickle 4 | 5 | from pred_necrosis_wsi import predict_slide 6 | 7 | slide_path = sys.argv[1]; 8 | classification_model_file = sys.argv[2] + '/cnn_nec_model.pkl'; 9 | heat_map_out = sys.argv[3]; 10 | 11 | def load_model_value(model_file): 12 | loaded_var = pickle.load(open(model_file, 'rb')); 13 | mu = loaded_var[0]; 14 | sigma = loaded_var[1]; 15 | 16 | param_values = loaded_var[2]; 17 | return mu, sigma, param_values; 18 | 19 | 20 | def Segment_Necrosis_WholeSlide(): 21 | mu, sigma, param_values = load_model_value(classification_model_file); 22 | output_path = slide_path; 23 | predict_slide(slide_path, mu, sigma, param_values, output_path, heat_map_out); 24 | 25 | 26 | if __name__ == "__main__": 27 | Segment_Necrosis_WholeSlide(); 28 | 29 | 30 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/necrosis/pred_thread_nec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../../conf/variables.sh 4 | 5 | FOLDER=$1 6 | # PARAL = [0, MAX_PARAL-1] 7 | PARAL=$2 8 | MAX_PARAL=$3 9 | DEVICE=$4 10 | 11 | DATA_FILE=patch-level-necrosis.txt 12 | DONE_FILE=extraction_done.txt 13 | EXEC_FILE=pred_necrosis.py 14 | 15 | #PRE_FILE_NUM=0 16 | #while [ 1 ]; do 17 | # LINE_N=0 18 | # FILE_NUM=0 19 | # EXTRACTING=0 20 | # for files in ${FOLDER}/*/; do 21 | # FILE_NUM=$((FILE_NUM+1)) 22 | # if [ ! -f ${files}/${DONE_FILE} ]; then EXTRACTING=1; fi 23 | # 24 | # LINE_N=$((LINE_N+1)) 25 | # if [ $((LINE_N % MAX_PARAL)) -ne ${PARAL} ]; then continue; fi 26 | # 27 | # if [ -f ${files}/${DONE_FILE} ]; then 28 | # if [ ! -f ${files}/${DATA_FILE} ]; then 29 | # echo ${files}/${DATA_FILE} generating 30 | # THEANO_FLAGS="device=${DEVICE}" python -u ${EXEC_FILE} \ 31 | # ${files} ${LYM_NECRO_CNN_MODEL_PATH} ${DATA_FILE} 32 | # fi 33 | # fi 34 | # done 35 | # 36 | # if [ ${EXTRACTING} -eq 0 ] && [ ${PRE_FILE_NUM} -eq ${FILE_NUM} ]; then break; fi 37 | # PRE_FILE_NUM=${FILE_NUM} 38 | #done 39 | 40 | exit 0 41 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/pytorch_resnet34_model/model_til_resnet34_external_input.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | #from torchvision import models 5 | import collections 6 | from distutils.util import strtobool; 7 | #import cv2 8 | import numpy as np 9 | from torchvision import transforms 10 | from PIL import Image 11 | 12 | 13 | 14 | class TILClassifierExternalInputModel(): 15 | def __init__(self, model, device, is_binary_output=False, threshold=0.5, max_side=100, resize_side=100): 16 | super(TILClassifierExternalInputModel,self).__init__() 17 | 18 | 19 | self.device = device 20 | self.model = model 21 | self.is_binary_output = is_binary_output 22 | self.threshold = threshold 23 | self.max_side = max_side 24 | self.resize_side = resize_side 25 | 26 | self.sig_layer = torch.nn.Sigmoid() 27 | 28 | 29 | 30 | def predict(self,x): 31 | if(len(x.shape) > 4): 32 | x = x.squeeze() 33 | x_tensor = self.preprocess_input(x) 34 | x_tensor=x_tensor.to(self.device) 35 | y = self.model(x_tensor) 36 | y = self.sig_layer(y).detach().cpu().numpy().squeeze() 37 | if(self.is_binary_output): 38 | y = (y > self.threshold).astype(np.float); 39 | y = y[:, np.newaxis] 40 | return y 41 | 42 | def preprocess_input(self, inputs): 43 | #print('inputs.shape a',inputs.shape) 44 | #np.clip(inputs, 0, 255, inputs); 45 | #print('inputs.shape c',inputs.shape) 46 | #inputs /= 255; 47 | #if(self.max_side > 0): 48 | # h = inputs.shape[-3] 49 | # w = inputs.shape[-2] 50 | # h2 = h 51 | # w2 = w 52 | # crop = False 53 | # if(h > self.max_side): 54 | # h2 = self.max_side 55 | # crop = True 56 | # if(w > self.max_side): 57 | # w2 = self.max_side 58 | # crop = True 59 | # if(crop): 60 | # print('cropping') 61 | # y = (h - h2)// 2 62 | # x = (w - w2)// 2 63 | # #y=0 64 | # #x=0 65 | # #if(not (h2 ==h)): 66 | # # y = np.random.randint(0, high = h-h2) 67 | # #if(not (w2 ==w)): 68 | # # x = np.random.randint(0, high = w-w2) 69 | # inputs = inputs[:,y:y+h2, x:x+w2, :] 70 | # #gt_dmap = gt_dmap[y:y+h2, x:x+w2] 71 | 72 | #if(self.resize_side > 0 and (inputs.shape[1] != self.resize_side or inputs.shape[2] != self.resize_side )): 73 | if(self.resize_side > 0 ): 74 | #print('resizing') 75 | inputs2 = np.zeros((inputs.shape[0], self.resize_side,self.resize_side, inputs.shape[3])) 76 | for i in range(inputs.shape[0]): 77 | #img = cv2.resize(inputs[i].squeeze(), (self.resize_side,self.resize_side)) 78 | img = np.array(Image.fromarray(inputs[i].squeeze().astype(np.uint8)).resize((int(self.resize_side), int(self.resize_side)), Image.ANTIALIAS)) 79 | inputs2[i] = img 80 | inputs = inputs2 81 | inputs /= 255; 82 | inputs=inputs.transpose((0,3,1,2)) 83 | #print('inputs.max() a',inputs.max()) 84 | #print('inputs.shape d',inputs.shape) 85 | inputs_tensor=torch.tensor(inputs,dtype=torch.float) 86 | for i in range(inputs_tensor.shape[0]): 87 | x=transforms.functional.normalize(inputs_tensor[i],mean=[0.485, 0.456, 0.406], 88 | std=[0.229, 0.224, 0.225]) 89 | #print('transform x shape',x.shape) 90 | inputs_tensor[i] = x 91 | #print('inputs_tensor.max() b',inputs_tensor.max().item()) 92 | return inputs_tensor; 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | cd lymphocyte 6 | nohup bash pred_thread_lym.sh \ 7 | ${PATCH_PATH} 0 1 ${LYM_CNN_PRED_DEVICE} \ 8 | &> ${LOG_OUTPUT_FOLDER}/log.pred_thread_lym_0.txt & 9 | cd .. 10 | 11 | cd necrosis 12 | nohup bash pred_thread_nec.sh \ 13 | ${PATCH_PATH} 0 1 ${NEC_CNN_PRED_DEVICE} \ 14 | &> ${LOG_OUTPUT_FOLDER}/log.pred_thread_nec_0.txt & 15 | cd .. 16 | 17 | cd color 18 | nohup bash color_stats.sh ${PATCH_PATH} 0 2 \ 19 | &> ${LOG_OUTPUT_FOLDER}/log.color_stats_0.txt & 20 | nohup bash color_stats.sh ${PATCH_PATH} 1 2 \ 21 | &> ${LOG_OUTPUT_FOLDER}/log.color_stats_1.txt & 22 | cd .. 23 | 24 | wait 25 | 26 | exit 0 27 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/start_color_only.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | 6 | cd color 7 | nohup bash color_stats.sh ${PATCH_PATH} 0 2 \ 8 | &> ${LOG_OUTPUT_FOLDER}/log.color_stats_0.txt & 9 | nohup bash color_stats.sh ${PATCH_PATH} 1 2 \ 10 | &> ${LOG_OUTPUT_FOLDER}/log.color_stats_1.txt & 11 | cd .. 12 | 13 | wait 14 | 15 | exit 0 16 | -------------------------------------------------------------------------------- /u24_lymphocyte/prediction/start_lym_only.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ../conf/variables.sh 4 | 5 | cd lymphocyte 6 | nohup bash pred_thread_lym.sh \ 7 | ${PATCH_PATH} 0 1 ${LYM_CNN_PRED_DEVICE} \ 8 | &> ${LOG_OUTPUT_FOLDER}/log.pred_thread_lym_0.txt & 9 | cd .. 10 | 11 | 12 | wait 13 | 14 | exit 0 15 | -------------------------------------------------------------------------------- /u24_lymphocyte/scripts/cleanup_heatmap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -n $BASE_DIR ]]; then 4 | cd $BASE_DIR 5 | else 6 | cd ../ 7 | fi 8 | 9 | source ./conf/variables.sh 10 | 11 | in_dir=${BASE_DIR} 12 | 13 | rm ${HEATMAP_TXT_OUTPUT_FOLDER}/* 14 | rm ${JSON_OUTPUT_FOLDER}/* 15 | rm ${BINARY_HEATMAP_TXT_OUTPUT_FOLDER}/* 16 | rm ${BINARY_JSON_OUTPUT_FOLDER}/* 17 | rm ${LOG_OUTPUT_FOLDER}/* 18 | 19 | delete_pattern=patch-level-lym.txt 20 | for folder in ${PATCH_PATH}/*; do 21 | echo $folder 22 | find $folder -name $delete_pattern -delete 23 | #rm "$folder/$delete_pattern" 24 | done 25 | 26 | wait; 27 | 28 | exit 0 29 | 30 | -------------------------------------------------------------------------------- /u24_lymphocyte/scripts/patch_2_heatmap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -n $BASE_DIR ]]; then 4 | cd $BASE_DIR 5 | else 6 | cd ../ 7 | fi 8 | source ./conf/variables.sh 9 | 10 | 11 | cd prediction 12 | nohup bash start.sh & 13 | cd .. 14 | 15 | wait; 16 | 17 | cd heatmap_gen 18 | nohup bash start.sh & 19 | cd .. 20 | 21 | wait; 22 | 23 | exit 0 24 | -------------------------------------------------------------------------------- /u24_lymphocyte/scripts/svs_2_heatmap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ../ 4 | 5 | source ./conf/variables.sh 6 | 7 | # create missing output directories 8 | for i in ${OUT_FOLDERS}; do 9 | if [ ! -d $i ]; then 10 | mkdir -p $i; 11 | fi 12 | done 13 | 14 | cd patch_extraction 15 | nohup bash start.sh & 16 | cd .. 17 | 18 | cd prediction 19 | nohup bash start.sh & 20 | cd .. 21 | 22 | wait; 23 | 24 | cd prediction 25 | nohup bash start.sh & 26 | cd .. 27 | 28 | wait; 29 | 30 | cd heatmap_gen 31 | nohup bash start.sh & 32 | cd .. 33 | 34 | wait; 35 | 36 | exit 0 37 | -------------------------------------------------------------------------------- /u24_lymphocyte/scripts/threshold_probability_heatmaps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # reads the threshold value from the model config file and applies the threshold on heatmap json files located in $JSON_OUTPUT_FOLDER and heatmap txt files located in $HEATMAP_TXT_OUTPUT_FOLDER 4 | # the thresholded versions are stored in $BINARY_JSON_OUTPUT_FOLDER and $BINARY_HEATMAP_TXT_OUTPUT_FOLDER respectively with the heatmap version name given by $BINARY_HEATMAP_VERSION 5 | 6 | if [[ -n $BASE_DIR ]]; then 7 | cd $BASE_DIR 8 | else 9 | cd ../ 10 | fi 11 | 12 | source ./conf/variables.sh 13 | 14 | cd ./scripts 15 | 16 | python threshold_probability_heatmaps.py $JSON_OUTPUT_FOLDER $BINARY_JSON_OUTPUT_FOLDER $HEATMAP_TXT_OUTPUT_FOLDER $BINARY_HEATMAP_TXT_OUTPUT_FOLDER $LYM_NECRO_CNN_MODEL_PATH $BINARY_HEATMAP_VERSION 17 | 18 | 19 | wait; 20 | 21 | --------------------------------------------------------------------------------