├── DeepPATH_code ├── 02_testing │ ├── xClasses │ │ ├── WORKSPACE │ │ └── inception │ │ │ ├── __init__.py │ │ │ ├── slim │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── ops.cpython-35.pyc │ │ │ │ ├── ops.cpython-36.pyc │ │ │ │ ├── slim.cpython-35.pyc │ │ │ │ ├── slim.cpython-36.pyc │ │ │ │ ├── losses.cpython-35.pyc │ │ │ │ ├── losses.cpython-36.pyc │ │ │ │ ├── scopes.cpython-35.pyc │ │ │ │ ├── scopes.cpython-36.pyc │ │ │ │ ├── __init__.cpython-35.pyc │ │ │ │ ├── __init__.cpython-36.pyc │ │ │ │ ├── variables.cpython-35.pyc │ │ │ │ ├── variables.cpython-36.pyc │ │ │ │ ├── inception_model.cpython-35.pyc │ │ │ │ └── inception_model.cpython-36.pyc │ │ │ ├── slim.py │ │ │ └── BUILD │ │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-35.pyc │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── dataset.cpython-35.pyc │ │ │ ├── nc_dataset.cpython-35.pyc │ │ │ ├── nc_dataset.cpython-36.pyc │ │ │ ├── imagenet_data.cpython-35.pyc │ │ │ ├── inception_eval.cpython-35.pyc │ │ │ ├── image_processing.cpython-35.pyc │ │ │ ├── image_processing.cpython-36.pyc │ │ │ ├── image_processing.cpython-37.pyc │ │ │ ├── inception_model.cpython-35.pyc │ │ │ ├── inception_model.cpython-36.pyc │ │ │ ├── nc_imagenet_data.cpython-35.pyc │ │ │ ├── nc_imagenet_data.cpython-36.pyc │ │ │ ├── nc_inception_eval.cpython-35.pyc │ │ │ ├── nc_inception_eval.cpython-36.pyc │ │ │ └── nc_inception_eval.cpython-37.pyc │ │ │ ├── nc_imagenet_data.py │ │ │ ├── data │ │ │ ├── preprocess_imagenet_validation_data.py │ │ │ ├── download_and_preprocess_flowers.sh │ │ │ ├── download_and_preprocess_flowers_mac.sh │ │ │ ├── download_and_preprocess_imagenet.sh │ │ │ ├── nc_download_and_preprocess_imagenet.sh │ │ │ └── download_imagenet.sh │ │ │ ├── nc_dataset.py │ │ │ └── BUILD │ └── multiClasses │ │ ├── WORKSPACE │ │ ├── inception │ │ ├── __pycache__ │ │ │ ├── dataset.cpython-35.pyc │ │ │ ├── nc_dataset.cpython-35.pyc │ │ │ ├── imagenet_data.cpython-35.pyc │ │ │ ├── inception_eval.cpython-35.pyc │ │ │ ├── inception_model.cpython-35.pyc │ │ │ ├── image_processing.cpython-35.pyc │ │ │ ├── nc_imagenet_data.cpython-35.pyc │ │ │ └── nc_inception_eval.cpython-35.pyc │ │ ├── slim │ │ │ ├── __pycache__ │ │ │ │ ├── ops.cpython-35.pyc │ │ │ │ ├── slim.cpython-35.pyc │ │ │ │ ├── losses.cpython-35.pyc │ │ │ │ ├── scopes.cpython-35.pyc │ │ │ │ ├── variables.cpython-35.pyc │ │ │ │ └── inception_model.cpython-35.pyc │ │ │ ├── slim.py │ │ │ ├── BUILD │ │ │ └── inception_test.py │ │ ├── nc_imagenet_data.py │ │ ├── data │ │ │ ├── preprocess_imagenet_validation_data.py │ │ │ ├── download_and_preprocess_flowers.sh │ │ │ ├── download_and_preprocess_flowers_mac.sh │ │ │ ├── download_and_preprocess_imagenet.sh │ │ │ ├── nc_download_and_preprocess_imagenet.sh │ │ │ └── download_imagenet.sh │ │ ├── nc_dataset.py │ │ └── BUILD │ │ └── nc_imagenet_eval.py ├── 01_training │ └── xClasses │ │ ├── WORKSPACE │ │ ├── inception │ │ ├── slim │ │ │ ├── __pycache__ │ │ │ │ └── slim.cpython-36.pyc │ │ │ ├── slim.py │ │ │ ├── BUILD │ │ │ └── inception_test.py │ │ ├── imagenet_train.py │ │ ├── imagenet_eval.py │ │ ├── imagenet_data.py │ │ ├── data │ │ │ ├── preprocess_imagenet_validation_data.py │ │ │ ├── download_and_preprocess_flowers.sh │ │ │ ├── download_and_preprocess_flowers_mac.sh │ │ │ ├── download_and_preprocess_imagenet.sh │ │ │ ├── nc_download_and_preprocess_imagenet.sh │ │ │ └── download_imagenet.sh │ │ ├── dataset.py │ │ └── BUILD │ │ └── sb_compile.sh ├── example_TCGA_lung │ ├── labelref_r1.txt │ ├── labelref_r3.txt │ └── checkpoints │ │ └── README.md ├── 03_postprocessing │ ├── __pycache__ │ │ └── xml_writer.cpython-36.pyc │ ├── 0e_Get_svs_PixelSizeMag.py │ ├── sb_valid_example.sh │ ├── sb_TF_ROC_2.sh │ ├── net2048_to_tsv.py │ └── multiClasses │ │ └── 0f_ProbHistogram.py ├── example_Melanoma_MLR │ ├── README.md │ ├── Instructions_for_testing_an_independent_dataset.R │ └── Code_for_the_training_and_testing_process.R ├── 00_preprocessing │ ├── 0a_split_Omero_csv.py │ ├── Get_Labstats_From_jpeg.py │ ├── Get_pixel_size_dist.py │ ├── Normalize_Vahadane.py │ └── archive │ │ └── convert_jpeg2hdf5.py ├── legacy_yml │ ├── conda3_520_env_deepPath.yml_legacy_20230420 │ └── conda3_520_env_deepPath.yml_legacy_20230921_beforeXml ├── conda3_520_env_deepPath.yml ├── conda3_520_env_python376_ROC.yml └── requirements.txt ├── .gitignore └── README.md /DeepPATH_code/02_testing/xClasses/WORKSPACE: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/WORKSPACE: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | LungCancer/LungCancer/LungCancer_2classes 2 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "inception") 2 | -------------------------------------------------------------------------------- /DeepPATH_code/example_TCGA_lung/labelref_r1.txt: -------------------------------------------------------------------------------- 1 | Solid_Normal_Tissue 2 | TCGA-LUAD 3 | TCGA-LUSC 4 | -------------------------------------------------------------------------------- /DeepPATH_code/example_TCGA_lung/labelref_r3.txt: -------------------------------------------------------------------------------- 1 | EGFR 2 | FAT1 3 | FAT4 4 | KEAP1 5 | KRAS 6 | LRP1B 7 | NF1 8 | SETBP1 9 | STK11 10 | TP53 11 | -------------------------------------------------------------------------------- /DeepPATH_code/03_postprocessing/__pycache__/xml_writer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/03_postprocessing/__pycache__/xml_writer.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/dataset.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/dataset.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/ops.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/ops.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/ops.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/ops.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/slim.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/slim.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/slim.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/slim.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/slim/__pycache__/slim.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/01_training/xClasses/inception/slim/__pycache__/slim.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/dataset.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/dataset.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_dataset.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_dataset.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_dataset.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_dataset.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/losses.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/losses.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/losses.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/losses.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/scopes.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/scopes.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/scopes.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/scopes.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/nc_dataset.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/nc_dataset.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/ops.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/ops.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/slim.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/slim.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/imagenet_data.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/imagenet_data.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/inception_eval.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/inception_eval.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/variables.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/variables.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/variables.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/variables.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/losses.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/losses.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/scopes.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/scopes.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/image_processing.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/image_processing.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/image_processing.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/image_processing.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/image_processing.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/image_processing.cpython-37.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/inception_model.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/inception_model.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/inception_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/inception_model.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_imagenet_data.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_imagenet_data.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_imagenet_data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_imagenet_data.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/imagenet_data.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/imagenet_data.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/inception_eval.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/inception_eval.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/inception_model.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/inception_model.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/variables.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/variables.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_inception_eval.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_inception_eval.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_inception_eval.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_inception_eval.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_inception_eval.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/__pycache__/nc_inception_eval.cpython-37.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/image_processing.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/image_processing.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/nc_imagenet_data.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/nc_imagenet_data.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/__pycache__/nc_inception_eval.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/__pycache__/nc_inception_eval.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/inception_model.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/inception_model.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/inception_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/xClasses/inception/slim/__pycache__/inception_model.cpython-36.pyc -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/inception_model.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ncoudray/DeepPATH/HEAD/DeepPATH_code/02_testing/multiClasses/inception/slim/__pycache__/inception_model.cpython-35.pyc -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/sb_compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --partition=cpu_dev,gpu4_dev,gpu8_dev 3 | #SBATCH --job-name=Compile 4 | #SBATCH --ntasks=1 5 | #SBATCH --output=rq_compile_%A_%a.out 6 | #SBATCH --error=rq_compile_%A_%a.err 7 | #SBATCH --mem=50G 8 | # #SBATCH --gres=gpu:1 9 | #SBATCH --time=00:05:00 10 | module load python/gpu/3.6.5 11 | module load bazel/0.15.2 12 | 13 | bazel build inception/imagenet_train 14 | 15 | -------------------------------------------------------------------------------- /DeepPATH_code/example_Melanoma_MLR/README.md: -------------------------------------------------------------------------------- 1 | Code for Multivariable logistic regression. 2 | 3 | Developed with R 3.6.1 (Authors: P.Johannet, Y.Xia, J.Zhong). 4 | 5 | See annotations in the code for usage. 6 | 7 | For reference, please see the following paper(s): 8 | 9 | "Johannet, Paul, et al. "Using machine learning algorithms to predict immunotherapy response in patients with advanced melanoma." Clinical Cancer Research 27.1 (2021): 131-140." 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | The DeepPATH framework gathers the codes that have been used to study the use of a deep learning architecture (inception v3 from Google) to classify Lung cancer images. 2 | 3 | For more details and references, please check: 4 | 5 | Nicolas Coudray, Paolo Santiago Ocampo, Theodore Sakellaropoulos, Navneet Narula, Matija Snuderl, David Fenyö, Andre L. Moreira, Narges Razavian, Aristotelis Tsirigos. Classification and mutation prediction from non–small cell lung cancer histopathology images using deep learning. Nature Medicine, 2018; DOI: 10.1038/s41591-018-0177-5 6 | 7 | https://www.nature.com/articles/s41591-018-0177-5 8 | 9 | https://www.biorxiv.org/content/early/2017/10/03/197574 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/slim/slim.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """TF-Slim grouped API. Please see README.md for details and usage.""" 16 | # pylint: disable=unused-import 17 | 18 | # Collapse tf-slim into a single namespace. 19 | from inception.slim import inception_model as inception 20 | from inception.slim import losses 21 | from inception.slim import ops 22 | from inception.slim import scopes 23 | from inception.slim import variables 24 | from inception.slim.scopes import arg_scope 25 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/slim.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """TF-Slim grouped API. Please see README.md for details and usage.""" 16 | # pylint: disable=unused-import 17 | 18 | # Collapse tf-slim into a single namespace. 19 | from inception.slim import inception_model as inception 20 | from inception.slim import losses 21 | from inception.slim import ops 22 | from inception.slim import scopes 23 | from inception.slim import variables 24 | from inception.slim.scopes import arg_scope 25 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/slim.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """TF-Slim grouped API. Please see README.md for details and usage.""" 16 | # pylint: disable=unused-import 17 | 18 | # Collapse tf-slim into a single namespace. 19 | from inception.slim import inception_model as inception 20 | from inception.slim import losses 21 | from inception.slim import ops 22 | from inception.slim import scopes 23 | from inception.slim import variables 24 | from inception.slim.scopes import arg_scope 25 | -------------------------------------------------------------------------------- /DeepPATH_code/example_TCGA_lung/checkpoints/README.md: -------------------------------------------------------------------------------- 1 | The checkpoints different runs using 2168 WSI (Lung) from the TCGA are accessible at [https://genome.med.nyu.edu/public/tsirigoslab/DeepLearning/LungCancer_LUADLUSC_Mutations/]. If you use them, please cite the following paper: 2 | 3 | (Nicolas Coudray, Paolo Santiago Ocampo, Theodore Sakellaropoulos, Navneet Narula, Matija Snuderl, David Fenyö, Andre L. Moreira, Narges Razavian, Aristotelis Tsirigos. Classification and mutation prediction from non–small cell lung cancer histopathology images using deep learning. Nature Medicine, 2018; DOI: 10.1038/s41591-018-0177-5) 4 | [https://www.nature.com/articles/s41591-018-0177-5] 5 | 6 | 7 | `run1a_3D_classifier` was run using batch size of 400; checkpoints at 69k 8 | (AUC of validation for Normal/LUAD/LUSC: 0.9997/0.970/0.967; for test set: 0.991/0.949/0.942) 9 | 10 | `run1b_10way_MutationClassifier`: softmax classifier of gene mutations (selecting LUAD slides using checkpoints above) 11 | (Valid and test AUCs: EGFR (0.832/0.777); FAT1 (0.741/0.653); FAT4 (0.672/0.673); KEAP1 (0.864/0.699); KRAS (0.768/0.753); LRP1B (0.758/0.733); NF1 (0.738/0.619); SETBP1 (0.832/0.606); STK11 (0.837/0.828); TP53 (0.831/0.734)) 12 | 13 | `run2a_3D_classifier` was run with a batch size of 100 for 400k iterations (validation AUC peaking around 108k iterations). 14 | (AUC of validation for Normal/LUAD/LUSC: 0.9998/0.977/0.977; for test set: 0.990/9.968/0.953) 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/imagenet_train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A binary to train Inception on the ImageNet data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | import tensorflow as tf 24 | 25 | from inception import inception_train 26 | from inception.imagenet_data import ImagenetData 27 | 28 | FLAGS = tf.app.flags.FLAGS 29 | 30 | 31 | def main(_): 32 | dataset = ImagenetData(subset=FLAGS.subset) 33 | assert dataset.data_files() 34 | if tf.gfile.Exists(FLAGS.train_dir): 35 | tf.gfile.DeleteRecursively(FLAGS.train_dir) 36 | tf.gfile.MakeDirs(FLAGS.train_dir) 37 | inception_train.train(dataset) 38 | 39 | 40 | if __name__ == '__main__': 41 | tf.app.run() 42 | -------------------------------------------------------------------------------- /DeepPATH_code/00_preprocessing/0a_split_Omero_csv.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import argparse 3 | 4 | # Example to remove spaces for filenames, run firt: 5 | # for f in *csv; do mv -- "$f" "${f// /_}"; done 6 | 7 | 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('-i', '--input', 10 | type=str, 11 | help="cvs file", default="") 12 | 13 | args = parser.parse_args() 14 | 15 | print(args) 16 | csvdir = args.input 17 | # csvdir = "brain_20210111_pathROIs_test1.csv" 18 | 19 | 20 | f = open(csvdir, 'r') 21 | reader = csv.reader(f) 22 | headers = next(reader, None) 23 | xmlcontent = {} 24 | for ncol in headers: 25 | xmlcontent[ncol] = [] 26 | 27 | 28 | for nrow in reader: 29 | for h, r in zip(headers, nrow): 30 | xmlcontent[h].append(r) 31 | 32 | f.close() 33 | 34 | f = open(csvdir, 'r') 35 | Lines = f.readlines() 36 | f.close() 37 | 38 | 39 | ndict = {} 40 | uniqueIm = set(xmlcontent['image_name']) 41 | for kk in uniqueIm: 42 | kk2 = '['.join(kk.split('[')[:-1]) 43 | if kk2 not in ndict.keys(): 44 | ndict[kk2] = Lines[0] 45 | # print(uniqueIm) 46 | # print(ndict) 47 | 48 | for kk in range(len(xmlcontent['image_name'])): 49 | kk2 = xmlcontent['image_name'][kk] 50 | kk3 = '['.join(kk2.split('[')[:-1]) 51 | ndict[kk3] = ndict[kk3] + Lines[kk+1] 52 | 53 | 54 | for kk in uniqueIm: 55 | kk2 = '['.join(kk.split('[')[:-1]) 56 | fname = ' '.join(kk.split()[:-1]) 57 | fname = '.'.join(fname.split('.')[:-1]) 58 | f = open( fname + '.csv','w') 59 | f.write(ndict[kk2]) 60 | f.close() 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /DeepPATH_code/00_preprocessing/Get_Labstats_From_jpeg.py: -------------------------------------------------------------------------------- 1 | 2 | from PIL import Image, ImageDraw, ImageCms 3 | import numpy as np 4 | import sys 5 | import getopt 6 | from skimage import color, io 7 | 8 | def RGB_to_lab(tile): 9 | # srgb_p = ImageCms.createProfile("sRGB") 10 | # lab_p = ImageCms.createProfile("LAB") 11 | # rgb2lab = ImageCms.buildTransformFromOpenProfiles(srgb_p, lab_p, "RGB", "LAB") 12 | # Lab = ImageCms.applyTransform(tile, rgb2lab) 13 | # Lab = np.array(Lab) 14 | # Lab = Lab.astype('float') 15 | # Lab[:,:,0] = Lab[:,:,0] / 2.55 16 | # Lab[:,:,1] = Lab[:,:,1] - 128 17 | # Lab[:,:,2] = Lab[:,:,2] - 128 18 | Lab = color.rgb2lab(tile) 19 | return Lab 20 | 21 | 22 | 23 | def main(argv): 24 | try: 25 | opts, args = getopt.getopt(argv,"hi:o:",["ifile="]) 26 | except getopt.GetoptError: 27 | print("test.py -i \") 28 | sys.exit(2) 29 | for opt, arg in opts: 30 | if opt == '-h': 31 | print('test.py -i inputfile') 32 | sys.exit() 33 | elif opt in ("-i", "--ifile"): 34 | inputfile = arg 35 | try: 36 | tile = Image.open(inputfile) 37 | Lab = RGB_to_lab(tile) 38 | print("LAB values:") 39 | for i in range(3): 40 | print("Channel " + str(i) + " has mean of " + str(np.mean(Lab[:,:,i])) + " and std of " + str(np.std(Lab[:,:,i]))) 41 | except Exception as e: 42 | print(e) 43 | 44 | if __name__== "__main__": 45 | print(sys.argv[1:]) 46 | main(sys.argv[1:]) 47 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/imagenet_eval.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A binary to evaluate Inception on the flowers data set. 16 | 17 | Note that using the supplied pre-trained inception checkpoint, the eval should 18 | achieve: 19 | precision @ 1 = 0.7874 recall @ 5 = 0.9436 [50000 examples] 20 | 21 | See the README.md for more details. 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | 28 | import tensorflow as tf 29 | 30 | from inception import inception_eval 31 | from inception.imagenet_data import ImagenetData 32 | 33 | FLAGS = tf.app.flags.FLAGS 34 | 35 | 36 | def main(unused_argv=None): 37 | dataset = ImagenetData(subset=FLAGS.subset) 38 | assert dataset.data_files() 39 | if tf.gfile.Exists(FLAGS.eval_dir): 40 | tf.gfile.DeleteRecursively(FLAGS.eval_dir) 41 | tf.gfile.MakeDirs(FLAGS.eval_dir) 42 | inception_eval.evaluate(dataset) 43 | 44 | 45 | if __name__ == '__main__': 46 | tf.app.run() 47 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/slim/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Contains the operations and nets for building TensorFlow-Slim models. 3 | 4 | package(default_visibility = ["//inception:internal"]) 5 | 6 | licenses(["notice"]) # Apache 2.0 7 | 8 | exports_files(["LICENSE"]) 9 | 10 | py_library( 11 | name = "scopes", 12 | srcs = ["scopes.py"], 13 | ) 14 | 15 | py_test( 16 | name = "scopes_test", 17 | size = "small", 18 | srcs = ["scopes_test.py"], 19 | deps = [ 20 | ":scopes", 21 | ], 22 | ) 23 | 24 | py_library( 25 | name = "variables", 26 | srcs = ["variables.py"], 27 | deps = [ 28 | ":scopes", 29 | ], 30 | ) 31 | 32 | py_test( 33 | name = "variables_test", 34 | size = "small", 35 | srcs = ["variables_test.py"], 36 | deps = [ 37 | ":variables", 38 | ], 39 | ) 40 | 41 | py_library( 42 | name = "losses", 43 | srcs = ["losses.py"], 44 | ) 45 | 46 | py_test( 47 | name = "losses_test", 48 | size = "small", 49 | srcs = ["losses_test.py"], 50 | deps = [ 51 | ":losses", 52 | ], 53 | ) 54 | 55 | py_library( 56 | name = "ops", 57 | srcs = ["ops.py"], 58 | deps = [ 59 | ":losses", 60 | ":scopes", 61 | ":variables", 62 | ], 63 | ) 64 | 65 | py_test( 66 | name = "ops_test", 67 | size = "small", 68 | srcs = ["ops_test.py"], 69 | deps = [ 70 | ":ops", 71 | ":variables", 72 | ], 73 | ) 74 | 75 | py_library( 76 | name = "inception", 77 | srcs = ["inception_model.py"], 78 | deps = [ 79 | ":ops", 80 | ":scopes", 81 | ], 82 | ) 83 | 84 | py_test( 85 | name = "inception_test", 86 | size = "medium", 87 | srcs = ["inception_test.py"], 88 | deps = [ 89 | ":inception", 90 | ], 91 | ) 92 | 93 | py_library( 94 | name = "slim", 95 | srcs = ["slim.py"], 96 | deps = [ 97 | ":inception", 98 | ":losses", 99 | ":ops", 100 | ":scopes", 101 | ":variables", 102 | ], 103 | ) 104 | 105 | py_test( 106 | name = "collections_test", 107 | size = "small", 108 | srcs = ["collections_test.py"], 109 | deps = [ 110 | ":slim", 111 | ], 112 | ) 113 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/slim/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Contains the operations and nets for building TensorFlow-Slim models. 3 | 4 | package(default_visibility = ["//inception:internal"]) 5 | 6 | licenses(["notice"]) # Apache 2.0 7 | 8 | exports_files(["LICENSE"]) 9 | 10 | py_library( 11 | name = "scopes", 12 | srcs = ["scopes.py"], 13 | ) 14 | 15 | py_test( 16 | name = "scopes_test", 17 | size = "small", 18 | srcs = ["scopes_test.py"], 19 | deps = [ 20 | ":scopes", 21 | ], 22 | ) 23 | 24 | py_library( 25 | name = "variables", 26 | srcs = ["variables.py"], 27 | deps = [ 28 | ":scopes", 29 | ], 30 | ) 31 | 32 | py_test( 33 | name = "variables_test", 34 | size = "small", 35 | srcs = ["variables_test.py"], 36 | deps = [ 37 | ":variables", 38 | ], 39 | ) 40 | 41 | py_library( 42 | name = "losses", 43 | srcs = ["losses.py"], 44 | ) 45 | 46 | py_test( 47 | name = "losses_test", 48 | size = "small", 49 | srcs = ["losses_test.py"], 50 | deps = [ 51 | ":losses", 52 | ], 53 | ) 54 | 55 | py_library( 56 | name = "ops", 57 | srcs = ["ops.py"], 58 | deps = [ 59 | ":losses", 60 | ":scopes", 61 | ":variables", 62 | ], 63 | ) 64 | 65 | py_test( 66 | name = "ops_test", 67 | size = "small", 68 | srcs = ["ops_test.py"], 69 | deps = [ 70 | ":ops", 71 | ":variables", 72 | ], 73 | ) 74 | 75 | py_library( 76 | name = "inception", 77 | srcs = ["inception_model.py"], 78 | deps = [ 79 | ":ops", 80 | ":scopes", 81 | ], 82 | ) 83 | 84 | py_test( 85 | name = "inception_test", 86 | size = "medium", 87 | srcs = ["inception_test.py"], 88 | deps = [ 89 | ":inception", 90 | ], 91 | ) 92 | 93 | py_library( 94 | name = "slim", 95 | srcs = ["slim.py"], 96 | deps = [ 97 | ":inception", 98 | ":losses", 99 | ":ops", 100 | ":scopes", 101 | ":variables", 102 | ], 103 | ) 104 | 105 | py_test( 106 | name = "collections_test", 107 | size = "small", 108 | srcs = ["collections_test.py"], 109 | deps = [ 110 | ":slim", 111 | ], 112 | ) 113 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Contains the operations and nets for building TensorFlow-Slim models. 3 | 4 | package(default_visibility = ["//inception:internal"]) 5 | 6 | licenses(["notice"]) # Apache 2.0 7 | 8 | exports_files(["LICENSE"]) 9 | 10 | py_library( 11 | name = "scopes", 12 | srcs = ["scopes.py"], 13 | ) 14 | 15 | py_test( 16 | name = "scopes_test", 17 | size = "small", 18 | srcs = ["scopes_test.py"], 19 | deps = [ 20 | ":scopes", 21 | ], 22 | ) 23 | 24 | py_library( 25 | name = "variables", 26 | srcs = ["variables.py"], 27 | deps = [ 28 | ":scopes", 29 | ], 30 | ) 31 | 32 | py_test( 33 | name = "variables_test", 34 | size = "small", 35 | srcs = ["variables_test.py"], 36 | deps = [ 37 | ":variables", 38 | ], 39 | ) 40 | 41 | py_library( 42 | name = "losses", 43 | srcs = ["losses.py"], 44 | ) 45 | 46 | py_test( 47 | name = "losses_test", 48 | size = "small", 49 | srcs = ["losses_test.py"], 50 | deps = [ 51 | ":losses", 52 | ], 53 | ) 54 | 55 | py_library( 56 | name = "ops", 57 | srcs = ["ops.py"], 58 | deps = [ 59 | ":losses", 60 | ":scopes", 61 | ":variables", 62 | ], 63 | ) 64 | 65 | py_test( 66 | name = "ops_test", 67 | size = "small", 68 | srcs = ["ops_test.py"], 69 | deps = [ 70 | ":ops", 71 | ":variables", 72 | ], 73 | ) 74 | 75 | py_library( 76 | name = "inception", 77 | srcs = ["inception_model.py"], 78 | deps = [ 79 | ":ops", 80 | ":scopes", 81 | ], 82 | ) 83 | 84 | py_test( 85 | name = "inception_test", 86 | size = "medium", 87 | srcs = ["inception_test.py"], 88 | deps = [ 89 | ":inception", 90 | ], 91 | ) 92 | 93 | py_library( 94 | name = "slim", 95 | srcs = ["slim.py"], 96 | deps = [ 97 | ":inception", 98 | ":losses", 99 | ":ops", 100 | ":scopes", 101 | ":variables", 102 | ], 103 | ) 104 | 105 | py_test( 106 | name = "collections_test", 107 | size = "small", 108 | srcs = ["collections_test.py"], 109 | deps = [ 110 | ":slim", 111 | ], 112 | ) 113 | -------------------------------------------------------------------------------- /DeepPATH_code/03_postprocessing/0e_Get_svs_PixelSizeMag.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os.path 3 | from openslide import open_slide, ImageSlide 4 | import sys 5 | 6 | FLAGS = None 7 | 8 | def main(): 9 | slide = open_slide(FLAGS.WSI) 10 | # calculate the best window size before rescaling to reach desired final pizelsize 11 | try: 12 | Objective = float(slide.properties[openslide.PROPERTY_NAME_OBJECTIVE_POWER]) 13 | OrgPixelSizeX = float(slide.properties['openslide.mpp-x']) 14 | OrgPixelSizeY = float(slide.properties['openslide.mpp-y']) 15 | except: 16 | try: 17 | for nfields in slide.properties['tiff.ImageDescription'].split('|'): 18 | if 'AppMag' in nfields: 19 | Objective = float(nfields.split(' = ')[1]) 20 | if 'MPP' in nfields: 21 | OrgPixelSizeX = float(nfields.split(' = ')[1]) 22 | OrgPixelSizeY = OrgPixelSizeX 23 | except: 24 | Objective = 1. 25 | OrgPixelSizeX = 0 26 | print("Error: No information found in the header") 27 | print(OrgPixelSizeX, Objective) 28 | if FLAGS.PxsMag == "PixelSize": 29 | sys.exit(OrgPixelSizeX) 30 | elif FLAGS.PxsMag == "Mag": 31 | sys.exit(Objective) 32 | 33 | if __name__ == '__main__': 34 | parser = argparse.ArgumentParser() 35 | # classify_image_graph_def.pb: 36 | # Binary representation of the GraphDef protocol buffer. 37 | # imagenet_synset_to_human_label_map.txt: 38 | # Map from synset ID to a human readable string. 39 | # imagenet_2012_challenge_label_map_proto.pbtxt: 40 | # Text representation of a protocol buffer mapping a label to synset ID. 41 | parser.add_argument( 42 | '--WSI', 43 | type=str, 44 | default='', 45 | help='original whole slide image.' 46 | ) 47 | parser.add_argument( 48 | '--PxsMag', 49 | type=str, 50 | default='Mag', 51 | help='set to Mag or PixelSize depending on what output you want.' 52 | ) 53 | 54 | FLAGS, unparsed = parser.parse_known_args() 55 | print(FLAGS) 56 | main() 57 | 58 | -------------------------------------------------------------------------------- /DeepPATH_code/example_Melanoma_MLR/Instructions_for_testing_an_independent_dataset.R: -------------------------------------------------------------------------------- 1 | ####load libraries 2 | library(tidyverse) 3 | library(dplyr) 4 | library(survival) 5 | library(survminer) 6 | library(ggpubr) 7 | library(pROC) 8 | library(ggplot2) 9 | 10 | ####test set 11 | ##spreadsheet can be excel or csv 12 | 13 | ##required columns: dcnn prediction, baseline ecog, treatment category, response code, pfs, time to pfs 14 | #dcnn prediction: prediction score is generated by logistic regression 15 | #baseline ecog 16 | #treatment category 17 | #response to treatment (can call column "response_code"): 0 = response and 1 = progression 18 | 19 | ##before uploading to R, create a new column for the logistic regression prediction using the formula below. call this column "logistic" 20 | #Logit(p)= -0.4970 + (2.0966*DCNN output) + (1.2522*ECOG) – (1.8908*Anti-CTLA-4 and Anti-PD-1 status) – (1.2013*Anti-PD-1 status). 21 | 22 | ##after creating this new column with corresponding logistic regression, import the import test set spreadsheet 23 | test 24 | 25 | ##specify labels of the ROC curve 26 | labels <- c("DCNN","DCNN + Clinical Data") 27 | 28 | ##generate and run test set roc curve 29 | test_roc <- roc(test$response_code, test$logistic) 30 | test_roc 31 | 32 | #set optimal cut point, which we determined for our model using the training dataset and closest to the top left point 33 | CUT=0.7395212 #spec: 0.8372093 #sen: 0.6410256 34 | 35 | ####risk stratification 36 | ##patients whose prediction is above the optimal cut point are considered high risk 37 | ##patients whose prediction is below the optimal cut point are considered low risk 38 | test$risk = ifelse(test$logistic>=CUT, 'High', 'Low') 39 | 40 | ####create km curves 41 | test_fit <- survfit(Surv(time_to_pfs, pfs)~risk, data=test) 42 | km_test <-ggsurvplot(test_fit, data = test, 43 | title="DCNN + Clinical Data", 44 | pval = T, pval.coord = c(0,0.03), 45 | risk.table = F, 46 | legend.labs = c("High Risk", "Low Risk"), 47 | conf.int = F, 48 | xlim=c(0,500), 49 | xlab = c('Time (Days)'), 50 | ylab = c('PFS Probability'), 51 | font.legend = list(size=20), 52 | font.x=c(20), 53 | font.y=c(20), 54 | font.tickslab=c(15), 55 | pval.size = 6, 56 | font.title=c(25)) 57 | km_test -------------------------------------------------------------------------------- /DeepPATH_code/03_postprocessing/sb_valid_example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --partition=fn_short,gpu4_dev,gpu8_dev 3 | #SBATCH --job-name=ts_ML101_valid 4 | #SBATCH --ntasks=1 5 | #SBATCH --output=rq_ts_ML101_valid__%A_%a.out 6 | #SBATCH --error=rq_ts_ML101_valid_%A_%a.err 7 | #SBATCH --mem=10G 8 | 9 | module load python/gpu/3.6.5 10 | 11 | export CHECKPOINT_PATH='/path_to_checkpoints/train/ML101/resultsML101/' 12 | export OUTPUT_DIR='/path_to_save_outputs/valid_test/ML101/valid' 13 | export DATA_DIR='/path_to_TFRecord_files/preprocess/ML101_TFRecord_valid/' 14 | export BASENAME='valid_' 15 | export LABEL_FILE='labels.txt' 16 | export ExpName='ML101_valid_' 17 | export STAT_FILE_FILTER='FALSE' 18 | declare -i RefLabel=0 19 | # To run the validation only on a subset of tiles filtered according to a previous "out_filename_Stats.txt" file with a given class: 20 | # export STAT_FILE_FILTER='/gpfs/scratch/coudrn01/NN_test/Melanoma_TMB/validtest/4020_both_xml_Normalized_10x/segment_P086/test_33000k/out_filename_Stats.txt' 21 | # declare -i RefLabel=1 22 | 23 | # check if next checkpoint available 24 | declare -i count=2500 25 | declare -i step=2500 26 | declare -i NbClasses=2 27 | declare -i PatientID=10 28 | 29 | 30 | while true; do 31 | echo $count 32 | if [ -f $CHECKPOINT_PATH/model.ckpt-$count.meta ]; then 33 | echo $CHECKPOINT_PATH/model.ckpt-$count.meta " exists" 34 | export TEST_OUTPUT=$OUTPUT_DIR/test_$count'k' 35 | if [ ! -d $TEST_OUTPUT ]; then 36 | mkdir -p $TEST_OUTPUT 37 | # create temporary directory for checkpoints 38 | mkdir -p $TEST_OUTPUT/tmp_checkpoints 39 | export CUR_CHECKPOINT=$TEST_OUTPUT/tmp_checkpoints 40 | 41 | 42 | ln -s $CHECKPOINT_PATH/*-$count.* $CUR_CHECKPOINT/. 43 | touch $CUR_CHECKPOINT/checkpoint 44 | echo 'model_checkpoint_path: "'$CUR_CHECKPOINT'/model.ckpt-'$count'"' > $CUR_CHECKPOINT/checkpoint 45 | echo 'all_model_checkpoint_paths: "'$CUR_CHECKPOINT'/model.ckpt-'$count'"' >> $CUR_CHECKPOINT/checkpoint 46 | 47 | export OUTFILENAME=$TEST_OUTPUT/out_filename_Stats.txt 48 | 49 | sbatch --job-name=$ExpName$BASENAME$count --output=rq_$ExpName$BASENAME$count_%A.out --error=rq_$ExpName$BASENAME$count_%A.err sb_TF_ROC_2.sh $TEST_OUTPUT $DATA_DIR $BASENAME $NbClasses $OUTFILENAME $LABEL_FILE $CUR_CHECKPOINT $PatientID $STAT_FILE_FILTER $RefLabel 50 | 51 | else 52 | echo 'checkpoint '$TEST_OUTPUT' skipped' 53 | fi 54 | 55 | else 56 | echo $CHECKPOINT_PATH/model.ckpt-$count.meta " does not exist" 57 | break 58 | fi 59 | 60 | # next checkpoint 61 | count=`expr "$count" + "$step"` 62 | done 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/nc_imagenet_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to the ImageNet data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | from inception.nc_dataset import Dataset 24 | 25 | 26 | class ImagenetData(Dataset): 27 | """ImageNet data set.""" 28 | 29 | def __init__(self, subset): 30 | super(ImagenetData, self).__init__('ImageNet', subset) 31 | 32 | def num_classes(self): 33 | """Returns the number of classes in the data set.""" 34 | return 10 35 | 36 | def num_examples_per_epoch(self): 37 | """Returns the number of examples in the data set.""" 38 | # Bounding box data consists of 615299 bounding boxes for 544546 images. 39 | if self.subset == 'train': 40 | return 223185 41 | if self.subset == 'validation': 42 | return 121094 43 | 44 | def download_message(self): 45 | """Instruction to download and extract the tarball from Flowers website.""" 46 | 47 | print('Failed to find any ImageNet %s files'% self.subset) 48 | print('') 49 | print('If you have already downloaded and processed the data, then make ' 50 | 'sure to set --data_dir to point to the directory containing the ' 51 | 'location of the sharded TFRecords.\n') 52 | print('If you have not downloaded and prepared the ImageNet data in the ' 53 | 'TFRecord format, you will need to do this at least once. This ' 54 | 'process could take several hours depending on the speed of your ' 55 | 'computer and network connection\n') 56 | print('Please see README.md for instructions on how to build ' 57 | 'the ImageNet dataset using download_and_preprocess_imagenet.\n') 58 | print('Note that the raw data size is 300 GB and the processed data size ' 59 | 'is 150 GB. Please ensure you have at least 500GB disk space.') 60 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/nc_imagenet_data.py: -------------------------------------------------------------------------------- 1 | # Code slightly modified by ABL group, NYU from: 2 | 3 | # Copyright 2016 Google Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ============================================================================== 17 | """Small library that points to the ImageNet data set. 18 | """ 19 | from __future__ import absolute_import 20 | from __future__ import division 21 | from __future__ import print_function 22 | 23 | 24 | 25 | from inception.nc_dataset import Dataset 26 | 27 | 28 | import tensorflow as tf 29 | 30 | FLAGS = tf.app.flags.FLAGS 31 | 32 | tf.app.flags.DEFINE_integer('ClassNumber', 5, 33 | """Number of classes.""") 34 | 35 | 36 | 37 | 38 | class ImagenetData(Dataset): 39 | """ImageNet data set.""" 40 | 41 | def __init__(self, subset): 42 | super(ImagenetData, self).__init__('ImageNet', subset) 43 | 44 | def num_classes(self): 45 | """Returns the number of classes in the data set.""" 46 | return FLAGS.ClassNumber 47 | 48 | def num_examples_per_epoch(self): 49 | """Returns the number of examples in the data set.""" 50 | # Bounding box data consists of 615299 bounding boxes for 544546 images. 51 | if self.subset == 'train': 52 | return 688634 53 | if self.subset == 'valid': 54 | return 149835 55 | 56 | def download_message(self): 57 | """Instruction to download and extract the tarball from Flowers website.""" 58 | 59 | print('Failed to find any ImageNet %s files'% self.subset) 60 | print('') 61 | print('If you have already downloaded and processed the data, then make ' 62 | 'sure to set --data_dir to point to the directory containing the ' 63 | 'location of the sharded TFRecords.\n') 64 | print('If you have not downloaded and prepared the ImageNet data in the ' 65 | 'TFRecord format, you will need to do this at least once. This ' 66 | 'process could take several hours depending on the speed of your ' 67 | 'computer and network connection\n') 68 | print('Please see README.md for instructions on how to build ' 69 | 'the ImageNet dataset using download_and_preprocess_imagenet.\n') 70 | print('Note that the raw data size is 300 GB and the processed data size ' 71 | 'is 150 GB. Please ensure you have at least 500GB disk space.') 72 | -------------------------------------------------------------------------------- /DeepPATH_code/03_postprocessing/sb_TF_ROC_2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## #SBATCH --partition=gpu4_medium,gpu4_long,gpu8_long,gpu8_medium 3 | #SBATCH --partition=gpu4_short,gpu8_short,gpu4_medium,gpu4_long,gpu8_long,gpu8_medium 4 | #SBATCH --exclude=gpu-0007 5 | #SBATCH --job-name=F001_valid 6 | #SBATCH --ntasks=4 7 | # #SBATCH --output=rq_F001_v_%A_%a.out 8 | # #SBATCH --error=rq_F001_v_%A_%a.err 9 | #SBATCH --mem=40G 10 | #SBATCH --gres=gpu:2 11 | 12 | module unload python/gpu/3.6.5 13 | module load anaconda3/gpu/5.2.0 14 | if [[ $CONDA_SHLVL == 1 ]]; then conda deactivate; fi 15 | conda activate /gpfs/data/coudraylab/NN/env/env_deepPath 16 | unset PYTHONPATH 17 | 18 | TEST_OUTPUT=$1 19 | DATA_DIR=$2 20 | BASENAME=$3 21 | NbClasses=$4 22 | OUTFILENAME=$5 23 | LABEL_FILE=$6 24 | CUR_CHECKPOINT=$7 25 | PatientID=$8 26 | STAT_FILE_FILTER=$9 27 | RefLabel=${10} 28 | 29 | echo $CUR_CHECKPOINT 30 | echo $TEST_OUTPUT 31 | echo $DATA_DIR 32 | echo $BASENAME 33 | echo $NbClasses 34 | 35 | # Test 36 | python /gpfs/data/coudraylab/NN/github/DeepPATH_code/02_testing/xClasses/nc_imagenet_eval.py --checkpoint_dir=$CUR_CHECKPOINT --eval_dir=$TEST_OUTPUT --data_dir=$DATA_DIR --batch_size 300 --run_once --ImageSet_basename=$BASENAME --ClassNumber $NbClasses --mode='0_softmax' --TVmode='test' 37 | 38 | 39 | echo " ROC" 40 | echo $OUTFILENAME 41 | echo $TEST_OUTPUT 42 | echo $LABEL_FILE 43 | echo $PatientID 44 | # ROC 45 | export OUTFILENAME=$TEST_OUTPUT/out_filename_Stats.txt 46 | export OUTFILENAMEUNIQ=$TEST_OUTPUT/out_filename_Stats_unique.txt 47 | 48 | sort -u $OUTFILENAME > $OUTFILENAMEUNIQ 49 | 50 | 51 | 52 | conda deactivate 53 | conda activate /gpfs/data/coudraylab/NN/env/env_deepPath_4_376 54 | 55 | if [[ $RefLabel -eq "FALSE" ]] 56 | then 57 | python /gpfs/data/coudraylab/NN/github/DeepPATH_code/03_postprocessing/0h_ROC_MultiOutput_BootStrap_2.py --file_stats=$OUTFILENAMEUNIQ --output_dir=$TEST_OUTPUT --labels_names=$LABEL_FILE --PatientID=$PatientID --color="black,red,blue,blueviolet,yellow,darkolivegreen,green,cyan,royalblue,fuchsia" 58 | # --color="black,darkred,red,orange,gold,yellow,darkolivegreen,green,cyan,royalblue,blueviolet,fuchsia" 59 | else 60 | python /gpfs/data/coudraylab/NN/github/DeepPATH_code/03_postprocessing/0h_ROC_MultiOutput_BootStrap_2.py --file_stats=$OUTFILENAMEUNIQ --output_dir=$TEST_OUTPUT --labels_names=$LABEL_FILE --PatientID=$PatientID --ref_file=$STAT_FILE_FILTER --ref_label=$RefLabel --ref_thresh=-2 61 | fi 62 | 63 | # THRESH=`ls $TEST_OUTPUT | grep out1_roc_data_AvP | sed -e 's/_/ /g' | sed -e 's/.txt/ /g' | awk '{print $NF}' | grep t | sed -e 's/t/ /g'` 64 | THRESH=`ls $TEST_OUTPUT | grep out1_roc_data_AvP | sed -e 's/_/ /g' | sed -e 's/.txt/ /g' | awk '{print $(NF-1)}' | grep J | sed -e 's/J/ /g'` 65 | THRESH=`echo $THRESH | sed -e 's/ /,/g'` 66 | echo $THRESH 67 | 68 | python /gpfs/data/coudraylab/NN/github/DeepPATH_code/03_postprocessing/0i_Sensitivity_Specificity.py --threshold=$THRESH --labelFile $LABEL_FILE --PatientID $PatientID --files_stats $OUTFILENAMEUNIQ --outputPath=$TEST_OUTPUT 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/imagenet_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to the ImageNet data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | from inception.dataset import Dataset 24 | import tensorflow as tf 25 | 26 | FLAGS = tf.app.flags.FLAGS 27 | 28 | tf.app.flags.DEFINE_integer('ClassNumber', 5, 29 | """Number of classes.""") 30 | 31 | tf.app.flags.DEFINE_integer('NbrOfImages', 400000, 32 | """Number of images per epoch.""") 33 | 34 | class ImagenetData(Dataset): 35 | """ImageNet data set.""" 36 | 37 | def __init__(self, subset): 38 | super(ImagenetData, self).__init__('ImageNet', subset) 39 | 40 | def num_classes(self): 41 | """Returns the number of classes in the data set.""" 42 | # return 1000 43 | return FLAGS.ClassNumber 44 | #LUAD, LUSC or Normal 45 | 46 | def num_examples_per_epoch(self): 47 | """Returns the number of examples in the data set.""" 48 | # Bounding box data consists of 615299 bounding boxes for 544546 images. 49 | if self.subset == 'train': 50 | # return 1281167 51 | return FLAGS.NbrOfImages 52 | if self.subset == 'validation': 53 | #return 50000 54 | return 149835 55 | 56 | def download_message(self): 57 | """Instruction to download and extract the tarball from Flowers website.""" 58 | 59 | print('Failed to find any ImageNet %s files'% self.subset) 60 | print('') 61 | print('If you have already downloaded and processed the data, then make ' 62 | 'sure to set --data_dir to point to the directory containing the ' 63 | 'location of the sharded TFRecords.\n') 64 | print('If you have not downloaded and prepared the ImageNet data in the ' 65 | 'TFRecord format, you will need to do this at least once. This ' 66 | 'process could take several hours depending on the speed of your ' 67 | 'computer and network connection\n') 68 | print('Please see README.md for instructions on how to build ' 69 | 'the ImageNet dataset using download_and_preprocess_imagenet.\n') 70 | print('Note that the raw data size is 300 GB and the processed data size ' 71 | 'is 150 GB. Please ensure you have at least 500GB disk space.') 72 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/nc_imagenet_eval.py: -------------------------------------------------------------------------------- 1 | """A binary to evaluate Inception on the Lung data set. 2 | Output generated: 3 | ** information for ROC curves with 2 aggregation methods (out_FPTPrate_PcTiles.txt, out_FPTPrate_ScoreTiles.txt) 4 | ** probability associated with each tile and info whether the max is a true positive or not (out_filename_Stats.txt) 5 | 6 | 7 | """ 8 | 9 | from __future__ import absolute_import 10 | from __future__ import division 11 | from __future__ import print_function 12 | 13 | import pickle 14 | import csv 15 | 16 | import tensorflow as tf 17 | 18 | from inception import nc_inception_eval 19 | from inception.nc_imagenet_data import ImagenetData 20 | import numpy as np 21 | #from inception import inception_eval 22 | #from inception.imagenet_data import ImagenetData 23 | import os 24 | 25 | FLAGS = tf.app.flags.FLAGS 26 | 27 | tf.app.flags.DEFINE_string('ImageSet_basename', 'test_', 28 | """Either 'test_', 'valid_' or 'train_'.""") 29 | 30 | tf.app.flags.DEFINE_integer('nbr_of_classes', 10, 31 | """Number of possible classes.""") 32 | 33 | tf.app.flags.DEFINE_string('labels_names', '/ifs/home/coudrn01/NN/Lung/Test_All512pxTiled/9_10mutations/label_names.txt', 34 | 'Names of the possible output labels ordered as desired') 35 | 36 | def main(unused_argv=None): 37 | 38 | input_path = os.path.join(FLAGS.data_dir, FLAGS.ImageSet_basename + '*') 39 | print(input_path) 40 | #FLAGS.batch_size = 30 41 | data_files = tf.gfile.Glob(input_path) 42 | print(data_files) 43 | 44 | mydict={} 45 | count_slides = 0 46 | 47 | #with open(FLAGS.labels_names, "r") as f: 48 | # for line in f: 49 | # line = line.replace('\r','\n') 50 | # line = line.split('\n') 51 | # for eachline in line: 52 | # if len(eachline)>0: 53 | # unique_labels.append(eachline) 54 | if "test" in FLAGS.ImageSet_basename: 55 | for next_slide in data_files: 56 | print("New Slide ------------ %d" % (count_slides)) 57 | labelindex = int(next_slide.split('_')[-1].split('.')[0]) 58 | #if labelindex == 1: 59 | # labelname = 'normal' 60 | #elif labelindex == 2: 61 | # labelname = 'luad' 62 | #elif labelindex == 3: 63 | # labelname = 'lusc' 64 | #else: 65 | # labelname = 'error_label_name' 66 | # labelname = unique_labels[labelindex] 67 | 68 | # print("label %d: %s" % (labelindex, labelname)) 69 | 70 | FLAGS.data_dir = next_slide 71 | dataset = ImagenetData(subset=FLAGS.subset) 72 | assert dataset.data_files() 73 | #if tf.gfile.Exists(FLAGS.eval_dir): 74 | # tf.gfile.DeleteRecursively(FLAGS.eval_dir) 75 | #tf.gfile.MakeDirs(FLAGS.eval_dir) 76 | precision_at_1, current_score = nc_inception_eval.evaluate(dataset) 77 | elif "valid" in FLAGS.ImageSet_basename: 78 | #FLAGS.data_dir = FLAGS.data_dir + "/valid*" 79 | dataset = ImagenetData(subset=FLAGS.subset) 80 | assert dataset.data_files() 81 | nc_inception_eval.evaluate(dataset) 82 | 83 | 84 | 85 | if __name__ == '__main__': 86 | tf.app.run() 87 | -------------------------------------------------------------------------------- /DeepPATH_code/00_preprocessing/Get_pixel_size_dist.py: -------------------------------------------------------------------------------- 1 | """ 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2017, Nicolas Coudray and Aristotelis Tsirigos (NYU) 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 11 | """ 12 | 13 | 14 | import openslide 15 | from openslide import open_slide, ImageSlide 16 | import os 17 | from glob import glob 18 | from openslide.deepzoom import DeepZoomGenerator 19 | import sys 20 | 21 | #slidepath = "/gpfs/data/abl/deepomics/lung_cancer/TCGA_diagnostic_ffpe/Raw_Diagnostic/*/*svs" 22 | # Mag = 20 23 | 24 | def main(argv): 25 | slidepath = sys.argv[1] 26 | print(slidepath) 27 | Mag = float(sys.argv[2]) 28 | print(Mag) 29 | 30 | #try: 31 | # opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="]) 32 | #except getopt.GetoptError: 33 | # print 'Get_pixel_size_dist.py -i -m ' 34 | # sys.exit(2) 35 | 36 | 37 | files = glob(slidepath) 38 | files = sorted(files) 39 | 40 | with open('pixelsizes_at_' + str(Mag) + 'x.txt','w+') as f: 41 | for imgNb in range(len(files)): 42 | filename = files[imgNb] 43 | basenameJPG = os.path.splitext(os.path.basename(filename))[0].split('.')[0] 44 | _slide = open_slide(filename) 45 | dz = DeepZoomGenerator(_slide) 46 | try: 47 | OrgPixelSizeX = float(_slide.properties['openslide.mpp-x']) 48 | OrgPixelSizeY = float(_slide.properties['openslide.mpp-y']) 49 | except: 50 | f.write('%s\t%f\tunknown\tunknown\n' %(basenameJPG,ThisMag) ) 51 | continue 52 | Factors = _slide.level_downsamples 53 | Objective = float(_slide.properties[openslide.PROPERTY_NAME_OBJECTIVE_POWER]) 54 | Available = tuple(Objective / x for x in Factors) 55 | for level in range(dz.level_count-1,-1,-1): 56 | ThisMag = Available[0]/pow(2,dz.level_count-(level+1)) 57 | if ThisMag != Mag: 58 | continue 59 | else: 60 | print("%s\t%f\t%f\t%f\n" %(basenameJPG,ThisMag, OrgPixelSizeX* pow(2,dz.level_count-(level+1)), OrgPixelSizeY* pow(2,dz.level_count-(level+1)))) 61 | f.write('%s\t%f\t%f\t%f\n' %(basenameJPG,ThisMag, OrgPixelSizeX* pow(2,dz.level_count-(level+1)), OrgPixelSizeY* pow(2,dz.level_count-(level+1)))) 62 | if __name__== "__main__": 63 | main(sys.argv[1:]) 64 | 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/data/preprocess_imagenet_validation_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | """Process the ImageNet Challenge bounding boxes for TensorFlow model training. 17 | 18 | Associate the ImageNet 2012 Challenge validation data set with labels. 19 | 20 | The raw ImageNet validation data set is expected to reside in JPEG files 21 | located in the following directory structure. 22 | 23 | data_dir/ILSVRC2012_val_00000001.JPEG 24 | data_dir/ILSVRC2012_val_00000002.JPEG 25 | ... 26 | data_dir/ILSVRC2012_val_00050000.JPEG 27 | 28 | This script moves the files into a directory structure like such: 29 | data_dir/n01440764/ILSVRC2012_val_00000293.JPEG 30 | data_dir/n01440764/ILSVRC2012_val_00000543.JPEG 31 | ... 32 | where 'n01440764' is the unique synset label associated with 33 | these images. 34 | 35 | This directory reorganization requires a mapping from validation image 36 | number (i.e. suffix of the original file) to the associated label. This 37 | is provided in the ImageNet development kit via a Matlab file. 38 | 39 | In order to make life easier and divorce ourselves from Matlab, we instead 40 | supply a custom text file that provides this mapping for us. 41 | 42 | Sample usage: 43 | ./preprocess_imagenet_validation_data.py ILSVRC2012_img_val \ 44 | imagenet_2012_validation_synset_labels.txt 45 | """ 46 | 47 | from __future__ import absolute_import 48 | from __future__ import division 49 | from __future__ import print_function 50 | 51 | import os 52 | import os.path 53 | import sys 54 | 55 | 56 | if __name__ == '__main__': 57 | if len(sys.argv) < 3: 58 | print('Invalid usage\n' 59 | 'usage: preprocess_imagenet_validation_data.py ' 60 | ' ') 61 | sys.exit(-1) 62 | data_dir = sys.argv[1] 63 | validation_labels_file = sys.argv[2] 64 | 65 | # Read in the 50000 synsets associated with the validation data set. 66 | labels = [l.strip() for l in open(validation_labels_file).readlines()] 67 | unique_labels = set(labels) 68 | 69 | # Make all sub-directories in the validation data dir. 70 | for label in unique_labels: 71 | labeled_data_dir = os.path.join(data_dir, label) 72 | os.makedirs(labeled_data_dir) 73 | 74 | # Move all of the image to the appropriate sub-directory. 75 | for i in range(len(labels)): 76 | basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1) 77 | original_filename = os.path.join(data_dir, basename) 78 | if not os.path.exists(original_filename): 79 | print('Failed to find: ' % original_filename) 80 | sys.exit(-1) 81 | new_filename = os.path.join(data_dir, labels[i], basename) 82 | os.rename(original_filename, new_filename) 83 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/data/preprocess_imagenet_validation_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | """Process the ImageNet Challenge bounding boxes for TensorFlow model training. 17 | 18 | Associate the ImageNet 2012 Challenge validation data set with labels. 19 | 20 | The raw ImageNet validation data set is expected to reside in JPEG files 21 | located in the following directory structure. 22 | 23 | data_dir/ILSVRC2012_val_00000001.JPEG 24 | data_dir/ILSVRC2012_val_00000002.JPEG 25 | ... 26 | data_dir/ILSVRC2012_val_00050000.JPEG 27 | 28 | This script moves the files into a directory structure like such: 29 | data_dir/n01440764/ILSVRC2012_val_00000293.JPEG 30 | data_dir/n01440764/ILSVRC2012_val_00000543.JPEG 31 | ... 32 | where 'n01440764' is the unique synset label associated with 33 | these images. 34 | 35 | This directory reorganization requires a mapping from validation image 36 | number (i.e. suffix of the original file) to the associated label. This 37 | is provided in the ImageNet development kit via a Matlab file. 38 | 39 | In order to make life easier and divorce ourselves from Matlab, we instead 40 | supply a custom text file that provides this mapping for us. 41 | 42 | Sample usage: 43 | ./preprocess_imagenet_validation_data.py ILSVRC2012_img_val \ 44 | imagenet_2012_validation_synset_labels.txt 45 | """ 46 | 47 | from __future__ import absolute_import 48 | from __future__ import division 49 | from __future__ import print_function 50 | 51 | import os 52 | import os.path 53 | import sys 54 | 55 | 56 | if __name__ == '__main__': 57 | if len(sys.argv) < 3: 58 | print('Invalid usage\n' 59 | 'usage: preprocess_imagenet_validation_data.py ' 60 | ' ') 61 | sys.exit(-1) 62 | data_dir = sys.argv[1] 63 | validation_labels_file = sys.argv[2] 64 | 65 | # Read in the 50000 synsets associated with the validation data set. 66 | labels = [l.strip() for l in open(validation_labels_file).readlines()] 67 | unique_labels = set(labels) 68 | 69 | # Make all sub-directories in the validation data dir. 70 | for label in unique_labels: 71 | labeled_data_dir = os.path.join(data_dir, label) 72 | os.makedirs(labeled_data_dir) 73 | 74 | # Move all of the image to the appropriate sub-directory. 75 | for i in range(len(labels)): 76 | basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1) 77 | original_filename = os.path.join(data_dir, basename) 78 | if not os.path.exists(original_filename): 79 | print('Failed to find: ' % original_filename) 80 | sys.exit(-1) 81 | new_filename = os.path.join(data_dir, labels[i], basename) 82 | os.rename(original_filename, new_filename) 83 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/data/preprocess_imagenet_validation_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | """Process the ImageNet Challenge bounding boxes for TensorFlow model training. 17 | 18 | Associate the ImageNet 2012 Challenge validation data set with labels. 19 | 20 | The raw ImageNet validation data set is expected to reside in JPEG files 21 | located in the following directory structure. 22 | 23 | data_dir/ILSVRC2012_val_00000001.JPEG 24 | data_dir/ILSVRC2012_val_00000002.JPEG 25 | ... 26 | data_dir/ILSVRC2012_val_00050000.JPEG 27 | 28 | This script moves the files into a directory structure like such: 29 | data_dir/n01440764/ILSVRC2012_val_00000293.JPEG 30 | data_dir/n01440764/ILSVRC2012_val_00000543.JPEG 31 | ... 32 | where 'n01440764' is the unique synset label associated with 33 | these images. 34 | 35 | This directory reorganization requires a mapping from validation image 36 | number (i.e. suffix of the original file) to the associated label. This 37 | is provided in the ImageNet development kit via a Matlab file. 38 | 39 | In order to make life easier and divorce ourselves from Matlab, we instead 40 | supply a custom text file that provides this mapping for us. 41 | 42 | Sample usage: 43 | ./preprocess_imagenet_validation_data.py ILSVRC2012_img_val \ 44 | imagenet_2012_validation_synset_labels.txt 45 | """ 46 | 47 | from __future__ import absolute_import 48 | from __future__ import division 49 | from __future__ import print_function 50 | 51 | import os 52 | import os.path 53 | import sys 54 | 55 | 56 | if __name__ == '__main__': 57 | if len(sys.argv) < 3: 58 | print('Invalid usage\n' 59 | 'usage: preprocess_imagenet_validation_data.py ' 60 | ' ') 61 | sys.exit(-1) 62 | data_dir = sys.argv[1] 63 | validation_labels_file = sys.argv[2] 64 | 65 | # Read in the 50000 synsets associated with the validation data set. 66 | labels = [l.strip() for l in open(validation_labels_file).readlines()] 67 | unique_labels = set(labels) 68 | 69 | # Make all sub-directories in the validation data dir. 70 | for label in unique_labels: 71 | labeled_data_dir = os.path.join(data_dir, label) 72 | os.makedirs(labeled_data_dir) 73 | 74 | # Move all of the image to the appropriate sub-directory. 75 | for i in range(len(labels)): 76 | basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1) 77 | original_filename = os.path.join(data_dir, basename) 78 | if not os.path.exists(original_filename): 79 | print('Failed to find: ' % original_filename) 80 | sys.exit(-1) 81 | new_filename = os.path.join(data_dir, labels[i], basename) 82 | os.rename(original_filename, new_filename) 83 | -------------------------------------------------------------------------------- /DeepPATH_code/example_Melanoma_MLR/Code_for_the_training_and_testing_process.R: -------------------------------------------------------------------------------- 1 | ####load libraries 2 | library(tidyverse) 3 | library(dplyr) 4 | library(survival) 5 | library(survminer) 6 | library(ggpubr) 7 | library(ggplot2) 8 | library(PredictABEL) 9 | library(pROC) 10 | 11 | ####impute 12 | impute<-function(x){ 13 | y<-x 14 | y[is.na(x)]<-sample(x[!is.na(x)], size=sum(is.na(x)), replace=T) 15 | return(y) 16 | } 17 | 18 | ####import training and test set data files 19 | ##file format 20 | #excel or csv 21 | #columns: dcnn prediction, baseline ecog, treatment category, response to treatment, pfs, time to pfs 22 | #file 1 = training dataset; file 2 = test dataset 23 | training 24 | test 25 | 26 | ####reformat variables 27 | ##training 28 | set.seed(1998) 29 | training$baseline_ecog.i<-as.numeric(as.character(training$baseline_ecog)) 30 | training$baseline_ecog.i<-impute(training$baseline_ecog.i) 31 | 32 | ####dcnn model 33 | ##response_code should is a binary (0=response; 1=no response) 34 | ##dcnn_prediction is generated by the neural network classifier 35 | model1<-glm(response_code~dcnn_prediction,data=training,family="binomial") 36 | 37 | ####multivaribale model 38 | model2<-glm(response_code~dcnn_prediction+baseline_ecog.i+treatment_category,data=training,family="binomial") 39 | 40 | #obtain predicted risks 41 | training_predRisk1 <- predRisk(model1) 42 | training_predRisk2 <- predRisk(model2) 43 | 44 | ####create new columns 45 | training$dcnn = training_predRisk1 46 | training$logistic = training_predRisk2 47 | 48 | #####set outcome 49 | trainingOutcome <- which(names(training)=="response_code") 50 | 51 | #specify labels of the ROC curve 52 | labels <- c("DCNN","DCNN + Clinical Data" ) 53 | 54 | #produce ROC curve 55 | plotROC(data=training, cOutcome=trainingOutcome, 56 | predrisk=cbind(training_predRisk1,training_predRisk2), labels=labels) 57 | 58 | ####validation on test dataset 59 | ####reformat variables 60 | set.seed(1998) 61 | test$baseline_ecog.i<-as.numeric(as.character(test$baseline_ecog)) 62 | test$baseline_ecog.i<-impute(test$baseline_ecog.i) 63 | 64 | #### 65 | test_predRisk1 <- predict(model1, newdata = test, type = 'response') 66 | test_predRisk2 <- predict(model2, newdata = test, type = 'response') 67 | 68 | ####create new columns 69 | test$dcnn = test_predRisk1 70 | test$logistic = test_predRisk2 71 | 72 | ####set outcome 73 | testOutcome <- which(names(test)=="response_code") 74 | 75 | ####plot roc 76 | plotROC(data=test, cOutcome=testOutcome, 77 | predrisk=cbind(test_predRisk1,test_predRisk2), labels=labels) 78 | 79 | ####use the optimal cut point from the training roc curve to stratify patients into low versus high risk for progression 80 | ##identify optimal cut point 81 | training_roc <- roc(training$response_code, training$logistic) 82 | coords(training_roc, x="best", input="threshold", best.method="closest.topleft") 83 | CUT=0.7395212 #spec: 0.8372093 #sen: 0.6410256 84 | 85 | ####generate km curves 86 | test$risk = ifelse(test$logistic>=CUT, 'High', 'Low') 87 | test_fit <- survfit(Surv(time_to_pfs, pfs)~risk, data=test) 88 | km_test <-ggsurvplot(test_fit, data = test, 89 | title="DCNN + Clinical Data", 90 | pval = T, pval.coord = c(0,0.03), 91 | risk.table = F, 92 | legend.labs = c("High Risk", "Low Risk"), 93 | conf.int = F, 94 | xlim=c(0,500), 95 | xlab = c('Time (Days)'), 96 | ylab = c('PFS Probability'), 97 | font.legend = list(size=20), 98 | font.x=c(20), 99 | font.y=c(20), 100 | font.tickslab=c(15), 101 | pval.size = 6, 102 | font.title=c(25)) 103 | km_test -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/data/download_and_preprocess_flowers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "Usage: download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | curl -o ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | shuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/data/download_and_preprocess_flowers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "Usage: download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | curl -o ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | shuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/data/download_and_preprocess_flowers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "Usage: download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | curl -o ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | shuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/data/download_and_preprocess_flowers_mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "Usage: download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | curl -o ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | gshuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/data/download_and_preprocess_flowers_mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "Usage: download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | curl -o ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | gshuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/data/download_and_preprocess_flowers_mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "Usage: download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | curl -o ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | gshuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to a data set. 16 | 17 | Methods of Data class: 18 | data_files: Returns a python list of all (sharded) data set files. 19 | num_examples_per_epoch: Returns the number of examples in the data set. 20 | num_classes: Returns the number of classes in the data set. 21 | reader: Return a reader for a single entry from the data set. 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | from abc import ABCMeta 28 | from abc import abstractmethod 29 | import os 30 | 31 | 32 | import tensorflow as tf 33 | 34 | FLAGS = tf.app.flags.FLAGS 35 | 36 | # Basic model parameters. 37 | #tf.app.flags.DEFINE_string('data_dir', '/tmp/mydata', 38 | # """Path to the processed data, i.e. """ 39 | # """TFRecord of Example protos.""") 40 | 41 | tf.app.flags.DEFINE_string('data_dir', '/ifs/home/coudrn01/NN/Lung/Test_All512pxTiled/6_Healthy_Cancer', 42 | """Path to the processed data, i.e. """ 43 | """TFRecord of Example protos.""") 44 | 45 | 46 | class Dataset(object): 47 | """A simple class for handling data sets.""" 48 | __metaclass__ = ABCMeta 49 | 50 | def __init__(self, name, subset): 51 | """Initialize dataset using a subset and the path to the data.""" 52 | assert subset in self.available_subsets(), self.available_subsets() 53 | self.name = name 54 | self.subset = subset 55 | 56 | @abstractmethod 57 | def num_classes(self): 58 | """Returns the number of classes in the data set.""" 59 | pass 60 | # return 10 61 | 62 | @abstractmethod 63 | def num_examples_per_epoch(self): 64 | """Returns the number of examples in the data subset.""" 65 | pass 66 | # if self.subset == 'train': 67 | # return 10000 68 | # if self.subset == 'validation': 69 | # return 1000 70 | 71 | @abstractmethod 72 | def download_message(self): 73 | """Prints a download message for the Dataset.""" 74 | pass 75 | 76 | def available_subsets(self): 77 | """Returns the list of available subsets.""" 78 | return ['train', 'valid'] 79 | # return ['train', 'validation'] 80 | 81 | def data_files(self): 82 | """Returns a python list of all (sharded) data subset files. 83 | 84 | Returns: 85 | python list of all (sharded) data set files. 86 | Raises: 87 | ValueError: if there are not data_files matching the subset. 88 | """ 89 | tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset) 90 | print('!!!!!!!!!!!!!!') 91 | print(tf_record_pattern) 92 | data_files = tf.gfile.Glob(tf_record_pattern) 93 | if not data_files: 94 | print('No files found for dataset %s/%s at %s' % (self.name, 95 | self.subset, 96 | FLAGS.data_dir)) 97 | 98 | self.download_message() 99 | exit(-1) 100 | return data_files 101 | 102 | def reader(self): 103 | """Return a reader for a single entry from the data set. 104 | 105 | See io_ops.py for details of Reader class. 106 | 107 | Returns: 108 | Reader object that reads the data set. 109 | """ 110 | return tf.TFRecordReader() 111 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/nc_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to a data set. 16 | 17 | Methods of Data class: 18 | data_files: Returns a python list of all (sharded) data set files. 19 | num_examples_per_epoch: Returns the number of examples in the data set. 20 | num_classes: Returns the number of classes in the data set. 21 | reader: Return a reader for a single entry from the data set. 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | from abc import ABCMeta 28 | from abc import abstractmethod 29 | import os 30 | 31 | 32 | import tensorflow as tf 33 | 34 | FLAGS = tf.app.flags.FLAGS 35 | 36 | # Basic model parameters. 37 | #tf.app.flags.DEFINE_string('data_dir', '/tmp/mydata', 38 | # """Path to the processed data, i.e. """ 39 | # """TFRecord of Example protos.""") 40 | 41 | tf.app.flags.DEFINE_string('data_dir', '/ifs/home/coudrn01/NN/Lung/Test_All512pxTiled/4_2Types', 42 | """Path to the processed data, i.e. """ 43 | """TFRecord of Example protos.""") 44 | 45 | 46 | class Dataset(object): 47 | """A simple class for handling data sets.""" 48 | __metaclass__ = ABCMeta 49 | 50 | def __init__(self, name, subset): 51 | """Initialize dataset using a subset and the path to the data.""" 52 | assert subset in self.available_subsets(), self.available_subsets() 53 | self.name = name 54 | self.subset = subset 55 | 56 | @abstractmethod 57 | def num_classes(self): 58 | """Returns the number of classes in the data set.""" 59 | pass 60 | # return 10 61 | 62 | @abstractmethod 63 | def num_examples_per_epoch(self): 64 | """Returns the number of examples in the data subset.""" 65 | pass 66 | # if self.subset == 'train': 67 | # return 10000 68 | # if self.subset == 'validation': 69 | # return 1000 70 | 71 | @abstractmethod 72 | def download_message(self): 73 | """Prints a download message for the Dataset.""" 74 | pass 75 | 76 | def available_subsets(self): 77 | """Returns the list of available subsets.""" 78 | return ['train', 'valid'] 79 | # return ['train', 'validation'] 80 | 81 | def data_files(self): 82 | """Returns a python list of all (sharded) data subset files. 83 | 84 | Returns: 85 | python list of all (sharded) data set files. 86 | Raises: 87 | ValueError: if there are not data_files matching the subset. 88 | """ 89 | #tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset) 90 | #tf_record_pattern = os.path.join(FLAGS.data_dir, 'test_*') 91 | tf_record_pattern = FLAGS.data_dir 92 | print("DEBUG:") 93 | print(tf_record_pattern) 94 | data_files = tf.gfile.Glob(tf_record_pattern) 95 | print(data_files) 96 | if not data_files: 97 | print('No files found for dataset %s/%s at %s' % (self.name, 98 | self.subset, 99 | FLAGS.data_dir)) 100 | 101 | self.download_message() 102 | exit(-1) 103 | return data_files 104 | 105 | def reader(self): 106 | """Return a reader for a single entry from the data set. 107 | 108 | See io_ops.py for details of Reader class. 109 | 110 | Returns: 111 | Reader object that reads the data set. 112 | """ 113 | return tf.TFRecordReader() 114 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/data/download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-01023-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "Usage: download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort > "${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/data/download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-01023-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "Usage: download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort > "${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/data/nc_download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-01023-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "Usage: download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort > "${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/data/download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-01023-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "Usage: download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort > "${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/data/nc_download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-01023-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "Usage: download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort > "${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/data/nc_download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-01023-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "Usage: download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort > "${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/nc_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to a data set. 16 | 17 | Methods of Data class: 18 | data_files: Returns a python list of all (sharded) data set files. 19 | num_examples_per_epoch: Returns the number of examples in the data set. 20 | num_classes: Returns the number of classes in the data set. 21 | reader: Return a reader for a single entry from the data set. 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | from abc import ABCMeta 28 | from abc import abstractmethod 29 | import os 30 | 31 | 32 | import tensorflow as tf 33 | 34 | FLAGS = tf.app.flags.FLAGS 35 | 36 | # Basic model parameters. 37 | #tf.app.flags.DEFINE_string('data_dir', '/tmp/mydata', 38 | # """Path to the processed data, i.e. """ 39 | # """TFRecord of Example protos.""") 40 | 41 | tf.app.flags.DEFINE_string('data_dir', '/ifs/home/coudrn01/NN/Lung/Test_All512pxTiled/4_2Types', 42 | """Path to the processed data, i.e. """ 43 | """TFRecord of Example protos.""") 44 | 45 | 46 | 47 | class Dataset(object): 48 | """A simple class for handling data sets.""" 49 | __metaclass__ = ABCMeta 50 | 51 | def __init__(self, name, subset): 52 | """Initialize dataset using a subset and the path to the data.""" 53 | assert subset in self.available_subsets(), self.available_subsets() 54 | self.name = name 55 | self.subset = subset 56 | 57 | @abstractmethod 58 | def num_classes(self): 59 | """Returns the number of classes in the data set.""" 60 | pass 61 | # return 10 62 | 63 | @abstractmethod 64 | def num_examples_per_epoch(self): 65 | """Returns the number of examples in the data subset.""" 66 | pass 67 | # if self.subset == 'train': 68 | # return 10000 69 | # if self.subset == 'validation': 70 | # return 1000 71 | 72 | @abstractmethod 73 | def download_message(self): 74 | """Prints a download message for the Dataset.""" 75 | pass 76 | 77 | def available_subsets(self): 78 | """Returns the list of available subsets.""" 79 | return ['train', 'valid'] 80 | # return ['train', 'validation'] 81 | 82 | def data_files(self): 83 | """Returns a python list of all (sharded) data subset files. 84 | 85 | Returns: 86 | python list of all (sharded) data set files. 87 | Raises: 88 | ValueError: if there are not data_files matching the subset. 89 | """ 90 | #tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset) 91 | #tf_record_pattern = os.path.join(FLAGS.data_dir, 'test_*') 92 | tf_record_pattern = FLAGS.data_dir 93 | #tf_record_pattern = os.path.join(FLAGS.data_dir, FLAGS.ImageSet_basename + '*') 94 | print("tf_record_pattern:") 95 | print(tf_record_pattern) 96 | data_files = tf.gfile.Glob(tf_record_pattern) 97 | print(data_files) 98 | if not data_files: 99 | print('No files found for dataset %s/%s at %s' % (self.name, 100 | self.subset, 101 | FLAGS.data_dir)) 102 | 103 | self.download_message() 104 | exit(-1) 105 | return data_files 106 | 107 | def reader(self): 108 | """Return a reader for a single entry from the data set. 109 | 110 | See io_ops.py for details of Reader class. 111 | 112 | Returns: 113 | Reader object that reads the data set. 114 | """ 115 | return tf.TFRecordReader() 116 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Example TensorFlow models for ImageNet. 3 | 4 | package(default_visibility = [":internal"]) 5 | 6 | licenses(["notice"]) # Apache 2.0 7 | 8 | exports_files(["LICENSE"]) 9 | 10 | package_group( 11 | name = "internal", 12 | packages = ["//inception/..."], 13 | ) 14 | 15 | py_library( 16 | name = "dataset", 17 | srcs = [ 18 | "dataset.py", 19 | ], 20 | ) 21 | 22 | py_library( 23 | name = "imagenet_data", 24 | srcs = [ 25 | "imagenet_data.py", 26 | ], 27 | deps = [ 28 | ":dataset", 29 | ], 30 | ) 31 | 32 | py_library( 33 | name = "flowers_data", 34 | srcs = [ 35 | "flowers_data.py", 36 | ], 37 | deps = [ 38 | ":dataset", 39 | ], 40 | ) 41 | 42 | py_library( 43 | name = "image_processing", 44 | srcs = [ 45 | "image_processing.py", 46 | ], 47 | ) 48 | 49 | py_library( 50 | name = "inception", 51 | srcs = [ 52 | "inception_model.py", 53 | ], 54 | visibility = ["//visibility:public"], 55 | deps = [ 56 | ":dataset", 57 | "//inception/slim", 58 | ], 59 | ) 60 | 61 | py_binary( 62 | name = "imagenet_eval", 63 | srcs = [ 64 | "imagenet_eval.py", 65 | ], 66 | deps = [ 67 | ":imagenet_data", 68 | ":inception_eval", 69 | ], 70 | ) 71 | 72 | py_binary( 73 | name = "flowers_eval", 74 | srcs = [ 75 | "flowers_eval.py", 76 | ], 77 | deps = [ 78 | ":flowers_data", 79 | ":inception_eval", 80 | ], 81 | ) 82 | 83 | py_library( 84 | name = "inception_eval", 85 | srcs = [ 86 | "inception_eval.py", 87 | ], 88 | deps = [ 89 | ":image_processing", 90 | ":inception", 91 | ], 92 | ) 93 | 94 | py_binary( 95 | name = "imagenet_train", 96 | srcs = [ 97 | "imagenet_train.py", 98 | ], 99 | deps = [ 100 | ":imagenet_data", 101 | ":inception_train", 102 | ], 103 | ) 104 | 105 | py_binary( 106 | name = "imagenet_distributed_train", 107 | srcs = [ 108 | "imagenet_distributed_train.py", 109 | ], 110 | deps = [ 111 | ":imagenet_data", 112 | ":inception_distributed_train", 113 | ], 114 | ) 115 | 116 | py_binary( 117 | name = "flowers_train", 118 | srcs = [ 119 | "flowers_train.py", 120 | ], 121 | deps = [ 122 | ":flowers_data", 123 | ":inception_train", 124 | ], 125 | ) 126 | 127 | py_library( 128 | name = "inception_train", 129 | srcs = [ 130 | "inception_train.py", 131 | ], 132 | deps = [ 133 | ":image_processing", 134 | ":inception", 135 | ], 136 | ) 137 | 138 | py_library( 139 | name = "inception_distributed_train", 140 | srcs = [ 141 | "inception_distributed_train.py", 142 | ], 143 | deps = [ 144 | ":image_processing", 145 | ":inception", 146 | ], 147 | ) 148 | 149 | py_binary( 150 | name = "build_image_data", 151 | srcs = ["data/build_image_data.py"], 152 | ) 153 | 154 | sh_binary( 155 | name = "download_and_preprocess_flowers", 156 | srcs = ["data/download_and_preprocess_flowers.sh"], 157 | data = [ 158 | ":build_image_data", 159 | ], 160 | ) 161 | 162 | sh_binary( 163 | name = "download_and_preprocess_imagenet", 164 | srcs = ["data/download_and_preprocess_imagenet.sh"], 165 | data = [ 166 | "data/download_imagenet.sh", 167 | "data/imagenet_2012_validation_synset_labels.txt", 168 | "data/imagenet_lsvrc_2015_synsets.txt", 169 | "data/imagenet_metadata.txt", 170 | "data/preprocess_imagenet_validation_data.py", 171 | "data/process_bounding_boxes.py", 172 | ":build_imagenet_data", 173 | ], 174 | ) 175 | 176 | py_binary( 177 | name = "build_imagenet_data", 178 | srcs = ["data/build_imagenet_data.py"], 179 | ) 180 | 181 | filegroup( 182 | name = "srcs", 183 | srcs = glob( 184 | [ 185 | "**/*.py", 186 | "BUILD", 187 | ], 188 | ), 189 | ) 190 | 191 | filegroup( 192 | name = "imagenet_metadata", 193 | srcs = [ 194 | "data/imagenet_lsvrc_2015_synsets.txt", 195 | "data/imagenet_metadata.txt", 196 | ], 197 | visibility = ["//visibility:public"], 198 | ) 199 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/xClasses/inception/data/download_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download ImageNet Challenge 2012 training and validation data set. 18 | # 19 | # Downloads and decompresses raw images and bounding boxes. 20 | # 21 | # **IMPORTANT** 22 | # To download the raw images, the user must create an account with image-net.org 23 | # and generate a username and access_key. The latter two are required for 24 | # downloading the raw images. 25 | # 26 | # usage: 27 | # ./download_imagenet.sh [dir name] [synsets file] 28 | set -e 29 | 30 | if [ "x$IMAGENET_ACCESS_KEY" == x -o "x$IMAGENET_USERNAME" == x ]; then 31 | cat <= WIDTH and image.shape[1] >= HEIGHT: 87 | hdf5_file[subsets[subset] + '_img'][IndX, ...] = image[:WIDTH,:HEIGHT,:] 88 | hdf5_file[subsets[subset] + '_labels'][IndX, ...] = lab 89 | # hdf5_file[subsets[subset] + '_name'][IndX, ...] = img 90 | img_list_file.write(str(IndX) + "\t" + img + "\n") 91 | IndX += 1 92 | if IndX % 1000 == 0: 93 | print(str(IndX)+ " imgs done") 94 | print(str(IndX)+ " imgs done") 95 | hdf5_file[subsets[subset] + '_img'].resize((IndX, WIDTH, HEIGHT, 3)) 96 | hdf5_file[subsets[subset] + '_labels'].resize((IndX, )) 97 | #hdf5_file[subsets[subset] + '_name'].resize((IndX, )) 98 | img_list_file.close() 99 | 100 | ''' 101 | for subset in range(len(subsets)): 102 | img_db_shape = (Count[subsets[subset]], WIDTH, HEIGHT, 3) 103 | # img_storage[subset] = 104 | hdf5_file.create_dataset(name=subsets[subset] + '_img',shape=img_db_shape, dtype=np.uint8) 105 | labels_db_shape = (Count[subsets[subset]],) 106 | # label_storage[subset] = 107 | hdf5_file.create_dataset(name=subsets[subset]+'_labels', shape=labels_db_shape, dtype=np.float32) 108 | IndX = 0 109 | for img, lab in zip(All_imgs[subsets[subset]], All_labels[subsets[subset]]): 110 | image = cv2.imread(img) 111 | if image.shape[0] >= WIDTH and image.shape[1] >= HEIGHT: 112 | hdf5_file[subsets[subset] + '_img'][Indx, ...] = image[:WIDTH,:HEIGHT,:] 113 | hdf5_file[subsets[subset] + '_labels'][Indx, ...] = lab 114 | IndX += 1 115 | ''' 116 | 117 | if __name__ == '__main__': 118 | 119 | # generate parser / parse parameters 120 | args = get_parser() 121 | 122 | # run experiment 123 | main(args) 124 | 125 | -------------------------------------------------------------------------------- /DeepPATH_code/03_postprocessing/multiClasses/0f_ProbHistogram.py: -------------------------------------------------------------------------------- 1 | """ NYU modifications: 2 | Author: Nicolas Coudray 3 | Date created: August/2017 4 | Python Version: 3.5.3 5 | 6 | Use it with Lung images stat file: 7 | Generate class probability histograms for each slide 8 | """ 9 | from __future__ import absolute_import 10 | from __future__ import division 11 | from __future__ import print_function 12 | 13 | import argparse 14 | import os.path 15 | import re 16 | import sys 17 | 18 | 19 | import matplotlib 20 | matplotlib.use('Agg') 21 | import matplotlib.pyplot as plt 22 | import scipy.misc 23 | from scipy.misc import imsave 24 | from scipy.misc import imread 25 | from scipy.interpolate import spline 26 | import numpy as np 27 | 28 | def get_stats_from_file(tiles_stats, ctype, filter_file): 29 | 30 | 31 | dict = {} 32 | print(ctype) 33 | with open(tiles_stats) as f: 34 | for line in f: 35 | line = line.replace('[','').replace(']','').split() 36 | basename = "_".join(".".join(line[0].split(".")[:-1]).split("_")[:-2]) 37 | if ctype =='Lung3Classes': 38 | is_TP = line[1] 39 | class_1 = float(line[3]) 40 | class_2 = float(line[4]) 41 | class_3 = float(line[5]) 42 | sum_class = class_1 + class_2 + class_3 43 | class_1 = class_1 / sum_class 44 | class_2 = class_2 / sum_class 45 | class_3 = class_3 / sum_class 46 | current_score = max(class_1, class_2, class_3) 47 | if current_score == class_1: 48 | oClass = 1 49 | elif current_score == class_2: 50 | oClass = 2 51 | else: 52 | oClass = 3 53 | #print(line) 54 | #print(basename + " " + is_TP) 55 | if basename in dict.keys(): 56 | dict[basename].append([class_1, class_2 ,class_3]) 57 | else: 58 | dict[basename] = [[class_1, class_2, class_3]] 59 | elif ctype == 'Mutations': 60 | analyze = True 61 | if os.path.isfile(filter_file): 62 | corr = 'corrected_' 63 | with open(filter_file) as fstat2: 64 | for line2 in fstat2: 65 | if ".".join(line[0].split(".")[:-1]) in line2: 66 | ref = line2.replace('[','').replace(']','').split() 67 | nMax = max([float(ref[3]), float(ref[4]), float(ref[5])]) 68 | LUAD = float(ref[4]) 69 | #print("Found:") 70 | #print(line2, nMax, LUAD) 71 | if LUAD != nMax: 72 | analyze = False 73 | #print(analyze) 74 | break 75 | if analyze == False: 76 | #print("continue") 77 | continue 78 | 79 | EGFR = float(line[13]) 80 | FAT1 = float(line[14]) 81 | FAT4 = float(line[15]) 82 | KEAP1 = float(line[16]) 83 | KRAS = float(line[17]) 84 | LRP1B = float(line[18]) 85 | NF1 = float(line[19]) 86 | SETBP1 = float(line[20]) 87 | STK11 = float(line[21]) 88 | TP53 = float(line[22]) 89 | #print(line) 90 | #print([EGFR, FAT1, KRAS, SETBP1, STK11, TP53]) 91 | if basename in dict.keys(): 92 | # dict[basename].append([EGFR, FAT1, FAT4, KEAP1, KRAS, LRP1B, NF1, SETBP1, STK11, TP53]) 93 | dict[basename].append([EGFR, FAT1, KRAS, SETBP1, STK11, TP53]) 94 | else: 95 | # dict[basename] = [[EGFR, FAT1, FAT4, KEAP1, KRAS, LRP1B, NF1, SETBP1, STK11, TP53]] 96 | dict[basename] = [[EGFR, FAT1, KRAS, SETBP1, STK11, TP53]] 97 | 98 | return dict 99 | 100 | def main(tiles_stats, output_dir, ctype, filter_file): 101 | 102 | 103 | 104 | dict = get_stats_from_file(tiles_stats, ctype, filter_file) 105 | for img in dict.keys(): 106 | NbrOfClasses = np.array(dict[img]).shape[1] 107 | plt.clf() 108 | ymax = 0 109 | lineL = {} 110 | if ctype =='Lung3Classes': 111 | color = ['k', 'r', 'b'] 112 | labels = ['Normal', 'LUAD', 'LUSC'] 113 | elif ctype =='Mutations': 114 | color = ['r', 'y', 'g', 'b', 'm', 'k'] 115 | labels = ['EGFR', 'FAT1', 'KRAS', 'SETBP1', 'STK11', 'TP53'] 116 | 117 | else: 118 | print("type of classification not recognized !!!!") 119 | continue 120 | 121 | with open(os.path.join(output_dir, img + "_" + ctype + "_histo.txt"), "a") as myfile: 122 | for nCl in range(NbrOfClasses): 123 | y,x = np.histogram( np.array(dict[img])[:,nCl], np.arange(0,1.1,0.01) ) 124 | ymax = max(ymax, max(y/sum(y))) 125 | meanV = np.mean(np.array(dict[img])[:,nCl])*100 126 | lineL[nCl] = plt.plot(x[:-1]*100,y/sum(y)*100, color[nCl], label="%s (%.3f)" % (labels[nCl], meanV) ) 127 | plt.plot([meanV, meanV],[0, 100], ':'+color[nCl]) 128 | 129 | myfile.write(labels[nCl] + "\t") 130 | myfile.write(" ".join(str(y/sum(y)*100).splitlines()) + "\n") 131 | 132 | myfile.write("x-axis\t") 133 | myfile.write(" ".join(str(x[:-1]*100).splitlines()) + "\n") 134 | 135 | #plt.legend([lineL[0], lineL[1], lineL[2]], ['Normal', 'LUAD', 'LUSC']) 136 | lgd = plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1)) 137 | plt.xlabel('Probability') 138 | plt.ylabel('Percentage of tiles') 139 | plt.title(img) 140 | plt.axis([0, 100, 0, ymax*100]) 141 | plt.xticks(np.arange(0, 105, 5.0)) 142 | plt.savefig(os.path.join(output_dir, img + "_" + ctype + "_histo.jpeg"), bbox_extra_artists=(lgd,), bbox_inches='tight') 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | if __name__ == '__main__': 155 | 156 | 157 | 158 | parser = argparse.ArgumentParser() 159 | parser.add_argument( 160 | '--output_dir', 161 | type=str, 162 | default='mustbedefined', 163 | help='Output directory.' 164 | ) 165 | parser.add_argument( 166 | '--tiles_stats', 167 | type=str, 168 | default='', 169 | help='text file where tile statistics are saved.' 170 | ) 171 | parser.add_argument( 172 | '--ctype', 173 | type=str, 174 | default='Lung3Classes', 175 | help='Lung3Classes or Mutations' 176 | ) 177 | parser.add_argument( 178 | '--filter_file', 179 | type=str, 180 | default='', 181 | help='text file to filter LUAD Mutations.' 182 | ) 183 | args = parser.parse_args() 184 | main(args.tiles_stats, args.output_dir, args.ctype, args.filter_file) 185 | 186 | -------------------------------------------------------------------------------- /DeepPATH_code/01_training/xClasses/inception/slim/inception_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for slim.inception.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from inception.slim import inception_model as inception 23 | 24 | 25 | class InceptionTest(tf.test.TestCase): 26 | 27 | def testBuildLogits(self): 28 | batch_size = 5 29 | height, width = 299, 299 30 | num_classes = 1000 31 | with self.test_session(): 32 | inputs = tf.random_uniform((batch_size, height, width, 3)) 33 | logits, _ = inception.inception_v3(inputs, num_classes) 34 | self.assertTrue(logits.op.name.startswith('logits')) 35 | self.assertListEqual(logits.get_shape().as_list(), 36 | [batch_size, num_classes]) 37 | 38 | def testBuildEndPoints(self): 39 | batch_size = 5 40 | height, width = 299, 299 41 | num_classes = 1000 42 | with self.test_session(): 43 | inputs = tf.random_uniform((batch_size, height, width, 3)) 44 | _, end_points = inception.inception_v3(inputs, num_classes) 45 | self.assertTrue('logits' in end_points) 46 | logits = end_points['logits'] 47 | self.assertListEqual(logits.get_shape().as_list(), 48 | [batch_size, num_classes]) 49 | self.assertTrue('aux_logits' in end_points) 50 | aux_logits = end_points['aux_logits'] 51 | self.assertListEqual(aux_logits.get_shape().as_list(), 52 | [batch_size, num_classes]) 53 | pre_pool = end_points['mixed_8x8x2048b'] 54 | self.assertListEqual(pre_pool.get_shape().as_list(), 55 | [batch_size, 8, 8, 2048]) 56 | 57 | def testVariablesSetDevice(self): 58 | batch_size = 5 59 | height, width = 299, 299 60 | num_classes = 1000 61 | with self.test_session(): 62 | inputs = tf.random_uniform((batch_size, height, width, 3)) 63 | # Force all Variables to reside on the device. 64 | with tf.variable_scope('on_cpu'), tf.device('/cpu:0'): 65 | inception.inception_v3(inputs, num_classes) 66 | with tf.variable_scope('on_gpu'), tf.device('/gpu:0'): 67 | inception.inception_v3(inputs, num_classes) 68 | for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'): 69 | self.assertDeviceEqual(v.device, '/cpu:0') 70 | for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'): 71 | self.assertDeviceEqual(v.device, '/gpu:0') 72 | 73 | def testHalfSizeImages(self): 74 | batch_size = 5 75 | height, width = 150, 150 76 | num_classes = 1000 77 | with self.test_session(): 78 | inputs = tf.random_uniform((batch_size, height, width, 3)) 79 | logits, end_points = inception.inception_v3(inputs, num_classes) 80 | self.assertTrue(logits.op.name.startswith('logits')) 81 | self.assertListEqual(logits.get_shape().as_list(), 82 | [batch_size, num_classes]) 83 | pre_pool = end_points['mixed_8x8x2048b'] 84 | self.assertListEqual(pre_pool.get_shape().as_list(), 85 | [batch_size, 3, 3, 2048]) 86 | 87 | def testUnknowBatchSize(self): 88 | batch_size = 1 89 | height, width = 299, 299 90 | num_classes = 1000 91 | with self.test_session() as sess: 92 | inputs = tf.placeholder(tf.float32, (None, height, width, 3)) 93 | logits, _ = inception.inception_v3(inputs, num_classes) 94 | self.assertTrue(logits.op.name.startswith('logits')) 95 | self.assertListEqual(logits.get_shape().as_list(), 96 | [None, num_classes]) 97 | images = tf.random_uniform((batch_size, height, width, 3)) 98 | sess.run(tf.global_variables_initializer()) 99 | output = sess.run(logits, {inputs: images.eval()}) 100 | self.assertEquals(output.shape, (batch_size, num_classes)) 101 | 102 | def testEvaluation(self): 103 | batch_size = 2 104 | height, width = 299, 299 105 | num_classes = 1000 106 | with self.test_session() as sess: 107 | eval_inputs = tf.random_uniform((batch_size, height, width, 3)) 108 | logits, _ = inception.inception_v3(eval_inputs, num_classes, 109 | is_training=False) 110 | predictions = tf.argmax(logits, 1) 111 | sess.run(tf.global_variables_initializer()) 112 | output = sess.run(predictions) 113 | self.assertEquals(output.shape, (batch_size,)) 114 | 115 | def testTrainEvalWithReuse(self): 116 | train_batch_size = 5 117 | eval_batch_size = 2 118 | height, width = 150, 150 119 | num_classes = 1000 120 | with self.test_session() as sess: 121 | train_inputs = tf.random_uniform((train_batch_size, height, width, 3)) 122 | inception.inception_v3(train_inputs, num_classes) 123 | tf.get_variable_scope().reuse_variables() 124 | eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3)) 125 | logits, _ = inception.inception_v3(eval_inputs, num_classes, 126 | is_training=False) 127 | predictions = tf.argmax(logits, 1) 128 | sess.run(tf.global_variables_initializer()) 129 | output = sess.run(predictions) 130 | self.assertEquals(output.shape, (eval_batch_size,)) 131 | 132 | 133 | if __name__ == '__main__': 134 | tf.test.main() 135 | -------------------------------------------------------------------------------- /DeepPATH_code/02_testing/multiClasses/inception/slim/inception_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for slim.inception.""" 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from inception.slim import inception_model as inception 23 | 24 | 25 | class InceptionTest(tf.test.TestCase): 26 | 27 | def testBuildLogits(self): 28 | batch_size = 5 29 | height, width = 299, 299 30 | num_classes = 1000 31 | with self.test_session(): 32 | inputs = tf.random_uniform((batch_size, height, width, 3)) 33 | logits, _ = inception.inception_v3(inputs, num_classes) 34 | self.assertTrue(logits.op.name.startswith('logits')) 35 | self.assertListEqual(logits.get_shape().as_list(), 36 | [batch_size, num_classes]) 37 | 38 | def testBuildEndPoints(self): 39 | batch_size = 5 40 | height, width = 299, 299 41 | num_classes = 1000 42 | with self.test_session(): 43 | inputs = tf.random_uniform((batch_size, height, width, 3)) 44 | _, end_points = inception.inception_v3(inputs, num_classes) 45 | self.assertTrue('logits' in end_points) 46 | logits = end_points['logits'] 47 | self.assertListEqual(logits.get_shape().as_list(), 48 | [batch_size, num_classes]) 49 | self.assertTrue('aux_logits' in end_points) 50 | aux_logits = end_points['aux_logits'] 51 | self.assertListEqual(aux_logits.get_shape().as_list(), 52 | [batch_size, num_classes]) 53 | pre_pool = end_points['mixed_8x8x2048b'] 54 | self.assertListEqual(pre_pool.get_shape().as_list(), 55 | [batch_size, 8, 8, 2048]) 56 | 57 | def testVariablesSetDevice(self): 58 | batch_size = 5 59 | height, width = 299, 299 60 | num_classes = 1000 61 | with self.test_session(): 62 | inputs = tf.random_uniform((batch_size, height, width, 3)) 63 | # Force all Variables to reside on the device. 64 | with tf.variable_scope('on_cpu'), tf.device('/cpu:0'): 65 | inception.inception_v3(inputs, num_classes) 66 | with tf.variable_scope('on_gpu'), tf.device('/gpu:0'): 67 | inception.inception_v3(inputs, num_classes) 68 | for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'): 69 | self.assertDeviceEqual(v.device, '/cpu:0') 70 | for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'): 71 | self.assertDeviceEqual(v.device, '/gpu:0') 72 | 73 | def testHalfSizeImages(self): 74 | batch_size = 5 75 | height, width = 150, 150 76 | num_classes = 1000 77 | with self.test_session(): 78 | inputs = tf.random_uniform((batch_size, height, width, 3)) 79 | logits, end_points = inception.inception_v3(inputs, num_classes) 80 | self.assertTrue(logits.op.name.startswith('logits')) 81 | self.assertListEqual(logits.get_shape().as_list(), 82 | [batch_size, num_classes]) 83 | pre_pool = end_points['mixed_8x8x2048b'] 84 | self.assertListEqual(pre_pool.get_shape().as_list(), 85 | [batch_size, 3, 3, 2048]) 86 | 87 | def testUnknowBatchSize(self): 88 | batch_size = 1 89 | height, width = 299, 299 90 | num_classes = 1000 91 | with self.test_session() as sess: 92 | inputs = tf.placeholder(tf.float32, (None, height, width, 3)) 93 | logits, _ = inception.inception_v3(inputs, num_classes) 94 | self.assertTrue(logits.op.name.startswith('logits')) 95 | self.assertListEqual(logits.get_shape().as_list(), 96 | [None, num_classes]) 97 | images = tf.random_uniform((batch_size, height, width, 3)) 98 | sess.run(tf.global_variables_initializer()) 99 | output = sess.run(logits, {inputs: images.eval()}) 100 | self.assertEquals(output.shape, (batch_size, num_classes)) 101 | 102 | def testEvaluation(self): 103 | batch_size = 2 104 | height, width = 299, 299 105 | num_classes = 1000 106 | with self.test_session() as sess: 107 | eval_inputs = tf.random_uniform((batch_size, height, width, 3)) 108 | logits, _ = inception.inception_v3(eval_inputs, num_classes, 109 | is_training=False) 110 | predictions = tf.argmax(logits, 1) 111 | sess.run(tf.global_variables_initializer()) 112 | output = sess.run(predictions) 113 | self.assertEquals(output.shape, (batch_size,)) 114 | 115 | def testTrainEvalWithReuse(self): 116 | train_batch_size = 5 117 | eval_batch_size = 2 118 | height, width = 150, 150 119 | num_classes = 1000 120 | with self.test_session() as sess: 121 | train_inputs = tf.random_uniform((train_batch_size, height, width, 3)) 122 | inception.inception_v3(train_inputs, num_classes) 123 | tf.get_variable_scope().reuse_variables() 124 | eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3)) 125 | logits, _ = inception.inception_v3(eval_inputs, num_classes, 126 | is_training=False) 127 | predictions = tf.argmax(logits, 1) 128 | sess.run(tf.global_variables_initializer()) 129 | output = sess.run(predictions) 130 | self.assertEquals(output.shape, (eval_batch_size,)) 131 | 132 | 133 | if __name__ == '__main__': 134 | tf.test.main() 135 | --------------------------------------------------------------------------------