├── data ├── .keep ├── eyepacs │ └── .keep ├── messidor │ └── .keep └── messidor2 │ └── .keep ├── lib ├── __init__.py ├── common.py ├── metrics.py ├── dataset.py ├── evaluation.py └── preprocess.py ├── .gitmodules ├── vendor ├── eyepacs │ └── testLabels.csv.zip └── messidor │ ├── abramoff-messidor-2-refstandard-jul16.csv │ └── messidor_gradability_grades.csv ├── .gitignore ├── LICENSE.md ├── preprocess_messidor2.py ├── preprocess_eyepacs.py ├── preprocess_messidor.py ├── messidor2.sh ├── benchmarks.yaml.jinja.example ├── messidor.sh ├── README.zh.md ├── grading_tool.py ├── README.md ├── eyepacs.sh ├── evaluate.py └── train.py /data/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/eyepacs/.keep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /data/messidor/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/messidor2/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "create_tfrecords"] 2 | path = create_tfrecords 3 | url = https://github.com/mikevoets/create_tfrecords.git 4 | -------------------------------------------------------------------------------- /vendor/eyepacs/testLabels.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikevoets/jama16-retina-replication/HEAD/vendor/eyepacs/testLabels.csv.zip -------------------------------------------------------------------------------- /lib/common.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | _cnt = 0 4 | 5 | 6 | def print_status(msg): 7 | global _cnt 8 | CURSOR_UP_ONE = '\x1b[1A' 9 | ERASE_LINE = '\x1b[2K' 10 | print(ERASE_LINE + CURSOR_UP_ONE) 11 | msg = "\r[{0:>2}] - {1}".format(_cnt, msg) 12 | sys.stdout.write(msg) 13 | sys.stdout.flush() 14 | _cnt += 1 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | downloads/ 11 | data/* 12 | !data/.keep 13 | !data/*/ 14 | data/*/* 15 | !data/*/.keep 16 | logs/ 17 | tmp/ 18 | 19 | # Grading tool 20 | .gt/ 21 | 22 | # Other dotfiles 23 | .python-version 24 | -------------------------------------------------------------------------------- /lib/metrics.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def generate_thresholds(num_thresholds, kepsilon=1e-7): 5 | thresholds = [ 6 | (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2) 7 | ] 8 | return [0.0 - kepsilon] + thresholds + [1.0 - kepsilon] 9 | 10 | 11 | def create_reset_metric(metric, scope='reset_metrics', **metric_args): 12 | with tf.variable_scope(scope) as s: 13 | metric_op, update_op = metric(**metric_args) 14 | vars = tf.contrib.framework.get_variables( 15 | s, collection=tf.GraphKeys.LOCAL_VARIABLES) 16 | reset_op = tf.variables_initializer(vars, name='reset') 17 | return metric_op, update_op, reset_op 18 | 19 | 20 | def confusion_matrix(tp, fp, fn, tn, num_labels=1, scope='confusion_matrix'): 21 | with tf.variable_scope(scope) as s: 22 | return tf.cast(tf.reshape(tf.stack([tp, fp, fn, tn], 0), 23 | [num_labels, 2, 2], 24 | name='confusion_matrix'), dtype=tf.int32) 25 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Mike Voets 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /preprocess_messidor2.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import sys 4 | from shutil import rmtree 5 | from PIL import Image 6 | from glob import glob 7 | from os import makedirs, rename 8 | from os.path import join, splitext, basename, exists 9 | from lib.preprocess import resize_and_center_fundus 10 | 11 | parser = argparse.ArgumentParser(description='Preprocess Messidor-2 data set.') 12 | parser.add_argument("--data_dir", help="Directory where Messidor-2 resides.", 13 | default="data/messidor2") 14 | 15 | args = parser.parse_args() 16 | data_dir = str(args.data_dir) 17 | 18 | labels = join(data_dir, 'labels.csv') 19 | 20 | # Create directories for grades. 21 | [makedirs(join(data_dir, str(i))) for i in [0, 1] 22 | if not exists(join(data_dir, str(i)))] 23 | 24 | # Create a tmp directory for saving temporary preprocessing files. 25 | tmp_path = join(data_dir, 'tmp') 26 | if exists(tmp_path): 27 | rmtree(tmp_path) 28 | makedirs(tmp_path) 29 | 30 | failed_images = [] 31 | 32 | with open(labels, 'r') as f: 33 | reader = csv.reader(f, delimiter=',') 34 | next(reader) 35 | 36 | for i, row in enumerate(reader): 37 | basename, grade = row 38 | 39 | im_paths = glob(join(data_dir, "Messidor-2/{}*".format(basename))) 40 | 41 | # Find contour of eye fundus in image, and scale 42 | # diameter of fundus to 299 pixels and crop the edges. 43 | res = resize_and_center_fundus(save_path=tmp_path, 44 | image_paths=im_paths, 45 | diameter=299, verbosity=0) 46 | 47 | # Status message. 48 | msg = "\r- Preprocessing pair of image: {0:>7}".format(i+1) 49 | sys.stdout.write(msg) 50 | sys.stdout.flush() 51 | 52 | if res != 2: 53 | failed_images.append(basename) 54 | continue 55 | 56 | # Move the files from the tmp folder to the right grade folder. 57 | for j in range(2): 58 | new_filename = "{0}.00{1}.jpg".format(basename, j) 59 | 60 | rename(join(tmp_path, new_filename), 61 | join(data_dir, str(int(grade)), new_filename)) 62 | 63 | # Clean tmp folder. 64 | rmtree(tmp_path) 65 | 66 | print("Could not preprocess {} images.".format(len(failed_images))) 67 | print(", ".join(failed_images)) 68 | -------------------------------------------------------------------------------- /preprocess_eyepacs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import sys 4 | from shutil import rmtree 5 | from PIL import Image 6 | from glob import glob 7 | from os import makedirs, rename 8 | from os.path import join, splitext, basename, exists 9 | from lib.preprocess import resize_and_center_fundus 10 | 11 | parser = argparse.ArgumentParser(description='Preprocess EyePACS data set.') 12 | parser.add_argument("--data_dir", help="Directory where EyePACS resides.", 13 | default="data/eyepacs") 14 | 15 | args = parser.parse_args() 16 | data_dir = str(args.data_dir) 17 | 18 | train_labels = join(data_dir, 'trainLabels.csv') 19 | test_labels = join(data_dir, 'testLabels.csv') 20 | 21 | # Create directories for grades. 22 | [makedirs(join(data_dir, str(i))) for i in [0, 1, 2, 3, 4] 23 | if not exists(join(data_dir, str(i)))] 24 | 25 | # Create a tmp directory for saving temporary preprocessing files. 26 | tmp_path = join(data_dir, 'tmp') 27 | if exists(tmp_path): 28 | rmtree(tmp_path) 29 | makedirs(tmp_path) 30 | 31 | failed_images = [] 32 | 33 | for labels in [train_labels, test_labels]: 34 | with open(labels, 'r') as f: 35 | reader = csv.reader(f, delimiter=',') 36 | next(reader) 37 | 38 | for i, row in enumerate(reader): 39 | basename, grade = row[:2] 40 | 41 | im_path = glob(join(data_dir, "{}*".format(basename)))[0] 42 | 43 | # Find contour of eye fundus in image, and scale 44 | # diameter of fundus to 299 pixels and crop the edges. 45 | res = resize_and_center_fundus(save_path=tmp_path, 46 | image_path=im_path, 47 | diameter=299, verbosity=0) 48 | 49 | # Status message. 50 | msg = "\r- Preprocessing image: {0:>7}".format(i+1) 51 | sys.stdout.write(msg) 52 | sys.stdout.flush() 53 | 54 | if res != 1: 55 | failed_images.append(basename) 56 | continue 57 | 58 | new_filename = "{0}.jpg".format(basename) 59 | 60 | # Move the file from the tmp folder to the right grade folder. 61 | rename(join(tmp_path, new_filename), 62 | join(data_dir, str(int(grade)), new_filename)) 63 | 64 | # Clean tmp folder. 65 | rmtree(tmp_path) 66 | 67 | print("Could not preprocess {} images.".format(len(failed_images))) 68 | print(", ".join(failed_images)) 69 | -------------------------------------------------------------------------------- /preprocess_messidor.py: -------------------------------------------------------------------------------- 1 | import xlrd 2 | import zipfile 3 | import argparse 4 | import sys 5 | from shutil import rmtree 6 | from PIL import Image 7 | from glob import glob 8 | from os import makedirs, rename 9 | from os.path import join, splitext, basename, exists 10 | from lib.preprocess import resize_and_center_fundus 11 | 12 | parser = argparse.ArgumentParser(description='Preprocess Messidor-Original data set.') 13 | parser.add_argument("--data_dir", help="Directory where Messidor-Original resides.", 14 | default="data/messidor") 15 | 16 | args = parser.parse_args() 17 | data_dir = str(args.data_dir) 18 | 19 | # Create directories for grades. 20 | [makedirs(join(data_dir, str(i))) for i in [0, 1, 2, 3] 21 | if not exists(join(data_dir, str(i)))] 22 | 23 | # Create a tmp directory for saving temporary preprocessing files. 24 | tmp_path = join(data_dir, 'tmp') 25 | if exists(tmp_path): 26 | rmtree(tmp_path) 27 | makedirs(tmp_path) 28 | 29 | # Find shard zip files. 30 | shards_paths = glob(join(data_dir, "*.zip")) 31 | 32 | for shard in shards_paths: 33 | shard_name = splitext(basename(shard))[0] 34 | shard_unpack_dir = join(data_dir, shard_name) 35 | 36 | # Unzip shard. 37 | print(f"Unzipping {shard_name}...") 38 | if exists(shard_unpack_dir): 39 | rmtree(shard_unpack_dir) 40 | 41 | zip_ref = zipfile.ZipFile(shard, 'r') 42 | zip_ref.extractall(shard_unpack_dir) 43 | zip_ref.close() 44 | 45 | # Open annotations file for shard. 46 | annotations_path = join( 47 | data_dir, f"Annotation_{shard_name}.xls") 48 | workbook = xlrd.open_workbook(annotations_path) 49 | worksheet = workbook.sheet_by_index(0) 50 | 51 | # Parse annotations file. 52 | for num, row in enumerate(range(1, worksheet.nrows)): 53 | filename = worksheet.cell(row, 0).value 54 | grade = worksheet.cell(row, 2).value 55 | 56 | im_path = glob(join(shard_unpack_dir, "**/{}".format(filename)), 57 | recursive=True)[0] 58 | 59 | # Find contour of eye fundus in image, and scale 60 | # diameter of fundus to 299 pixels and crop the edges. 61 | res = resize_and_center_fundus(save_path=tmp_path, image_path=im_path, 62 | diameter=299, verbosity=0) 63 | 64 | # Status-message. 65 | msg = "\r- Preprocessing image: {0:>6} / {1}".format( 66 | num+1, worksheet.nrows-1) 67 | 68 | # Print the status message. 69 | sys.stdout.write(msg) 70 | sys.stdout.flush() 71 | 72 | if res != 1: 73 | continue 74 | 75 | new_filename = "{0}.jpg".format(splitext(basename(im_path))[0]) 76 | 77 | # Move the file from the tmp folder to the right grade folder. 78 | rename(join(tmp_path, new_filename), 79 | join(data_dir, str(int(grade)), new_filename)) 80 | 81 | print() 82 | rmtree(shard_unpack_dir) 83 | 84 | # Clean tmp folder. 85 | rmtree(tmp_path) 86 | -------------------------------------------------------------------------------- /lib/dataset.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from random import shuffle 3 | import os 4 | 5 | BRIGHTNESS_MAX_DELTA = 0.125 6 | SATURATION_LOWER = 0.5 7 | SATURATION_UPPER = 1.5 8 | HUE_MAX_DELTA = 0.2 9 | CONTRAST_LOWER = 0.5 10 | CONTRAST_UPPER = 1.5 11 | 12 | 13 | def _tfrecord_dataset_from_folder(folder, ext='.tfrecord'): 14 | tfrecords = [os.path.join(folder, n) 15 | for n in os.listdir(folder) if n.endswith(ext)] 16 | return tf.data.TFRecordDataset(tfrecords) 17 | 18 | 19 | def _parse_example(proto, num_channels, normalization_fn, augmentation=True): 20 | features = {"image/encoded": tf.FixedLenFeature((), tf.string), 21 | "image/format": tf.FixedLenFeature((), tf.string), 22 | "image/class/label": tf.FixedLenFeature((), tf.int64), 23 | "image/height": tf.FixedLenFeature((), tf.int64), 24 | "image/width": tf.FixedLenFeature((), tf.int64)} 25 | parsed = tf.parse_single_example(proto, features) 26 | 27 | image = tf.image.decode_jpeg(parsed["image/encoded"], num_channels) 28 | image = normalization_fn(image) 29 | 30 | # Apply data augmentations randomly. 31 | augmentations = [ 32 | {'fn': tf.image.random_flip_left_right}, 33 | {'fn': tf.image.random_brightness, 34 | 'args': [BRIGHTNESS_MAX_DELTA]}, 35 | {'fn': tf.image.random_saturation, 36 | 'args': [SATURATION_LOWER, SATURATION_UPPER]}, 37 | {'fn': tf.image.random_hue, 38 | 'args': [HUE_MAX_DELTA]}, 39 | {'fn': tf.image.random_contrast, 40 | 'args': [CONTRAST_LOWER, CONTRAST_UPPER]}] 41 | 42 | shuffle(augmentations) 43 | 44 | if augmentation: 45 | for aug in augmentations: 46 | if 'args' in aug: 47 | image = aug['fn'](image, *aug['args']) 48 | else: 49 | image = aug['fn'](image) 50 | 51 | # Transpose images to channels first since training is done on GPU(s) 52 | # and cuDNN works more efficiently with channels in first dimension. 53 | image = tf.transpose(image, [2, 0, 1]) 54 | 55 | label = tf.cast( 56 | tf.reshape(parsed["image/class/label"], [-1]), 57 | tf.float32) 58 | 59 | return image, label 60 | 61 | 62 | def initialize_dataset(image_dir, batch_size, num_epochs=1, 63 | num_workers=1, prefetch_buffer_size=None, 64 | shuffle_buffer_size=None, 65 | normalization_fn=tf.image.per_image_standardization, 66 | num_channels=3, image_dim=[299, 299], 67 | augmentation=True): 68 | # Retrieve data set from pattern. 69 | dataset = _tfrecord_dataset_from_folder(image_dir) 70 | 71 | dataset = dataset.map( 72 | lambda e: _parse_example(e, num_channels, normalization_fn, augmentation), 73 | num_parallel_calls=num_workers) 74 | 75 | if shuffle_buffer_size is not None: 76 | dataset = dataset.shuffle(shuffle_buffer_size) 77 | 78 | dataset = dataset.repeat(num_epochs) 79 | dataset = dataset.batch(batch_size) 80 | 81 | if prefetch_buffer_size is not None: 82 | dataset = dataset.prefetch(prefetch_buffer_size) 83 | 84 | return dataset 85 | -------------------------------------------------------------------------------- /lib/evaluation.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def _get_operations_by_names(graph, names): 5 | return [graph.get_operation_by_name(name) for name in names] 6 | 7 | 8 | def _get_tensors_by_names(graph, names): 9 | return [graph.get_tensor_by_name(name) for name in names] 10 | 11 | 12 | def perform_test(sess, init_op, summary_writer=None, epoch=None, 13 | feed_dict_fn=None, feed_dict_args={}, custom_tensors=[]): 14 | tf.keras.backend.set_learning_phase(False) 15 | sess.run(init_op) 16 | 17 | if len(custom_tensors) == 0: 18 | # Retrieve all default tensors and operations. 19 | graph = tf.get_default_graph() 20 | reset_tp, reset_fp, reset_fn, reset_tn, reset_brier, reset_auc = \ 21 | _get_operations_by_names( 22 | graph, ['tp/reset', 'fp/reset', 'fn/reset', 'tn/reset', 23 | 'brier/reset', 'auc/reset']) 24 | 25 | update_tp, update_fp, update_fn, update_tn, update_brier, update_auc, \ 26 | brier, auc, confusion_matrix, summaries_op = \ 27 | _get_tensors_by_names( 28 | graph, ['tp/true_positives/AssignAdd:0', 29 | 'fp/false_positives/AssignAdd:0', 30 | 'fn/false_negatives/AssignAdd:0', 31 | 'tn/true_negatives/AssignAdd:0', 32 | 'brier/mean_squared_error/update_op:0', 33 | 'auc/auc/update_op:0', 34 | 'brier/mean_squared_error/value:0', 35 | 'auc/auc/value:0', 36 | 'confusion_matrix/Cast:0', 37 | 'Merge/MergeSummary:0']) 38 | 39 | # Reset all streaming variables. 40 | sess.run([reset_tp, reset_fp, reset_fn, reset_tn, reset_brier, reset_auc]) 41 | 42 | # Create an array with tensors to run for each batch. 43 | tensors = [update_tp, update_fp, update_fn, 44 | update_tn, update_brier, update_auc] 45 | else: 46 | tensors = custom_tensors 47 | 48 | try: 49 | batch_results = [] 50 | while True: 51 | if feed_dict_fn is not None: 52 | feed_dict = feed_dict_fn(**feed_dict_args) 53 | else: 54 | feed_dict = None 55 | 56 | # Retrieve the validation set confusion metrics. 57 | batch_results.append(sess.run(tensors, feed_dict)) 58 | 59 | except tf.errors.OutOfRangeError: 60 | pass 61 | 62 | # Yield the result if custom tensors were defined. 63 | if len(custom_tensors) > 0: 64 | return [np.vstack(x) for x in zip(*batch_results)] 65 | 66 | # Retrieve confusion matrix and estimated roc auc score. 67 | test_conf_matrix, test_brier, test_auc, summaries = sess.run( 68 | [confusion_matrix, brier, auc, summaries_op]) 69 | 70 | # Write summary. 71 | if summary_writer is not None: 72 | summary_writer.add_summary(summaries, epoch) 73 | 74 | # Print total roc auc score for validation. 75 | print(f"Brier score: {test_brier:6.4}, AUC: {test_auc:10.8}") 76 | 77 | # Print confusion matrix. 78 | print(f"Confusion matrix:") 79 | print(test_conf_matrix[0]) 80 | 81 | return test_auc 82 | -------------------------------------------------------------------------------- /messidor2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Preprocess script for the Messidor-Original data set. 3 | 4 | # Assumes that the data set resides in ./data/messidor. 5 | 6 | messidor_download_url="http://webeye.ophth.uiowa.edu/abramoff/messidor-2.zip" 7 | messidor_dir="./data/messidor2" 8 | messidor_path="$messidor_dir/messidor-2.zip" 9 | grades_path="./vendor/messidor/abramoff-messidor-2-refstandard-jul16.csv" 10 | default_output_dir="$messidor_dir/bin2" 11 | 12 | check_parameters() 13 | { 14 | if [ "$1" -ge 3 ]; then 15 | echo "Illegal number of parameters". 16 | exit 1 17 | fi 18 | if [ "$1" -ge 1 ]; then 19 | for param in $2; do 20 | if [ $(echo "$3" | grep -c -- "$param") -eq 0 ]; then 21 | echo "Unknown parameter $param." 22 | exit 1 23 | fi 24 | done 25 | fi 26 | return 0 27 | } 28 | 29 | strip_params=$(echo "$@" | sed "s/--\([a-z]\+\)\(=\(.\+\)\)\?/\1/g") 30 | check_parameters "$#" "$strip_params" "output" 31 | 32 | # Get output directory from parameters. 33 | output_dir=$(echo "$@" | sed "s/.*--output=\([^ ]\+\).*/\1/g") 34 | 35 | # Check if output directory is valid. 36 | if ! [[ "$output_dir" =~ ^[^-]+$ ]]; then 37 | output_dir=$default_output_dir 38 | fi 39 | 40 | if ls "$output_dir" >/dev/null 2>&1; then 41 | echo "Dataset is already located in $output_dir." 42 | echo "Specify another output directory with the --output flag." 43 | exit 1 44 | fi 45 | 46 | messidor_size=$(ls -s $messidor_path 2>/dev/null | cut -d " " -f1) 47 | 48 | if [ !ls "$messidor_path" >/dev/null 2>&1 ] || \ 49 | [[ $messidor_size -ne 3398280 ]]; then 50 | echo "Downloading messidor-2.zip. This may take a while..." 51 | curl -L0 "$messidor_download_url" --output "$messidor_path" 52 | fi 53 | 54 | count_files=$(ls $messidor_dir/Messidor-2 2>/dev/null | wc -l) 55 | 56 | if [ !ls "$messidor_dir/Messidor-2" >/dev/null 2>&1 ] || \ 57 | [[ $count_files -ne 1757 ]]; then 58 | # Check if unzip has been installed. 59 | dpkg -l | grep unzip 60 | if [ $? -gt 0 ]; then 61 | echo "Please install unzip: apt-get/yum install unzip" >&2 62 | exit 1 63 | fi 64 | 65 | if [[ $count_files -ne 1757 ]]; then 66 | echo "Messidor-2 wasn't unpacked properly before" 67 | fi 68 | 69 | echo "Unpacking messidor-2.zip" 70 | unzip "$messidor_path" -d "$messidor_dir" || exit 1 71 | fi 72 | 73 | # Copying labels file from vendor to data directory. 74 | cp "$grades_path" "$messidor_dir/labels.csv" 75 | 76 | # Preprocess the data set and categorize the images by labels into 77 | # subdirectories. 78 | python preprocess_messidor2.py --data_dir="$messidor_dir" || exit 1 79 | 80 | echo "Preparing data set..." 81 | mkdir -p "$output_dir/0" "$output_dir/1" 82 | 83 | echo "Moving images to new directories..." 84 | find "$messidor_dir/0" -iname "*.jpg" -exec mv {} "$output_dir/0/." \; 85 | find "$messidor_dir/1" -iname "*.jpg" -exec mv {} "$output_dir/1/." \; 86 | 87 | # Convert the data set to tfrecords. 88 | echo "Converting data set to tfrecords..." 89 | git submodule update --init 90 | 91 | python ./create_tfrecords/create_tfrecord.py --dataset_dir="$output_dir" \ 92 | --num_shards=2 || \ 93 | { echo "Submodule not initialized. Run git submodule update --init"; 94 | exit 1; } 95 | 96 | echo "Cleaning up..." 97 | rm -r "$messidor_path" "$messidor_dir/Messidor-2" "$messidor_dir/labels.csv" 98 | 99 | echo "Done!" 100 | exit 101 | 102 | # References: 103 | # [1] http://www.adcis.net/en/Download-Third-Party/Messidor.html 104 | -------------------------------------------------------------------------------- /benchmarks.yaml.jinja.example: -------------------------------------------------------------------------------- 1 | {%- set name = "benchmark" -%} 2 | {%- set image = "maikovich/tf_cnn_benchmarks:latest" -%} 3 | {%- set worker_replicas = 2 -%} 4 | {%- set ps_replicas = 2 -%} 5 | {%- set batch_size = 64 -%} 6 | {%- set data_dir = "/data" -%} 7 | {%- set train_dir = "/data/model" -%} 8 | 9 | {%- set port = 5000 -%} 10 | {%- set replicas = {"worker": worker_replicas, "ps": ps_replicas} -%} 11 | 12 | {%- set gpu_per_node = "2" -%} 13 | {%- set cpu_per_node = "3" -%} 14 | {%- set mem_per_node = "4Gi" -%} 15 | 16 | {%- set volume_mount_path = "/data" -%} 17 | 18 | {%- set namespace = "namespace" -%} 19 | {%- set volume_mount_name = "volume_name" -%} 20 | {%- set volume_claim_name = "volume_claim_name" -%} 21 | 22 | {%- macro worker_hosts() -%} 23 | {%- for i in range(worker_replicas) -%} 24 | {%- if not loop.first -%},{%- endif -%} 25 | {{ name }}-worker-{{ i }}:{{ port }} 26 | {%- endfor -%} 27 | {%- endmacro -%} 28 | 29 | {%- macro ps_hosts() -%} 30 | {%- for i in range(ps_replicas) -%} 31 | {%- if not loop.first -%},{%- endif -%} 32 | {{ name }}-ps-{{ i }}:{{ port }} 33 | {%- endfor -%} 34 | {%- endmacro -%} 35 | 36 | {%- for job in ["ps", "worker"] -%} 37 | {%- for i in range(replicas[job]) -%} 38 | kind: Service 39 | apiVersion: v1 40 | metadata: 41 | name: {{ name }}-{{ job }}-{{ i }} 42 | namespace: {{ namespace }} 43 | labels: 44 | task: {{ name }}-{{ i }} 45 | spec: 46 | selector: 47 | name: {{ name }} 48 | job: {{ job }} 49 | task: "{{ i }}" 50 | ports: 51 | - port: {{ port }} 52 | --- 53 | kind: ReplicaSet 54 | apiVersion: extensions/v1beta1 55 | metadata: 56 | name: {{ name }}-{{ job }}-{{ i }} 57 | namespace: {{ namespace }} 58 | spec: 59 | replicas: 1 60 | template: 61 | metadata: 62 | labels: 63 | name: {{ name }} 64 | job: {{ job }} 65 | task: "{{ i }}" 66 | {% if gpu_per_node != "" %} 67 | driver: nvidia-gpu 68 | {% endif %} 69 | spec: 70 | containers: 71 | - name: tensorflow 72 | image: {{ image }} 73 | imagePullPolicy: Always 74 | ports: 75 | - containerPort: {{ port }} 76 | args: 77 | - "--data_dir={{ data_dir }}" 78 | - "--train_dir={{ train_dir }}" 79 | - "--task_index={{ i }}" 80 | - "--job_name={{ job }}" 81 | - "--worker_hosts={{ worker_hosts() }}" 82 | - "--ps_hosts={{ ps_hosts() }}" 83 | - "--num_gpus={{ gpu_per_node }}" 84 | - "--batch_size={{ batch_size }}" 85 | - "--model=inception3" 86 | - "--variable_update=parameter_server" 87 | - "--local_parameter_device=cpu" 88 | - "--optimizer=sgd" 89 | - "--data_format=NCHW" 90 | - "--data_name=retina" 91 | {% endif %} 92 | {% if gpu_per_node != "" %} 93 | resources: 94 | requests: 95 | {% if job != "ps" %} 96 | alpha.kubernetes.io/nvidia-gpu: {{ gpu_per_node }} 97 | {% endif %} 98 | cpu: {{ cpu_per_node }} 99 | memory: {{ mem_per_node }} 100 | limits: 101 | {% if job != "ps" %} 102 | alpha.kubernetes.io/nvidia-gpu: {{ gpu_per_node }} 103 | {% endif %} 104 | cpu: {{ cpu_per_node }} 105 | memory: {{ mem_per_node }} 106 | {% endif %} 107 | {% if volume_mount_name != "" %} 108 | volumeMounts: 109 | - name: {{ volume_mount_name }} 110 | mountPath: {{ volume_mount_path }} 111 | volumes: 112 | - name: {{ volume_mount_name }} 113 | persistentVolumeClaim: 114 | claimName: {{ volume_claim_name }} 115 | {% endif %} 116 | --- 117 | {% endfor %} 118 | {%- endfor -%} 119 | -------------------------------------------------------------------------------- /messidor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Preprocess script for the Messidor-Original data set. 3 | 4 | # Assumes that the data set resides in ./data/messidor. 5 | 6 | messidor_dir="./data/messidor" 7 | default_output_dir="$messidor_dir/bin2" 8 | grad_grades="./vendor/messidor/messidor_gradability_grades.csv" 9 | 10 | check_parameters() 11 | { 12 | if [ "$1" -ge 3 ]; then 13 | echo "Illegal number of parameters". 14 | exit 1 15 | fi 16 | if [ "$1" -ge 1 ]; then 17 | for param in $2; do 18 | if [ $(echo "$3" | grep -c -- "$param") -eq 0 ]; then 19 | echo "Unknown parameter $param." 20 | exit 1 21 | fi 22 | done 23 | fi 24 | return 0 25 | } 26 | 27 | strip_params=$(echo "$@" | sed "s/--\([a-z]\+\)\(=\(.\+\)\)\?/\1/g") 28 | check_parameters "$#" "$strip_params" "output only_gradable" 29 | 30 | # Get output directory from parameters. 31 | output_dir=$(echo "$@" | sed "s/.*--output=\([^ ]\+\).*/\1/g") 32 | 33 | # Check if output directory is valid. 34 | if ! [[ "$output_dir" =~ ^[^-]+$ ]]; then 35 | output_dir=$default_output_dir 36 | fi 37 | 38 | if ls "$output_dir" >/dev/null 2>&1; then 39 | echo "Dataset is already located in $output_dir." 40 | echo "Specify another output directory with the --output flag." 41 | exit 1 42 | fi 43 | 44 | # Confirm the Basexx .zip files and annotations .xls files are present. 45 | xls_count=$(find "$messidor_dir" -maxdepth 1 -iname "Annotation_Base*.xls" | wc -l) 46 | zip_count=$(find "$messidor_dir" -maxdepth 1 -iname "Base*.zip" | wc -l) 47 | 48 | if [ $xls_count -ne 12 ]; then 49 | echo "$messidor_dir does not contain any all annotation files!" 50 | exit 1 51 | fi 52 | 53 | if [ $zip_count -ne 12 ]; then 54 | echo "$messidor_dir does not contain all Basexx zip files!" 55 | exit 1 56 | fi 57 | 58 | # Preprocess the data set and categorize the images by labels into 59 | # subdirectories. 60 | python preprocess_messidor.py --data_dir="$messidor_dir" || exit 1 61 | 62 | # Remove ungradable images if needed. 63 | if echo "$@" | grep -F -c -- "--only_gradable" >/dev/null; then 64 | echo "Remove ungradable images" 65 | cat "$grad_grades" | while read tbl; do 66 | if [[ "$tbl" =~ ^.*0$ ]]; then 67 | file=$(echo "$tbl" | sed "s/\(.*\) 0/\1/") 68 | find "$messidor_dir"/[0-3] -iname "$file*" -delete 69 | fi 70 | done 71 | fi 72 | 73 | # According to [1], we have to correct some duplicate images and 74 | # grades in the data set. 75 | 76 | echo "Correcting data set..." 77 | # 16 August 2017: Image duplicates in Base33 78 | echo "20051202_54744_0400_PP.jpg 20051202_40508_0400_PP.jpg 79 | 20051202_41238_0400_PP.jpg 20051202_41260_0400_PP.jpg 80 | 20051202_54530_0400_PP.jpg 20051205_33025_0400_PP.jpg 81 | 20051202_55607_0400_PP.jpg 20051202_41034_0400_PP.jpg 82 | 20051205_35099_0400_PP.jpg 20051202_54555_0400_PP.jpg 83 | 20051205_35110_0400_PP.jpg 20051202_54611_0400_PP.jpg 84 | 20051202_55498_0400_PP.jpg" | tr " " "\n" | xargs -I% find "$messidor_dir" -name % -delete 85 | 86 | # 31 August 2016: Erratum in Base11 Excel file 87 | find "$messidor_dir/3" -name "20051020_63045_0100_PP.jpg" -exec mv {} "$messidor_dir/0/." \; 88 | 89 | # 24 October 2016: Erratum in Base11 and Base 13 Excel files 90 | find "$messidor_dir/1" -name "20051020_64007_0100_PP.jpg" -exec mv {} "$messidor_dir/3/." \; 91 | find "$messidor_dir/3" -name "20051020_63936_0100_PP.jpg" -exec mv {} "$messidor_dir/1/." \; 92 | find "$messidor_dir/2" -name "20060523_48477_0100_PP.jpg" -exec mv {} "$messidor_dir/3/." \; 93 | 94 | echo "Preparing data set..." 95 | mkdir -p "$output_dir/0" "$output_dir/1" 96 | 97 | echo "Moving images to new directories..." 98 | find "$messidor_dir/"[0-1] -iname "*.jpg" -exec mv {} "$output_dir/0/." \; 99 | find "$messidor_dir/"[2-3] -iname "*.jpg" -exec mv {} "$output_dir/1/." \; 100 | 101 | echo "Removing old directories..." 102 | rmdir "$messidor_dir/"[0-3] 103 | 104 | # Convert the data set to tfrecords. 105 | echo "Converting data set to tfrecords..." 106 | git submodule update --init 107 | 108 | python ./create_tfrecords/create_tfrecord.py --dataset_dir="$output_dir" \ 109 | --num_shards=2 || \ 110 | { echo "Submodule not initialized. Run git submodule update --init"; 111 | exit 1; } 112 | 113 | echo "Done!" 114 | exit 115 | 116 | # References: 117 | # [1] http://www.adcis.net/en/Download-Third-Party/Messidor.html 118 | -------------------------------------------------------------------------------- /README.zh.md: -------------------------------------------------------------------------------- 1 | 中文 - [English](https://github.com/mikevoets/jama16-retina-replication/blob/master/README.md) 2 | 3 | # 代码给 JAMA 2016年; 316(22) 复制研究 4 | 5 | 发表文章链接：[doi:10.1371/journal.pone.0217541](https://doi.org/10.1371/journal.pone.0217541)。 6 | 7 | 发布训练的神经网络模型：[doi:10.6084/m9.figshare.8312183](https://doi.org/10.6084/m9.figshare.8312183). 8 | 9 | ## 摘要 10 | 11 | 我们试图复制开发以及验证深度学习算法的主要方法，是用来检测在 JAMA 2016 上所发表的视网膜眼底照片中的糖尿病视网膜病变（Development and Validation of a Deep Learning Algorithm for Detection of Diabetic Retinopathy in Retinal Fundus Photographs; JAMA 2016年, 316(22); [链接](https://jamanetwork.com/journals/jama/fullarticle/2588763))。我们重新实现了该方法，但是源代码不可使用，因此我们使用了公开可使用的数据集。 12 | 13 | 最初的研究是使用来自 EyePACS 和印度三家医院的非公共眼底图像而进行培训。我们使用了 Kaggle 的不同 EyePACS 数据集。最初的研究使用了基准数据集 Messidor-2 来评估算法的性能。我们使用了Messidor-2数据集的另一个分布，因为原始数据集不再可用。在起初的研究中，眼科医生重新评估了糖尿病视网膜病变，黄斑水肿和图像可分级性的所有图像。我们的数据集中每个图像有一个糖尿病视网膜病变等级，我们自己评估了图像的可分级性。 14 | 15 | 我们无法复制原版的研究。我们的算法的接收器操作曲线在 Kaggle EyePACS 测试集（AUC）为0.951 (95% CI, 0.947-0.956)，在 Messidor-2 为0.853 (95% CI, 0.835-0.871)，但是在原版的研究中，未接近报告中的AUC，0.99。这可能是由于每个图像使用一个等级或不同的超参数的设置而引起的。 16 | 17 | ## 要求 18 | 19 | Python要求： 20 | 21 | - Python >= 3.6 22 | - Tensorflow >= 1.4 23 | - OpenCV >= 1.3 24 | - Pillow 25 | - h5py 26 | - xlrd 27 | - matplotlib >= 2.1 28 | 29 | 其他需求： 30 | 31 | - p7zip-full 32 | - unzip 33 | 34 | ## 训练前的预先处理 35 | 36 | 1.执行 `$ git submodule update --init` 来加载 [create_tfrecords](https://github.com/mikevoets/create_tfrecords) 存储库。这工具会将数据集转换为 TFRecord 文件。 37 | 38 | 2.下载压缩的 [_Kaggle_ EyePACS数据集](https://www.kaggle.com/c/diabetic-retinopathy-detection) 并将所有的文件（即训练和测试集和标签）放在 `data/eyepacs` 文件夹。我们建议您使用 [Kaggle API](https://github.com/Kaggle/kaggle-api)。 39 | 40 | 3.执行 `$ ./eyepacs.sh` 来解压缩并预先处理 _Kaggle_ EyePACS 数据集，和将此集重新分配到训练和测试集中。如果您只想使用可分级图像来进行训练和评估，使用 `--only_gradable` 标志来执行。注意：这是一个大型的数据集，因此可能需要数小时才能完成。 41 | 42 | 对于 Messidor-2： 43 | 44 | 4.执行 `$ ./messidor2.sh` 来下载，解压缩和预先处理 Messidor-2 的数据集。该数据集可从Michael D. Abramoff的页面 [此处](https://medicine.uiowa.edu/eye/abramoff) 上的数据集和算法部分下载。 45 | 46 | 对于 Messidor-Original： 47 | 48 | 5.下载 [Messidor-Original 数据集](http://www.adcis.net/en/Download-Third-Party/Messidor.html) 并将所有的文件放在 `data/messidor` 的文件夹中。 49 | 50 | 6.执行 `$ ./messidor.sh` 来预先处理 Messidor-Original 数据集。如果只要可分级图像而进行评估，使用 `--only_gradable` 标志执行。 51 | 52 | ## 培训 53 | 54 | 使用默认设置来开始训练，并执行 `$ python train.py`。可选使用 `-sm` 参数来指定模型检查点应保存到的路径。 55 | 56 | 运行 `$ python train.py -h` 以查看用于使用您自己的数据集进行培训的其他可选参数，或者保存摘要或操作阈值指标的位置。 57 | 58 | ## 评价 59 | 60 | 要在 _Kaggle_ EyePACS 测试集上评估或者测试训练神经网络，执行 `$ python evaluate.py -e`。要评估 Messidor-Original，要使用 `-m` 标志来执行它。要评估 Messidor-2，使用 `-m2` 标志。 61 | 62 | 为创建网络集合和评估预测的线性平均值，要使用 `-lm` 参数。要指定要评估为集合的多个模型，模型路径应要以逗号分隔或者满足正则的表达式。例如：`-lm =./tmp/model-1，./tmp/model-2，./tmp/model-3` 或者 `-lm =./tmp/model-？`。 63 | 64 | 注意：本研究中使用的训练模型可公开访问，可以下载; [链接](https://doi.org/10.6084/m9.figshare.8312183)。 65 | 66 | 评估脚本通过使用操作阈值而输出混淆的矩阵，特异性和灵敏度。默认的操作阈值为0.5，也可以使用 `-op` 参数来进行更改。 67 | 68 | 执行 `$ python evaluate.py -h` 以获取其他参数的选项。 69 | 70 | ### 评估自定义数据集 71 | 72 | 要在不同的数据集上来评估训练的神经网络，请按照下列的步骤操作： 73 | 74 | 1.创建 Python 的脚本或启动 Python 的会期，使用 `lib/preprocess.py` 中的 `resize_and_center_fundus` 来预先处理所有的图像并把其调整为299x299的像素。 75 | 76 | 2.创建一个目录包含着两个子目录 `image_dir/0` 和 `image_dir/1`。把诊断为可接受糖尿病视网膜病变的预先处理图像移动到 `image_dir/1` ，把没有rDR的图像移动到 `image_dir/0`。 77 | 78 | 3.创建 TFRecords： `$ python ./create_tfrecords/create_tfrecord.py --dataset_dir=image_dir`（执行 `-h` 来使用可选参数)。 79 | 80 | 4.为了评估，执行 `$ python evaluate.py -o=image_dir`。执行 `-h` 来使用可选参数。 81 | 82 | ## 基准 83 | 84 | 我们将存储库分为 TensorFlow 基准测试（[tensorflow/benchmarks](https://github.com/tensorflow/benchmarks)）来执行本研究中所使用的视网膜眼底图像和标签的基准（[链接到 fork](https://github.com/mikevoets/benchmarks))。我们还提供了一个 Docker 图像 [maikovich/tf_cnn_benchmarks](https://hub.docker.com/r/maikovich/tf_cnn_benchmarks/) 是为了更加轻松地执行基准测试。 85 | 86 | 要使用本研究的 GPU 数据来执行基准训练，使用Docker 来执行以下指令。将 `path/to/train_dir` 替换成为带有测试数据的 TFRecord 分片集目录的路径。可以根据环境的功能，而修改 `--num_gpu` ，`--batch_size` 标志。对于其它的标志，请参阅分叉存储库。 87 | 88 | ``` 89 | $ nvidia-docker run --mount type=bind，source=/path/to/train_dir，target=/data maikovich/tf_cnn_benchmarks:latest --data_dir=/data --model=inception3 --data_name=retina --num_gpus=2 --batch_size=64 90 | ``` 91 | 92 | 执行基准来测量评估测试数据集的性能，使用 `--eval` 标志来执行以上的指令，并用带有测试数据的 TFRecord 分片集目录的路径，来替换安装源目录。 93 | 94 | 在这存储库中，我们也提供了一个例子 _jinja_ file（请参阅 _benchmarks.yaml.jinja.example_）以产生 Kubernetes yaml 的描述。这可以使用在 Kubernetes 集群上的多个节点上以分布式方式，来执行基准测试。要查看更多有关如何将 jinja 文件编译为 yaml 的信息，请参阅 [自述文件](https://github.com/tensorflow/ecosystem/tree/master/kubernetes)。 95 | 96 | 原版基准的概述结果，例如：ImageNet 的数据可以在 [此处](https://www.tensorflow.org/performance/benchmarks) 找到。 97 | -------------------------------------------------------------------------------- /grading_tool.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | from tkinter import * 3 | from tkinter import filedialog 4 | import PIL.ImageTk 5 | import os 6 | import sys 7 | from glob import glob 8 | import csv 9 | from shutil import copyfile 10 | 11 | class GradingTool(Frame): 12 | def chg_image(self): 13 | self.image = PIL.ImageTk.PhotoImage(self.im) 14 | self.label.config(image=self.image, bg="#000000", 15 | width=self.image.width(), height=self.image.height()) 16 | 17 | def open_image(self, i): 18 | filename = self.im_paths[i] 19 | self.im = PIL.Image.open(filename) 20 | self.page_num = i 21 | self.chg_image() 22 | self.page_num_strf.set("{}/{}".format(self.page_num + 1, self.im_count)) 23 | self.image_name_strf.set(os.path.basename(filename)) 24 | 25 | def open(self): 26 | d = filedialog.askdirectory(initialdir=os.getcwd(), 27 | title='Select folder', mustexist=1) 28 | if d: 29 | self.dataset_name = "_".join(d.split("/")[-2:]) 30 | self.im_paths = sorted( 31 | glob(os.path.join(d, "**/*.jp*g"), 32 | recursive=True)) 33 | self.im_count = len(self.im_paths) 34 | self.gradable_dir = "./.gt/{}_gradable".format(self.dataset_name) 35 | self.csv_filename = "./.gt/{}.csv".format(self.dataset_name) 36 | self.checkp_filename = "./.gt/{}_checkp.txt".format(self.dataset_name) 37 | 38 | if os.path.exists(self.gradable_dir): 39 | checkpoint = self.get_checkpoint() 40 | self.page_num = checkpoint + 1 41 | 42 | if self.page_num == self.im_count: 43 | print("Already graded entire data set!") 44 | sys.exit(2) 45 | 46 | print("Continue at {}".format(self.page_num + 1)) 47 | else: 48 | os.makedirs(self.gradable_dir) 49 | self.page_num = 0 50 | 51 | self.csvfile = open(self.csv_filename, 'a') 52 | self.csv = csv.writer(self.csvfile, delimiter=' ') 53 | 54 | self.open_image(self.page_num) 55 | 56 | def copy_images(self): 57 | with open(self.csv_filename, 'r') as csvfile: 58 | reader = csv.reader(csvfile, delimiter=' ') 59 | 60 | for row in reader: 61 | im_path, gradable = row[:2] 62 | im_path = os.path.relpath(im_path) 63 | 64 | if gradable == "1": 65 | directory = "/".join(im_path.split("/")[:-1]) 66 | 67 | if not os.path.exists(os.path.join( 68 | self.gradable_dir, directory)): 69 | os.makedirs(os.path.join(self.gradable_dir, directory)) 70 | 71 | copyfile(im_path, os.path.join(self.gradable_dir, im_path)) 72 | 73 | def continue_later(self): 74 | self.csvfile.close() 75 | print("Progress saved. Continue grading later.") 76 | sys.exit(0) 77 | 78 | def finalize(self): 79 | self.csvfile.close() 80 | self.copy_images() 81 | print("Done grading! Gradable images are copied to {}" 82 | .format(self.gradable_dir)) 83 | sys.exit(0) 84 | 85 | def write_checkpoint(self, i): 86 | with open(self.checkp_filename, 'w') as f: 87 | f.write(str(i)) 88 | 89 | def get_checkpoint(self): 90 | with open(self.checkp_filename, 'r') as f: 91 | checkpoint = f.read() 92 | return int(checkpoint) 93 | 94 | def get_next(self): 95 | self.write_checkpoint(self.page_num) 96 | try: 97 | self.open_image(self.page_num + 1) 98 | except IndexError: 99 | self.finalize() 100 | 101 | def gradable(self): 102 | self.csv.writerow([self.im_paths[self.page_num], '1']) 103 | self.get_next() 104 | 105 | def not_gradable(self): 106 | self.csv.writerow([self.im_paths[self.page_num], '0']) 107 | self.get_next() 108 | 109 | def __init__(self): 110 | Frame.__init__(self) 111 | self.master.title('Grading Tool') 112 | 113 | self.page_num = 0 114 | self.page_num_strf = StringVar() 115 | self.image_name_strf = StringVar() 116 | 117 | frame = Frame(self) 118 | Button(frame, text="Open Folder", command=self.open).pack(side=LEFT) 119 | Button(frame, text="Gradable", command=self.gradable).pack(side=LEFT) 120 | Button(frame, text="Not Gradable", command=self.not_gradable).pack(side=LEFT) 121 | frame.pack(side=TOP, fill=BOTH) 122 | 123 | frame2 = Frame(self) 124 | Label(frame2, textvariable=self.image_name_strf).pack(side=LEFT) 125 | Label(frame2, textvariable=self.page_num_strf).pack(side=RIGHT) 126 | frame2.pack(side=TOP, fill=BOTH) 127 | 128 | self.label = Label(self) 129 | self.label.pack() 130 | 131 | frame3 = Frame(self) 132 | Button(frame3, text="Continue Later", command=self.continue_later).pack(side=LEFT) 133 | frame3.pack(side=TOP, fill=BOTH) 134 | 135 | self.pack() 136 | 137 | 138 | if __name__ == "__main__": 139 | tool = GradingTool(); tool.mainloop() 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | English - [中文](https://github.com/mikevoets/jama16-retina-replication/blob/master/README.zh.md) 2 | 3 | # Code for JAMA 2016; 316(22) Replication Study 4 | 5 | Published article link: [doi:10.1371/journal.pone.0217541](https://doi.org/10.1371/journal.pone.0217541). 6 | 7 | Published trained neural network models: [doi:10.6084/m9.figshare.8312183](https://doi.org/10.6084/m9.figshare.8312183). 8 | 9 | ## Abstract 10 | 11 | We have attempted to replicate the methods and reproduce the results in _Development and validation of a deep learning algorithm for detection of diabetic retinopathy in retinal fundus photographs_, published in JAMA 2016; 316(22) ([link](https://jamanetwork.com/journals/jama/fullarticle/2588763)), using publicly available data sets. We re-implemented the main method in the original study since the source code is not available. 12 | 13 | The original study used non-public fundus images from EyePACS and three hospitals in India for training. We used a different EyePACS data set from Kaggle. The original study used the benchmark data set Messidor-2 to evaluate the algorithm’s performance. We used another distribution of the Messidor-2 data set, since the original data set is no longer available. In the original study, ophthalmologists re-graded all images for diabetic retinopathy, macular edema, and image gradability. We have one diabetic retinopathy grade per image for our data sets, and we assessed image gradability ourselves. 14 | 15 | We were not able to reproduce the original study’s results with publicly available data. Our algorithm’s area under the receiver operating characteristic curve (AUC) of 0.951 (95% CI, 0.947-0.956) on the Kaggle EyePACS test set and 0.853 (95% CI, 0.835-0.871) on Messidor-2 did not come close to the reported AUC of 0.99 on both test sets in the original study. This may be caused by the use of a single grade per image, or different data. This study shows the challenges of reproducing deep learning method results, and the need for more replication and reproduction studies to validate deep learning methods, especially for medical image analysis. 16 | 17 | ## Requirements 18 | 19 | Python requirements: 20 | 21 | - Python >= 3.6 22 | - Tensorflow >= 1.4 23 | - OpenCV >= 1.3 24 | - Pillow 25 | - h5py 26 | - xlrd 27 | - matplotlib >= 2.1 28 | 29 | Other requirements: 30 | 31 | - p7zip-full 32 | - unzip 33 | 34 | ## Preprocessing before training 35 | 36 | 1. Run `$ git submodule update --init` to load the [create_tfrecords](https://github.com/mikevoets/create_tfrecords) repository. This tool will convert the data sets into TFRecord files. 37 | 38 | 2. Download the compressed [_Kaggle_ EyePACS data set](https://www.kaggle.com/c/diabetic-retinopathy-detection) and place all files (i.e. train and test set and labels) in the `data/eyepacs` folder. We recommend you to use the [Kaggle API](https://github.com/Kaggle/kaggle-api). 39 | 40 | 3. Run `$ ./eyepacs.sh` to decompress and preprocess the _Kaggle_ EyePACS data set, and redistribute this set into a training and test set. Run with the `--only_gradable` flag if you want to train and evaluate with gradable images only. NB: This is a large data set, so this may take hours to finish. 41 | 42 | For Messidor-2: 43 | 44 | 4. Run `$ ./messidor2.sh` to download, unpack, and preprocess the Messidor-2 data set. This data set is downloaded from the Datasets and Algorithms' section on Michael D. Abramoff's page [here](https://medicine.uiowa.edu/eye/abramoff). 45 | 46 | For Messidor-Original: 47 | 48 | 5. Download the [Messidor-Original data set](http://www.adcis.net/en/Download-Third-Party/Messidor.html) and place all files in the `data/messidor` folder. 49 | 50 | 6. Run `$ ./messidor.sh` to preprocess the Messidor-Original data set. Run with the `--only_gradable` flag if you want to evaluate with gradable images only. 51 | 52 | ## Training 53 | 54 | To start training with default settings, run `$ python train.py`. Optionally specify the path to where models checkpoints should be saved to with the `-sm` parameter. 55 | 56 | Run `$ python train.py -h` to see additional optional parameters for training with your own data set, or where to save summaries or operating threshold metrics. 57 | 58 | ## Evaluation 59 | 60 | To evaluate or test the trained neural network on the _Kaggle_ EyePACS test set, run `$ python evaluate.py -e`. To evaluate on Messidor-2, use the `-m2` flag. To evaluate on Messidor-Original, run it with the `-m` flag. 61 | 62 | To create an ensemble of networks and evaluate the linear average of predictions, use the `-lm` parameter. To specify multiple models to evaluate as an ensemble, the model paths should be comma-separated or satisfy a regular expression. For example: `-lm=./tmp/model-1,./tmp/model-2,./tmp/model-3` or `-lm=./tmp/model-?`. 63 | 64 | NB: The trained models used in this study are publicly accessible and can be downloaded from [here](https://doi.org/10.6084/m9.figshare.8312183). 65 | 66 | The evaluation script outputs a confusion matrix, and specificity and sensitivity by using an operating threshold. The default operating threshold is 0.5, and can be changed with the `-op` parameter. 67 | 68 | Run `$ python evaluate.py -h` for additional parameter options. 69 | 70 | ### Evaluate on a custom data set 71 | 72 | To evaluate the trained neural network on a different data set, follow these steps: 73 | 74 | 1. Create a Python script or start a Python session, and preprocess and resize all images to 299x299 pixels with `resize_and_center_fundus` from `lib/preprocess.py`. 75 | 76 | 2. Create a directory with two subdirectories `image_dir/0` and `image_dir/1`. Move the preprocessed images diagnosed with referable diabetic retinopathy to `image_dir/1` and the images without rDR to `image_dir/0`. 77 | 78 | 3. Create TFRecords: `$ python ./create_tfrecords/create_tfrecord.py --dataset_dir=image_dir` (run with `-h` for optional parameters). 79 | 80 | 4. To evaluate, run `$ python evaluate.py -o=image_dir`. Run with `-h` for optional parameters. 81 | 82 | ## Benchmarks 83 | 84 | We forked the repository for TensorFlow benchmarks ([tensorflow/benchmarks](https://github.com/tensorflow/benchmarks)) to run the benchmarks with the retinal fundus images and labels used in this study ([link to fork](https://github.com/mikevoets/benchmarks)). We further provide a Docker image [maikovich/tf_cnn_benchmarks](https://hub.docker.com/r/maikovich/tf_cnn_benchmarks/) to run the benchmarks easily. 85 | 86 | To run the benchmarks for training with this study's data on GPUs, run the following command with Docker. Substitute `path/to/train_dir` with the path to the directory that contains the TFRecord shards with the training set. The `--num_gpu`, `--batch_size` flags can be modified according to your environment's capabilities. For other flags, see the forked repository. 87 | 88 | ``` 89 | $ nvidia-docker run --mount type=bind,source=/path/to/train_dir,target=/data maikovich/tf_cnn_benchmarks:latest --data_dir=/data --model=inception3 --data_name=retina --num_gpus=2 --batch_size=64 90 | ``` 91 | 92 | As a reference, our training speed was 91.1 (± 0.4) images per second with one NVIDIA GTX 1080. For our training set of 57 146 images, that means approximately a minute per epoch of training. Add half a minute validation after each epoch due to initialization and loading into memory. Total training took typically 80 to 100 epochs, so it took around 2 to 2.5 hours to train one model. 93 | 94 | To run the benchmarks to measure performance of evaluating test data sets, run the above command with the `--eval` flag and substitute the mount source directory with the path to the directory containing the TFRecord shards with test data. 95 | 96 | In this repository, we also provide a example _jinja_ file (see _benchmarks.yaml.jinja.example_) to generate a Kubernetes yaml description. This can be used to run the benchmarks in a distributed fashion on multiple nodes on a Kubernetes cluster. To see more information about how to compile the jinja file into yaml, see this [README](https://github.com/tensorflow/ecosystem/tree/master/kubernetes). 97 | 98 | An overview of the original benchmark results with e.g. ImageNet data can be found [here](https://www.tensorflow.org/performance/benchmarks). 99 | -------------------------------------------------------------------------------- /eyepacs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Preprocess script for the EyePACS data set from Kaggle. 3 | 4 | # Assumes that the data set resides in ./data/eyepacs. 5 | 6 | eyepacs_dir="./data/eyepacs" 7 | default_pool_dir="$eyepacs_dir/pool" 8 | default_shuffle_seed=42 9 | default_output_dir="$eyepacs_dir/bin2" 10 | grad_grades="./vendor/eyepacs/eyepacs_gradability_grades.csv" 11 | 12 | # From [1]. 13 | get_seeded_random() 14 | { 15 | seed="$1" 16 | openssl enc -aes-256-ctr -pass pass:"$seed" -nosalt /dev/null 17 | } 18 | 19 | print_usage() 20 | { 21 | echo "" 22 | echo "Extracting and preprocessing script for Kaggle EyePACS." 23 | echo "" 24 | echo "Optional parameters: --redistribute, --pool_dir, --seed, --only_gradable" 25 | echo "--redistribute Redistribute the data set from pool (default: false)" 26 | echo "--pool_dir Path to pool folder (default: $default_pool_dir)" 27 | echo "--seed Seed number for shuffling before distributing the data set (default: $default_shuffle_seed)" 28 | echo "--only_gradable Skip ungradable images. (default: false)" 29 | echo "--output_dir Path to output directory (default: $default_output_dir)" 30 | exit 1 31 | } 32 | 33 | check_parameters() 34 | { 35 | if [ "$1" -ge 6 ]; then 36 | echo "Illegal number of parameters". 37 | print_usage 38 | fi 39 | if [ "$1" -ge 1 ]; then 40 | for param in $2; do 41 | if [ $(echo "$3" | grep -c -- "$param") -eq 0 ]; then 42 | echo "Unknown parameter $param." 43 | print_usage 44 | fi 45 | done 46 | fi 47 | return 0 48 | } 49 | 50 | if echo "$@" | grep -c -- "-h" >/dev/null; then 51 | print_usage 52 | fi 53 | 54 | strip_params=$(echo "$@" | sed "s/--\([a-z_]\+\)\(=\([^ ]\+\)\)\?/\1/g") 55 | check_parameters "$#" "$strip_params" "redistribute seed pool_dir only_gradable output_dir" 56 | 57 | # Get seed from parameters. 58 | shuffle_seed=$(echo "$@" | sed "s/.*--seed=\([0-9]\+\).*/\1/") 59 | 60 | # Replace seed with default seed number if no seed number. 61 | if ! [[ "$shuffle_seed" =~ ^-?[0-9]+$ ]]; then 62 | shuffle_seed=$default_shuffle_seed 63 | fi 64 | 65 | # Get pool directory from parameters. 66 | pool_dir=$(echo "$@" | sed "s/.*--pool_dir=\([^ ]\+\).*/\1/") 67 | 68 | # Check if output directory is valid. 69 | if ! [[ "$pool_dir" =~ ^[^-]+$ ]]; then 70 | pool_dir=$default_pool_dir 71 | fi 72 | 73 | # Get output directory from parameters. 74 | output_dir=$(echo "$@" | sed "s/.*--output_dir=\([^ ]\+\).*/\1/") 75 | 76 | if ! [[ "$output_dir" =~ ^[^-]+$ ]]; then 77 | output_dir=$default_output_dir 78 | fi 79 | 80 | if ls "$pool_dir" >/dev/null 2>&1 && ! echo "$@" | grep -c -- "--redistribute" >/dev/null; then 81 | echo "Path already exists: $pool_dir." 82 | echo "" 83 | echo "If you want to redistribute data sets from the pool, run this " 84 | echo " with the --redistribute flag." 85 | echo "If you want to extract and preprocess the images to another pool " 86 | echo " directory, specify --pool_dir with a non-existing directory." 87 | exit 1 88 | fi 89 | 90 | if ls "$output_dir" >/dev/null 2>&1; then 91 | echo "Path already exists: $output_dir." 92 | echo "" 93 | echo "Specify a non-existing --output_dir if you want to redistribute" 94 | echo " from the existing pool to another directory, along with " 95 | echo " the --redistribute flag." 96 | exit 1 97 | fi 98 | 99 | # Skip unpacking if --redistribute parameter is defined. 100 | if ! echo "$@" | grep -c -- "--redistribute" >/dev/null; then 101 | # Confirm the Basexx .zip files and annotations .xls files are present. 102 | train_zip_count=$(find "$eyepacs_dir" -maxdepth 1 -iname "train.zip.00*" | wc -l) 103 | test_zip_count=$(find "$eyepacs_dir" -maxdepth 1 -iname "test.zip.00*" | wc -l) 104 | train_csv_zip=$(find "$eyepacs_dir" -maxdepth 1 -iname "trainLabels.csv.zip" | wc -l) 105 | 106 | if [ $train_zip_count -ne 5 ]; then 107 | echo "$eyepacs_dir does not contain all train.zip files!" 108 | exit 1 109 | fi 110 | 111 | if [ $test_zip_count -ne 7 ]; then 112 | echo "$eyepacs_dir does not contain all test.zip files!" 113 | exit 1 114 | fi 115 | 116 | if [ $train_csv_zip -ne 1 ]; then 117 | echo "$eyepacs_dir does not contain trainLabels.csv.zip file!" 118 | exit 1 119 | fi 120 | 121 | # Test preprocess script. 122 | error=$(python preprocess_eyepacs.py -h 2>&1 1>/dev/null) 123 | if [ $? -ne 0 ]; then 124 | echo "$error" >&2 125 | exit 1 126 | fi 127 | 128 | echo "Unzip the data set (0/2)..." 129 | 130 | # Check if p7zip is installed. 131 | dpkg -l | grep p7zip-full 132 | if [ $? -gt 0 ]; then 133 | echo "Please install p7zip-full: apt-get/yum install p7zip-full" >&2 134 | exit 1 135 | fi 136 | 137 | # Unzip training set. 138 | 7z e "$eyepacs_dir/train.zip.001" -o"$pool_dir" || exit 1 139 | 140 | echo "Unzip the data set (1/2)..." 141 | 142 | # Unzip test set. 143 | 7z e "$eyepacs_dir/test.zip.001" -o"$pool_dir" || exit 1 144 | 145 | # Copy test labels from vendor to data set folder. 146 | cp vendor/eyepacs/testLabels.csv.zip "$eyepacs_dir/." 147 | 148 | # Unzip labels. 149 | 7z e "$eyepacs_dir/trainLabels.csv.zip" -o"$pool_dir" || exit 1 150 | 7z e "$eyepacs_dir/testLabels.csv.zip" -o"$pool_dir" || exit 1 151 | 152 | python preprocess_eyepacs.py --data_dir="$pool_dir" 153 | 154 | # Remove images in pool. 155 | find "$pool_dir" -maxdepth 1 -iname "*.jpeg" -delete 156 | 157 | # Remove ungradable images if needed. 158 | if echo "$@" | grep -c -- "--only_gradable" >/dev/null; then 159 | echo "Remove ungradable images." 160 | cat "$grad_grades" | while read tbl; do 161 | if [[ "$tbl" =~ ^.*0$ ]]; then 162 | file=$(echo "$tbl" | sed "s/\(.*\) 0/\1/") 163 | find "$pool_dir" -iname "$file*" -delete 164 | fi 165 | done 166 | fi 167 | fi 168 | 169 | # Distribution numbers for data sets with ungradable images. 170 | if echo "$@" | grep -c -- "--only_gradable" >/dev/null; then 171 | bin2_0_cnt=39202 172 | bin2_0_tr_cnt=31106 173 | bin2_1_tr_cnt=12582 174 | else 175 | bin2_0_cnt=48784 176 | bin2_0_tr_cnt=40688 177 | bin2_1_tr_cnt=16458 178 | fi 179 | 180 | echo "Finding images..." 181 | for i in {0..4}; do 182 | k=$(find "$pool_dir/$i" -iname "*.jpg" | wc -l) 183 | echo "Found $k images in class $i." 184 | done 185 | 186 | # Define distributions for data sets. 187 | bin2_0=$( 188 | find "$pool_dir/"[0-1] -iname "*.jpg" | 189 | shuf --random-source=<(get_seeded_random "$shuffle_seed") | 190 | head -n "$bin2_0_cnt" 191 | ) 192 | 193 | bin2_1=$(find "$pool_dir/"[2-4] -iname "*.jpg") 194 | bin2_1_cnt=$(echo $bin2_1 | tr " " "\n" | wc -l) 195 | 196 | echo "Creating directories for data sets" 197 | mkdir -p "$output_dir/train/0" "$output_dir/train/1" 198 | mkdir -p "$output_dir/test/0" "$output_dir/test/1" 199 | mkdir -p "$output_dir/validation" 200 | 201 | distribute_images() 202 | { 203 | echo "$1" | 204 | tr " " "\n" | 205 | $2 -n "$3" | 206 | xargs -I{} cp "{}" "$4" 207 | } 208 | 209 | echo "Gathering images for train set (0/2)" 210 | distribute_images "$bin2_0" head "$bin2_0_tr_cnt" "$output_dir/train/0/." 211 | 212 | echo "Gathering images for train set (1/2)" 213 | distribute_images "$bin2_1" head "$bin2_1_tr_cnt" "$output_dir/train/1/." 214 | 215 | echo "Gathering images for test set (0/2)" 216 | distribute_images "$bin2_0" tail "$(expr $bin2_0_cnt - $bin2_0_tr_cnt)" "$output_dir/test/0/." 217 | 218 | echo "Gathering images for test set (1/2)" 219 | distribute_images "$bin2_1" tail "$(expr $bin2_1_cnt - $bin2_1_tr_cnt)" "$output_dir/test/1/." 220 | 221 | echo "Converting data set to tfrecords..." 222 | git submodule update --init 223 | 224 | python ./create_tfrecords/create_tfrecord.py --dataset_dir="$output_dir/train" \ 225 | --num_shards=8 --validation_size=0.2 || \ 226 | { echo "Submodule not initialized. Run git submodule update --init"; 227 | exit 1; } 228 | 229 | echo "Moving validation tfrecords to separate folder." 230 | find "$output_dir/train" -name "validation*.tfrecord" -exec mv {} "$output_dir/validation/." \; 231 | find "$output_dir/train" -maxdepth 1 -iname "*.txt" -exec cp {} "$output_dir/validation/." \; 232 | 233 | python ./create_tfrecords/create_tfrecord.py --dataset_dir="$output_dir/test" \ 234 | --num_shards=4 || exit 1 235 | 236 | echo "Done!" 237 | exit 238 | 239 | # References: 240 | # [1] https://stackoverflow.com/questions/41962359/shuffling-numbers-in-bash-using-seed 241 | -------------------------------------------------------------------------------- /lib/preprocess.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Functions for preprocessing data-set. 4 | # 5 | # Implemented in Python 3.5 6 | # 7 | ######################################################################## 8 | 9 | import tensorflow as tf 10 | import os 11 | import sys 12 | import cv2 13 | import matplotlib 14 | matplotlib.use('agg') 15 | from pylab import array, arange, uint8 16 | 17 | ######################################################################## 18 | 19 | 20 | def _increase_contrast(image): 21 | """ 22 | Helper function for increasing contrast of image. 23 | """ 24 | # Create a local copy of the image. 25 | copy = image.copy() 26 | 27 | maxIntensity = 255.0 28 | x = arange(maxIntensity) 29 | 30 | # Parameters for manipulating image data. 31 | phi = 1.3 32 | theta = 1.5 33 | y = (maxIntensity/phi)*(x/(maxIntensity/theta))**0.5 34 | 35 | # Decrease intensity such that dark pixels become much darker, 36 | # and bright pixels become slightly dark. 37 | copy = (maxIntensity/phi)*(copy/(maxIntensity/theta))**2 38 | copy = array(copy, dtype=uint8) 39 | 40 | return copy 41 | 42 | 43 | def _find_contours(image): 44 | """ 45 | Helper function for finding contours of image. 46 | 47 | Returns coordinates of contours. 48 | """ 49 | # Increase constrast in image to increase changes of finding 50 | # contours. 51 | processed = _increase_contrast(image) 52 | 53 | # Get the gray-scale of the image. 54 | gray = cv2.cvtColor(processed, cv2.COLOR_BGR2GRAY) 55 | 56 | # Detect contour(s) in the image. 57 | cnts = cv2.findContours( 58 | gray, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2] 59 | center = None 60 | 61 | # At least ensure that some contours were found. 62 | if len(cnts) > 0: 63 | # Find the largest contour in the mask. 64 | c = max(cnts, key=cv2.contourArea) 65 | ((x, y), radius) = cv2.minEnclosingCircle(c) 66 | 67 | # Assume the radius is of a certain size. 68 | if radius > 100: 69 | M = cv2.moments(c) 70 | center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) 71 | 72 | return (center, radius) 73 | 74 | 75 | def _get_filename(file_path): 76 | """ 77 | Helper function to get filename including extension. 78 | """ 79 | return file_path.split("/")[-1] 80 | 81 | 82 | def _resize_and_center_fundus(image, diameter): 83 | """ 84 | Helper function for scale normalizing image. 85 | """ 86 | copy = image.copy() 87 | 88 | # Find largest contour in image. 89 | contours = _find_contours(image) 90 | 91 | # Return unless we have gotten some result contours. 92 | if contours is None: 93 | return None 94 | 95 | center, radius = contours 96 | 97 | # Calculate the min and max-boundaries for cropping the image. 98 | x_min = max(0, int(center[0] - radius)) 99 | y_min = max(0, int(center[1] - radius)) 100 | z = int(radius*2) 101 | x_max = x_min + z 102 | y_max = y_min + z 103 | 104 | # Crop the image. 105 | copy = copy[y_min:y_max, x_min:x_max] 106 | 107 | # Scale the image. 108 | fx = fy = (diameter / 2) / radius 109 | copy = cv2.resize(copy, (0, 0), fx=fx, fy=fy) 110 | 111 | # Add padding to image. 112 | shape = copy.shape 113 | 114 | # Get the border shape size. 115 | top = bottom = int((diameter - shape[0])/2) 116 | left = right = int((diameter - shape[1])/2) 117 | 118 | # Add 1 pixel if necessary. 119 | if shape[0] + top + bottom == diameter - 1: 120 | top += 1 121 | 122 | if shape[1] + left + right == diameter - 1: 123 | left += 1 124 | 125 | # Define border of the image. 126 | border = [top, bottom, left, right] 127 | 128 | # Add border. 129 | copy = cv2.copyMakeBorder(copy, *border, 130 | borderType=cv2.BORDER_CONSTANT, 131 | value=[0, 0, 0]) 132 | # Return the image. 133 | return copy 134 | 135 | 136 | def _get_image_paths(images_path): 137 | """ 138 | Helper function for getting file paths to images. 139 | """ 140 | return [os.path.join(images_path, fn) for fn in os.listdir(images_path)] 141 | 142 | 143 | def _resize_and_center_fundus_all(image_paths, save_path, diameter, verbosity): 144 | # Get the total amount of images. 145 | num_images = len(image_paths) 146 | success = 0 147 | 148 | # For each image in the specified directory. 149 | for i, image_path in enumerate(image_paths): 150 | if verbosity > 0: 151 | # Status-message. 152 | msg = "\r- Preprocessing image: {0:>6} / {1}".format( 153 | i+1, num_images) 154 | 155 | # Print the status message. 156 | sys.stdout.write(msg) 157 | sys.stdout.flush() 158 | 159 | try: 160 | # Load the image and clone it for output. 161 | image = cv2.imread(os.path.abspath(image_path), -1) 162 | 163 | processed = _resize_and_center_fundus(image, diameter=diameter) 164 | 165 | if processed is None: 166 | print("Could not preprocess {}...".format(image_path)) 167 | else: 168 | # Get the save path for the processed image. 169 | image_filename = _get_filename(image_path) 170 | image_jpeg_filename = "{0}.jpg".format(os.path.splitext( 171 | os.path.basename(image_filename))[0]) 172 | output_path = os.path.join(save_path, image_jpeg_filename) 173 | 174 | # Save the image. 175 | cv2.imwrite(output_path, processed, 176 | [int(cv2.IMWRITE_JPEG_QUALITY), 100]) 177 | 178 | success += 1 179 | except AttributeError as e: 180 | print(e) 181 | print("Could not preprocess {}...".format(image_path)) 182 | 183 | return success 184 | 185 | 186 | ######################################################################## 187 | 188 | 189 | def resize_and_center_fundus(save_path=None, images_path=None, image_paths=None, 190 | image_path=None, diameter=299, verbosity=1): 191 | """ 192 | Function for resizing and centering fundus in image. 193 | 194 | :param save_path: 195 | Required. Saves preprocessed image to the given path. 196 | 197 | :param images_path: 198 | Optional. Path to directory in where images reside. 199 | 200 | :param image_path: 201 | Optional. Single path to image. 202 | 203 | :param image_paths: 204 | Optional. List of paths to images. 205 | 206 | :param diameter: 207 | Optional. Result diameter of fundus. Defaults to 299. 208 | 209 | :return: 210 | Nothing. 211 | """ 212 | if save_path is None: 213 | raise ValueError("Save path not specified!") 214 | 215 | save_path = os.path.abspath(save_path) 216 | 217 | if image_paths is not None: 218 | return _resize_and_center_fundus_all(image_paths=image_paths, 219 | save_path=save_path, 220 | diameter=diameter, 221 | verbosity=verbosity) 222 | 223 | elif images_path is not None: 224 | # Get the paths to all images. 225 | image_paths = _get_image_paths(images_path) 226 | # Scale all images. 227 | return _resize_and_center_fundus_all(image_paths=image_paths, 228 | save_path=save_path, 229 | diameter=diameter, 230 | verbosity=verbosity) 231 | 232 | elif image_path is not None: 233 | return _resize_and_center_fundus_all(image_paths=[image_path], 234 | save_path=save_path, 235 | diameter=diameter, 236 | verbosity=verbosity) 237 | 238 | 239 | def resize(images_paths, size=299): 240 | """ 241 | Function for resizing images. 242 | 243 | :param images_paths: 244 | Required. Paths to images. 245 | 246 | :param size: 247 | Optional. Size to which resize to. Defaults to 299. 248 | 249 | :return: 250 | Nothing. 251 | """ 252 | for image_path in images_paths: 253 | image = cv2.imread(image_path) 254 | 255 | # Resize the image. 256 | image = cv2.resize(image, (size, size)) 257 | 258 | # Save the image. 259 | cv2.imwrite(image_path, image, [int(cv2.IMWRITE_JPEG_QUALITY), 100]) 260 | 261 | 262 | def rescale_min_1_to_1(image): 263 | """ 264 | Rescale image to [-1, 1]. 265 | 266 | :param image: 267 | Required. Image tensor. 268 | 269 | :return: 270 | Scaled image. 271 | """ 272 | # Image must be casted to float32 first. 273 | image = tf.cast(image, tf.float32) 274 | # Rescale image from [0, 255] to [0, 2]. 275 | image = tf.multiply(image, 1. / 127.5) 276 | # Rescale to [-1, 1]. 277 | return tf.subtract(image, 1.0) 278 | 279 | 280 | def rescale_0_to_1(image): 281 | """ 282 | Rescale image to [0, 1]. 283 | 284 | :param image: 285 | Required. Image tensor. 286 | 287 | :return: 288 | Scaled image. 289 | """ 290 | return tf.image.convert_image_dtype(image, tf.float32) 291 | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import sys 4 | import argparse 5 | import random 6 | import tensorflow as tf 7 | import numpy as np 8 | import lib.dataset 9 | import lib.evaluation 10 | import lib.metrics 11 | import csv 12 | from glob import glob 13 | 14 | print(f"Numpy version: {np.__version__}") 15 | print(f"Tensorflow version: {tf.__version__}") 16 | 17 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 18 | random.seed(432) 19 | 20 | # Default settings. 21 | default_eyepacs_dir = "./data/eyepacs/bin2/test" 22 | default_messidor_dir = "./data/messidor/bin2" 23 | default_messidor2_dir = "./data/messidor2/bin2" 24 | default_load_model_path = "./tmp/model" 25 | default_save_operating_thresholds_path = "./tmp/test_op_pts.csv" 26 | default_batch_size = 32 27 | 28 | parser = argparse.ArgumentParser( 29 | description="Evaluate performance of trained graph " 30 | "on test data set. " 31 | "Specify --data_dir if you use the -o param.") 32 | parser.add_argument("-m", "--messidor", action="store_true", 33 | help="evaluate performance on Messidor-Original") 34 | parser.add_argument("-m2", "--messidor2", action="store_true", 35 | help="evaluate performance on Messidor-2") 36 | parser.add_argument("-e", "--eyepacs", action="store_true", 37 | help="evaluate performance on EyePacs set") 38 | parser.add_argument("-o", "--other", action="store_true", 39 | help="evaluate performance on your own dataset") 40 | parser.add_argument("--data_dir", help="directory where data set resides") 41 | parser.add_argument("-lm", "--load_model_path", 42 | help="path to where graph model should be loaded from " 43 | "creates an ensemble if paths are comma separated " 44 | "or a regexp", 45 | default=default_load_model_path) 46 | parser.add_argument("-so", "--save_operating_thresholds_path", 47 | help="path to where operating points metrics should be saved", 48 | default=default_save_operating_thresholds_path) 49 | parser.add_argument("-b", "--batch_size", 50 | help="batch size", default=default_batch_size) 51 | parser.add_argument("-op", "--operating_threshold", 52 | help="operating threshold", default=0.5) 53 | 54 | args = parser.parse_args() 55 | 56 | if bool(args.eyepacs) == bool(args.messidor) == bool(args.messidor2) == bool(args.other): 57 | print("Can only evaluate one data set at once!") 58 | parser.print_help() 59 | sys.exit(2) 60 | 61 | if args.data_dir is not None: 62 | data_dir = str(args.data_dir) 63 | elif args.eyepacs: 64 | data_dir = default_eyepacs_dir 65 | elif args.messidor: 66 | data_dir = default_messidor_dir 67 | elif args.messidor2: 68 | data_dir = default_messidor2_dir 69 | elif args.other and args.data_dir is None: 70 | print("Please specify --data_dir.") 71 | parser.print_help() 72 | sys.exit(2) 73 | 74 | load_model_path = str(args.load_model_path) 75 | batch_size = int(args.batch_size) 76 | save_operating_thresholds_path = str(args.save_operating_thresholds_path) 77 | operating_threshold = float(args.operating_threshold) 78 | 79 | # Check if the model path has comma-separated entries. 80 | if "," in load_model_path: 81 | load_model_paths = load_model_path.split(",") 82 | # Check if the model path has a regexp character. 83 | elif any(char in load_model_path for char in '*+?'): 84 | load_model_paths = [".".join(x.split(".")[:-1]) 85 | for x in glob("{}*".format(load_model_path))] 86 | load_model_paths = list(set(load_model_paths)) 87 | else: 88 | load_model_paths = [load_model_path] 89 | 90 | print(""" 91 | Evaluating: {}, 92 | Saving operating thresholds metrics at: {}, 93 | Using operating treshold: {}, 94 | """.format(data_dir, save_operating_thresholds_path, operating_threshold)) 95 | print("Trying to load model(s):\n{}".format("\n".join(load_model_paths))) 96 | 97 | # Other setting variables. 98 | num_channels = 3 99 | num_workers = 8 100 | prefetch_buffer_size = 2 * batch_size 101 | num_thresholds = 200 102 | kepsilon = 1e-7 103 | 104 | # Define thresholds. 105 | thresholds = lib.metrics.generate_thresholds(num_thresholds, kepsilon) \ 106 | + [operating_threshold] 107 | 108 | got_all_y = False 109 | all_y = [] 110 | 111 | 112 | def feed_images(sess, x_tensor, y_tensor, x_batcher, y_batcher): 113 | _x, _y = sess.run([x_batcher, y_batcher]) 114 | if not got_all_y: 115 | all_y.append(_y) 116 | return {x_tensor: _x, y_tensor: _y} 117 | 118 | 119 | eval_graph = tf.Graph() 120 | with eval_graph.as_default() as g: 121 | # Variable for average predictions. 122 | average_predictions = tf.placeholder(tf.float32, shape=[None, 1]) 123 | all_labels = tf.placeholder(tf.float32, shape=[None, 1]) 124 | 125 | # Metrics for finding best validation set. 126 | tp, update_tp, reset_tp = lib.metrics.create_reset_metric( 127 | tf.metrics.true_positives_at_thresholds, scope='tp', 128 | labels=all_labels, predictions=average_predictions, 129 | thresholds=thresholds) 130 | 131 | fp, update_fp, reset_fp = lib.metrics.create_reset_metric( 132 | tf.metrics.false_positives_at_thresholds, scope='fp', 133 | labels=all_labels, predictions=average_predictions, 134 | thresholds=thresholds) 135 | 136 | fn, update_fn, reset_fn = lib.metrics.create_reset_metric( 137 | tf.metrics.false_negatives_at_thresholds, scope='fn', 138 | labels=all_labels, predictions=average_predictions, 139 | thresholds=thresholds) 140 | 141 | tn, update_tn, reset_tn = lib.metrics.create_reset_metric( 142 | tf.metrics.true_negatives_at_thresholds, scope='tn', 143 | labels=all_labels, predictions=average_predictions, 144 | thresholds=thresholds) 145 | 146 | # Last element presents the metrics at operating threshold. 147 | confusion_matrix = lib.metrics.confusion_matrix( 148 | tp[-1], fp[-1], fn[-1], tn[-1], scope='confusion_matrix') 149 | 150 | brier, update_brier, reset_brier = lib.metrics.create_reset_metric( 151 | tf.metrics.mean_squared_error, scope='brier', 152 | labels=all_labels, predictions=average_predictions) 153 | 154 | auc, update_auc, reset_auc = lib.metrics.create_reset_metric( 155 | tf.metrics.auc, scope='auc', 156 | labels=all_labels, predictions=average_predictions) 157 | 158 | specificities = tf.div(tn, tn + fp + kepsilon) 159 | sensitivities = tf.div(tp, tp + fn + kepsilon) 160 | 161 | 162 | all_predictions = [] 163 | 164 | for model_path in load_model_paths: 165 | # Start session. 166 | with tf.Session(graph=tf.Graph()) as sess: 167 | tf.keras.backend.set_session(sess) 168 | tf.keras.backend.set_learning_phase(False) 169 | 170 | # Load the meta graph and restore variables from training. 171 | saver = tf.train.import_meta_graph("{}.meta".format(model_path)) 172 | saver.restore(sess, model_path) 173 | 174 | graph = tf.get_default_graph() 175 | x = graph.get_tensor_by_name("x:0") 176 | y = graph.get_tensor_by_name("y:0") 177 | 178 | try: 179 | predictions = graph.get_tensor_by_name("predictions:0") 180 | except KeyError: 181 | predictions = graph.get_tensor_by_name("predictions/Sigmoid:0") 182 | 183 | # Initialize the test set. 184 | test_dataset = lib.dataset.initialize_dataset( 185 | data_dir, batch_size, num_workers=num_workers, 186 | prefetch_buffer_size=prefetch_buffer_size, 187 | num_channels=num_channels, augmentation=False) 188 | 189 | # Create an iterator. 190 | iterator = tf.data.Iterator.from_structure( 191 | test_dataset.output_types, test_dataset.output_shapes) 192 | 193 | test_images, test_labels = iterator.get_next() 194 | 195 | test_init_op = iterator.make_initializer(test_dataset) 196 | 197 | # Perform the evaluation. 198 | test_predictions = lib.evaluation.perform_test( 199 | sess=sess, init_op=test_init_op, 200 | feed_dict_fn=feed_images, 201 | feed_dict_args={"sess": sess, "x_tensor": x, "y_tensor": y, 202 | "x_batcher": test_images, "y_batcher": test_labels}, 203 | custom_tensors=[predictions]) 204 | 205 | all_predictions.append(test_predictions[0]) 206 | 207 | tf.reset_default_graph() 208 | got_all_y = True 209 | 210 | # Convert the predictions to a numpy array. 211 | all_predictions = np.array(all_predictions) 212 | 213 | # Calculate the linear average of all predictions. 214 | avg_pred = np.mean(all_predictions, axis=0) 215 | 216 | # Convert all labels to numpy array. 217 | all_y = np.vstack(all_y) 218 | 219 | # Use these predictions for printing evaluation results. 220 | with tf.Session(graph=eval_graph) as sess: 221 | # Reset all streaming variables. 222 | sess.run([reset_tp, reset_fp, reset_fn, reset_tn, reset_brier, reset_auc]) 223 | 224 | # Update all streaming variables with predictions. 225 | sess.run([update_tp, update_fp, update_fn, update_tn, 226 | update_brier, update_auc], 227 | feed_dict={average_predictions: avg_pred, all_labels: all_y}) 228 | 229 | # Retrieve confusion matrix and estimated roc auc score. 230 | test_conf_matrix, test_brier, test_auc, test_specificities, \ 231 | test_sensitivities = \ 232 | sess.run([confusion_matrix, brier, auc, specificities, 233 | sensitivities]) 234 | 235 | # Print total roc auc score for validation. 236 | print(f"Brier score: {test_brier:6.4}, AUC: {test_auc:10.8}") 237 | 238 | # Print confusion matrix. 239 | print(f"Confusion matrix at operating threshold {operating_threshold:0.3f}") 240 | print(test_conf_matrix[0]) 241 | 242 | # Print sensitivity and specificity at operating threshold. 243 | print("Specificity: {0:0.4f}, Sensitivity: {1:0.4f} at " \ 244 | "Operating Threshold {2:0.4f}." \ 245 | .format(test_specificities[-1], test_sensitivities[-1], 246 | thresholds[-1])) 247 | 248 | # Write sensitivities and specificities to file. 249 | with open(save_operating_thresholds_path, 'w') as csvfile: 250 | writer = csv.writer(csvfile, delimiter=' ') 251 | writer.writerow(['threshold', 'specificity', 'sensitivity']) 252 | 253 | for idx in range(num_thresholds): 254 | writer.writerow([ 255 | "{:0.4f}".format(x) for x in [ 256 | thresholds[idx], test_specificities[idx], 257 | test_sensitivities[idx]]]) 258 | 259 | sys.exit(0) 260 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import os 4 | import random 5 | import sys 6 | import argparse 7 | import csv 8 | from glob import glob 9 | import lib.metrics 10 | import lib.dataset 11 | import lib.evaluation 12 | from lib.preprocess import rescale_min_1_to_1, rescale_0_to_1 13 | 14 | print(f"Numpy version: {np.__version__}") 15 | print(f"Tensorflow version: {tf.__version__}") 16 | 17 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 18 | random.seed(432) 19 | 20 | # Various loading and saving constants. 21 | default_train_dir = "./data/eyepacs/bin2/train" 22 | default_val_dir = "./data/eyepacs/bin2/validation" 23 | default_save_model_path = "./tmp/model" 24 | default_save_summaries_dir = "./tmp/logs" 25 | default_save_operating_thresholds_path = "./tmp/op_pts.csv" 26 | 27 | parser = argparse.ArgumentParser( 28 | description="Trains and saves neural network for " 29 | "detection of diabetic retinopathy.") 30 | parser.add_argument("-t", "--train_dir", 31 | help="path to folder that contains training tfrecords", 32 | default=default_train_dir) 33 | parser.add_argument("-v", "--val_dir", 34 | help="path to folder that contains validation tfrecords", 35 | default=default_val_dir) 36 | parser.add_argument("-sm", "--save_model_path", 37 | help="path to where graph model should be saved", 38 | default=default_save_model_path) 39 | parser.add_argument("-ss", "--save_summaries_dir", 40 | help="path to folder where summaries should be saved", 41 | default=default_save_summaries_dir) 42 | parser.add_argument("-so", "--save_operating_thresholds_path", 43 | help="path to where operating points should be saved", 44 | default=default_save_operating_thresholds_path) 45 | 46 | args = parser.parse_args() 47 | train_dir = str(args.train_dir) 48 | val_dir = str(args.val_dir) 49 | save_model_path = str(args.save_model_path) 50 | save_summaries_dir = str(args.save_summaries_dir) 51 | save_operating_thresholds_path = str(args.save_operating_thresholds_path) 52 | 53 | print(""" 54 | Training images folder: {}, 55 | Validation images folder: {}, 56 | Saving model and graph checkpoints at: {}, 57 | Saving summaries at: {}, 58 | Saving operating points at: {}, 59 | """.format(train_dir, val_dir, save_model_path, save_summaries_dir, 60 | save_operating_thresholds_path)) 61 | 62 | # Various constants. 63 | num_channels = 3 64 | num_workers = 8 65 | normalization_fn = rescale_min_1_to_1 66 | 67 | # Hyper-parameters for training. 68 | learning_rate = 1e-3 69 | decay = 4e-5 70 | train_batch_size = 32 71 | 72 | # Hyper-parameters for validation. 73 | num_epochs = 200 74 | wait_epochs = 10 75 | min_delta_auc = 0.01 76 | val_batch_size = 32 77 | num_thresholds = 200 78 | kepsilon = 1e-7 79 | 80 | # Define thresholds. 81 | thresholds = lib.metrics.generate_thresholds(num_thresholds, kepsilon) + [0.5] 82 | 83 | # Buffer size for image shuffling. 84 | shuffle_buffer_size = 2048 85 | prefetch_buffer_size = 2 * train_batch_size 86 | 87 | # Set image datas format to channels first. 88 | image_data_format = 'channels_first' 89 | 90 | # Set up a session and bind it to Keras. 91 | sess = tf.Session() 92 | tf.keras.backend.set_session(sess) 93 | tf.keras.backend.set_learning_phase(True) 94 | tf.keras.backend.set_image_data_format(image_data_format) 95 | 96 | # Initialize each data set. 97 | train_dataset = lib.dataset.initialize_dataset( 98 | train_dir, train_batch_size, 99 | num_workers=num_workers, prefetch_buffer_size=prefetch_buffer_size, 100 | shuffle_buffer_size=shuffle_buffer_size, num_channels=num_channels, 101 | normalization_fn=normalization_fn) 102 | 103 | val_dataset = lib.dataset.initialize_dataset( 104 | val_dir, val_batch_size, 105 | num_workers=num_workers, prefetch_buffer_size=prefetch_buffer_size, 106 | shuffle_buffer_size=shuffle_buffer_size, num_channels=num_channels, 107 | normalization_fn=normalization_fn, augmentation=False) 108 | 109 | # Create initializable iterators. 110 | iterator = tf.data.Iterator.from_structure( 111 | train_dataset.output_types, train_dataset.output_shapes) 112 | 113 | images, labels = iterator.get_next() 114 | x = tf.placeholder_with_default(images, images.shape, name='x') 115 | y = tf.placeholder_with_default(labels, labels.shape, name='y') 116 | 117 | train_init_op = iterator.make_initializer(train_dataset) 118 | val_init_op = iterator.make_initializer(val_dataset) 119 | 120 | # Base model InceptionV3 without top and global average pooling. 121 | base_model = tf.keras.applications.InceptionV3( 122 | include_top=False, weights='imagenet', pooling='avg', input_tensor=x) 123 | 124 | # Add dense layer with the same amount of neurons as labels. 125 | logits = tf.layers.dense(base_model.output, units=1) 126 | 127 | # Get the predictions with a sigmoid activation function. 128 | predictions = tf.sigmoid(logits, name='predictions') 129 | 130 | # Retrieve loss of network using cross entropy. 131 | mean_xentropy = tf.reduce_mean( 132 | tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)) 133 | 134 | # Define optimizer. 135 | global_step = tf.Variable(0, dtype=tf.int32) 136 | 137 | train_op = tf.train.RMSPropOptimizer( 138 | learning_rate=learning_rate, decay=decay) \ 139 | .minimize(loss=mean_xentropy, global_step=global_step) 140 | 141 | # Metrics for finding best validation set. 142 | tp, update_tp, reset_tp = lib.metrics.create_reset_metric( 143 | tf.metrics.true_positives_at_thresholds, scope='tp', 144 | labels=y, predictions=predictions, thresholds=thresholds) 145 | 146 | fp, update_fp, reset_fp = lib.metrics.create_reset_metric( 147 | tf.metrics.false_positives_at_thresholds, scope='fp', 148 | labels=y, predictions=predictions, thresholds=thresholds) 149 | 150 | fn, update_fn, reset_fn = lib.metrics.create_reset_metric( 151 | tf.metrics.false_negatives_at_thresholds, scope='fn', 152 | labels=y, predictions=predictions, thresholds=thresholds) 153 | 154 | tn, update_tn, reset_tn = lib.metrics.create_reset_metric( 155 | tf.metrics.true_negatives_at_thresholds, scope='tn', 156 | labels=y, predictions=predictions, thresholds=thresholds) 157 | 158 | confusion_matrix = lib.metrics.confusion_matrix( 159 | tp[-1], fp[-1], fn[-1], tn[-1], scope='confusion_matrix') 160 | 161 | brier, update_brier, reset_brier = lib.metrics.create_reset_metric( 162 | tf.metrics.mean_squared_error, scope='brier', 163 | labels=y, predictions=predictions) 164 | 165 | auc, update_auc, reset_auc = lib.metrics.create_reset_metric( 166 | tf.metrics.auc, scope='auc', labels=y, predictions=predictions) 167 | tf.summary.scalar('auc', auc) 168 | 169 | specificities = tf.div(tn, tn + fp + kepsilon) 170 | sensitivities = tf.div(tp, tp + fn + kepsilon) 171 | 172 | # Merge all the summaries and write them out. 173 | summaries_op = tf.summary.merge_all() 174 | train_writer = tf.summary.FileWriter(save_summaries_dir + "/train") 175 | test_writer = tf.summary.FileWriter(save_summaries_dir + "/test") 176 | 177 | def print_training_status(epoch, num_epochs, batch_num, xent, i_step=None): 178 | def length(x): return len(str(x)) 179 | 180 | m = [] 181 | m.append( 182 | f"Epoch: {{0:>{length(num_epochs)}}}/{{1:>{length(num_epochs)}}}" 183 | .format(epoch, num_epochs)) 184 | m.append(f"Batch: {batch_num:>4}, Xent: {xent:6.4}") 185 | 186 | if i_step is not None: 187 | m.append(f"Step: {i_step:>10}") 188 | 189 | print(", ".join(m), end="\r") 190 | 191 | 192 | # Add ops for saving and restoring all variables. 193 | saver = tf.train.Saver() 194 | 195 | # Initialize variables. 196 | sess.run(tf.global_variables_initializer()) 197 | sess.run(tf.local_variables_initializer()) 198 | 199 | 200 | # Train for the specified amount of epochs. 201 | # Can be stopped early if peak of validation auc (Area under curve) 202 | # is reached. 203 | latest_peak_auc = 0 204 | waited_epochs = 0 205 | 206 | for epoch in range(num_epochs): 207 | # Start training. 208 | tf.keras.backend.set_learning_phase(True) 209 | sess.run(train_init_op) 210 | batch_num = 0 211 | 212 | # Track brier score for an indication on convergance. 213 | sess.run(reset_brier) 214 | 215 | try: 216 | while True: 217 | # Optimize cross entropy. 218 | i_global, batch_xent, *_ = sess.run( 219 | [global_step, mean_xentropy, train_op, update_brier]) 220 | 221 | # Print a nice training status. 222 | print_training_status( 223 | epoch, num_epochs, batch_num, batch_xent, i_global) 224 | 225 | # Report summaries. 226 | batch_num += 1 227 | 228 | except tf.errors.OutOfRangeError: 229 | # Retrieve training brier score. 230 | train_brier = sess.run(brier) 231 | print("\nEnd of epoch {0}! (Brier: {1:8.6})".format(epoch, train_brier)) 232 | 233 | # Perform validation. 234 | val_auc = lib.evaluation.perform_test( 235 | sess=sess, init_op=val_init_op, 236 | summary_writer=train_writer, epoch=epoch) 237 | 238 | if val_auc < latest_peak_auc + min_delta_auc: 239 | # Stop early if peak of val auc has been reached. 240 | # If it is lower than the previous auc value, wait up to `wait_epochs` 241 | # to see if it does not increase again. 242 | 243 | if wait_epochs == waited_epochs: 244 | print("Stopped early at epoch {0} with saved peak auc {1:10.8}" 245 | .format(epoch+1, latest_peak_auc)) 246 | break 247 | 248 | waited_epochs += 1 249 | else: 250 | latest_peak_auc = val_auc 251 | print(f"New peak auc reached: {val_auc:10.8}") 252 | 253 | # Save the model weights. 254 | saver.save(sess, save_model_path) 255 | 256 | # Reset waited epochs. 257 | waited_epochs = 0 258 | 259 | # Load the saved best meta graph and restore variables from that checkpoint. 260 | saver = tf.train.import_meta_graph("{}.meta".format(save_model_path)) 261 | saver.restore(sess, save_model_path) 262 | 263 | # Get predictions of all data of our training set. 264 | tf.keras.backend.set_learning_phase(False) 265 | sess.run([train_init_op, reset_tp, reset_fp, reset_fn, reset_tn]) 266 | 267 | try: 268 | while True: 269 | # Update all confusion metrics for each batch. 270 | sess.run([update_tp, update_fp, update_fn, update_tn]) 271 | 272 | except tf.errors.OutOfRangeError: 273 | pass 274 | 275 | # Write sensitivities and specificities to file. 276 | with open(save_operating_thresholds_path, 'w') as csvfile: 277 | writer = csv.writer(csvfile, delimiter=' ') 278 | writer.writerow(['threshold', 'specificity', 'sensitivity']) 279 | 280 | train_specificities, train_sensitivities = \ 281 | sess.run([specificities, sensitivities]) 282 | 283 | for idx in range(num_thresholds): 284 | writer.writerow([ 285 | "{:0.4f}".format(x) for x in [ 286 | thresholds[idx], train_specificities[idx], 287 | train_sensitivities[idx]]]) 288 | 289 | # Close the session. 290 | sess.close() 291 | sys.exit(0) 292 | -------------------------------------------------------------------------------- /vendor/messidor/abramoff-messidor-2-refstandard-jul16.csv: -------------------------------------------------------------------------------- 1 | examid,rDR 2 | IM0001,0 3 | IM0002,0 4 | IM0003,0 5 | IM0004,0 6 | IM0005,0 7 | IM0006,0 8 | IM0007,0 9 | IM0008,0 10 | IM0009,0 11 | IM0010,0 12 | IM0011,0 13 | IM0012,0 14 | IM0013,0 15 | IM0014,0 16 | IM0015,0 17 | IM0016,0 18 | IM0017,0 19 | IM0018,0 20 | IM0019,1 21 | IM0020,0 22 | IM0021,1 23 | IM0022,0 24 | IM0023,0 25 | IM0024,0 26 | IM0025,1 27 | IM0026,1 28 | IM0027,1 29 | IM0028,1 30 | IM0029,0 31 | IM0030,0 32 | IM0031,0 33 | IM0032,0 34 | IM0033,0 35 | IM0034,0 36 | IM0035,0 37 | IM0036,0 38 | IM0037,0 39 | IM0038,0 40 | IM0039,0 41 | IM0040,0 42 | IM0041,0 43 | IM0042,0 44 | IM0043,0 45 | IM0044,0 46 | IM0045,0 47 | IM0046,0 48 | IM0047,0 49 | IM0048,0 50 | IM0049,0 51 | IM0050,0 52 | IM0051,0 53 | IM0052,0 54 | IM0053,0 55 | IM0054,0 56 | IM0055,0 57 | IM0056,0 58 | IM0057,0 59 | IM0058,0 60 | IM0059,0 61 | IM0060,0 62 | IM0061,0 63 | IM0062,0 64 | IM0063,0 65 | IM0064,0 66 | IM0065,0 67 | IM0066,0 68 | IM0067,0 69 | IM0068,0 70 | IM0069,0 71 | IM0070,0 72 | IM0071,0 73 | IM0072,0 74 | IM0073,0 75 | IM0074,0 76 | IM0075,0 77 | IM0076,0 78 | IM0077,0 79 | IM0078,0 80 | IM0079,0 81 | IM0080,0 82 | IM0081,0 83 | IM0082,0 84 | IM0083,0 85 | IM0085,0 86 | IM0087,0 87 | IM0089,0 88 | IM0090,1 89 | IM0091,0 90 | IM0092,1 91 | IM0093,0 92 | IM0094,1 93 | IM0096,0 94 | IM0097,1 95 | IM0098,1 96 | IM0099,0 97 | IM0100,0 98 | IM0101,1 99 | IM0102,1 100 | IM0103,1 101 | IM0104,1 102 | IM0105,1 103 | IM0106,1 104 | IM0107,1 105 | IM0108,1 106 | IM0109,1 107 | IM0110,1 108 | IM0111,0 109 | IM0112,0 110 | IM0113,0 111 | IM0114,0 112 | IM0115,0 113 | IM0116,0 114 | IM0117,0 115 | IM0118,0 116 | IM0119,0 117 | IM0120,0 118 | IM0121,0 119 | IM0122,0 120 | IM0123,0 121 | IM0124,0 122 | IM0125,0 123 | IM0126,0 124 | IM0127,0 125 | IM0128,0 126 | IM0129,0 127 | IM0130,0 128 | IM0131,0 129 | IM0132,0 130 | IM0133,0 131 | IM0134,1 132 | IM0135,1 133 | IM0136,0 134 | IM0137,1 135 | IM0138,1 136 | IM0139,1 137 | IM0140,1 138 | IM0141,1 139 | IM0142,0 140 | IM0143,0 141 | IM0144,0 142 | IM0145,0 143 | IM0146,0 144 | IM0147,1 145 | IM0148,0 146 | IM0254,0 147 | IM0255,1 148 | IM0256,0 149 | IM0257,1 150 | IM0258,0 151 | IM0259,1 152 | IM0260,0 153 | IM0261,0 154 | IM0263,1 155 | IM0264,0 156 | IM0265,1 157 | IM0266,1 158 | IM0268,1 159 | IM0270,1 160 | IM0271,0 161 | IM0272,0 162 | IM0273,1 163 | IM0274,1 164 | IM0275,0 165 | IM0276,1 166 | IM0278,1 167 | IM0280,0 168 | IM0281,1 169 | IM0282,0 170 | IM0283,0 171 | IM0284,1 172 | IM0285,1 173 | IM0287,0 174 | IM0288,0 175 | IM0289,1 176 | IM0290,0 177 | IM0291,1 178 | IM0292,0 179 | IM0293,0 180 | IM0294,0 181 | IM0295,1 182 | IM0296,1 183 | IM0297,0 184 | IM0298,1 185 | IM0299,1 186 | IM0300,1 187 | IM0301,1 188 | IM0302,1 189 | IM0303,0 190 | IM0304,1 191 | IM0305,0 192 | IM0306,0 193 | IM0307,1 194 | IM0308,0 195 | IM0309,1 196 | IM0310,0 197 | IM0311,0 198 | IM0312,0 199 | IM0313,0 200 | IM0314,1 201 | IM0315,1 202 | IM0316,1 203 | IM0318,0 204 | IM0319,0 205 | IM0320,0 206 | IM0321,0 207 | IM0322,1 208 | IM0323,0 209 | IM0324,1 210 | IM0325,0 211 | IM0326,0 212 | IM0327,0 213 | IM0328,1 214 | IM0329,0 215 | IM0330,0 216 | IM0331,1 217 | IM0332,1 218 | IM0333,0 219 | IM0334,0 220 | IM0336,0 221 | IM0337,1 222 | IM0338,0 223 | IM0339,0 224 | IM0340,0 225 | IM0341,1 226 | IM0342,1 227 | IM0343,1 228 | IM0344,1 229 | IM0346,1 230 | IM0350,0 231 | IM0353,0 232 | IM0354,0 233 | IM0355,1 234 | IM0356,0 235 | IM0357,1 236 | IM0358,1 237 | IM0359,1 238 | IM0360,1 239 | IM0361,1 240 | IM0362,0 241 | IM0365,0 242 | IM0367,1 243 | IM0368,0 244 | IM0369,1 245 | IM0370,1 246 | IM0371,0 247 | IM0372,0 248 | IM0373,0 249 | IM0374,0 250 | IM0376,0 251 | IM0377,0 252 | IM0378,0 253 | IM0379,0 254 | IM0380,1 255 | IM0381,0 256 | IM0382,0 257 | IM0383,0 258 | IM0384,0 259 | IM0385,1 260 | IM0386,0 261 | IM0387,0 262 | IM0388,1 263 | IM0389,1 264 | IM0390,0 265 | IM0391,1 266 | IM0392,0 267 | IM0393,1 268 | IM0395,0 269 | IM0396,1 270 | IM0397,1 271 | IM0398,0 272 | IM0399,0 273 | IM0400,0 274 | IM0401,0 275 | IM0402,0 276 | IM0403,0 277 | IM0404,0 278 | IM0405,0 279 | IM0406,0 280 | IM0407,0 281 | IM0408,0 282 | IM0409,0 283 | IM0410,1 284 | IM0411,1 285 | IM0412,1 286 | IM0413,1 287 | IM0414,0 288 | IM0415,1 289 | IM0416,0 290 | IM0417,1 291 | IM0418,1 292 | IM0419,1 293 | IM0420,0 294 | IM0421,1 295 | IM0422,1 296 | IM0423,0 297 | IM0424,1 298 | IM0425,0 299 | IM0426,0 300 | IM0427,0 301 | IM0428,1 302 | IM0429,0 303 | IM0430,1 304 | IM0431,1 305 | IM0432,0 306 | IM0433,0 307 | IM0434,1 308 | IM0436,1 309 | IM0437,0 310 | IM0438,1 311 | IM0440,0 312 | IM0441,0 313 | IM0442,1 314 | IM0443,1 315 | IM0444,1 316 | IM0445,0 317 | IM0447,0 318 | IM0448,0 319 | IM0449,0 320 | IM0450,0 321 | IM0451,1 322 | IM0452,1 323 | IM0453,1 324 | IM0457,1 325 | IM0458,0 326 | IM0460,0 327 | IM0462,0 328 | IM0463,0 329 | IM0464,0 330 | IM0465,1 331 | IM0466,1 332 | IM0467,1 333 | IM0468,0 334 | IM0469,0 335 | IM0470,0 336 | IM0471,0 337 | IM0472,0 338 | IM0473,1 339 | IM0474,1 340 | IM0475,0 341 | IM0476,0 342 | IM0477,0 343 | IM0478,0 344 | IM0479,0 345 | IM0480,0 346 | IM0481,0 347 | IM0482,1 348 | IM0483,1 349 | IM0484,0 350 | IM0485,1 351 | IM0486,0 352 | IM0487,1 353 | IM0488,1 354 | IM0489,0 355 | IM0490,0 356 | IM0491,0 357 | IM0492,0 358 | IM0493,0 359 | IM0494,0 360 | IM0495,0 361 | IM0496,0 362 | IM0497,0 363 | IM0498,0 364 | IM0499,0 365 | IM0500,0 366 | IM0501,0 367 | IM0502,0 368 | IM0503,1 369 | IM0504,1 370 | IM0505,0 371 | IM0506,0 372 | IM0507,1 373 | IM0508,1 374 | IM0509,1 375 | IM0510,1 376 | IM0511,0 377 | IM0512,1 378 | IM0513,0 379 | IM0514,0 380 | IM0515,1 381 | IM0516,0 382 | IM0517,1 383 | IM0518,0 384 | IM0519,0 385 | IM0520,0 386 | IM0521,1 387 | IM0522,1 388 | IM0523,1 389 | IM0524,0 390 | IM0525,1 391 | IM0526,0 392 | IM0527,1 393 | IM0528,0 394 | IM0529,0 395 | IM0530,0 396 | IM0531,0 397 | IM0532,0 398 | IM0533,1 399 | IM0534,1 400 | IM0535,0 401 | IM0536,0 402 | IM0537,1 403 | IM0538,0 404 | IM0539,0 405 | IM0540,0 406 | IM0541,0 407 | IM0542,1 408 | IM0543,1 409 | IM0544,0 410 | IM0545,0 411 | IM0546,0 412 | IM0547,0 413 | IM0548,0 414 | IM0549,1 415 | IM0550,0 416 | IM0552,0 417 | IM0553,0 418 | IM0554,1 419 | IM0555,0 420 | IM0556,1 421 | IM0557,1 422 | IM0558,1 423 | IM0559,1 424 | IM0560,0 425 | IM0561,0 426 | IM0563,1 427 | IM0564,0 428 | IM0565,0 429 | IM0566,0 430 | IM0567,0 431 | IM0568,0 432 | IM0569,1 433 | IM0570,0 434 | IM0571,0 435 | IM0572,0 436 | IM0573,0 437 | IM0574,0 438 | IM0575,1 439 | IM0576,1 440 | IM0577,0 441 | IM0578,1 442 | IM0579,0 443 | IM0580,1 444 | IM0581,0 445 | IM0582,0 446 | IM0583,0 447 | IM0584,0 448 | IM0585,1 449 | IM0586,0 450 | IM0587,0 451 | IM0588,0 452 | IM0589,0 453 | IM0590,0 454 | IM0591,0 455 | IM0592,1 456 | IM0593,0 457 | IM0594,1 458 | IM0595,0 459 | IM0596,0 460 | IM0597,1 461 | IM0598,0 462 | IM0599,1 463 | IM0600,0 464 | IM0601,0 465 | IM0602,0 466 | IM0603,0 467 | IM0604,0 468 | IM0605,0 469 | IM0606,0 470 | IM0607,1 471 | IM0608,0 472 | IM0609,0 473 | IM0610,0 474 | IM0611,0 475 | IM0612,0 476 | IM0613,0 477 | IM0614,1 478 | IM0615,0 479 | IM0616,0 480 | IM0617,0 481 | IM0618,0 482 | IM0619,0 483 | IM0620,1 484 | IM0621,0 485 | IM0622,0 486 | IM0623,0 487 | IM0624,0 488 | IM0625,0 489 | IM0626,0 490 | IM0627,0 491 | IM0628,0 492 | IM0629,0 493 | IM0630,1 494 | IM0631,0 495 | IM0632,0 496 | IM0633,1 497 | IM0634,0 498 | IM0635,0 499 | IM0636,0 500 | IM0637,0 501 | IM0638,0 502 | IM0639,0 503 | IM0640,1 504 | IM0641,0 505 | IM0642,0 506 | IM0643,1 507 | IM0644,0 508 | IM0645,1 509 | IM0646,0 510 | IM0647,0 511 | IM0648,1 512 | IM0649,0 513 | IM0650,0 514 | IM0651,0 515 | IM0652,0 516 | IM0653,1 517 | IM0654,1 518 | IM0655,0 519 | IM0656,0 520 | IM0657,0 521 | IM0658,1 522 | IM0659,0 523 | IM0660,0 524 | IM0661,0 525 | IM0662,0 526 | IM0663,0 527 | IM0664,0 528 | IM0665,0 529 | IM0666,0 530 | IM0667,0 531 | IM1002,0 532 | IM1005,0 533 | IM1006,0 534 | IM1007,0 535 | IM1008,1 536 | IM1009,0 537 | IM1010,0 538 | IM1011,0 539 | IM1012,0 540 | IM1013,0 541 | IM1014,0 542 | IM1015,0 543 | IM1016,0 544 | IM1017,1 545 | IM1019,0 546 | IM1020,0 547 | IM1023,0 548 | IM1024,0 549 | IM1025,0 550 | IM1026,0 551 | IM1027,0 552 | IM1030,0 553 | IM1031,1 554 | IM1033,0 555 | IM1036,0 556 | IM1037,0 557 | IM1038,0 558 | IM1039,0 559 | IM1040,0 560 | IM1041,0 561 | IM1044,0 562 | IM1045,0 563 | IM1047,0 564 | IM1048,0 565 | IM1051,0 566 | IM1052,0 567 | IM1053,0 568 | IM1055,0 569 | IM1058,1 570 | IM1059,0 571 | IM1062,0 572 | IM1064,0 573 | IM1066,0 574 | IM1067,0 575 | IM1068,0 576 | IM1069,0 577 | IM1070,0 578 | IM1071,0 579 | IM1072,0 580 | IM1073,0 581 | IM1074,0 582 | IM1075,0 583 | IM1082,0 584 | IM1083,0 585 | IM1084,0 586 | IM1085,0 587 | IM1086,0 588 | IM1087,0 589 | IM1088,0 590 | IM1089,0 591 | IM1091,0 592 | IM1092,0 593 | IM1096,0 594 | IM1097,0 595 | IM1098,0 596 | IM1100,0 597 | IM1101,0 598 | IM1103,0 599 | IM1104,0 600 | IM1105,0 601 | IM1106,0 602 | IM1107,0 603 | IM1110,0 604 | IM1111,0 605 | IM1112,1 606 | IM1114,0 607 | IM1115,0 608 | IM1117,0 609 | IM1122,1 610 | IM1123,0 611 | IM1124,0 612 | IM1125,0 613 | IM1127,0 614 | IM1128,0 615 | IM1133,0 616 | IM1134,0 617 | IM1135,0 618 | IM1136,0 619 | IM1137,0 620 | IM1138,0 621 | IM1140,0 622 | IM1141,0 623 | IM1142,0 624 | IM1143,0 625 | IM1144,0 626 | IM1145,0 627 | IM1146,0 628 | IM1148,0 629 | IM1149,0 630 | IM1150,0 631 | IM1151,0 632 | IM1152,1 633 | IM1154,0 634 | IM1155,0 635 | IM1158,0 636 | IM1161,0 637 | IM1162,0 638 | IM1163,0 639 | IM1164,0 640 | IM1165,0 641 | IM1166,0 642 | IM1168,0 643 | IM1170,1 644 | IM1172,0 645 | IM1174,0 646 | IM1176,0 647 | IM1177,0 648 | IM1179,0 649 | IM1180,0 650 | IM1181,0 651 | IM1182,0 652 | IM1183,0 653 | IM1184,0 654 | IM1185,0 655 | IM1188,0 656 | IM1190,0 657 | IM1192,0 658 | IM1195,0 659 | IM1197,0 660 | IM1199,0 661 | IM1200,0 662 | IM1201,0 663 | IM1202,0 664 | IM1205,0 665 | IM1206,0 666 | IM1207,0 667 | IM1208,0 668 | IM1210,0 669 | IM1212,0 670 | IM1213,0 671 | IM1214,1 672 | IM1215,0 673 | IM1216,0 674 | IM1219,0 675 | IM1220,0 676 | IM1222,0 677 | IM1223,0 678 | IM1224,0 679 | IM1226,0 680 | IM1228,0 681 | IM1230,0 682 | IM1232,0 683 | IM1234,0 684 | IM1235,0 685 | IM1236,0 686 | IM1238,0 687 | IM1240,0 688 | IM1241,0 689 | IM1242,0 690 | IM1243,0 691 | IM1244,0 692 | IM1246,0 693 | IM1252,0 694 | IM1253,0 695 | IM1254,0 696 | IM1259,0 697 | IM1260,0 698 | IM1261,0 699 | IM1262,0 700 | IM1266,0 701 | IM1268,1 702 | IM1269,0 703 | IM1270,0 704 | IM1272,0 705 | IM1275,0 706 | IM1277,0 707 | IM1280,0 708 | IM1282,0 709 | IM1284,0 710 | IM1286,0 711 | IM1289,0 712 | IM1290,0 713 | IM1291,0 714 | IM1292,0 715 | IM1293,0 716 | IM1294,0 717 | IM1295,1 718 | IM1296,0 719 | IM1297,0 720 | IM1298,0 721 | IM1299,0 722 | IM1300,0 723 | IM1301,0 724 | IM1303,0 725 | IM1306,0 726 | IM1307,0 727 | IM1309,0 728 | IM1310,0 729 | IM1312,0 730 | IM1313,0 731 | IM1314,0 732 | IM1315,0 733 | IM1316,0 734 | IM1319,0 735 | IM1320,0 736 | IM1322,0 737 | IM1323,0 738 | IM1324,0 739 | IM1325,0 740 | IM1327,0 741 | IM1329,0 742 | IM1331,0 743 | IM1332,0 744 | IM1334,0 745 | IM1335,0 746 | IM1336,0 747 | IM1340,0 748 | IM1341,0 749 | IM1342,0 750 | IM1344,0 751 | IM1346,0 752 | IM1347,0 753 | IM1348,1 754 | IM1350,0 755 | IM1351,1 756 | IM1352,0 757 | IM1353,0 758 | IM1355,0 759 | IM1356,0 760 | IM1357,0 761 | IM1360,0 762 | IM1362,0 763 | IM1366,0 764 | IM1368,0 765 | IM1371,0 766 | IM1372,0 767 | IM1374,0 768 | IM1377,0 769 | IM1378,0 770 | IM1379,0 771 | IM1380,0 772 | IM1381,0 773 | IM1384,0 774 | IM1386,0 775 | IM1390,0 776 | IM1391,0 777 | IM1392,0 778 | IM1395,1 779 | IM1397,0 780 | IM1398,0 781 | IM1400,0 782 | IM1401,0 783 | IM1402,0 784 | IM1403,0 785 | IM1406,0 786 | IM1408,0 787 | IM1409,0 788 | IM1410,1 789 | IM1414,0 790 | IM1419,0 791 | IM1420,0 792 | IM1421,0 793 | IM1423,0 794 | IM1424,0 795 | IM1426,0 796 | IM1427,0 797 | IM1428,0 798 | IM1433,0 799 | IM1434,0 800 | IM1436,0 801 | IM1437,0 802 | IM1440,0 803 | IM1441,0 804 | IM1442,0 805 | IM1443,1 806 | IM1444,0 807 | IM1445,0 808 | IM1447,0 809 | IM1449,0 810 | IM1450,0 811 | IM1452,0 812 | IM1454,0 813 | IM1456,0 814 | IM1458,0 815 | IM1460,0 816 | IM1463,0 817 | IM1465,0 818 | IM1467,0 819 | IM1468,0 820 | IM1470,0 821 | IM1471,0 822 | IM1473,0 823 | IM1474,0 824 | IM1475,0 825 | IM1479,0 826 | IM1480,0 827 | IM1485,1 828 | IM1487,0 829 | IM1488,0 830 | IM1489,0 831 | IM1490,0 832 | IM1491,1 833 | IM1492,0 834 | IM1494,1 835 | IM1496,0 836 | IM1498,0 837 | IM1499,0 838 | IM1500,0 839 | IM1501,0 840 | IM1502,0 841 | IM1504,0 842 | IM1505,0 843 | IM1506,0 844 | IM1507,0 845 | IM1510,0 846 | IM1511,1 847 | IM1515,0 848 | IM1516,0 849 | IM1517,0 850 | IM1519,0 851 | IM1521,0 852 | IM1524,0 853 | IM1525,0 854 | IM1527,0 855 | IM1529,1 856 | IM1532,0 857 | IM1533,0 858 | IM1535,0 859 | IM1537,0 860 | IM1540,0 861 | IM1541,0 862 | IM1543,0 863 | IM1544,0 864 | IM1545,0 865 | IM1546,0 866 | IM1548,0 867 | IM1550,0 868 | IM1555,0 869 | IM1556,0 870 | IM1558,0 871 | IM1560,0 872 | IM1562,0 873 | IM1563,0 874 | IM1564,0 875 | IM1565,0 876 | -------------------------------------------------------------------------------- /vendor/messidor/messidor_gradability_grades.csv: -------------------------------------------------------------------------------- 1 | image_name gradability 2 | 20051020_43808_0100_PP 1 3 | 20051020_44261_0100_PP 1 4 | 20051020_44284_0100_PP 1 5 | 20051020_44338_0100_PP 1 6 | 20051020_44400_0100_PP 1 7 | 20051020_44431_0100_PP 1 8 | 20051020_44692_0100_PP 1 9 | 20051020_44714_0100_PP 1 10 | 20051020_44762_0100_PP 1 11 | 20051020_44982_0100_PP 1 12 | 20051020_56791_0100_PP 1 13 | 20051020_57157_0100_PP 1 14 | 20051020_57967_0100_PP 1 15 | 20051020_58065_0100_PP 1 16 | 20051020_61757_0100_PP 1 17 | 20051020_61804_0100_PP 1 18 | 20051020_61907_0100_PP 1 19 | 20051020_62510_0100_PP 1 20 | 20051020_63045_0100_PP 1 21 | 20051020_63269_0100_PP 1 22 | 20051020_64249_0100_PP 1 23 | 20051020_64388_0100_PP 1 24 | 20051020_64653_0100_PP 1 25 | 20051020_64703_0100_PP 1 26 | 20051020_64775_0100_PP 1 27 | 20051020_64836_0100_PP 1 28 | 20051021_36476_0100_PP 1 29 | 20051021_39482_0100_PP 1 30 | 20051021_39845_0100_PP 1 31 | 20051021_39914_0100_PP 1 32 | 20051021_40018_0100_PP 1 33 | 20051021_51418_0100_PP 1 34 | 20051021_51476_0100_PP 1 35 | 20051021_51936_0100_PP 1 36 | 20051021_51994_0100_PP 1 37 | 20051021_57798_0100_PP 1 38 | 20051021_57862_0100_PP 1 39 | 20051021_57975_0100_PP 1 40 | 20051021_58035_0100_PP 1 41 | 20051021_58316_0100_PP 1 42 | 20051021_58522_0100_PP 1 43 | 20051021_59589_0100_PP 1 44 | 20051109_57451_0400_PP 1 45 | 20051109_57843_0400_PP 1 46 | 20051109_59582_0400_PP 1 47 | 20051109_59751_0400_PP 1 48 | 20051109_59864_0400_PP 1 49 | 20051109_59969_0400_PP 1 50 | 20051109_60031_0400_PP 1 51 | 20051109_60136_0400_PP 1 52 | 20051110_33960_0400_PP 1 53 | 20051110_34013_0400_PP 1 54 | 20051110_34071_0400_PP 1 55 | 20051110_34493_0400_PP 1 56 | 20051110_35239_0400_PP 1 57 | 20051110_35279_0400_PP 1 58 | 20051110_35332_0400_PP 1 59 | 20051110_35926_0400_PP 1 60 | 20051110_37992_0400_PP 1 61 | 20051110_38111_0400_PP 1 62 | 20051110_38156_0400_PP 1 63 | 20051110_38194_0400_PP 1 64 | 20051110_38239_0400_PP 1 65 | 20051110_38280_0400_PP 1 66 | 20051110_38313_0400_PP 1 67 | 20051110_38352_0400_PP 1 68 | 20051110_38372_0400_PP 1 69 | 20051110_38411_0400_PP 1 70 | 20051110_38442_0400_PP 1 71 | 20051110_38472_0400_PP 1 72 | 20051110_38507_0400_PP 1 73 | 20051110_38644_0400_PP 1 74 | 20051110_38702_0400_PP 1 75 | 20051116_43388_0400_PP 1 76 | 20051116_43585_0400_PP 1 77 | 20051116_43679_0400_PP 1 78 | 20051116_43710_0400_PP 1 79 | 20051116_43721_0400_PP 1 80 | 20051116_43785_0400_PP 1 81 | 20051116_43801_0400_PP 1 82 | 20051117_37042_0400_PP 1 83 | 20051117_37051_0400_PP 1 84 | 20051117_37078_0400_PP 1 85 | 20051117_37100_0400_PP 1 86 | 20051117_37130_0400_PP 1 87 | 20051117_37155_0400_PP 1 88 | 20051117_37185_0400_PP 1 89 | 20051117_37213_0400_PP 1 90 | 20051117_37232_0400_PP 1 91 | 20051117_37321_0400_PP 1 92 | 20051130_53586_0400_PP 1 93 | 20051130_53623_0400_PP 1 94 | 20051130_53663_0400_PP 1 95 | 20051130_53991_0400_PP 1 96 | 20051130_54030_0400_PP 1 97 | 20051130_54077_0400_PP 1 98 | 20051130_54121_0400_PP 1 99 | 20051130_54257_0400_PP 1 100 | 20051130_54301_0400_PP 1 101 | 20051130_54323_0400_PP 1 102 | 20051130_54333_0400_PP 1 103 | 20051130_54364_0400_PP 1 104 | 20051130_54401_0400_PP 1 105 | 20051130_54476_0400_PP 1 106 | 20051130_54498_0400_PP 1 107 | 20051130_54595_0400_PP 1 108 | 20051130_54616_0400_PP 1 109 | 20051130_54802_0400_PP 1 110 | 20051130_54846_0400_PP 1 111 | 20051130_54943_0400_PP 1 112 | 20051130_54956_0400_PP 1 113 | 20051130_55004_0400_PP 1 114 | 20051130_55012_0400_PP 1 115 | 20051130_55037_0400_PP 1 116 | 20051130_55048_0400_PP 1 117 | 20051130_55074_0400_PP 1 118 | 20051130_55086_0400_PP 1 119 | 20051130_55107_0400_PP 1 120 | 20051130_55124_0400_PP 1 121 | 20051130_55160_0400_PP 1 122 | 20051130_55269_0400_PP 1 123 | 20051130_55311_0400_PP 1 124 | 20051130_55323_0400_PP 1 125 | 20051130_55381_0400_PP 1 126 | 20051130_55393_0400_PP 1 127 | 20051130_58716_0400_PP 1 128 | 20051130_58824_0400_PP 1 129 | 20051130_58874_0400_PP 1 130 | 20051130_58888_0400_PP 1 131 | 20051130_58921_0400_PP 1 132 | 20051130_58962_0400_PP 1 133 | 20051130_58998_0400_PP 1 134 | 20051130_59029_0400_PP 1 135 | 20051130_59092_0400_PP 1 136 | 20051130_59186_0400_PP 1 137 | 20051130_59284_0400_PP 1 138 | 20051130_59374_0400_PP 1 139 | 20051130_59404_0400_PP 1 140 | 20051130_59432_0400_PP 1 141 | 20051130_59457_0400_PP 1 142 | 20051130_59464_0400_PP 1 143 | 20051130_59749_0400_PP 1 144 | 20051130_59775_0400_PP 1 145 | 20051130_59793_0400_PP 1 146 | 20051130_59802_0400_PP 1 147 | 20051130_59825_0400_PP 1 148 | 20051130_59850_0400_PP 1 149 | 20051130_59869_0400_PP 1 150 | 20051130_59916_0400_PP 1 151 | 20051130_59930_0400_PP 1 152 | 20051130_59964_0400_PP 1 153 | 20051130_59993_0400_PP 1 154 | 20051130_60040_0400_PP 1 155 | 20051130_60066_0400_PP 1 156 | 20051130_60140_0400_PP 1 157 | 20051130_60186_0400_PP 1 158 | 20051130_60234_0400_PP 1 159 | 20051130_60448_0400_PP 1 160 | 20051130_60490_0400_PP 1 161 | 20051130_60532_0400_PP 1 162 | 20051130_60562_0400_PP 1 163 | 20051130_60585_0400_PP 1 164 | 20051130_60601_0400_PP 1 165 | 20051130_60630_0400_PP 1 166 | 20051130_60658_0400_PP 1 167 | 20051130_60691_0400_PP 1 168 | 20051130_60715_0400_PP 1 169 | 20051130_60740_0400_PP 1 170 | 20051130_60760_0400_PP 1 171 | 20051130_60794_0400_PP 1 172 | 20051130_60822_0400_PP 1 173 | 20051201_37312_0400_PP 1 174 | 20051201_37337_0400_PP 1 175 | 20051201_37362_0400_PP 1 176 | 20051201_37382_0400_PP 1 177 | 20051201_37390_0400_PP 1 178 | 20051201_37409_0400_PP 1 179 | 20051201_37462_0400_PP 1 180 | 20051201_37534_0400_PP 1 181 | 20051201_37557_0400_PP 1 182 | 20051201_38041_0400_PP 1 183 | 20051201_38081_0400_PP 1 184 | 20051201_38103_0400_PP 1 185 | 20051201_38137_0400_PP 1 186 | 20051201_38155_0400_PP 1 187 | 20051201_38164_0400_PP 1 188 | 20051201_38182_0400_PP 1 189 | 20051201_38199_0400_PP 1 190 | 20051201_38211_0400_PP 1 191 | 20051201_38246_0400_PP 1 192 | 20051201_38262_0400_PP 1 193 | 20051201_38280_0400_PP 1 194 | 20051201_38291_0400_PP 1 195 | 20051201_38321_0400_PP 1 196 | 20051201_38349_0400_PP 1 197 | 20051201_38411_0400_PP 1 198 | 20051201_38466_0400_PP 1 199 | 20051201_38475_0400_PP 1 200 | 20051201_38506_0400_PP 1 201 | 20051201_38537_0400_PP 1 202 | 20051205_57548_0400_PP 1 203 | 20051205_57651_0400_PP 1 204 | 20051205_57704_0400_PP 1 205 | 20051205_57723_0400_PP 1 206 | 20051205_57757_0400_PP 1 207 | 20051205_57780_0400_PP 1 208 | 20051205_57799_0400_PP 1 209 | 20051205_57809_0400_PP 1 210 | 20051205_57838_0400_PP 1 211 | 20051205_57879_0400_PP 1 212 | 20051205_57942_0400_PP 1 213 | 20051205_57988_0400_PP 1 214 | 20051205_58010_0400_PP 1 215 | 20051205_58037_0400_PP 1 216 | 20051205_58060_0400_PP 1 217 | 20051205_58080_0400_PP 1 218 | 20051205_58091_0400_PP 1 219 | 20051205_58109_0400_PP 1 220 | 20051205_58125_0400_PP 1 221 | 20051205_58150_0400_PP 1 222 | 20051205_58177_0400_PP 1 223 | 20051205_58354_0400_PP 1 224 | 20051205_58373_0400_PP 1 225 | 20051205_58409_0400_PP 1 226 | 20051205_58431_0400_PP 1 227 | 20051205_58458_0400_PP 1 228 | 20051205_58502_0400_PP 1 229 | 20051205_58577_0400_PP 1 230 | 20051205_58613_0400_PP 1 231 | 20051205_59240_0400_PP 1 232 | 20051205_59300_0400_PP 1 233 | 20051205_59337_0400_PP 1 234 | 20051205_59351_0400_PP 1 235 | 20051205_59367_0400_PP 1 236 | 20051205_59379_0400_PP 1 237 | 20051205_59510_0400_PP 1 238 | 20051205_59538_0400_PP 1 239 | 20051205_59617_0400_PP 1 240 | 20051205_59651_0400_PP 1 241 | 20051205_59702_0400_PP 1 242 | 20051205_59715_0400_PP 1 243 | 20051205_59738_0400_PP 1 244 | 20051205_59754_0400_PP 1 245 | 20051205_59804_0400_PP 1 246 | 20051205_59833_0400_PP 1 247 | 20051205_59858_0400_PP 1 248 | 20051205_59879_0400_PP 1 249 | 20051205_59901_0400_PP 1 250 | 20051205_59923_0400_PP 1 251 | 20051205_60093_0400_PP 1 252 | 20051205_60119_0400_PP 1 253 | 20051213_61406_0100_PP 1 254 | 20051213_61802_0100_PP 1 255 | 20051213_61892_0100_PP 1 256 | 20051213_61951_0100_PP 1 257 | 20051213_62251_0100_PP 1 258 | 20051213_62314_0100_PP 1 259 | 20051214_40529_0100_PP 1 260 | 20051214_40994_0100_PP 1 261 | 20051214_41055_0100_PP 1 262 | 20051214_42133_0100_PP 1 263 | 20051214_42197_0100_PP 1 264 | 20051214_42323_0100_PP 1 265 | 20051214_42363_0100_PP 1 266 | 20051214_51178_0100_PP 1 267 | 20051214_51211_0100_PP 1 268 | 20051214_51308_0100_PP 1 269 | 20051214_51342_0100_PP 1 270 | 20051214_52115_0100_PP 1 271 | 20051214_56242_0100_PP 1 272 | 20051214_56392_0100_PP 1 273 | 20051214_56438_0100_PP 1 274 | 20051214_56636_0100_PP 1 275 | 20051214_56688_0100_PP 1 276 | 20051214_57761_0100_PP 1 277 | 20051214_57817_0100_PP 1 278 | 20051216_43814_0200_PP 1 279 | 20051216_43913_0200_PP 1 280 | 20051216_44066_0200_PP 1 281 | 20051216_44092_0200_PP 1 282 | 20051216_44221_0200_PP 1 283 | 20051216_44252_0200_PP 1 284 | 20051216_44398_0200_PP 1 285 | 20051216_44420_0200_PP 1 286 | 20051216_44635_0200_PP 1 287 | 20051216_44660_0200_PP 1 288 | 20051216_44811_0200_PP 1 289 | 20051216_44832_0200_PP 1 290 | 20051216_45076_0200_PP 1 291 | 20051216_45478_0200_PP 1 292 | 20051216_45499_0200_PP 1 293 | 20051216_45595_0200_PP 1 294 | 20051216_45619_0200_PP 1 295 | 20051216_45781_0200_PP 1 296 | 20051216_45873_0200_PP 1 297 | 20051216_45992_0200_PP 1 298 | 20051216_46018_0200_PP 1 299 | 20051216_46156_0200_PP 1 300 | 20051216_46183_0200_PP 1 301 | 20051216_47514_0200_PP 1 302 | 20051216_47536_0200_PP 1 303 | 20060407_38711_0200_PP 1 304 | 20060407_38970_0200_PP 1 305 | 20060407_39155_0200_PP 1 306 | 20060407_39184_0200_PP 1 307 | 20060407_39306_0200_PP 1 308 | 20060407_39328_0200_PP 1 309 | 20060407_39402_0200_PP 1 310 | 20060407_39435_0200_PP 1 311 | 20060407_39546_0200_PP 1 312 | 20060407_39567_0200_PP 1 313 | 20060407_39667_0200_PP 1 314 | 20060407_39687_0200_PP 1 315 | 20060407_39761_0200_PP 1 316 | 20060407_39780_0200_PP 1 317 | 20060407_40740_0200_PP 1 318 | 20060407_40761_0200_PP 1 319 | 20060407_41149_0200_PP 1 320 | 20060407_41177_0200_PP 1 321 | 20060407_41312_0200_PP 1 322 | 20060407_41330_0200_PP 1 323 | 20060407_41980_0200_PP 1 324 | 20060407_42099_0200_PP 1 325 | 20060407_42133_0200_PP 1 326 | 20060407_42290_0200_PP 1 327 | 20060407_42308_0200_PP 1 328 | 20060407_43223_0200_PP 1 329 | 20060407_43258_0200_PP 1 330 | 20060407_43594_0200_PP 1 331 | 20060407_43618_0200_PP 1 332 | 20060407_43726_0200_PP 1 333 | 20060407_43746_0200_PP 1 334 | 20060407_43834_0200_PP 1 335 | 20060407_43856_0200_PP 1 336 | 20060407_44042_0200_PP 1 337 | 20060407_44061_0200_PP 1 338 | 20060407_44173_0200_PP 1 339 | 20060407_44192_0200_PP 1 340 | 20060407_44400_0200_PP 1 341 | 20060407_44422_0200_PP 1 342 | 20060407_44510_0200_PP 1 343 | 20060407_44529_0200_PP 1 344 | 20060407_44743_0200_PP 1 345 | 20060407_44766_0200_PP 1 346 | 20060407_44866_0200_PP 1 347 | 20060407_44889_0200_PP 1 348 | 20060407_45087_0200_PP 1 349 | 20060407_45107_0200_PP 1 350 | 20060407_45230_0200_PP 1 351 | 20060407_45247_0200_PP 1 352 | 20060407_45362_0200_PP 1 353 | 20060407_45379_0200_PP 1 354 | 20060407_45718_0200_PP 1 355 | 20060407_45737_0200_PP 1 356 | 20060407_45834_0200_PP 1 357 | 20060407_45852_0200_PP 1 358 | 20060407_45932_0200_PP 1 359 | 20060407_45949_0200_PP 1 360 | 20060407_46050_0200_PP 1 361 | 20060407_46070_0200_PP 1 362 | 20060407_46235_0200_PP 1 363 | 20060407_46257_0200_PP 1 364 | 20060410_39047_0200_PP 1 365 | 20060410_39075_0200_PP 1 366 | 20060410_39203_0200_PP 1 367 | 20060410_39229_0200_PP 1 368 | 20060410_39334_0200_PP 1 369 | 20060410_39476_0200_PP 1 370 | 20060410_39500_0200_PP 1 371 | 20060410_39586_0200_PP 1 372 | 20060410_39695_0200_PP 1 373 | 20060410_39726_0200_PP 1 374 | 20060410_39859_0200_PP 1 375 | 20060410_39879_0200_PP 1 376 | 20060410_39976_0200_PP 1 377 | 20060410_39994_0200_PP 1 378 | 20060410_40381_0200_PP 1 379 | 20060410_40403_0200_PP 1 380 | 20060410_40499_0200_PP 1 381 | 20060410_40633_0200_PP 1 382 | 20060410_40846_0200_PP 1 383 | 20060410_40953_0200_PP 1 384 | 20060410_40979_0200_PP 1 385 | 20060410_41077_0200_PP 1 386 | 20060410_41098_0200_PP 1 387 | 20060410_41209_0200_PP 1 388 | 20060410_41330_0200_PP 1 389 | 20060410_41603_0200_PP 1 390 | 20060410_41880_0200_PP 1 391 | 20060410_41901_0200_PP 1 392 | 20060410_43847_0200_PP 1 393 | 20060410_43868_0200_PP 1 394 | 20060410_43959_0200_PP 1 395 | 20060410_43979_0200_PP 1 396 | 20060410_44345_0200_PP 1 397 | 20060410_44364_0200_PP 1 398 | 20060410_44464_0200_PP 1 399 | 20060410_44504_0200_PP 1 400 | 20060410_44601_0200_PP 1 401 | 20060410_44770_0200_PP 1 402 | 20060410_44802_0200_PP 1 403 | 20060410_44934_0200_PP 1 404 | 20060410_45167_0200_PP 1 405 | 20060410_45188_0200_PP 1 406 | 20060410_47042_0200_PP 1 407 | 20060410_47186_0200_PP 1 408 | 20060411_58071_0200_PP 1 409 | 20060411_58392_0200_PP 1 410 | 20060411_58413_0200_PP 1 411 | 20060411_58469_0200_PP 1 412 | 20060411_58494_0200_PP 1 413 | 20060411_58550_0200_PP 1 414 | 20060411_58571_0200_PP 1 415 | 20060411_58644_0200_PP 1 416 | 20060411_58808_0200_PP 1 417 | 20060411_58829_0200_PP 1 418 | 20060411_58891_0200_PP 1 419 | 20060411_58913_0200_PP 1 420 | 20060411_60347_0200_PP 1 421 | 20060411_60368_0200_PP 1 422 | 20060411_60511_0200_PP 1 423 | 20060411_61029_0200_PP 1 424 | 20060411_61564_0200_PP 1 425 | 20060411_61702_0200_PP 1 426 | 20060411_61869_0200_PP 1 427 | 20060411_61901_0200_PP 1 428 | 20060411_62036_0200_PP 1 429 | 20060411_62058_0200_PP 1 430 | 20060411_62142_0200_PP 1 431 | 20060411_62162_0200_PP 1 432 | 20060412_51835_0200_PP 1 433 | 20060412_51860_0200_PP 1 434 | 20060412_52098_0200_PP 1 435 | 20060412_52351_0200_PP 1 436 | 20060412_52371_0200_PP 1 437 | 20060412_52425_0200_PP 1 438 | 20060412_52446_0200_PP 1 439 | 20060412_52606_0200_PP 1 440 | 20060412_57874_0200_PP 1 441 | 20060412_57952_0200_PP 1 442 | 20060412_57974_0200_PP 1 443 | 20060412_58032_0200_PP 1 444 | 20060412_58054_0200_PP 1 445 | 20060412_58137_0200_PP 1 446 | 20060412_58368_0200_PP 1 447 | 20060412_58723_0200_PP 1 448 | 20060412_58802_0200_PP 1 449 | 20060412_58819_0200_PP 1 450 | 20060412_59175_0200_PP 1 451 | 20060412_59196_0200_PP 1 452 | 20060412_59259_0200_PP 1 453 | 20060412_59338_0200_PP 1 454 | 20060412_59355_0200_PP 1 455 | 20060412_59423_0200_PP 1 456 | 20060412_60751_0200_PP 1 457 | 20060412_60773_0200_PP 1 458 | 20060412_61365_0200_PP 1 459 | 20060412_61383_0200_PP 1 460 | 20060412_61433_0200_PP 1 461 | 20060412_61450_0200_PP 1 462 | 20060412_61525_0200_PP 1 463 | 20060412_61665_0200_PP 1 464 | 20060522_43643_0100_PP 1 465 | 20060522_43758_0100_PP 1 466 | 20060522_45069_0100_PP 1 467 | 20060522_45111_0100_PP 1 468 | 20060522_45777_0100_PP 1 469 | 20060522_45796_0100_PP 1 470 | 20060522_45853_0100_PP 1 471 | 20060522_45873_0100_PP 1 472 | 20060522_46104_0100_PP 1 473 | 20060522_46149_0100_PP 1 474 | 20060522_46379_0100_PP 1 475 | 20060522_46400_0100_PP 1 476 | 20060522_46454_0100_PP 1 477 | 20060523_42953_0100_PP 1 478 | 20060523_43105_0100_PP 1 479 | 20060523_43123_0100_PP 1 480 | 20060523_43248_0100_PP 1 481 | 20060523_43333_0100_PP 1 482 | 20060523_43354_0100_PP 1 483 | 20060523_45300_0100_PP 1 484 | 20060523_45316_0100_PP 1 485 | 20060523_45449_0100_PP 1 486 | 20060523_45467_0100_PP 1 487 | 20060523_45524_0100_PP 1 488 | 20060523_45548_0100_PP 1 489 | 20060523_47987_0100_PP 1 490 | 20060523_48005_0100_PP 1 491 | 20060523_48136_0100_PP 1 492 | 20060523_48161_0100_PP 1 493 | 20060523_48357_0100_PP 1 494 | 20060523_48406_0100_PP 1 495 | 20060523_48591_0100_PP 1 496 | 20060523_48643_0100_PP 1 497 | 20060523_48659_0100_PP 1 498 | 20060523_48709_0100_PP 1 499 | 20060523_48728_0100_PP 1 500 | 20060523_48890_0100_PP 1 501 | 20060523_48931_0100_PP 1 502 | 20060523_49100_0100_PP 1 503 | 20060523_49120_0100_PP 1 504 | 20060523_49176_0100_PP 1 505 | 20060523_49191_0100_PP 1 506 | 20060523_49449_0100_PP 1 507 | 20060523_49515_0100_PP 1 508 | 20060523_49573_0100_PP 1 509 | 20060523_49591_0100_PP 1 510 | 20060523_49663_0100_PP 1 511 | 20060523_49681_0100_PP 1 512 | 20060523_49726_0100_PP 1 513 | 20060523_49745_0100_PP 1 514 | 20060523_50153_0100_PP 1 515 | 20060523_50325_0100_PP 1 516 | 20060523_50342_0100_PP 1 517 | 20060523_50616_0100_PP 1 518 | 20060523_50631_0100_PP 1 519 | 20060529_56013_0100_PP 1 520 | 20060529_56039_0100_PP 1 521 | 20060529_56420_0100_PP 1 522 | 20060529_56442_0100_PP 1 523 | 20060529_56563_0100_PP 1 524 | 20060529_56592_0100_PP 1 525 | 20060529_56948_0100_PP 1 526 | 20060529_56968_0100_PP 1 527 | 20060529_57261_0100_PP 1 528 | 20060529_57287_0100_PP 1 529 | 20060529_57351_0100_PP 1 530 | 20060529_57372_0100_PP 1 531 | 20060530_51254_0100_PP 1 532 | 20060530_51279_0100_PP 1 533 | 20060530_53042_0100_PP 1 534 | 20060530_53062_0100_PP 1 535 | 20060530_53132_0100_PP 1 536 | 20060530_53152_0100_PP 1 537 | 20060530_53597_0100_PP 1 538 | 20060530_53617_0100_PP 1 539 | 20060530_53742_0100_PP 1 540 | 20060530_53761_0100_PP 0 541 | 20060530_53816_0100_PP 1 542 | 20060530_53836_0100_PP 1 543 | 20060530_53928_0100_PP 0 544 | 20060530_53954_0100_PP 1 545 | 20060530_54485_0100_PP 1 546 | 20060530_55180_0100_PP 1 547 | 20060530_55607_0100_PP 1 548 | 20060530_55628_0100_PP 1 549 | 20051020_43832_0100_PP 1 550 | 20051020_44782_0100_PP 1 551 | 20051020_55701_0100_PP 1 552 | 20051020_56592_0100_PP 1 553 | 20051020_61998_0100_PP 1 554 | 20051020_62014_0100_PP 0 555 | 20051020_62461_0100_PP 1 556 | 20051020_63337_0100_PP 1 557 | 20051020_63936_0100_PP 1 558 | 20051020_64945_0100_PP 1 559 | 20051020_64993_0100_PP 1 560 | 20051021_36380_0100_PP 1 561 | 20051021_39661_0100_PP 1 562 | 20051116_43954_0400_PP 1 563 | 20051116_43995_0400_PP 1 564 | 20051116_44026_0400_PP 1 565 | 20051116_44051_0400_PP 1 566 | 20051116_44070_0400_PP 1 567 | 20051116_44083_0400_PP 1 568 | 20051116_44126_0400_PP 1 569 | 20051116_44482_0400_PP 1 570 | 20051116_44586_0400_PP 1 571 | 20051116_44615_0400_PP 1 572 | 20051116_44648_0400_PP 1 573 | 20051116_44690_0400_PP 1 574 | 20051116_44718_0400_PP 1 575 | 20051116_44750_0400_PP 1 576 | 20051116_44785_0400_PP 1 577 | 20051116_44804_0400_PP 1 578 | 20051116_44816_0400_PP 1 579 | 20051202_36970_0400_PP 1 580 | 20051202_37003_0400_PP 1 581 | 20051202_37011_0400_PP 1 582 | 20051202_37162_0400_PP 1 583 | 20051202_37199_0400_PP 1 584 | 20051202_37227_0400_PP 1 585 | 20051202_37349_0400_PP 1 586 | 20051202_38513_0400_PP 1 587 | 20051202_39630_0400_PP 1 588 | 20051202_39708_0400_PP 1 589 | 20051202_39797_0400_PP 1 590 | 20051202_41076_0400_PP 1 591 | 20051202_41143_0400_PP 1 592 | 20051202_48232_0400_PP 1 593 | 20051202_48264_0400_PP 1 594 | 20051202_48287_0400_PP 1 595 | 20051202_48310_0400_PP 1 596 | 20051202_48329_0400_PP 1 597 | 20051202_48393_0400_PP 1 598 | 20051202_48421_0400_PP 1 599 | 20051202_48443_0400_PP 1 600 | 20051202_48575_0400_PP 1 601 | 20051202_48586_0400_PP 1 602 | 20051207_54614_0400_PP 1 603 | 20051207_55266_0400_PP 1 604 | 20051207_62252_0400_PP 1 605 | 20051207_62266_0400_PP 1 606 | 20051207_62275_0400_PP 1 607 | 20051207_62320_0400_PP 1 608 | 20051212_36525_0400_PP 1 609 | 20051212_36548_0400_PP 1 610 | 20051213_62518_0100_PP 1 611 | 20051213_62572_0100_PP 1 612 | 20051214_40767_0100_PP 1 613 | 20051214_51469_0100_PP 1 614 | 20051216_45100_0200_PP 1 615 | 20051216_45375_0200_PP 1 616 | 20051216_45899_0200_PP 1 617 | 20060119_41191_0200_PP 1 618 | 20060407_40937_0200_PP 1 619 | 20060407_40957_0200_PP 1 620 | 20060407_45592_0200_PP 1 621 | 20060410_39355_0200_PP 1 622 | 20060410_39606_0200_PP 1 623 | 20060410_40254_0200_PP 1 624 | 20060410_40274_0200_PP 1 625 | 20060410_40481_0200_PP 1 626 | 20060410_40830_0200_PP 1 627 | 20060410_41626_0200_PP 1 628 | 20060410_41767_0200_PP 1 629 | 20060410_44953_0200_PP 1 630 | 20060410_46893_0200_PP 1 631 | 20060410_47016_0200_PP 1 632 | 20060410_47166_0200_PP 1 633 | 20060410_47331_0200_PP 1 634 | 20060411_57962_0200_PP 1 635 | 20060411_57986_0200_PP 1 636 | 20060411_58051_0200_PP 1 637 | 20060411_58155_0200_PP 1 638 | 20060411_58718_0200_PP 1 639 | 20060411_58736_0200_PP 1 640 | 20060411_58971_0200_PP 1 641 | 20060411_58993_0200_PP 1 642 | 20060411_60028_0200_PP 1 643 | 20060411_60283_0200_PP 1 644 | 20060411_60533_0200_PP 1 645 | 20060411_61060_0200_PP 1 646 | 20060411_61548_0200_PP 1 647 | 20060411_61687_0200_PP 1 648 | 20060411_61952_0200_PP 1 649 | 20060411_61979_0200_PP 1 650 | 20060411_62228_0200_PP 1 651 | 20060411_62250_0200_PP 1 652 | 20060412_51656_0200_PP 1 653 | 20060412_51775_0200_PP 1 654 | 20060412_52587_0200_PP 1 655 | 20060412_52758_0200_PP 1 656 | 20060412_52840_0200_PP 1 657 | 20060412_52978_0200_PP 1 658 | 20060412_52997_0200_PP 1 659 | 20060412_57846_0200_PP 1 660 | 20060412_58171_0200_PP 1 661 | 20060412_58387_0200_PP 1 662 | 20060412_58668_0200_PP 1 663 | 20060412_58747_0200_PP 1 664 | 20060412_58945_0200_PP 1 665 | 20060412_58965_0200_PP 1 666 | 20060412_59037_0200_PP 1 667 | 20060412_59100_0200_PP 1 668 | 20060412_59400_0200_PP 1 669 | 20060412_59636_0200_PP 1 670 | 20060412_59658_0200_PP 1 671 | 20060412_60337_0200_PP 1 672 | 20060412_60396_0200_PP 1 673 | 20060412_60475_0200_PP 1 674 | 20060412_60573_0200_PP 1 675 | 20060412_61251_0200_PP 1 676 | 20060412_61299_0200_PP 1 677 | 20060412_61316_0200_PP 1 678 | 20060412_61593_0200_PP 1 679 | 20060412_61615_0200_PP 1 680 | 20060412_61681_0200_PP 1 681 | 20060412_63009_0200_PP 1 682 | 20060412_63032_0200_PP 1 683 | 20060412_63088_0200_PP 1 684 | 20060412_63104_0200_PP 1 685 | 20060522_45402_0100_PP 1 686 | 20060522_45935_0100_PP 1 687 | 20060522_45967_0100_PP 1 688 | 20060523_43016_0100_PP 1 689 | 20060523_48425_0100_PP 1 690 | 20060523_48816_0100_PP 1 691 | 20060523_50408_0100_PP 1 692 | 20060530_54718_0100_PP 1 693 | 20060530_55203_0100_PP 1 694 | 20060530_55370_0100_PP 1 695 | 20060530_55390_0100_PP 1 696 | 20060530_55451_0100_PP 1 697 | 20060530_55468_0100_PP 1 698 | 20051020_43882_0100_PP 1 699 | 20051020_44901_0100_PP 1 700 | 20051020_44923_0100_PP 1 701 | 20051020_45004_0100_PP 1 702 | 20051020_62577_0100_PP 1 703 | 20051020_62615_0100_PP 1 704 | 20051020_62878_0100_PP 1 705 | 20051020_63711_0100_PP 1 706 | 20051020_63829_0100_PP 1 707 | 20051021_40180_0100_PP 0 708 | 20051021_40248_0100_PP 1 709 | 20051021_58178_0100_PP 1 710 | 20051021_58388_0100_PP 1 711 | 20051116_54343_0400_PP 1 712 | 20051116_54429_0400_PP 1 713 | 20051116_54454_0400_PP 1 714 | 20051116_54484_0400_PP 1 715 | 20051116_54587_0400_PP 1 716 | 20051116_54642_0400_PP 1 717 | 20051116_54663_0400_PP 1 718 | 20051116_54692_0400_PP 1 719 | 20051116_54825_0400_PP 1 720 | 20051116_54832_0400_PP 1 721 | 20051116_54839_0400_PP 1 722 | 20051116_54865_0400_PP 1 723 | 20051116_55573_0400_PP 1 724 | 20051116_55593_0400_PP 1 725 | 20051116_55617_0400_PP 1 726 | 20051116_55643_0400_PP 1 727 | 20051116_55649_0400_PP 1 728 | 20051116_55656_0400_PP 1 729 | 20051202_51488_0400_PP 1 730 | 20051202_51518_0400_PP 1 731 | 20051202_51574_0400_PP 1 732 | 20051202_51599_0400_PP 1 733 | 20051202_51616_0400_PP 1 734 | 20051202_51655_0400_PP 1 735 | 20051202_51677_0400_PP 1 736 | 20051202_54197_0400_PP 1 737 | 20051202_54208_0400_PP 1 738 | 20051202_54477_0400_PP 1 739 | 20051202_54498_0400_PP 1 740 | 20051202_54547_0400_PP 1 741 | 20051202_54576_0400_PP 1 742 | 20051202_54783_0400_PP 1 743 | 20051202_55457_0400_PP 1 744 | 20051202_55484_0400_PP 1 745 | 20051202_55525_0400_PP 1 746 | 20051202_55562_0400_PP 1 747 | 20051202_55582_0400_PP 1 748 | 20051202_55626_0400_PP 1 749 | 20051202_55650_0400_PP 1 750 | 20051202_55669_0400_PP 1 751 | 20051202_55697_0400_PP 1 752 | 20051202_55735_0400_PP 1 753 | 20051202_55742_0400_PP 1 754 | 20051202_55767_0400_PP 1 755 | 20051202_55777_0400_PP 1 756 | 20051202_55797_0400_PP 1 757 | 20051202_55816_0400_PP 1 758 | 20051202_55834_0400_PP 1 759 | 20051202_55846_0400_PP 1 760 | 20051202_55869_0400_PP 1 761 | 20051202_55883_0400_PP 1 762 | 20051202_55921_0400_PP 1 763 | 20051202_55944_0400_PP 1 764 | 20051202_55969_0400_PP 1 765 | 20051202_55984_0400_PP 1 766 | 20051202_56000_0400_PP 1 767 | 20051202_56012_0400_PP 1 768 | 20051202_56028_0400_PP 1 769 | 20051202_56050_0400_PP 1 770 | 20051208_39243_0400_PP 1 771 | 20051208_39254_0400_PP 1 772 | 20051208_39297_0400_PP 1 773 | 20051208_39404_0400_PP 1 774 | 20051208_39438_0400_PP 1 775 | 20051208_39462_0400_PP 1 776 | 20051208_39484_0400_PP 1 777 | 20051208_39498_0400_PP 1 778 | 20051208_39524_0400_PP 1 779 | 20051208_39532_0400_PP 1 780 | 20051208_39546_0400_PP 1 781 | 20051208_39572_0400_PP 1 782 | 20051212_36605_0400_PP 1 783 | 20051212_36640_0400_PP 1 784 | 20051212_37442_0400_PP 1 785 | 20051212_38000_0400_PP 1 786 | 20051212_38043_0400_PP 1 787 | 20051212_38076_0400_PP 1 788 | 20051212_38106_0400_PP 1 789 | 20051212_38130_0400_PP 1 790 | 20051212_38161_0400_PP 1 791 | 20051213_62046_0100_PP 1 792 | 20051213_62188_0100_PP 1 793 | 20051214_40361_0100_PP 1 794 | 20051214_40456_0100_PP 1 795 | 20051214_40596_0100_PP 1 796 | 20051214_40719_0100_PP 1 797 | 20051214_41289_0100_PP 1 798 | 20051214_41358_0100_PP 1 799 | 20051214_41582_0100_PP 1 800 | 20051214_50895_0100_PP 1 801 | 20051214_50933_0100_PP 1 802 | 20051214_51071_0100_PP 1 803 | 20051214_51442_0100_PP 1 804 | 20051214_52695_0100_PP 1 805 | 20051214_53168_0100_PP 1 806 | 20051214_56534_0100_PP 1 807 | 20051214_56565_0100_PP 1 808 | 20051214_57404_0100_PP 1 809 | 20051216_44939_0200_PP 1 810 | 20051216_44961_0200_PP 1 811 | 20051216_45351_0200_PP 1 812 | 20051216_45757_0200_PP 1 813 | 20051216_46847_0200_PP 1 814 | 20051216_47383_0200_PP 1 815 | 20051216_47420_0200_PP 1 816 | 20051216_47668_0200_PP 1 817 | 20051216_47703_0200_PP 1 818 | 20060407_40582_0200_PP 1 819 | 20060407_40624_0200_PP 1 820 | 20060407_41809_0200_PP 1 821 | 20060407_41831_0200_PP 1 822 | 20060407_41937_0200_PP 1 823 | 20060407_43460_0200_PP 1 824 | 20060407_44304_0200_PP 1 825 | 20060407_45477_0200_PP 1 826 | 20060407_45494_0200_PP 1 827 | 20060407_45611_0200_PP 1 828 | 20060407_46422_0200_PP 1 829 | 20060410_40146_0200_PP 1 830 | 20060410_40165_0200_PP 1 831 | 20060410_40605_0200_PP 1 832 | 20060410_41191_0200_PP 1 833 | 20060410_41356_0200_PP 1 834 | 20060410_41747_0200_PP 1 835 | 20060410_43675_0200_PP 1 836 | 20060410_43698_0200_PP 1 837 | 20060410_44106_0200_PP 1 838 | 20060410_44124_0200_PP 1 839 | 20060410_44224_0200_PP 1 840 | 20060410_44623_0200_PP 1 841 | 20060410_46874_0200_PP 1 842 | 20060410_47351_0200_PP 1 843 | 20060411_58134_0200_PP 1 844 | 20060411_58624_0200_PP 1 845 | 20060411_59062_0200_PP 1 846 | 20060411_59087_0200_PP 1 847 | 20060411_59176_0200_PP 1 848 | 20060411_59190_0200_PP 1 849 | 20060411_59549_0200_PP 1 850 | 20060411_59587_0200_PP 1 851 | 20060411_59638_0200_PP 1 852 | 20060411_59657_0200_PP 1 853 | 20060411_59728_0200_PP 1 854 | 20060411_59747_0200_PP 1 855 | 20060411_59941_0200_PP 1 856 | 20060411_60009_0200_PP 1 857 | 20060411_60263_0200_PP 1 858 | 20060411_60426_0200_PP 1 859 | 20060411_60446_0200_PP 1 860 | 20060411_61402_0200_PP 1 861 | 20060411_61419_0200_PP 1 862 | 20060411_61781_0200_PP 1 863 | 20060411_62319_0200_PP 1 864 | 20060412_51677_0200_PP 1 865 | 20060412_51746_0200_PP 1 866 | 20060412_51932_0200_PP 1 867 | 20060412_52020_0200_PP 1 868 | 20060412_52038_0200_PP 1 869 | 20060412_52122_0200_PP 1 870 | 20060412_52174_0200_PP 1 871 | 20060412_52194_0200_PP 1 872 | 20060412_52271_0200_PP 1 873 | 20060412_52668_0200_PP 1 874 | 20060412_52685_0200_PP 1 875 | 20060412_52777_0200_PP 1 876 | 20060412_52860_0200_PP 1 877 | 20060412_57650_0200_PP 1 878 | 20060412_58287_0200_PP 1 879 | 20060412_58314_0200_PP 1 880 | 20060412_58471_0200_PP 1 881 | 20060412_58497_0200_PP 1 882 | 20060412_58548_0200_PP 1 883 | 20060412_59012_0200_PP 1 884 | 20060412_59124_0200_PP 1 885 | 20060412_59242_0200_PP 1 886 | 20060412_60454_0200_PP 1 887 | 20060412_60545_0200_PP 1 888 | 20060412_60825_0200_PP 1 889 | 20060412_60845_0200_PP 1 890 | 20060412_60895_0200_PP 1 891 | 20060412_60978_0200_PP 1 892 | 20060412_61000_0200_PP 1 893 | 20060412_61064_0200_PP 1 894 | 20060412_61081_0200_PP 1 895 | 20060412_61151_0200_PP 1 896 | 20060412_61222_0200_PP 1 897 | 20060412_61501_0200_PP 1 898 | 20060412_61790_0200_PP 1 899 | 20060412_61808_0200_PP 1 900 | 20060522_45212_0100_PP 1 901 | 20060522_45248_0100_PP 1 902 | 20060522_45455_0100_PP 1 903 | 20060522_46266_0100_PP 1 904 | 20060523_43038_0100_PP 1 905 | 20060523_43267_0100_PP 1 906 | 20060523_45216_0100_PP 1 907 | 20060523_45235_0100_PP 1 908 | 20060523_45369_0100_PP 1 909 | 20060523_45389_0100_PP 1 910 | 20060523_45662_0100_PP 1 911 | 20060523_45697_0100_PP 1 912 | 20060523_45787_0100_PP 1 913 | 20060523_45812_0100_PP 1 914 | 20060523_48028_0100_PP 1 915 | 20060523_48102_0100_PP 1 916 | 20060523_48572_0100_PP 1 917 | 20060523_48787_0100_PP 1 918 | 20060523_48990_0100_PP 1 919 | 20060523_49010_0100_PP 1 920 | 20060523_49269_0100_PP 1 921 | 20060523_49288_0100_PP 1 922 | 20060523_50135_0100_PP 1 923 | 20060523_50392_0100_PP 1 924 | 20060523_50730_0100_PP 1 925 | 20060529_56834_0100_PP 1 926 | 20060529_56853_0100_PP 1 927 | 20060529_57156_0100_PP 1 928 | 20060529_57174_0100_PP 1 929 | 20060530_53233_0100_PP 1 930 | 20060530_53522_0100_PP 1 931 | 20060530_53540_0100_PP 1 932 | 20060530_54011_0100_PP 1 933 | 20060530_54030_0100_PP 1 934 | 20060530_54223_0100_PP 1 935 | 20060530_54443_0100_PP 1 936 | 20060530_54632_0100_PP 1 937 | 20060530_54653_0100_PP 1 938 | 20051019_38557_0100_PP 1 939 | 20051020_43906_0100_PP 1 940 | 20051020_44349_0100_PP 1 941 | 20051020_44598_0100_PP 1 942 | 20051020_44636_0100_PP 1 943 | 20051020_44843_0100_PP 1 944 | 20051020_45050_0100_PP 1 945 | 20051020_45068_0100_PP 1 946 | 20051020_45110_0100_PP 1 947 | 20051020_45137_0100_PP 1 948 | 20051020_52801_0100_PP 1 949 | 20051020_53062_0100_PP 1 950 | 20051020_53178_0100_PP 1 951 | 20051020_53997_0100_PP 0 952 | 20051020_54209_0100_PP 1 953 | 20051020_55346_0100_PP 1 954 | 20051020_57566_0100_PP 1 955 | 20051020_57622_0100_PP 1 956 | 20051020_57761_0100_PP 1 957 | 20051020_57844_0100_PP 1 958 | 20051020_58214_0100_PP 1 959 | 20051020_58276_0100_PP 1 960 | 20051020_61557_0100_PP 1 961 | 20051020_62337_0100_PP 1 962 | 20051020_62385_0100_PP 1 963 | 20051020_62709_0100_PP 1 964 | 20051020_62802_0100_PP 1 965 | 20051020_63141_0100_PP 1 966 | 20051020_64007_0100_PP 1 967 | 20051020_64518_0100_PP 1 968 | 20051020_64570_0100_PP 1 969 | 20051020_65166_0100_PP 0 970 | 20051020_65230_0100_PP 1 971 | 20051021_36097_0100_PP 1 972 | 20051021_36208_0100_PP 1 973 | 20051021_39222_0100_PP 1 974 | 20051021_39314_0100_PP 1 975 | 20051021_39552_0100_PP 1 976 | 20051021_39719_0100_PP 1 977 | 20051021_40074_0100_PP 1 978 | 20051021_40377_0100_PP 1 979 | 20051021_40450_0100_PP 1 980 | 20051021_51561_0100_PP 1 981 | 20051021_51625_0100_PP 1 982 | 20051021_51748_0100_PP 1 983 | 20051021_51804_0100_PP 1 984 | 20051021_52075_0100_PP 1 985 | 20051021_52127_0100_PP 1 986 | 20051021_58683_0100_PP 1 987 | 20051021_58802_0100_PP 1 988 | 20051021_59136_0100_PP 1 989 | 20051021_59243_0100_PP 1 990 | 20051021_59459_0100_PP 1 991 | 20051021_59504_0100_PP 1 992 | 20051116_58804_0400_PP 1 993 | 20051116_58816_0400_PP 1 994 | 20051116_58828_0400_PP 1 995 | 20051116_58835_0400_PP 1 996 | 20051116_58850_0400_PP 1 997 | 20051116_58864_0400_PP 1 998 | 20051116_58872_0400_PP 1 999 | 20051116_59150_0400_PP 1 1000 | 20051116_59168_0400_PP 1 1001 | 20051116_59193_0400_PP 1 1002 | 20051205_31388_0400_PP 1 1003 | 20051205_31396_0400_PP 1 1004 | 20051205_31934_0400_PP 1 1005 | 20051205_31949_0400_PP 1 1006 | 20051205_31994_0400_PP 1 1007 | 20051205_32966_0400_PP 1 1008 | 20051205_32981_0400_PP 1 1009 | 20051205_33006_0400_PP 1 1010 | 20051205_33012_0400_PP 1 1011 | 20051205_35142_0400_PP 1 1012 | 20051205_35162_0400_PP 1 1013 | 20051205_35201_0400_PP 1 1014 | 20051205_35224_0400_PP 1 1015 | 20051205_35246_0400_PP 1 1016 | 20051205_35276_0400_PP 1 1017 | 20051205_35305_0400_PP 1 1018 | 20051205_35323_0400_PP 1 1019 | 20051205_35339_0400_PP 1 1020 | 20051205_35354_0400_PP 1 1021 | 20051205_35392_0400_PP 1 1022 | 20051205_35408_0400_PP 1 1023 | 20051205_35417_0400_PP 1 1024 | 20051205_35432_0400_PP 1 1025 | 20051207_56219_0400_PP 1 1026 | 20051207_56227_0400_PP 1 1027 | 20051208_40972_0400_PP 1 1028 | 20051208_41318_0400_PP 1 1029 | 20051208_41373_0400_PP 1 1030 | 20051208_41570_0400_PP 1 1031 | 20051208_41707_0400_PP 1 1032 | 20051208_42314_0400_PP 1 1033 | 20051208_42322_0400_PP 1 1034 | 20051208_42546_0400_PP 1 1035 | 20051208_42552_0400_PP 1 1036 | 20051209_38404_0400_PP 1 1037 | 20051212_40940_0400_PP 1 1038 | 20051212_41009_0400_PP 1 1039 | 20051212_41144_0400_PP 1 1040 | 20051212_41169_0400_PP 1 1041 | 20051212_41432_0400_PP 1 1042 | 20051213_62383_0100_PP 1 1043 | 20051213_62437_0100_PP 1 1044 | 20051213_62648_0100_PP 1 1045 | 20051213_62705_0100_PP 1 1046 | 20051214_40849_0100_PP 1 1047 | 20051214_40912_0100_PP 1 1048 | 20051214_41429_0100_PP 1 1049 | 20051214_41490_0100_PP 1 1050 | 20051214_41949_0100_PP 1 1051 | 20051214_50812_0100_PP 1 1052 | 20051214_51039_0100_PP 1 1053 | 20051214_51569_0100_PP 1 1054 | 20051214_51609_0100_PP 1 1055 | 20051214_51701_0100_PP 1 1056 | 20051214_51733_0100_PP 1 1057 | 20051214_51811_0100_PP 1 1058 | 20051214_51840_0100_PP 1 1059 | 20051214_51921_0100_PP 1 1060 | 20051214_51953_0100_PP 1 1061 | 20051214_52204_0100_PP 1 1062 | 20051214_52242_0100_PP 1 1063 | 20051214_52349_0100_PP 1 1064 | 20051214_52492_0100_PP 1 1065 | 20051214_52611_0100_PP 1 1066 | 20051214_52707_0100_PP 1 1067 | 20051214_56169_0100_PP 1 1068 | 20051214_56269_0100_PP 1 1069 | 20051214_56778_0100_PP 0 1070 | 20051214_56821_0100_PP 1 1071 | 20051214_56944_0100_PP 1 1072 | 20051214_56971_0100_PP 1 1073 | 20051214_57125_0100_PP 1 1074 | 20051214_57154_0100_PP 1 1075 | 20051214_57230_0100_PP 1 1076 | 20051214_57260_0100_PP 1 1077 | 20051214_57377_0100_PP 1 1078 | 20051214_57940_0100_PP 1 1079 | 20051216_45226_0200_PP 1 1080 | 20051216_45245_0200_PP 1 1081 | 20051216_46624_0200_PP 1 1082 | 20051216_46660_0200_PP 1 1083 | 20051216_46827_0200_PP 1 1084 | 20051216_47000_0200_PP 1 1085 | 20051216_47024_0200_PP 1 1086 | 20060407_43436_0200_PP 1 1087 | 20060407_44285_0200_PP 1 1088 | 20060407_44617_0200_PP 1 1089 | 20060407_44636_0200_PP 1 1090 | 20060407_46394_0200_PP 1 1091 | 20060410_44248_0200_PP 1 1092 | 20060411_51958_0200_PP 1 1093 | 20060411_51985_0200_PP 1 1094 | 20060411_57853_0200_PP 1 1095 | 20060411_57879_0200_PP 1 1096 | 20060411_58221_0200_PP 1 1097 | 20060411_58238_0200_PP 1 1098 | 20060411_58303_0200_PP 1 1099 | 20060411_58328_0200_PP 1 1100 | 20060411_59812_0200_PP 1 1101 | 20060411_59859_0200_PP 1 1102 | 20060411_59919_0200_PP 1 1103 | 20060411_61196_0200_PP 1 1104 | 20060411_61214_0200_PP 1 1105 | 20060411_61478_0200_PP 1 1106 | 20060411_61494_0200_PP 1 1107 | 20060411_61624_0200_PP 1 1108 | 20060411_61641_0200_PP 1 1109 | 20060411_61808_0200_PP 1 1110 | 20060411_62298_0200_PP 1 1111 | 20060411_62373_0200_PP 1 1112 | 20060411_62390_0200_PP 1 1113 | 20060412_51952_0200_PP 1 1114 | 20060412_52245_0200_PP 1 1115 | 20060412_52497_0200_PP 1 1116 | 20060412_52520_0200_PP 1 1117 | 20060412_52910_0200_PP 1 1118 | 20060412_52931_0200_PP 1 1119 | 20060412_53045_0200_PP 1 1120 | 20060412_57680_0200_PP 1 1121 | 20060412_58563_0200_PP 1 1122 | 20060412_58638_0200_PP 1 1123 | 20060412_58869_0200_PP 1 1124 | 20060412_58889_0200_PP 1 1125 | 20060412_59717_0200_PP 1 1126 | 20060412_59735_0200_PP 1 1127 | 20060412_60913_0200_PP 1 1128 | 20060412_61169_0200_PP 1 1129 | 20060412_62936_0200_PP 1 1130 | 20060412_62960_0200_PP 1 1131 | 20060522_45541_0100_PP 1 1132 | 20060522_45583_0100_PP 1 1133 | 20060522_45691_0100_PP 1 1134 | 20060522_45718_0100_PP 1 1135 | 20060523_43174_0100_PP 1 1136 | 20060523_43196_0100_PP 1 1137 | 20060523_48182_0100_PP 1 1138 | 20060523_48199_0100_PP 1 1139 | 20060523_48477_0100_PP 1 1140 | 20060523_48499_0100_PP 1 1141 | 20060523_49791_0100_PP 1 1142 | 20060523_49809_0100_PP 1 1143 | 20060523_49859_0100_PP 1 1144 | 20060523_49875_0100_PP 1 1145 | 20060523_49928_0100_PP 1 1146 | 20060523_49942_0100_PP 1 1147 | 20060523_50003_0100_PP 1 1148 | 20060523_50019_0100_PP 1 1149 | 20060523_50234_0100_PP 1 1150 | 20060523_50262_0100_PP 1 1151 | 20060523_50469_0100_PP 1 1152 | 20060523_50489_0100_PP 1 1153 | 20060523_50539_0100_PP 1 1154 | 20060523_50556_0100_PP 1 1155 | 20060523_50707_0100_PP 1 1156 | 20060523_50790_0100_PP 1 1157 | 20060523_50806_0100_PP 1 1158 | 20060529_56255_0100_PP 1 1159 | 20060529_56338_0100_PP 1 1160 | 20060529_56700_0100_PP 1 1161 | 20060529_56730_0100_PP 1 1162 | 20060529_57030_0100_PP 1 1163 | 20060529_57063_0100_PP 1 1164 | 20060529_57430_0100_PP 1 1165 | 20060529_57447_0100_PP 1 1166 | 20060530_36788_0100_PP 1 1167 | 20060530_36860_0100_PP 1 1168 | 20060530_36895_0100_PP 1 1169 | 20060530_51377_0100_PP 1 1170 | 20060530_51399_0100_PP 1 1171 | 20060530_52988_0100_PP 1 1172 | 20060530_53271_0100_PP 1 1173 | 20060530_53421_0100_PP 1 1174 | 20060530_53455_0100_PP 1 1175 | 20060530_53702_0100_PP 1 1176 | 20060530_54107_0100_PP 1 1177 | 20060530_54117_0100_PP 1 1178 | 20060530_54242_0100_PP 1 1179 | 20060530_54312_0100_PP 1 1180 | 20060530_54332_0100_PP 1 1181 | 20060530_54390_0100_PP 1 1182 | 20060530_54529_0100_PP 1 1183 | 20060530_54556_0100_PP 1 1184 | 20060530_55304_0100_PP 1 1185 | 20060530_55724_0100_PP 1 1186 | 20060530_55746_0100_PP 1 1187 | 20060530_55816_0100_PP 1 1188 | 20060530_55837_0100_PP 1 1189 | --------------------------------------------------------------------------------