├── DREAM_DM_starter_tf.py ├── Dockerfile ├── README.md ├── sc1_infer.sh ├── test.sh └── train.sh /DREAM_DM_starter_tf.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import dicom 4 | import gzip 5 | import numpy as np 6 | from os import listdir, remove, mkdir 7 | from os.path import isfile, join, isdir 8 | import scipy.misc 9 | from sklearn.cross_validation import train_test_split 10 | import tensorflow as tf 11 | import tflearn 12 | import sys 13 | import time 14 | 15 | def super_print(statement, f): 16 | """ 17 | This basically prints everything in statement. 18 | We'll add a new line character for the output file. 19 | We'll just use print for the output. 20 | INPUTS: 21 | - statement: (string) the string to print. 22 | - f: (opened file) this is the output file object to print to 23 | """ 24 | sys.stdout.write(statement + '\n') 25 | sys.stdout.flush() 26 | f.write(statement + '\n') 27 | return 0 28 | 29 | def create_test_splits(path_csv_test): 30 | """ 31 | Goes through the data folder and divides for testing. 32 | INPUTS: 33 | - path_csv_test: (string) path to test csv 34 | """ 35 | X_tr = [] 36 | X_te = [] 37 | Y_tr = [] 38 | Y_te = [] 39 | # First, let's map examID and laterality to fileName 40 | dict_X_left = {} 41 | dict_X_right = {} 42 | counter = 0 43 | with open(path_csv_test, 'r') as file_crosswalk: 44 | reader_crosswalk = csv.reader(file_crosswalk, delimiter='\t') 45 | for row in reader_crosswalk: 46 | if counter == 0: 47 | counter += 1 48 | continue 49 | if row[3].strip()=='R': 50 | dict_X_right[row[0].strip()] = row[4].strip() 51 | X_te.append((row[0].strip(), 'R', row[4].strip())) 52 | elif row[3].strip()=='L': 53 | dict_X_left[row[0].strip()] = row[4].strip() 54 | X_te.append((row[0].strip(), 'L', row[4].strip())) 55 | #for key_X in set(dict_X_left.keys()) & set(dict_X_right.keys()): 56 | # X_te.append((dict_X_left[key_X], dict_X_right[key_X])) 57 | return X_tr, X_te, Y_tr, Y_te 58 | 59 | def create_data_splits(path_csv_crosswalk, path_csv_metadata): 60 | """ 61 | Goes through data folder and divides train/val. 62 | INPUTS: 63 | - path_csv_crosswalk: (string) path to first csv file 64 | - path_csv_metadata: (string) path to second csv file 65 | There should be two csv files. The first will relate the filename 66 | to the actual patient ID and L/R side, then the second csv file 67 | will relate this to whether we get the cancer. This is ridiculous. 68 | Very very very bad filesystem. Hope this gets better. 69 | """ 70 | # First, let's map the .dcm.gz file to a (patientID, examIndex, imageView) tuple. 71 | dict_img_to_patside = {} 72 | counter = 0 73 | with open(path_csv_crosswalk, 'r') as file_crosswalk: 74 | reader_crosswalk = csv.reader(file_crosswalk, delimiter='\t') 75 | for row in reader_crosswalk: 76 | if counter == 0: 77 | counter += 1 78 | continue 79 | dict_img_to_patside[row[5].strip()] = (row[0].strip(), row[4].strip()) 80 | # Now, let's map the tuple to cancer or non-cancer. 81 | dict_tuple_to_cancer = {} 82 | counter = 0 83 | with open(path_csv_metadata, 'r') as file_metadata: 84 | reader_metadata = csv.reader(file_metadata, delimiter='\t') 85 | for row in reader_metadata: 86 | if counter == 0: 87 | counter += 1 88 | continue 89 | cancerL = row[3] 90 | cancerR = row[4] 91 | if cancerL == '.': 92 | cancerL = '0' 93 | if cancerR == '.': 94 | cancerR = '0' 95 | dict_tuple_to_cancer[(row[0].strip(), 'L')] = int(cancerL) 96 | dict_tuple_to_cancer[(row[0].strip(), 'R')] = int(cancerR) 97 | # Alright, now, let's connect those dictionaries together... 98 | X_tot = [] 99 | Y_tot = [] 100 | for img_name in dict_img_to_patside: 101 | X_tot.append(img_name) 102 | Y_tot.append(dict_tuple_to_cancer[dict_img_to_patside[img_name]]) 103 | # Making train/val split and returning. 104 | X_tr, X_te, Y_tr, Y_te = train_test_split(X_tot, Y_tot, test_size=0.001) 105 | return X_tr, X_te, Y_tr, Y_te 106 | 107 | def read_in_one_image(path_img, name_img, matrix_size, data_aug=False): 108 | """ 109 | This is SUPER basic. This can be improved. 110 | Basically, all data is stored as a .dcm.gz. 111 | First, we'll uncompress and save as temp.dcm. 112 | Then we'll read in the dcm to get to the array. 113 | We'll resize the image to [matrix_size, matrix_size]. 114 | We'll also convert to a np.float32 and zero-center 1-scale the data. 115 | INPUTS: 116 | - path_img: (string) path to the data 117 | - name_img: (string) name of the image e.g. '123456.dcm' 118 | - matrix_size: (int) one dimension of the square image e.g. 224 119 | """ 120 | # Setting up the filepaths and opening up the format. 121 | #filepath_temp = join(path_img, 'temp.dcm') 122 | filepath_img = join(path_img, name_img) 123 | # Reading/uncompressing/writing 124 | #if isfile(filepath_temp): 125 | # remove(filepath_temp) 126 | #with gzip.open(filepath_img, 'rb') as f_gzip: 127 | # file_content = f_gzip.read() 128 | # with open(filepath_temp, 'w') as f_dcm: 129 | # f_dcm.write(file_content) 130 | # Reading in dicom file to ndarray and processing 131 | dicom_content = dicom.read_file(filepath_img) 132 | img = dicom_content.pixel_array 133 | img = scipy.misc.imresize(img, (matrix_size, matrix_size), interp='cubic') 134 | img = img.astype(np.float32) 135 | img -= np.mean(img) 136 | img /= np.std(img) 137 | # Removing temporary file. 138 | #remove(filepath_temp) 139 | # Let's do some stochastic data augmentation. 140 | if not data_aug: 141 | return img 142 | if np.random.rand() > 0.5: #flip left-right 143 | img = np.fliplr(img) 144 | num_rot = np.random.choice(4) #rotate 90 randomly 145 | img = np.rot90(img, num_rot) 146 | up_bound = np.random.choice(174) #zero out square 147 | right_bound = np.random.choice(174) 148 | img[up_bound:(up_bound+50), right_bound:(right_bound+50)] = 0.0 149 | return img 150 | 151 | def conv2d(l_input, filt_size, filt_num, stride=1, alpha=0.1, name="conv2d", norm="bn"): 152 | """ 153 | A simple 2-dimensional convolution layer. 154 | Layer Architecture: 2d-convolution - bias-addition - batch_norm - reLU 155 | All weights are created with a (hopefully) unique scope. 156 | INPUTS: 157 | - l_input: (tensor.4d) input of size [batch_size, layer_width, layer_height, channels] 158 | - filt_size: (int) size of the square filter to be made 159 | - filt_num: (int) number of filters to be made 160 | - stride: (int) stride of our convolution 161 | - alpha: (float) for the leaky ReLU. Do 0.0 for ReLU. 162 | - name: (string) unique name for this convolution layer 163 | - norm: (string) to decide which normalization to use ("bn", "lrn", None) 164 | """ 165 | # Creating and Doing the Convolution. 166 | input_size = l_input.get_shape().as_list() 167 | weight_shape = [filt_size, filt_size, input_size[3], filt_num] 168 | std = 0.01#np.sqrt(2.0 / (filt_size * filt_size * input_size[3])) 169 | with tf.variable_scope(name+"_conv_weights"): 170 | W = tf.get_variable("W", weight_shape, initializer=tf.random_normal_initializer(stddev=std)) 171 | tf.add_to_collection("reg_variables", W) 172 | conv_layer = tf.nn.conv2d(l_input, W, strides=[1, stride, stride, 1], padding='SAME') 173 | norm_layer = conv_layer 174 | # Normalization 175 | #if norm=="bn": 176 | # norm_layer = tflearn.layers.normalization.batch_normalization(conv_layer, name=(name+"_batch_norm"), decay=0.9) 177 | #elif norm=="lrn": 178 | # norm_layer = tflearn.layers.normalization.local_response_normalization(conv_layer) 179 | # ReLU 180 | relu_layer = tf.maximum(norm_layer, norm_layer*alpha) 181 | return relu_layer 182 | 183 | def max_pool(l_input, k=2, stride=None): 184 | """ 185 | A simple 2-dimensional max pooling layer. 186 | Strides and size of max pool kernel is constrained to be the same. 187 | INPUTS: 188 | - l_input: (tensor.4d) input of size [batch_size, layer_width, layer_height, channels] 189 | - k: (int) size of the max_filter to be made. also size of stride. 190 | """ 191 | if stride==None: 192 | stride=k 193 | # Doing the Max Pool 194 | max_layer = tf.nn.max_pool(l_input, ksize = [1, k, k, 1], strides = [1, stride, stride, 1], padding='SAME') 195 | return max_layer 196 | 197 | def incept(l_input, kSize=[16,16,16,16,16,16], name="incept", norm="bn"): 198 | """ 199 | So, this is the classical incept layer. 200 | INPUTS: 201 | - l_input: (tensor.4d) input of size [batch_size, layer_width, layer_height, channels] 202 | - ksize: (array (6,)) [1x1, 3x3reduce, 3x3, 5x5reduce, 5x5, poolproj] 203 | - name: (string) name of incept layer 204 | - norm: (string) to decide which normalization ("bn", "lrn", None) 205 | """ 206 | layer_1x1 = conv2d(l_input, 1, kSize[0], name=(name+"_1x1"), norm=norm) 207 | layer_3x3a = conv2d(l_input, 1, kSize[1], name=(name+"_3x3a"), norm=norm) 208 | layer_3x3b = conv2d(layer_3x3a, 3, kSize[2], name=(name+"_3x3b"), norm=norm) 209 | layer_5x5a = conv2d(l_input, 1, kSize[3], name=(name+"_5x5a"), norm=norm) 210 | layer_5x5b = conv2d(layer_5x5a, 5, kSize[4], name=(name+"_5x5b"), norm=norm) 211 | layer_poola = max_pool(l_input, k=3, stride=1) 212 | layer_poolb = conv2d(layer_poola, 1, kSize[5], name=(name+"_poolb"), norm=norm) 213 | return tf.concat(3, [layer_1x1, layer_3x3b, layer_5x5b, layer_poolb]) 214 | 215 | def dense(l_input, hidden_size, keep_prob, alpha=0.1, name="dense"): 216 | """ 217 | Dense (Fully Connected) layer. 218 | Architecture: reshape - Affine - batch_norm - dropout - relu 219 | WARNING: should not be the output layer. Use "output" for that. 220 | INPUTS: 221 | - l_input: (tensor.2d or more) basically, of size [batch_size, etc...] 222 | - hidden_size: (int) Number of hidden neurons. 223 | - keep_prob: (float) Probability to keep neuron during dropout layer. 224 | - alpha: (float) Slope for leaky ReLU. Set 0.0 for ReLU. 225 | - name: (string) unique name for layer. 226 | """ 227 | # Flatten Input Layer 228 | input_size = l_input.get_shape().as_list() 229 | reshape_size = 1 230 | for iter_size in range(1, len(input_size)): 231 | reshape_size *= input_size[iter_size] 232 | reshape_layer = tf.reshape(l_input, [-1, reshape_size]) 233 | # Creating and Doing Affine Transformation 234 | weight_shape = [reshape_layer.get_shape().as_list()[1], hidden_size] 235 | std = 0.01#np.sqrt(2.0 / reshape_layer.get_shape().as_list()[1]) 236 | with tf.variable_scope(name+"_dense_weights"): 237 | W = tf.get_variable("W", weight_shape, initializer=tf.random_normal_initializer(stddev=std)) 238 | tf.add_to_collection("reg_variables", W) 239 | affine_layer = tf.matmul(reshape_layer, W) 240 | norm_layer = affine_layer 241 | # Batch Normalization 242 | #norm_layer = tflearn.layers.normalization.batch_normalization(affine_layer, name=(name+"_batch_norm"), decay=0.9) 243 | # Dropout 244 | dropout_layer = tf.nn.dropout(norm_layer, keep_prob) 245 | # ReLU 246 | relu_layer = tf.maximum(dropout_layer, dropout_layer*alpha) 247 | return relu_layer 248 | 249 | def output(l_input, output_size, name="output"): 250 | """ 251 | Output layer. Just a simple affine transformation. 252 | INPUTS: 253 | - l_input: (tensor.2d or more) basically, of size [batch_size, etc...] 254 | - output_size: (int) basically, number of classes we're predicting 255 | - name: (string) unique name for layer. 256 | """ 257 | # Flatten Input Layer 258 | input_size = l_input.get_shape().as_list() 259 | reshape_size = 1 260 | for iter_size in range(1, len(input_size)): 261 | reshape_size *= input_size[iter_size] 262 | reshape_layer = tf.reshape(l_input, [-1, reshape_size]) 263 | # Creating and Doing Affine Transformation 264 | weight_shape = [reshape_layer.get_shape().as_list()[1], output_size] 265 | std = 0.01#np.sqrt(2.0 / reshape_layer.get_shape().as_list()[1]) 266 | with tf.variable_scope(name+"_output_weights"): 267 | W = tf.get_variable("W", weight_shape, initializer=tf.random_normal_initializer(stddev=std)) 268 | b = tf.get_variable("b", output_size, initializer=tf.constant_initializer(0.0)) 269 | tf.add_to_collection("reg_variables", W) 270 | affine_layer = tf.matmul(reshape_layer, W) + b 271 | return affine_layer 272 | 273 | def get_L2_loss(reg_param, key="reg_variables"): 274 | """ 275 | L2 Loss Layer. Usually will use "reg_variables" collection. 276 | INPUTS: 277 | - reg_param: (float) the lambda value for regularization. 278 | - key: (string) the key for the tf collection to get from. 279 | """ 280 | L2_loss = 0.0 281 | for W in tf.get_collection(key): 282 | L2_loss += reg_param * tf.nn.l2_loss(W) 283 | return L2_loss 284 | 285 | def get_CE_loss(logits, labels): 286 | """ 287 | This calculates the cross entropy loss. 288 | Modular function made just because tf program name is long. 289 | INPUTS: 290 | - logits: (tensor.2d) logit probability values. 291 | - labels: (array of ints) basically, label \in {0,...,L-1} 292 | """ 293 | return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels)) 294 | 295 | def get_accuracy(logits, labels): 296 | """ 297 | Calculates accuracy of predictions. Softmax based on largest. 298 | INPUTS: 299 | - logits: (tensor.2d) logit probability values. 300 | - labels: (array of ints) basically, label \in {0,...,L-1} 301 | """ 302 | pred_labels = tf.argmax(logits,1) 303 | correct_pred = tf.equal(pred_labels, labels) 304 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 305 | return accuracy 306 | 307 | def get_optimizer(cost, lr=0.001, decay=1.0, epoch_every=10): 308 | """ 309 | Creates an optimizer based on learning rate and loss. 310 | We will use Adam optimizer. This may have to change in the future. 311 | INPUTS: 312 | - cost: (tf value) usually sum of L2 loss and CE loss 313 | - lr: (float) the learning rate. 314 | - decay: (float) how much to decay each epoch. 315 | - epoch_every: (int) how many iterations is an epoch. 316 | """ 317 | global_step = tf.Variable(0, trainable=False) 318 | starter_learning_rate = float(lr) 319 | learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 320 | epoch_every, decay, staircase=True) 321 | optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost, global_step=global_step) 322 | return optimizer 323 | 324 | def Alex_conv(layer, b_name=""): 325 | """ 326 | The convolution part of the classic Alex Net. 327 | Everything has been hardcoded to show example of use. 328 | INPUT: 329 | - layer: (tensor.4d) input tensor. 330 | - b_name: (string) branch name. If not doing branch, doesn't matter. 331 | """ 332 | conv1 = conv2d(layer, 11, 96, stride=4, name=b_name+"conv1") 333 | pool1 = max_pool(conv1, k=2) 334 | conv2 = conv2d(pool1, 11, 256, name=b_name+"conv2") 335 | pool2 = max_pool(conv2, k=2) 336 | conv3 = conv2d(pool2, 3, 384, name=b_name+"conv3") 337 | conv4 = conv2d(conv3, 3, 384, name=b_name+"conv4") 338 | conv5 = conv2d(conv3, 3, 256, name=b_name+"conv5") 339 | pool5 = max_pool(conv5, k=2) 340 | return pool5 341 | 342 | def general_conv(layer, architecture_conv, b_name="", norm="bn"): 343 | """ 344 | A generalized convolution block that takes an architecture. 345 | INPUTS: 346 | - layer: (tensor.4d) input tensor. 347 | - architecture_conv: (list of lists) 348 | [[filt_size, filt_num, stride], ..., [0, poolSize], 349 | [filt_size, filt_num, stride], ..., [0, poolSize], 350 | ...] 351 | - b_name: (string) branch name. If not doing branch, doesn't matter. 352 | """ 353 | for conv_iter, conv_numbers in enumerate(architecture_conv): 354 | if conv_numbers[0]==0: 355 | layer = max_pool(layer, k=conv_numbers[1]) 356 | else: 357 | if len(conv_numbers)==2: 358 | conv_numbers.append(1) 359 | layer = conv2d(layer, conv_numbers[0], conv_numbers[1], stride=conv_numbers[2], 360 | name=(b_name+"conv"+str(conv_iter)), norm=norm) 361 | return layer 362 | 363 | def GoogLe_conv(layer, b_name="", norm="bn"): 364 | """ 365 | This should be the convolution layers of the GoogLe net. 366 | We follow the v1 architecture as laid out by 367 | http://www.cs.unc.edu/~wliu/papers/GoogLeNet.pdf 368 | INPUTS: 369 | - layer: (tensor.4d) input tensor 370 | - b_name: (string) branch name, if necessary. 371 | - norm: (string) which normalization to use. 372 | """ 373 | conv1 = conv2d(layer, 7, 64, stride=2, name=b_name+"conv1", norm=norm) 374 | pool1 = max_pool(conv1, k=3, stride=2) 375 | conv2a = conv2d(pool1, 1, 64, name=b_name+"conv2a", norm=norm) 376 | conv2b = conv2d(conv2a, 3, 192, name=b_name+"conv2b", norm=norm) 377 | pool2 = max_pool(conv2b, k=3, stride=2) 378 | incept3a = incept(pool2, kSize=[64,96,128,16,32,32], name=b_name+"incept3a", norm=norm) 379 | incept3b = incept(incept3a, kSize=[128,128,192,32,96,64], name=b_name+"incept3b", norm=norm) 380 | pool3 = max_pool(incept3b, k=3, stride=2) 381 | incept4a = incept(pool3, kSize=[192,96,208,16,48,64], name=b_name+"incept4a", norm=norm) 382 | incept4b = incept(incept4a, kSize=[160,112,224,24,64,64], name=b_name+"incept4b", norm=norm) 383 | incept4c = incept(incept4b, kSize=[128,128,256,24,64,64], name=b_name+"incept4c", norm=norm) 384 | incept4d = incept(incept4c, kSize=[112,144,288,32,64,64], name=b_name+"incept4d", norm=norm) 385 | incept4e = incept(incept4d, kSize=[256,160,320,32,128,128], name=b_name+"incept4e", norm=norm) 386 | pool4 = max_pool(incept4e, k=3, stride=2) 387 | incept5a = incept(pool4, kSize=[256,160,320,32,128,128], name=b_name+"incept5a", norm=norm) 388 | incept5b = incept(incept5a, kSize=[384,192,384,48,128,128], name=b_name+"incept5b", norm=norm) 389 | size_pool = incept5b.get_shape().as_list()[1] 390 | pool5 = tf.nn.avg_pool(incept5b, ksize=[1,size_pool,size_pool,1], strides=[1,1,1,1], padding='VALID') 391 | return pool5 392 | 393 | def Le_Net(X, output_size, keep_prob=1.0, name=""): 394 | """ 395 | Very Simple Lenet 396 | INPUTS: 397 | - X: (tensor.4d) input tensor. 398 | - output_size: (int) number of classes we're predicting 399 | - keep_prob: (float) probability to keep during dropout. should be 0.4 at train. 400 | """ 401 | layer = X 402 | conv1 = conv2d(layer, 5, 6, stride=1, name=name+"conv1", norm="bn") 403 | pool1 = max_pool(conv1, k=2, stride=2) 404 | conv2 = conv2d(pool1, 3, 16, stride=1, name=name+"conv2", norm="bn") 405 | pool2 = max_pool(conv2, k=2, stride=2) 406 | dense1 = dense(pool2, 120, keep_prob, name=name+"dense1") 407 | return output(dense1, output_size, name=name+"output") 408 | 409 | def GoogLe_Net(X, output_size, keep_prob=1.0, name=""): 410 | """ 411 | This is the famous GoogLeNet incarnation of the inception network. 412 | All the power is in the convs, so this is quite simple. 413 | INPUTS: 414 | - X: (tensor.4d) input tensor. 415 | - output_size: (int) number of classes we're predicting 416 | - keep_prob: (float) probability to keep during dropout. should be 0.4 at train. 417 | """ 418 | layer = GoogLe_conv(X, b_name=name) 419 | drop1 = tf.nn.dropout(layer, keep_prob) 420 | return output(layer, output_size, name=name+"output") 421 | 422 | def Alex_Net(X, output_size, keep_prob=1.0, name=""): 423 | """ 424 | The classic alex net architecture. 425 | INPUTS: 426 | - X: (tensor.4d) A tensor with dimensions (none, width, height, num_channels) 427 | - output_size: (int) The number of classes there are. 428 | - keep_prob: (float) Chance of keeping a neuron during dropout. 429 | """ 430 | layer = X 431 | layer = Alex_conv(layer, b_name=name) 432 | dense1 = dense(layer, 4096, keep_prob, name=name+"dense1") 433 | dense2 = dense(dense1, 4096, keep_prob, name=name+"dense2") 434 | return output(dense2, output_size, name=name+"output") 435 | 436 | def VGG16_Net(X, output_size, keep_prob=1.0): 437 | """ 438 | The classic VGG16 net architecture. 439 | INPUTS: 440 | - X: (tensor.4d) A tensor with dimensions (none, width, height, num_channels) 441 | - output_size: (int) The number of classes there are. 442 | - keep_prob: (float) Chance of keeping a neuron during dropout. 443 | """ 444 | architecture_conv=[[3, 64], [3, 64], [0, 2], 445 | [3, 128], [3, 128], [0, 2], 446 | [3, 256], [3, 256], [3, 256], [0, 2], 447 | [3, 512], [3, 512], [3, 512], [0, 2], 448 | [3, 512], [3, 512], [3, 512], [0, 2]] 449 | layer = general_conv(X, architecture_conv, b_name=name) 450 | layer = dense(layer, 4096, keep_prob, name=name+"dense1") 451 | layer = dense(layer, 4096, keep_prob, name=name+"dense2") 452 | return output(layer, output_size, name=name+"output") 453 | 454 | def test_out(sess, list_dims, list_placeholders, list_operations, X_te, opts): 455 | """ 456 | This code is to call a test on the validation set. 457 | INPUTS: 458 | - sess: (tf session) the session to run everything on 459 | - list_dim: (list of ints) list of dimensions 460 | - list_placeholders: (list of tensors) list of the placeholders for feed_dict 461 | - list_operations: (list of tensors) list of operations for graph access 462 | - X_tr: (list of strings) list of training sample names 463 | - opts: (parsed arguments) 464 | """ 465 | # Let's unpack the lists 466 | matrix_size, num_channels = list_dims 467 | x, y, keep_prob = list_placeholders 468 | prob, pred, saver, L2_loss, CE_loss, cost, optimizer, accuracy, init = list_operations 469 | # Initializing what to put in. 470 | dataXX = np.zeros((1, matrix_size, matrix_size, num_channels), dtype=np.float32) 471 | # Running through the images. 472 | f = open(opts.outtxt, 'w') 473 | statement = 'subjectID' + '\t' + 'laterality' + '\t' + 'confidence' 474 | super_print(statement, f) 475 | for iter_data in range(len(X_te)): 476 | id_iter, lat_iter, img_iter = X_te[iter_data] 477 | dataXX[0, :, :, 0] = read_in_one_image(opts.path_data, img_iter, matrix_size) 478 | tflearn.is_training(False) 479 | pred_iter = sess.run(prob, feed_dict={x: dataXX, keep_prob: 1.0}) 480 | statement = id_iter + '\t' + lat_iter + '\t' + str(pred_iter[0][1]) 481 | super_print(statement, f) 482 | #left_img, right_img = X_te[iter_data] 483 | #dataXX[0, :, :, 0] = read_in_one_image(opts.path_data, left_img, matrix_size) 484 | #tflearn.is_training(False) 485 | #pred_left = sess.run(pred, feed_dict={x: dataXX, keep_prob: 1.0}) 486 | #dataXX[0, :, :, 0] = read_in_one_image(opts.path_data, right_img, matrix_size) 487 | #pred_right = sess.run(pred, feed_dict={x: dataXX, keep_prob: 1.0}) 488 | #statement = str(pred_left) + '\t' + str(pred_right) 489 | #super_print(statement, f) 490 | f.close() 491 | 492 | def test_all(sess, list_dims, list_placeholders, list_operations, X_te, Y_te, opts): 493 | """ 494 | This code is to call a test on the validation set. 495 | INPUTS: 496 | - sess: (tf session) the session to run everything on 497 | - list_dim: (list of ints) list of dimensions 498 | - list_placeholders: (list of tensors) list of the placeholders for feed_dict 499 | - list_operations: (list of tensors) list of operations for graph access 500 | - X_tr: (list of strings) list of training sample names 501 | - Y_tr: (list of ints) list of lables for training samples 502 | - opts: (parsed arguments) 503 | """ 504 | # Let's unpack the lists. 505 | matrix_size, num_channels = list_dims 506 | x, y, keep_prob = list_placeholders 507 | prob, pred, saver, L2_loss, CE_loss, cost, optimizer, accuracy, init = list_operations 508 | # Initializing what to put in. 509 | loss_te = 0.0 510 | acc_te = 0.0 511 | dataXX = np.zeros((1, matrix_size, matrix_size, num_channels), dtype=np.float32) 512 | dataYY = np.zeros((1, ), dtype=np.int64) 513 | # Running through all test data points 514 | v_TP = 0.0 515 | v_FP = 0.0 516 | v_FN = 0.0 517 | v_TN = 0.0 518 | for iter_data in range(len(X_te)): 519 | # Reading in the data 520 | dataXX[0, :, :, 0] = read_in_one_image(opts.path_data, X_te[iter_data], matrix_size) 521 | dataYY[0] = Y_te[iter_data] 522 | tflearn.is_training(False) 523 | loss_iter, acc_iter = sess.run((cost, accuracy), feed_dict={x: dataXX, y: dataYY, keep_prob: 1.0}) 524 | # Figuring out the ROC stuff 525 | if Y_te[iter_data] == 1: 526 | if acc_iter == 1: 527 | v_TP += 1.0 / len(X_te) 528 | else: 529 | v_FN += 1.0 /len(X_te) 530 | else: 531 | if acc_iter == 1: 532 | v_TN += 1.0 /len(X_te) 533 | else: 534 | v_FP += 1.0 /len(X_te) 535 | # Adding to total accuracy and loss 536 | loss_te += loss_iter / len(X_te) 537 | acc_te += acc_iter / len(X_te) 538 | return (loss_te, acc_te, [v_TP, v_FP, v_TN, v_FN]) 539 | 540 | def train_one_iteration(sess, list_dims, list_placeholders, list_operations, X_tr, Y_tr, opts): 541 | """ 542 | Basically, run one iteration of the training. 543 | INPUTS: 544 | - sess: (tf session) the session to run everything on 545 | - list_dim: (list of ints) list of dimensions 546 | - list_placeholders: (list of tensors) list of the placeholders for feed_dict 547 | - list_operations: (list of tensors) list of operations for graph access 548 | - X_tr: (list of strings) list of training sample names 549 | - Y_tr: (list of ints) list of lables for training samples 550 | - opts: (parsed arguments) 551 | """ 552 | # Let's unpack the lists. 553 | matrix_size, num_channels = list_dims 554 | x, y, keep_prob = list_placeholders 555 | prob, pred, saver, L2_loss, CE_loss, cost, optimizer, accuracy, init = list_operations 556 | # Initializing what to put in. 557 | dataXX = np.zeros((opts.bs, matrix_size, matrix_size, num_channels), dtype=np.float32) 558 | dataYY = np.zeros((opts.bs, ), dtype=np.int64) 559 | ind_list = np.random.choice(range(len(X_tr)), opts.bs, replace=False) 560 | # Fill in our dataXX and dataYY for training one batch. 561 | for iter_data,ind in enumerate(ind_list): 562 | dataXX[iter_data, :, :, 0] = read_in_one_image(opts.path_data, X_tr[ind], matrix_size, data_aug=False) 563 | dataYY[iter_data] = Y_tr[ind] 564 | tflearn.is_training(True) 565 | _, loss_iter, acc_iter = sess.run((optimizer, cost, accuracy), feed_dict={x: dataXX, y: dataYY, keep_prob: opts.dropout}) 566 | return (loss_iter, acc_iter) 567 | 568 | def train_net(X_tr, X_te, Y_tr, Y_te, opts, f): 569 | """ 570 | Training of the net. All we need is data names and parameters. 571 | INPUTS: 572 | - X_tr: (list of strings) training image names 573 | - X_te: (list of strings) validation image names 574 | - Y_tr: (list of ints) training labels 575 | - Y_te: (list of ints) validation labels 576 | - opts: parsed argument thing 577 | - f: (opened file) for output writing 578 | """ 579 | # Setting the size and number of channels of input. 580 | matrix_size = opts.matrix_size 581 | num_channels = 1 582 | list_dims = [matrix_size, num_channels] 583 | # Finding out other constant values to be used. 584 | data_count = len(X_tr) 585 | iter_count = int(np.ceil(float(opts.epoch) * data_count / opts.bs)) 586 | epoch_every = int(np.ceil(float(iter_count) / opts.epoch)) 587 | print_every = min([100, epoch_every]) 588 | max_val_acc = 0.0 589 | # Creating Placeholders 590 | x = tf.placeholder(tf.float32, [None, matrix_size, matrix_size, num_channels]) 591 | y = tf.placeholder(tf.int64) 592 | keep_prob = tf.placeholder(tf.float32) 593 | list_placeholders = [x, y, keep_prob] 594 | # Create the network 595 | if opts.net == "Alex": 596 | pred = Alex_Net(x, 2, keep_prob=keep_prob) 597 | elif opts.net == "Le": 598 | pred = Le_Net(x, 2, keep_prob=keep_prob) 599 | elif opts.net == "VGG16": 600 | pred = VGG16_Net(x, 2, keep_prob=keep_prob) 601 | elif opts.net == "GoogLe": 602 | pred = GoogLe_Net(x, 2, keep_prob=keep_prob) 603 | else: 604 | statement = "Please specify valid network (e.g. Alex, VGG16, GoogLe)." 605 | super_print(statement, f) 606 | return 0 607 | # Define Operations in TF Graph 608 | saver = tf.train.Saver() 609 | L2_loss = get_L2_loss(opts.reg) 610 | CE_loss = get_CE_loss(pred, y) 611 | cost = L2_loss + CE_loss 612 | prob = tf.nn.softmax(pred) 613 | optimizer = get_optimizer(cost, lr=opts.lr, decay=opts.decay, epoch_every=epoch_every) 614 | accuracy = get_accuracy(pred, y) 615 | init = tf.initialize_all_variables() 616 | list_operations = [prob, pred, saver, L2_loss, CE_loss, cost, optimizer, accuracy, init] 617 | # Do the Training 618 | print "Training Started..." 619 | start_time = time.time() 620 | with tf.Session() as sess: 621 | sess.run(init) 622 | loss_tr = 0.0 623 | acc_tr = 0.0 624 | if opts.test: 625 | saver.restore(sess, opts.saver) 626 | test_out(sess, list_dims, list_placeholders, list_operations, X_te, opts) 627 | return 0 628 | for iter in range(iter_count): 629 | loss_temp, acc_temp = train_one_iteration(sess, list_dims, list_placeholders, list_operations, X_tr, Y_tr, opts) 630 | loss_tr += loss_temp / print_every 631 | acc_tr += acc_temp / print_every 632 | if ((iter)%print_every) == 0: 633 | current_time = time.time() 634 | loss_te, acc_te, ROC_values = test_all(sess, list_dims, list_placeholders, list_operations, X_te, Y_te, opts) 635 | # Printing out stuff 636 | statement = " Iter"+str(iter+1)+": "+str((current_time - start_time)/60) 637 | statement += ", Acc_tr: "+str(acc_tr) 638 | statement += ", Acc_val: "+str(acc_te) 639 | statement += ", Loss_tr: "+str(loss_tr) 640 | statement += ", Loss_val: "+str(loss_te) 641 | super_print(statement, f) 642 | statement = " True_Positive: "+str(ROC_values[0]) 643 | statement += ", False_Positive: "+str(ROC_values[1]) 644 | statement += ", True_Negative: "+str(ROC_values[2]) 645 | statement += ", False_Negative: "+str(ROC_values[3]) 646 | super_print(statement, f) 647 | loss_tr = 0.0 648 | acc_tr = 0.0 649 | if acc_te > max_val_acc: 650 | max_val_acc = acc_te 651 | saver.save(sess, opts.saver) 652 | if (current_time - start_time)/60 > opts.time: 653 | break 654 | statement = "Best you could do: " + str(max_val_acc) 655 | super_print(statement, f) 656 | return 0 657 | 658 | def main(args): 659 | """ 660 | Main Function to do deep learning using tensorflow on pilot. 661 | INPUTS: 662 | - args: (list of strings) command line arguments 663 | """ 664 | # Setting up reading of command line options, storing defaults if not provided. 665 | parser = argparse.ArgumentParser(description = "Do deep learning!") 666 | parser.add_argument("--pf", dest="path_data", type=str, default="/trainingData") 667 | parser.add_argument("--csv1", dest="csv1", type=str, default="/metadata/images_crosswalk.tsv") 668 | parser.add_argument("--csv2", dest="csv2", type=str, default="/metadata/exams_metadata.tsv") 669 | parser.add_argument("--csv3", dest="csv3", type=str, default="/scoringData/image_metadata.tsv") 670 | parser.add_argument("--net", dest="net", type=str, default="GoogLe") 671 | parser.add_argument("--lr", dest="lr", type=float, default=0.001) 672 | parser.add_argument("--reg", dest="reg", type=float, default=0.00001) 673 | parser.add_argument("--out", dest="output", type=str, default="/modelState/out_train.txt") 674 | parser.add_argument("--outtxt", dest="outtxt", type=str, default="/output/out.txt") 675 | parser.add_argument("--saver", dest="saver", type=str, default="/modelState/model.ckpt") 676 | parser.add_argument("--decay", dest="decay", type=float, default=1.0) 677 | parser.add_argument("--dropout", dest="dropout", type=float, default=0.5) 678 | parser.add_argument("--bs", dest="bs", type=int, default=10) 679 | parser.add_argument("--epoch", dest="epoch", type=int, default=10) 680 | parser.add_argument("--test", dest="test", type=int, default=0) 681 | parser.add_argument("--ms", dest="matrix_size", type=int, default=224) 682 | parser.add_argument("--time", dest="time", type=float, default=1000000) 683 | opts = parser.parse_args(args[1:]) 684 | # Setting up the output file. 685 | if isfile(opts.output): 686 | remove(opts.output) 687 | f = open(opts.output, 'w') 688 | # Finding list of data. 689 | statement = "Parsing the csv's." 690 | super_print(statement, f) 691 | path_csv_crosswalk = opts.csv1 692 | path_csv_metadata = opts.csv2 693 | path_csv_test = opts.csv1 694 | if opts.test: 695 | X_tr, X_te, Y_tr, Y_te = create_test_splits(path_csv_test) 696 | else: 697 | X_tr, X_te, Y_tr, Y_te = create_data_splits(path_csv_crosswalk, path_csv_metadata) 698 | # Train a network and print a bunch of information. 699 | statement = "Let's start the training!" 700 | super_print(statement, f) 701 | statement = "Network: " + opts.net + ", Dropout: " + str(opts.dropout) + ", Reg: " + str(opts.reg) + ", LR: " + str(opts.lr) + ", Decay: " + str(opts.decay) 702 | super_print(statement, f) 703 | train_net(X_tr, X_te, Y_tr, Y_te, opts, f) 704 | f.close() 705 | return 0 706 | 707 | if __name__ == '__main__': 708 | main(sys.argv) 709 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:0.9.0-gpu 2 | MAINTAINER Darvin Yi 3 | 4 | # Prepare for the Digital Mammography DREAM Challenge 5 | RUN pip install --upgrade pip 6 | RUN pip install pydicom 7 | RUN pip install -U scikit-learn 8 | RUN pip install tflearn #git+https://github.com/tflearn/tflearn.git 9 | 10 | WORKDIR / 11 | COPY DREAM_DM_starter_tf.py . 12 | COPY train.sh . 13 | COPY test.sh . 14 | COPY score_sc1.sh . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # DREAM_DM_starter_code 5 | 6 | This is some starter code for the DREAM Digital Mammography challenge. This directory will give examples of how to Dockerize your code, implement a working data I/O pipeline, and some basic examples of Deep Learning with the challenge data. The python code utilizes the Tensorflow 0.9.0 framework and provides four popular CNN architectures: 7 | 8 | - [Le Net](yann.lecun.com/exdb/lenet) 9 | - [Alex Net](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf) 10 | - [VGG16 Net](https://arxiv.org/pdf/1409.1556.pdf) 11 | - [GoogLe Net](www.cs.unc.edu/~wliu/papers/GoogLeNet.pdf) 12 | 13 | 14 | ## Dependencies 15 | 16 | You must have docker or nvidia-docker running on your system to be able to build the docker image. 17 | 18 | ## Docker 19 | 20 | As shown on the [submission wiki page](https://www.synapse.org/#!Synapse:syn5644795/wiki/392923) for the DREAM challenge, you can build and push the docker image with the following three lines: 21 | 22 | ``` 23 | docker build -t docker.synapse.org/syn123456/dm-trivial-model . 24 | docker login -u -p docker.synapse.org 25 | docker push docker.synapse.org/syn123456/dm-trivial-model 26 | ``` 27 | 28 | where "syn123456" is the ID of the Synapse project where your team collaborates on this challenge. You can find more information on the wiki page. 29 | 30 | ## Usage 31 | 32 | The python program takes care of training and testing a model (saving a model when necessary). We have strived to create an extremely modular program, and thus, just from the flags you should be able to capture most of the hyperparameter variance that a vanilla convolutional network can use. These are the flags: 33 | 34 | - `--pf`: The path to the data that must be read in. The default is set to `"/trainingData"`, but this must be changed to `"/scoringData"` for when you run the test script. 35 | - `--csv1`: The first metadata file for training. The default is correctly set to `"/metadata/images_crosswalk_SubChallenge1.tsv"`. 36 | - `--csv2`: The second metadata file for training. The default is correctly set to `"/metadata/exams_metadata_SubChallenge1.tsv"`. 37 | - `--csv3`: The first and only metadata file for testing. The default is correctly set to `"/scoringData/image_metdata.tsv"`. 38 | - `--net`: The network architecture to use. There are primarily four archtectures as listed above with the names: `Le`, `Alex`, `VGG16`, and `GoogLe`. The default is set to GoogLe Net. 39 | - `--lr`: The learning rate. The default is set to `0.001`. 40 | - `--reg`: The L2 regularization parameter. The default is set to `0.00001`. 41 | - `--out`: An output textfile. This is depreciated and should be taken out. 42 | - `--outtxt`: The path of your output textfile that will be scored by the pipeline. The default is correctly set to `"/output/out.txt"`. 43 | - `--saver`: The path of the model to save. The default is set to a correct path `"/modelState/model.ckpt"`, but you can change the name of the checkpoint file if you please. 44 | - `--decay`: The learning rate decay. Default is set to `1.0`, ergo, no decay. 45 | - `--dropout`: The chance of keeping a neuron during the layers of dropout. Default is set to `0.5`. 46 | - `--bs`: The batchsize. On a single 12GB GPU, Alex Net can handle more than 200, VGG16 can handle about 15, and GoogLe net can handle about 80. Le Net is small, so it can handle more than 1000. Default is set to `10`. 47 | - `--epoch`: This will set the maximum number of epochs (when each image has statistically been seen once. The default is set to `10`. 48 | - `--test`: If set to anything greater than 0, the program will run a test time loop, and do no training while producing the output file. Default is set to `0`. 49 | - `--ms`: The matrix size the input should be resized to. The default is set to `224`, but for Le Net, you should use a size of `32`. 50 | - `--time`: The maximum amount of time (in minutes) that you want the training to run for. The default is set to `1000000`, forcing the ending criteria to be decided by the epoch count. 51 | 52 | **Note:** The program will take into account both the time and epoch parameters and stop the program at the more conservative upper bound. 53 | 54 | To submit to the challenge, you need to submit a training and testing script. This can be done with a single line with our code. For example, the training line 55 | 56 | ``` 57 | python DREAM_DM_pilot_tf.py --net Le -- ms 32 --lr 0.0001 --decay 0.985 --bs 10 --epoch 2 --dropout 0.5 58 | ``` 59 | 60 | and the training line 61 | 62 | ``` 63 | python DREAM_DM_pilot_tf.py --net Le --ms 32 --test 1 64 | ``` 65 | 66 | **Note**: As the test script will depend on the model that is saved by the training script, you **MUST** define the same network and matrix size between both the training and testing scripts for this pipeline to work. 67 | 68 | ## Advanced Usage 69 | 70 | The module `general_conv` lets you define your own convolution layer. Basically, since every convolutional neural network can be defined by an extremely simple set of numbers, this program will take in those numbers and give you the corresponding network. You can feed in your customized CNN architecture to `general_conv` through the list of list variable `architecture_conv`. This will either be a list of 3 numbers (filter size, filter count, and stride) for a convolution layer or a list of two numbers (0 followed by the pooling size) for a max pool layer. For example, below, I can use these modules to quickly define and call Alex Net or VGG16 Net. 71 | 72 | ``` 73 | # AlexNet with an input tensor X and 2 outputs. 74 | architecture_Alex = [[11, 96, 4], [0, 2], 75 | [11, 256, 1], [0, 2], 76 | [3, 384, 1], [3, 384, 1], [3, 256, 1], [0, 2]] 77 | layer = general_conv(X, architecture_Alex) 78 | layer = dense(layer, 4096) 79 | layer = dense(layer, 4096) 80 | pred_Alex = output(layer, 2) 81 | 82 | # VGG16 Net with an input tensor X and 2 outputs. 83 | architecture_VGG16 = [[3, 64, 1], [3, 64, 1], [0, 2], 84 | [3, 128, 1], [3, 128, 1], [0, 2], 85 | [3, 256, 1], [3, 256, 1], [3, 256, 1], [0, 2], 86 | [3, 512, 1], [3, 512, 1], [3, 512, 1], [0, 2], 87 | [3, 512, 1], [3, 512, 1], [3, 512, 1], [0, 2]] 88 | layer = general_conv(X, architecture_VGG16) 89 | layer = dense(layer, 4096) 90 | layer = dense(layer, 4096) 91 | pred_VGG16 = output(layer, 2) 92 | ``` -------------------------------------------------------------------------------- /sc1_infer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Digital Mammography DREAM Challenge 4 | # Testing inference method 5 | 6 | # Run testing 7 | python DREAM_DM_starter_tf.py --net GoogLe --ms 224 --test 1 --out /output/out_1.txt --pf /scoringData &> /output/out_2.txt 8 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Digital Mammography DREAM Challenge 4 | # Testing inference method 5 | 6 | # Run testing 7 | python DREAM_DM_starter_tf.py --net GoogLe --ms 224 --test 1 --out /output/out_1.txt --pf /scoringData &> /output/out_2.txt 8 | -------------------------------------------------------------------------------- /train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Digital Mammography DREAM Challenge 4 | # Training inference method 5 | 6 | 7 | nvidia-smi 8 | python -V 9 | pip show tensorflow 10 | 11 | 12 | # Run training 13 | python DREAM_DM_starter_tf.py --lr 0.0001 --reg 0.0001 --decay 0.985 --bs 50 --time 240 --net GoogLe --ms 224 --dropout 0.6 14 | --------------------------------------------------------------------------------