├── CIFAR ├── Model.py ├── get_data.py ├── mc_drop_cifar.py └── train.py ├── Cifar10 ├── CIFAR10.py ├── Model.py ├── __init__.py ├── dataset.py ├── download.py ├── evaluate.py ├── mc_drop.py ├── poly_inverse_time_decay.py └── train.py ├── README.md ├── lenet-all-standard-dropout ├── MNIST_data │ ├── t10k-images-idx3-ubyte.gz │ ├── t10k-labels-idx1-ubyte.gz │ ├── train-images-idx3-ubyte.gz │ └── train-labels-idx1-ubyte.gz ├── Model.py ├── NOISE │ ├── Model.py │ ├── load_data.py │ ├── mc_drop_softmaxmean.py │ ├── mcdrop_accuracy.py │ ├── noisy.py │ ├── poly_inverse_time_decay.py │ └── stddrop_accuracy.py ├── __init__.py ├── __pycache__ │ ├── Model.cpython-36.pyc │ └── poly_inverse_time_decay.cpython-36.pyc ├── bias_variance │ ├── p-0.50 │ │ ├── Model.py │ │ ├── poly_inverse_time_decay.py │ │ ├── run_uncertainty.sh │ │ └── uncertainty.py │ └── p-0.75 │ │ ├── Model.py │ │ ├── poly_inverse_time_decay.py │ │ ├── run_uncertainty.sh │ │ └── uncertainty.py ├── evaluate.py ├── evaluate_mc_dropout.py ├── evaluate_mc_dropout_py3.py ├── evaluation.sh ├── mc_drop.py ├── poly_inverse_time_decay.py ├── subsets │ ├── 1-32 │ │ ├── Model.py │ │ ├── __init__.py │ │ ├── evaluate.py │ │ ├── mc_drop.py │ │ ├── poly_inverse_time_decay.py │ │ ├── train.py │ │ └── trainingStdDrop.log │ └── 1-4 │ │ ├── Model.py │ │ ├── __init__.py │ │ ├── evaluate.py │ │ ├── evaluation.sh │ │ ├── mc_drop.py │ │ ├── poly_inverse_time_decay.py │ │ └── train.py ├── train.py ├── trainednoise_02 │ ├── Model.py │ ├── __init__.py │ ├── evaluate.py │ ├── mc_drop_eval.py │ ├── mc_eval.sh │ ├── noisy.py │ ├── poly_inverse_time_decay.py │ ├── std_drop_eval.py │ ├── std_eval.sh │ └── train.py └── trainednoise_04 │ ├── Model.py │ ├── __init__.py │ ├── empty │ ├── evaluate.py │ ├── mc_drop.py │ ├── noisy.py │ ├── poly_inverse_time_decay.py │ └── train.py ├── lenet-ip ├── Model.py ├── poly_inverse_time_decay.py ├── subsets │ ├── 1-32 │ │ ├── Model.py │ │ ├── poly_inverse_time_decay.py │ │ └── train.py │ └── 1-4 │ │ ├── Model.py │ │ ├── poly_inverse_time_decay.py │ │ └── train.py ├── train.py └── uncertainty.py └── mnist_LeNet.py /CIFAR/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import tensorflow.contrib.layers as layers 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=128, learning_rate=0.01, num_labels=10, keep_prob=0.5, scope="model"): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | self._scope = scope 11 | self._keep_prob = keep_prob 12 | self._conv_hidden_dims = [192, 192] 13 | with tf.variable_scope(self._scope): 14 | self._build_model() 15 | 16 | def _build_net(self, x, reuse=False, trainable=True, scope="inference_net"): 17 | with tf.variable_scope(scope, reuse=reuse): 18 | out = x 19 | for i in range(len(self._conv_hidden_dims)): 20 | out = layers.conv2d(out, num_outputs=self._conv_hidden_dims[i], kernel_size=(5, 5), 21 | activation_fn=tf.nn.relu, trainable=trainable) 22 | out = layers.dropout(out, keep_prob=self._keep_prob, is_training=trainable) 23 | out = layers.max_pool2d(out, kernel_size=(2, 2)) 24 | 25 | out = layers.flatten(out) 26 | out = layers.fully_connected(out, num_outputs=1000, activation_fn=tf.nn.relu, trainable=trainable) 27 | out = layers.dropout(out, keep_prob=self._keep_prob, is_training=trainable) 28 | logits = layers.fully_connected(out, self._num_labels, trainable=trainable) 29 | 30 | return logits 31 | 32 | def _build_model(self): 33 | self.x_ = tf.placeholder(tf.float32, shape=[None, 3072], name='x_') # data gets loaded as a 32x32 vector 34 | x = tf.reshape(self.x_, [-1, 32, 32, 3], name='x') # CIFAR dataset is shape 32,32,3 35 | self.y = tf.placeholder(tf.float32, shape=[None, self._num_labels], name='y') # 10 labels 36 | # self.keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 37 | self.lr = tf.placeholder(tf.float32, shape=(), name='lr') 38 | 39 | self.logits = self._build_net(x) 40 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=self.y) 41 | self.loss = tf.reduce_mean(cross_entropy) 42 | optimizer = tf.train.MomentumOptimizer(self.lr, momentum=0.9, use_nesterov=True) 43 | self.train_op = optimizer.minimize(loss=self.loss) 44 | self.acc = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.y, 1)), dtype=tf.float32)) 45 | 46 | # for eval steps 47 | self.val_logits = self._build_net(x, reuse=True, trainable=False) 48 | self.val_acc = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(self.val_logits, 1), tf.argmax(self.y, 1)), dtype=tf.float32)) 49 | 50 | tf.summary.scalar('loss', self.loss) 51 | tf.summary.scalar('accuracy', self.acc) 52 | self.merged = tf.summary.merge_all() 53 | -------------------------------------------------------------------------------- /CIFAR/get_data.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | import os 4 | from urllib.request import urlretrieve 5 | import tarfile 6 | import zipfile 7 | import sys 8 | 9 | 10 | def get_data_set(name="train"): 11 | x = None 12 | y = None 13 | l = None 14 | 15 | maybe_download_and_extract() 16 | 17 | folder_name = "cifar_10" 18 | 19 | f = open('./data_set/'+folder_name+'/batches.meta', 'rb') 20 | datadict = pickle.load(f, encoding='latin1') 21 | f.close() 22 | l = datadict['label_names'] 23 | 24 | if name is "train": 25 | for i in range(5): 26 | f = open('./data_set/'+folder_name+'/data_batch_' + str(i + 1), 'rb') 27 | datadict = pickle.load(f, encoding='latin1') 28 | f.close() 29 | 30 | _X = datadict["data"] 31 | _Y = datadict['labels'] 32 | 33 | _X = np.array(_X, dtype=float) / 255.0 34 | _X = _X.reshape([-1, 3, 32, 32]) 35 | _X = _X.transpose([0, 2, 3, 1]) 36 | _X = _X.reshape(-1, 32*32*3) 37 | 38 | if x is None: 39 | x = _X 40 | y = _Y 41 | else: 42 | x = np.concatenate((x, _X), axis=0) 43 | y = np.concatenate((y, _Y), axis=0) 44 | 45 | elif name is "test": 46 | f = open('./data_set/'+folder_name+'/test_batch', 'rb') 47 | datadict = pickle.load(f, encoding='latin1') 48 | f.close() 49 | 50 | x = datadict["data"] 51 | y = np.array(datadict['labels']) 52 | 53 | x = np.array(x, dtype=float) / 255.0 54 | x = x.reshape([-1, 3, 32, 32]) 55 | x = x.transpose([0, 2, 3, 1]) 56 | x = x.reshape(-1, 32*32*3) 57 | 58 | def dense_to_one_hot(labels_dense, num_classes=10): 59 | num_labels = labels_dense.shape[0] 60 | index_offset = np.arange(num_labels) * num_classes 61 | labels_one_hot = np.zeros((num_labels, num_classes)) 62 | labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 63 | 64 | return labels_one_hot 65 | 66 | return x, dense_to_one_hot(y), l 67 | 68 | 69 | def _print_download_progress(count, block_size, total_size): 70 | pct_complete = float(count * block_size) / total_size 71 | msg = "\r- Download progress: {0:.1%}".format(pct_complete) 72 | sys.stdout.write(msg) 73 | sys.stdout.flush() 74 | 75 | 76 | def maybe_download_and_extract(): 77 | main_directory = "./data_set/" 78 | cifar_10_directory = main_directory+"cifar_10/" 79 | if not os.path.exists(main_directory): 80 | os.makedirs(main_directory) 81 | 82 | url = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 83 | filename = url.split('/')[-1] 84 | file_path = os.path.join(main_directory, filename) 85 | zip_cifar_10 = file_path 86 | file_path, _ = urlretrieve(url=url, filename=file_path, reporthook=_print_download_progress) 87 | 88 | print() 89 | print("Download finished. Extracting files.") 90 | if file_path.endswith(".zip"): 91 | zipfile.ZipFile(file=file_path, mode="r").extractall(main_directory) 92 | elif file_path.endswith((".tar.gz", ".tgz")): 93 | tarfile.open(name=file_path, mode="r:gz").extractall(main_directory) 94 | print("Done.") 95 | 96 | os.rename(main_directory+"./cifar-10-batches-py", cifar_10_directory) 97 | os.remove(zip_cifar_10) 98 | -------------------------------------------------------------------------------- /CIFAR/mc_drop_cifar.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | from get_data import get_data_set 6 | 7 | FLAGS = tf.app.flags.FLAGS 8 | 9 | 10 | def accuracy(logits, y): 11 | with tf.variable_scope('accuracy') as scope: 12 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), name=scope.name) #calculates how much % of our predictions is correct 13 | tf.summary.scalar('accuracy', accuracy) 14 | return accuracy 15 | 16 | def evaluate(): 17 | with tf.Graph().as_default(): 18 | model = Model()#keep_prob=FLAGS.prob) 19 | 20 | test_x, test_y, test_l = get_data_set("test") 21 | x_ = tf.placeholder(tf.float32, shape=[None, 3072], name='x_') 22 | x = tf.reshape(x_, [-1, 32, 32, 3], name='x') 23 | y = tf.placeholder(tf.float32, shape=[None, None], name='y') 24 | 25 | 26 | softmax_tensors = tf.placeholder(tf.float32, shape=[None, 100, 10]) #SECOND ARGUMENT IS BATCH SIZE, IDEALLY THIS IS A FLAG 27 | logits = model._build_net(x) 28 | softmax = tf.nn.softmax(logits) 29 | shape = tf.shape(softmax) 30 | saver = tf.train.Saver(max_to_keep = None) 31 | mean = tf.reduce_mean(softmax_tensors,0) 32 | 33 | acc = accuracy(mean,y) 34 | 35 | with tf.Session() as sess: 36 | tf.global_variables_initializer().run() 37 | saver.restore(sess, FLAGS.checkpoint_file_path) 38 | accuracy_list = np.zeros(FLAGS.T) 39 | num_batches=0 40 | for test_feature_batch, test_label_batch in batch_features_labels(test_x, test_y, batch_size=100): 41 | num_batches+=1 42 | softmax_list = [] 43 | batch_accuracy_list=[] 44 | for i in range(FLAGS.T): 45 | batch_softmax_i = sess.run([softmax], feed_dict={model.x_: test_feature_batch, model.y: test_label_batch}) 46 | softmax_list.append(batch_softmax_i) 47 | if i>1: 48 | arr=np.squeeze(np.array(softmax_list)[:i,:,:]) 49 | else : 50 | arr=np.array(softmax_list)[0,:,:] 51 | #accuracy for this batch for first i trials 52 | accuracy_i = sess.run([acc], feed_dict={softmax_tensors: arr, y: test_label_batch}) 53 | #list from 1-T of accuracy after 1<=i<=T trials 54 | batch_accuracy_list.append(accuracy_i) 55 | accuracy_list += batch_accuracy_list 56 | accuracy_list /= num_batches 57 | print (accuracy_list) 58 | #f=open('evals2.out','a') 59 | #f.write(str(FLAGS.prob)+ ","+str(i+1)+","+str(total_accuracy)+'\n') 60 | #f.close() 61 | 62 | 63 | def main(argv=None): 64 | evaluate() 65 | 66 | if __name__ == '__main__': 67 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checks/model-99000', 'path to checkpoint file') 68 | tf.app.flags.DEFINE_integer('T', 10, 'Number of Forward Passes') 69 | tf.app.flags.DEFINE_float('prob', 0.5, 'probability of dropout') 70 | tf.app.run() 71 | -------------------------------------------------------------------------------- /CIFAR/train.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import tensorflow as tf 3 | import numpy as np 4 | import tensorflow.contrib.layers as layers 5 | from get_data import get_data_set 6 | from Model import Model 7 | 8 | logging.basicConfig(filename='log.out', level=logging.INFO) 9 | logger = logging.getLogger('train') 10 | 11 | 12 | def batch_features_labels(features, labels, batch_size): 13 | """ 14 | Split features and labels into batches 15 | """ 16 | for start in range(0, len(features), batch_size): 17 | end = min(start + batch_size, len(features)) 18 | yield features[start:end], labels[start:end] 19 | 20 | 21 | if __name__ == '__main__': 22 | 23 | batch_size = 128 24 | num_iter = 10 ** 5 25 | lr = 0.01 26 | decay_steps = 1 27 | decay_rate = 0.0001 28 | power = 0.75 29 | checkpoint_freq = 1000 30 | 31 | train_x, train_y, train_l = get_data_set("train") 32 | test_x, test_y, test_l = get_data_set("test") 33 | 34 | # hard coding for now, replace these 35 | checkpoint_path = "./checkpoints/model" 36 | summary_dir = "./summaries/" 37 | 38 | with tf.Session() as sess: 39 | model = Model() 40 | 41 | global_step = 0 42 | saver = tf.train.Saver(max_to_keep = None) 43 | summary_writer = tf.summary.FileWriter(summary_dir, sess.graph) 44 | 45 | sess.run(tf.global_variables_initializer()) 46 | saver.save(sess, checkpoint_path, global_step=global_step) 47 | 48 | while global_step < num_iter: 49 | randidx = np.random.randint(len(train_x), size=batch_size) 50 | batch_xs = train_x[randidx] 51 | batch_ys = train_y[randidx] 52 | 53 | summary, loss, acc, _ = sess.run([model.merged, model.loss, model.acc, model.train_op], feed_dict={ 54 | model.x_: batch_xs, 55 | model.y: batch_ys, 56 | model.lr: lr 57 | }) 58 | 59 | # decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / decay_step) 60 | 61 | global_step += 1 62 | lr = lr / ((1 + decay_rate * (global_step / decay_steps)) ** power) 63 | summary_writer.add_summary(summary, global_step) 64 | summary_writer.flush() 65 | 66 | logger.info("Global step: {} Loss: {} Accuracy: {}".format(global_step, loss, acc)) 67 | 68 | if global_step % checkpoint_freq == 0: 69 | 70 | test_batch_acc_total = 0 71 | test_batch_count = 0 72 | 73 | for test_feature_batch, test_label_batch in batch_features_labels(test_x, test_y, batch_size=100): 74 | test_batch_acc_total += sess.run([model.val_acc], feed_dict={ 75 | model.x_: test_feature_batch, 76 | model.y: test_label_batch 77 | keep_prob: 1.0 78 | }) 79 | test_batch_count += 1 80 | 81 | val_acc = test_batch_acc_total/test_batch_count 82 | 83 | logger.info("EVALUATION STEP:\tTest Accuracy: {}".format(val_acc)) 84 | saver.save(sess, checkpoint_path, global_step=global_step) 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /Cifar10/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=128, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | #each layer has 192 output in CIFAR10 14 | kernel = self._create_weights([5, 5, 3, 192]) #[filter_height, filter_width, in_channels, out_channels] 15 | conv = self._create_conv2d(images, kernel) #create the conv layer 16 | bias = self._create_bias([192]) #create bias variable 17 | preactivation = tf.nn.bias_add(conv, bias) #add bias 18 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 32,32,192] 19 | self._activation_summary(conv1) #this is for TensorBoard visualization 20 | 21 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 22 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 16,16,192] 23 | 24 | with tf.variable_scope('conv2') as scope: 25 | #each layer has 192 units in CIFAR10 26 | kernel = self._create_weights([5, 5, 192, 192]) 27 | conv = self._create_conv2d(h_pool1, kernel) 28 | bias = self._create_bias([192]) 29 | preactivation = tf.nn.bias_add(conv, bias) 30 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 16,16,192] 31 | self._activation_summary(conv2) 32 | 33 | 34 | dropout2 = tf.nn.dropout(conv2, keep_prob) 35 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 8, 8, 192] 36 | 37 | with tf.variable_scope('dense') as scope: 38 | reshape = tf.reshape(h_pool2, [-1, 8 * 8 * 192]) 39 | #last inner-product layer has 1000 units instead of 500 in MNIST 40 | W_dense = self._create_weights([8 * 8 * 192, 1000]) 41 | b_dense = self._create_bias([1000]) 42 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 43 | self._activation_summary(dense) 44 | 45 | with tf.variable_scope('logit') as scope: 46 | #last inner-product layer has 1000 units instead of 500 in MNIST 47 | W_logit = self._create_weights([1000, self._num_labels]) 48 | b_logit = self._create_bias([self._num_labels]) 49 | dense_drop = tf.nn.dropout(dense, keep_prob) 50 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 51 | self._activation_summary(logit) 52 | return logit 53 | 54 | def train(self, loss, global_step): 55 | #should probably make these variables arguements but cba 56 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 57 | learning_rate = td.poly_inverse_time_decay(learning_rate = self._learning_rate, global_step = global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 58 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 59 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 60 | train_op = optimizer.minimize( 61 | loss=loss, 62 | global_step=tf.train.get_global_step()) 63 | return train_op 64 | 65 | def loss(self, logits, labels): 66 | with tf.variable_scope('loss') as scope: 67 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 68 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 69 | tf.add_to_collection('losses', cross_entropy_mean) 70 | 71 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 72 | 73 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 74 | 75 | def accuracy(self, logits, y): 76 | with tf.variable_scope('accuracy') as scope: 77 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 78 | name=scope.name) #calculates how much % of our predictions is correct 79 | tf.summary.scalar('accuracy', accuracy) 80 | return accuracy 81 | 82 | def _create_conv2d(self, x, W): 83 | return tf.nn.conv2d(input=x, 84 | filter=W, #4d tensor that includes the size of the filter and number of filters 85 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 86 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 87 | 88 | def _create_max_pool_2x2(self, input): 89 | return tf.nn.max_pool(value=input, 90 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 91 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 92 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 93 | 94 | def _create_weights(self, shape): 95 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 96 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 97 | tf.add_to_collection('losses', weight_decay) 98 | return var 99 | 100 | 101 | def _create_bias(self, shape): 102 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 103 | 104 | def _activation_summary(self, x): 105 | #This is simply to catch information for visualization using tensorboard 106 | tensor_name = x.op.name 107 | tf.summary.histogram(tensor_name + '/activations', x) 108 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 109 | -------------------------------------------------------------------------------- /Cifar10/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Cifar10/download.py: -------------------------------------------------------------------------------- 1 | 2 | ######################################################################## 3 | # 4 | # Functions for downloading and extracting data-files from the internet. 5 | # 6 | # Implemented in Python 3.5 7 | # 8 | ######################################################################## 9 | # 10 | # This file is part of the TensorFlow Tutorials available at: 11 | # 12 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 13 | # 14 | # Published under the MIT License. See the file LICENSE for details. 15 | # 16 | # Copyright 2016 by Magnus Erik Hvass Pedersen 17 | # 18 | ######################################################################## 19 | 20 | import sys 21 | import os 22 | import urllib.request 23 | import tarfile 24 | import zipfile 25 | 26 | ######################################################################## 27 | 28 | 29 | def _print_download_progress(count, block_size, total_size): 30 | """ 31 | Function used for printing the download progress. 32 | Used as a call-back function in maybe_download_and_extract(). 33 | """ 34 | 35 | # Percentage completion. 36 | pct_complete = float(count * block_size) / total_size 37 | 38 | # Status-message. Note the \r which means the line should overwrite itself. 39 | msg = "\r- Download progress: {0:.1%}".format(pct_complete) 40 | 41 | # Print it. 42 | sys.stdout.write(msg) 43 | sys.stdout.flush() 44 | 45 | 46 | ######################################################################## 47 | 48 | 49 | def maybe_download_and_extract(url, download_dir): 50 | """ 51 | Download and extract the data if it doesn't already exist. 52 | Assumes the url is a tar-ball file. 53 | :param url: 54 | Internet URL for the tar-file to download. 55 | Example: "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 56 | :param download_dir: 57 | Directory where the downloaded file is saved. 58 | Example: "data/CIFAR-10/" 59 | :return: 60 | Nothing. 61 | """ 62 | 63 | # Filename for saving the file downloaded from the internet. 64 | # Use the filename from the URL and add it to the download_dir. 65 | filename = url.split('/')[-1] 66 | file_path = os.path.join(download_dir, filename) 67 | 68 | # Check if the file already exists. 69 | # If it exists then we assume it has also been extracted, 70 | # otherwise we need to download and extract it now. 71 | if not os.path.exists(file_path): 72 | # Check if the download directory exists, otherwise create it. 73 | if not os.path.exists(download_dir): 74 | os.makedirs(download_dir) 75 | 76 | # Download the file from the internet. 77 | file_path, _ = urllib.request.urlretrieve(url=url, 78 | filename=file_path, 79 | reporthook=_print_download_progress) 80 | 81 | print() 82 | print("Download finished. Extracting files.") 83 | 84 | if file_path.endswith(".zip"): 85 | # Unpack the zip-file. 86 | zipfile.ZipFile(file=file_path, mode="r").extractall(download_dir) 87 | elif file_path.endswith((".tar.gz", ".tgz")): 88 | # Unpack the tar-ball. 89 | tarfile.open(name=file_path, mode="r:gz").extractall(download_dir) 90 | 91 | print("Done.") 92 | else: 93 | print("Data has apparently already been downloaded and unpacked.") 94 | 95 | -------------------------------------------------------------------------------- /Cifar10/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | #images, labels = mnist.load_test_data(FLAGS.test_data) 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 13 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 14 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=1.0) 18 | accuracy = model.accuracy(logits, y) 19 | 20 | saver = tf.train.Saver() 21 | 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | 26 | total_accuracy = sess.run([accuracy], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | print('Test accuracy: {}'.format(total_accuracy)) 29 | 30 | 31 | def main(argv=None): 32 | evaluate() 33 | 34 | if __name__ == '__main__': 35 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 36 | tf.app.run() -------------------------------------------------------------------------------- /Cifar10/mc_drop.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | 9 | def evaluate(): 10 | with tf.Graph().as_default(): 11 | 12 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 13 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 14 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 15 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 16 | softmax_tensors = tf.placeholder(tf.float32, shape=[50, 10000, 10]) 17 | 18 | model = Model() 19 | logits = model.inference(x, keep_prob=0.5) 20 | softmax = tf.nn.softmax(logits) 21 | shape = tf.shape(softmax) 22 | saver = tf.train.Saver(max_to_keep = None) 23 | 24 | mean = tf.reduce_mean(softmax_tensors,0) 25 | accuracy = model.accuracy(mean,y) 26 | 27 | softmax_list = [] 28 | with tf.Session() as sess: 29 | tf.global_variables_initializer().run() 30 | saver.restore(sess, FLAGS.checkpoint_file_path) 31 | for i in range(50): 32 | softmaxi = sess.run([softmax], 33 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 34 | softmax_list.append(softmaxi) 35 | 36 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 37 | 38 | total_accuracy = sess.run([accuracy], 39 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: mnist.test.labels}) 40 | print('Test accuracy: {}'.format(total_accuracy)) 41 | 42 | 43 | 44 | 45 | 46 | def main(argv=None): 47 | evaluate() 48 | 49 | if __name__ == '__main__': 50 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000000', 'path to checkpoint file') 51 | tf.app.run() 52 | -------------------------------------------------------------------------------- /Cifar10/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | #This is a minor modification TensorFlow Code 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | 6 | import math 7 | 8 | from tensorflow.python.framework import constant_op 9 | from tensorflow.python.framework import dtypes 10 | from tensorflow.python.framework import ops 11 | from tensorflow.python.ops import control_flow_ops 12 | from tensorflow.python.ops import math_ops 13 | from tensorflow.python.ops import random_ops 14 | 15 | def poly_inverse_time_decay(learning_rate, 16 | global_step, 17 | decay_steps, 18 | decay_rate, 19 | power, 20 | staircase=False, 21 | name=None): 22 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 23 | When training a model, it is often recommended to lower the learning rate as 24 | the training progresses. This function applies an inverse decay function 25 | to a provided initial learning rate. It requires an `global_step` value to 26 | compute the decayed learning rate. You can just pass a TensorFlow variable 27 | that you increment at each training step. 28 | The function returns the decayed learning rate. It is computed as: 29 | ```python 30 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 31 | decay_step) 32 | ``` 33 | or, if `staircase` is `True`, as: 34 | ```python 35 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 36 | decay_step)) 37 | ``` 38 | Example: decay 1/t with a rate of 0.5: 39 | ```python 40 | ... 41 | global_step = tf.Variable(0, trainable=False) 42 | learning_rate = 0.1 43 | decay_steps = 1.0 44 | decay_rate = 0.5 45 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 46 | decay_steps, decay_rate) 47 | # Passing global_step to minimize() will increment it at each step. 48 | learning_step = ( 49 | tf.train.GradientDescentOptimizer(learning_rate) 50 | .minimize(...my loss..., global_step=global_step) 51 | ) 52 | ``` 53 | Args: 54 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 55 | Python number. The initial learning rate. 56 | global_step: A Python number. 57 | Global step to use for the decay computation. Must not be negative. 58 | decay_steps: How often to apply decay. 59 | decay_rate: A Python number. The decay rate. 60 | staircase: Whether to apply decay in a discrete staircase, as opposed to 61 | continuous, fashion. 62 | name: String. Optional name of the operation. Defaults to 63 | 'InverseTimeDecay'. 64 | Returns: 65 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 66 | learning rate. 67 | Raises: 68 | ValueError: if `global_step` is not supplied. 69 | """ 70 | if global_step is None: 71 | raise ValueError("global_step is required for inverse_time_decay.") 72 | with ops.name_scope(name, "InverseTimeDecay", 73 | [learning_rate, global_step, decay_rate]) as name: 74 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 75 | dtype = learning_rate.dtype 76 | global_step = math_ops.cast(global_step, dtype) 77 | decay_steps = math_ops.cast(decay_steps, dtype) 78 | decay_rate = math_ops.cast(decay_rate, dtype) 79 | power = math_ops.cast(power, dtype) 80 | p = global_step / decay_steps 81 | if staircase: 82 | p = math_ops.floor(p) 83 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 84 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 85 | 86 | return math_ops.div(learning_rate, denom, name=name) 87 | -------------------------------------------------------------------------------- /Cifar10/train.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from Model import Model 4 | import CIFAR10 5 | import numpy as np 6 | import math 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | def train(): 11 | model = Model() 12 | 13 | with tf.Graph().as_default(): 14 | 15 | CIFAR10.data_path = 'data/CIFAR-10/' 16 | CIFAR10.maybe_download_and_extract() 17 | 18 | class_names = CIFAR10.load_class_names() 19 | print (class_names) 20 | 21 | #load 50,000 train images (np array) 22 | images_train, cls_train, labels_train = CIFAR10.load_training_data() #cls is the label, labels_train is one hot encoded 23 | #load 10,000 test images (np array) 24 | images_test, cls_test, labels_test = CIFAR10.load_test_data() 25 | # print (images_test.shape) 26 | # print (images_test.shape) 27 | 28 | x = tf.placeholder(shape = [None, 32,32,3], dtype = tf.float32, name = 'x') #mnist dataset is shape 28,28,1 29 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 30 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 31 | global_step = tf.contrib.framework.get_or_create_global_step() 32 | 33 | logits = model.inference(x, keep_prob=keep_prob) 34 | loss = model.loss(logits=logits, labels=y) 35 | 36 | accuracy = model.accuracy(logits, y) 37 | summary_op = tf.summary.merge_all() 38 | train_op = model.train(loss, global_step=global_step) 39 | 40 | init = tf.global_variables_initializer() 41 | saver = tf.train.Saver(max_to_keep = None) 42 | 43 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 44 | input_size = 50000 #50,000 training images 45 | porp = int(math.ceil(input_size/batch_size)) 46 | 47 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 48 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 49 | sess.run(init) 50 | 51 | for i in range(FLAGS.num_iter): 52 | if i%(porp) == 0: 53 | permutation=np.random.permutation(input_size) #create a list with random indexes 54 | increment = 0 #restart the increment variable 55 | 56 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 57 | increment += 1 58 | image_batch=images_train[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 59 | label_batch=labels_train[batch_idx] #this is a list with batch size number of elements. Each element is a 10 dimensional vector (1 hot encode) 60 | 61 | 62 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 63 | feed_dict={x: image_batch, y: label_batch, keep_prob: 0.5}) 64 | writer.add_summary(summary, i) 65 | 66 | f = open('trainingStdDrop.log', 'a+') 67 | 68 | if i % 10 == 0: 69 | # Get accuracy in batches for memory limitations 70 | test_batch_acc_total = 0 71 | test_batch_count = 0 72 | 73 | for test_feature_batch, test_label_batch in batch_features_labels(images_test, labels_test, batch_size=100): 74 | test_batch_acc_total += accuracy.eval(feed_dict={x: test_feature_batch, y: test_label_batch, keep_prob: 1.0}) 75 | test_batch_count += 1 76 | 77 | validation_accuracy = test_batch_acc_total/test_batch_count 78 | #validation_accuracy = accuracy.eval(feed_dict={x: images_test, y: labels_test, keep_prob: 1.0}) 79 | print("Iteration: {}\tLoss: {}\tValidation Accuracy: {}".format(i, cur_loss, validation_accuracy)) 80 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 81 | 82 | if i % 1000 == 0: 83 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 84 | 85 | f.close() 86 | 87 | def batch_features_labels(features, labels, batch_size): 88 | """ 89 | Split features and labels into batches 90 | """ 91 | for start in range(0, len(features), batch_size): 92 | end = min(start + batch_size, len(features)) 93 | yield features[start:end], labels[start:end] 94 | 95 | 96 | def main(argv=None): 97 | train() 98 | 99 | 100 | if __name__ == '__main__': 101 | tf.app.flags.DEFINE_integer('batch_size', 128, 'size of training batches') 102 | tf.app.flags.DEFINE_integer('num_iter', 100000, 'number of training iterations') 103 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 104 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 105 | 106 | tf.app.run() 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bayesian CNNs 2 | This repository contains a TensorFlow implementation of "[Bayesian Convolutional Neural Networks with Bernoulli Approximate Variational Inference](https://arxiv.org/abs/1506.02158)". 3 | 4 | (Part of Cambridge ML Coursework) 5 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/MNIST_data/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raj-shah/BayesianCNN/40ead64274d0456b0bed5649e28e31d2e62a3f92/lenet-all-standard-dropout/MNIST_data/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /lenet-all-standard-dropout/MNIST_data/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raj-shah/BayesianCNN/40ead64274d0456b0bed5649e28e31d2e62a3f92/lenet-all-standard-dropout/MNIST_data/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /lenet-all-standard-dropout/MNIST_data/train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raj-shah/BayesianCNN/40ead64274d0456b0bed5649e28e31d2e62a3f92/lenet-all-standard-dropout/MNIST_data/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /lenet-all-standard-dropout/MNIST_data/train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raj-shah/BayesianCNN/40ead64274d0456b0bed5649e28e31d2e62a3f92/lenet-all-standard-dropout/MNIST_data/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /lenet-all-standard-dropout/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=global_step) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=global_step) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/load_data.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | noiseLevel= [0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5] 9 | 10 | dataset_list = [] 11 | for i in range(10): 12 | dataset_list.append(np.load(str(noiseLevel[i])+'.npy')) 13 | 14 | dataset5 = dataset_list[4] 15 | 16 | 17 | plt.gray() # use this line if you don't want to see it in color 18 | plt.imshow(dataset5[53]) 19 | plt.show() 20 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/mc_drop_softmaxmean.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | 9 | 10 | def evaluate(): 11 | with tf.Graph().as_default(): 12 | 13 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 14 | labels = mnist.test.labels 15 | 16 | noiseLevel= [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1] 17 | 18 | # plt.gray() 19 | # plt.imshow(dataset5[53]) 20 | # plt.show() 21 | 22 | x = tf.placeholder(shape=[10000, 28,28,1], dtype=tf.float32, name='x') 23 | y = tf.placeholder(shape=[10000, 10], dtype=tf.float32, name='y') #10 labels 24 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 25 | 26 | model = Model() 27 | logits = model.inference(x, keep_prob=0.5) 28 | softmax = tf.nn.softmax(logits) 29 | shape = tf.shape(softmax) 30 | saver = tf.train.Saver(max_to_keep = None) 31 | 32 | mean = tf.reduce_mean(softmax_tensors,0) 33 | accuracy = model.accuracy(mean,y) 34 | 35 | for l in range(len(noiseLevel)): 36 | data = np.load(str(noiseLevel[l])+'.npy') 37 | data = data.reshape([10000,28,28,1]) 38 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 39 | tf.global_variables_initializer().run() 40 | saver.restore(sess, FLAGS.checkpoint_file_path) 41 | softmax_list = [] 42 | for i in range(FLAGS.T): 43 | softmaxi = sess.run([softmax], 44 | feed_dict={x: data, y: labels}) 45 | softmax_list.append(softmaxi) 46 | 47 | mean_softmax = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 48 | 49 | print (np.squeeze(mean_softmax).shape) 50 | softmax_max = np.max(np.squeeze(mean_softmax), axis = 1) 51 | print (softmax_max.shape) 52 | #softmax_correct = np.multiply(np.squeeze(mean_softmax), labels) 53 | summ = (np.sum(softmax_max)) 54 | print (np.divide(summ, 10000.0)) 55 | 56 | 57 | 58 | 59 | def main(argv=None): 60 | evaluate() 61 | 62 | if __name__ == '__main__': 63 | tf.app.flags.DEFINE_string('checkpoint_file_path', '/home/si318/Desktop/MLSALT4/lenet-all-standard-dropout/checkpoints/model.ckpt-5000000', 'path to checkpoint file') 64 | tf.app.flags.DEFINE_integer('T', 50, 'number of forward passes') 65 | tf.app.run() 66 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/mcdrop_accuracy.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | labels = mnist.test.labels 13 | 14 | noiselevel = [0, 0.2, 0.4, 0.6, 0.8, 1] 15 | 16 | x = tf.placeholder(shape=[10000, 28,28,1], dtype=tf.float32, name='x') 17 | y = tf.placeholder(shape=[10000, 10], dtype=tf.float32, name='y') #10 labels 18 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 19 | 20 | model = Model() 21 | logits = model.inference(x, keep_prob=0.5) 22 | softmax = tf.nn.softmax(logits) 23 | shape = tf.shape(softmax) 24 | saver = tf.train.Saver(max_to_keep = None) 25 | 26 | mean = tf.reduce_mean(softmax_tensors,0) 27 | accuracy = model.accuracy(mean,y) 28 | 29 | #this is for each forward pass accuracy 30 | softmax_each=tf.placeholder(tf.float32, shape=[10000, 10]) 31 | accuracyi=model.accuracy(softmax_each,y) 32 | 33 | 34 | for j in range(len(noiselevel)): 35 | data = np.load(str(noiselevel[j])+'.npy') 36 | data = data.reshape([10000,28,28,1]) 37 | 38 | with tf.Session() as sess: 39 | tf.global_variables_initializer().run() 40 | saver.restore(sess, FLAGS.checkpoint_file_path) 41 | softmax_list = [] 42 | accuracy_list = [] 43 | for i in range(FLAGS.T): 44 | softmaxi = sess.run([softmax], 45 | feed_dict={x: data, y: labels}) 46 | softmax_list.append(softmaxi) 47 | 48 | #added for accuracy of each forward pass: 49 | #print(softmaxi[0]) 50 | accuracyi1=sess.run([accuracyi], 51 | feed_dict={softmax_each:np.squeeze(np.array(softmaxi[0])), y: labels}) 52 | accuracy_list.append(accuracyi1) 53 | 54 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 55 | 56 | total_accuracy = sess.run([accuracy], 57 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: labels}) 58 | #print (softmax_list[0].shape) 59 | standard_deviation=np.std(np.array(accuracy_list)) 60 | f = open('out_mcdrop.log', 'a+') 61 | f.write('noise intensity: {}\n'.format(noiselevel[j])) 62 | f.write('test accuracy: {}\n'.format(total_accuracy)) 63 | f.write('standard deviation of 50 forward passes: {}\n'.format(standard_deviation)) 64 | f.write('accuracy list: {}\n'.format(accuracy_list)) 65 | f.close() 66 | 67 | 68 | 69 | def main(argv=None): 70 | evaluate() 71 | 72 | if __name__ == '__main__': 73 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-5000000', 'path to checkpoint file') 74 | tf.app.flags.DEFINE_integer('T', 50, 'number of forward passes') 75 | tf.app.run() 76 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/noisy.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | noiseLevel= [0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5] 9 | 10 | tf.logging.set_verbosity(tf.logging.INFO) 11 | 12 | mnist = tf.contrib.learn.datasets.load_dataset("mnist") 13 | # train_data = mnist.train.images # np.array 14 | # train_labels = np.asarray(mnist.train.labels, dtype=np.int32) 15 | eval_data = mnist.test.images # np.array 16 | eval_labels = np.asarray(mnist.test.labels, dtype=np.int32) 17 | eval_data = eval_data.reshape(10000,28,28) 18 | 19 | 20 | 21 | 22 | for i in range(len(noiseLevel)): 23 | dataset = np.zeros([10000,28,28]) 24 | count2 = 0 25 | for image in eval_data: 26 | image = image.reshape(28,28) 27 | noisePlaces=np.random.choice([0, 1], size=(28,28), p=[1-noiseLevel[i], noiseLevel[i]]) 28 | noiseScale=np.random.rand(28,28) 29 | noise=noiseScale*noisePlaces 30 | noisy_image= np.mod(image + noise, 1) 31 | #print (noisy_image.shape) 32 | dataset[count2,:,:] = noisy_image 33 | count2 += 1 34 | np.save(str(noiseLevel[i]), dataset) 35 | 36 | 37 | 38 | 39 | 40 | # plt.gray() # use this line if you don't want to see it in color 41 | # plt.imshow(eval_data[9999]) 42 | # plt.show() 43 | # plt.gray() # use this line if you don't want to see it in color 44 | # plt.imshow(dataset[9999]) 45 | # plt.show() 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) 86 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/NOISE/stddrop_accuracy.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | labels = mnist.test.labels 13 | 14 | noiselevel = [0, 0.2, 0.4, 0.6, 0.8, 1] 15 | 16 | x = tf.placeholder(shape=[10000, 28,28,1], dtype=tf.float32, name='x') 17 | y = tf.placeholder(shape=[10000, 10], dtype=tf.float32, name='y') #10 labels 18 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 19 | 20 | model = Model() 21 | logits = model.inference(x, keep_prob=1.0) 22 | softmax = tf.nn.softmax(logits) 23 | shape = tf.shape(softmax) 24 | saver = tf.train.Saver(max_to_keep = None) 25 | 26 | mean = tf.reduce_mean(softmax_tensors,0) 27 | accuracy = model.accuracy(mean,y) 28 | 29 | #this is for each forward pass accuracy 30 | softmax_each=tf.placeholder(tf.float32, shape=[10000, 10]) 31 | accuracyi=model.accuracy(softmax_each,y) 32 | 33 | 34 | for j in range(len(noiselevel)): 35 | data = np.load(str(noiselevel[j])+'.npy') 36 | data = data.reshape([10000,28,28,1]) 37 | 38 | with tf.Session() as sess: 39 | tf.global_variables_initializer().run() 40 | saver.restore(sess, FLAGS.checkpoint_file_path) 41 | softmax_list = [] 42 | accuracy_list = [] 43 | for i in range(FLAGS.T): 44 | softmaxi = sess.run([softmax], 45 | feed_dict={x: data, y: labels}) 46 | softmax_list.append(softmaxi) 47 | 48 | #added for accuracy of each forward pass: 49 | #print(softmaxi[0]) 50 | accuracyi1=sess.run([accuracyi], 51 | feed_dict={softmax_each:np.squeeze(np.array(softmaxi[0])), y: labels}) 52 | accuracy_list.append(accuracyi1) 53 | 54 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 55 | 56 | total_accuracy = sess.run([accuracy], 57 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: labels}) 58 | #print (softmax_list[0].shape) 59 | standard_deviation=np.std(np.array(accuracy_list)) 60 | f = open('out_stddrop.log', 'a+') 61 | f.write('noise intensity: {}\n'.format(noiselevel[j])) 62 | f.write('test accuracy: {}\n'.format(total_accuracy)) 63 | f.close() 64 | 65 | 66 | 67 | 68 | def main(argv=None): 69 | evaluate() 70 | 71 | if __name__ == '__main__': 72 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-5000000', 'path to checkpoint file') 73 | tf.app.flags.DEFINE_integer('T', 2, 'number of forward passes') 74 | tf.app.run() 75 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/__pycache__/Model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raj-shah/BayesianCNN/40ead64274d0456b0bed5649e28e31d2e62a3f92/lenet-all-standard-dropout/__pycache__/Model.cpython-36.pyc -------------------------------------------------------------------------------- /lenet-all-standard-dropout/__pycache__/poly_inverse_time_decay.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raj-shah/BayesianCNN/40ead64274d0456b0bed5649e28e31d2e62a3f92/lenet-all-standard-dropout/__pycache__/poly_inverse_time_decay.cpython-36.pyc -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.50/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=tf.train.get_global_step()) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.50/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) 86 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.50/run_uncertainty.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for f in 1 2 3 4 5 3 | do 4 | for t in 50 5 | do 6 | for p in 0.25 .5 .6 .75 .9 7 | do 8 | 9 | python uncertainty.py --checkpoint_file_path checkpoints/model.ckpt-5000000 --T $t --prob $p 10 | done 11 | done 12 | done 13 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.50/uncertainty.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | FLAGS = tf.app.flags.FLAGS 6 | def evaluate(): 7 | with tf.Graph().as_default(): 8 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 9 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 10 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 11 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 12 | softmax_tensors = tf.placeholder(tf.float32, shape=[None, 10000, 10]) 13 | 14 | model = Model() 15 | logits = model.inference(x, keep_prob=FLAGS.prob) 16 | softmax = tf.nn.softmax(logits) 17 | shape = tf.shape(softmax) 18 | saver = tf.train.Saver(max_to_keep = None) 19 | mean = tf.reduce_mean(softmax_tensors,0) 20 | accuracy = model.accuracy(mean,y) 21 | softmax_list = [] 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | for i in range(FLAGS.T): 26 | softmaxi = sess.run([softmax], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | softmax_list.append(softmaxi) 29 | for i in range(FLAGS.T): 30 | if i>1: 31 | arr=np.squeeze(np.array(softmax_list)[:i,:,:]) 32 | else : 33 | arr=np.array(softmax_list)[0,:,:] 34 | total_accuracy, soft = sess.run([accuracy,mean], 35 | feed_dict={softmax_tensors: arr, y: mnist.test.labels}) 36 | f=open('bias_var.log','a') 37 | f.write(str(FLAGS.prob)+ ","+str(i+1)+","+str(total_accuracy)+'\n') 38 | f.close() 39 | #L=np.array([np.squeeze(np.array(soft)).flatten(), mnist.test.labels.flatten()]) 40 | #L=np.transpose(L) 41 | #L=L[L[:,0].argsort()] 42 | #np.savetxt(str(i)+'uncertainty2.out', L, delimiter=',') 43 | 44 | 45 | 46 | def main(argv=None): 47 | evaluate() 48 | if __name__ == '__main__': 49 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-5000000', 'path to checkpoint file') 50 | tf.app.flags.DEFINE_integer('T', 2, 'Number of Forward Passes') 51 | tf.app.flags.DEFINE_float('prob', 1.0, 'probability of dropout') 52 | tf.app.run() 53 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.75/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=tf.train.get_global_step()) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.75/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) 86 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.75/run_uncertainty.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for f in 1 2 3 4 5 3 | do 4 | for t in 50 5 | do 6 | for p in 0.25 .5 .6 .75 .9 7 | do 8 | 9 | python uncertainty.py --checkpoint_file_path checkpoints/model.ckpt-240000 --T $t --prob $p 10 | done 11 | done 12 | done 13 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/bias_variance/p-0.75/uncertainty.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | FLAGS = tf.app.flags.FLAGS 6 | def evaluate(): 7 | with tf.Graph().as_default(): 8 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 9 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 10 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 11 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 12 | softmax_tensors = tf.placeholder(tf.float32, shape=[None, 10000, 10]) 13 | 14 | model = Model() 15 | logits = model.inference(x, keep_prob=FLAGS.prob) 16 | softmax = tf.nn.softmax(logits) 17 | shape = tf.shape(softmax) 18 | saver = tf.train.Saver(max_to_keep = None) 19 | mean = tf.reduce_mean(softmax_tensors,0) 20 | accuracy = model.accuracy(mean,y) 21 | softmax_list = [] 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | for i in range(FLAGS.T): 26 | softmaxi = sess.run([softmax], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | softmax_list.append(softmaxi) 29 | for i in range(FLAGS.T): 30 | if i>1: 31 | arr=np.squeeze(np.array(softmax_list)[:i,:,:]) 32 | else : 33 | arr=np.array(softmax_list)[0,:,:] 34 | total_accuracy, soft = sess.run([accuracy,mean], 35 | feed_dict={softmax_tensors: arr, y: mnist.test.labels}) 36 | f=open('bias_var.log','a') 37 | f.write(str(FLAGS.prob)+ ","+str(i+1)+","+str(total_accuracy)+'\n') 38 | f.close() 39 | #L=np.array([np.squeeze(np.array(soft)).flatten(), mnist.test.labels.flatten()]) 40 | #L=np.transpose(L) 41 | #L=L[L[:,0].argsort()] 42 | #np.savetxt(str(i)+'uncertainty2.out', L, delimiter=',') 43 | 44 | 45 | 46 | def main(argv=None): 47 | evaluate() 48 | if __name__ == '__main__': 49 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-240000', 'path to checkpoint file') 50 | tf.app.flags.DEFINE_integer('T', 2, 'Number of Forward Passes') 51 | tf.app.flags.DEFINE_float('prob', 1.0, 'probability of dropout') 52 | tf.app.run() 53 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | #images, labels = mnist.load_test_data(FLAGS.test_data) 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 13 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 14 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=1.0) 18 | accuracy = model.accuracy(logits, y) 19 | 20 | saver = tf.train.Saver() 21 | 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | 26 | total_accuracy = sess.run([accuracy], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | print('Test accuracy: {}'.format(total_accuracy)) 29 | 30 | 31 | def main(argv=None): 32 | evaluate() 33 | 34 | if __name__ == '__main__': 35 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 36 | tf.app.run() -------------------------------------------------------------------------------- /lenet-all-standard-dropout/evaluate_mc_dropout.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | #function for MC dropout evaluation 8 | def mc_dropout_logits(x,keep_prob,T): 9 | model= Model() 10 | prediction_list=[] 11 | for i in xrange(T): 12 | prediction=model.inference(x,keep_prob) 13 | prediction_list=[prediction_list,prediction] 14 | print prediction 15 | #logits=tf.reduce_mean(prediction_list,0) 16 | #return logits 17 | #print prediction_list 18 | 19 | def evaluate(): 20 | with tf.Graph().as_default(): 21 | #images, labels = mnist.load_test_data(FLAGS.test_data) 22 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 23 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 24 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 25 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 26 | 27 | model = Model() 28 | softmax_prob_layer = [] 29 | #logits with drop out 30 | logits = model.inference(x, keep_prob=0.5) 31 | #normalized probabilities 32 | softmax_prob = tf.nn.softmax(logits) 33 | 34 | #accuracy = model.accuracy(logits, y) 35 | saver = tf.train.Saver() 36 | with tf.Session() as sess: 37 | tf.global_variables_initializer().run() 38 | 39 | #print(sess.run([logits_layer],feed_dict={x_: mnist.test.images, y: mnist.test.labels})) 40 | #total_accuracy = sess.run([accuracy], 41 | #feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 42 | 43 | saver.restore(sess, FLAGS.checkpoint_file_path) 44 | #Monte Carlo stocastic forward pass: 45 | for i in range(50): 46 | softmax_prob_layer.append(sess.run([softmax_prob],feed_dict={x_: mnist.test.images, y: mnist.test.labels})) 47 | 48 | softmax_prob_sum = tf.convert_to_tensor(np.array(softmax_prob_layer)) 49 | softmax_prob_average = tf.reduce_mean(softmax_prob_sum,0) 50 | softmax_prob_average = tf.squeeze(softmax_prob_average) 51 | #obtain the test accuracy: 52 | accuracy = model.accuracy(softmax_prob_average, y) 53 | total_accuracy = sess.run([accuracy],feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 54 | print('Test accuracy: {}'.format(total_accuracy)) 55 | #print(sess.run([softmax_prob_average],feed_dict={x_: mnist.test.images, y: mnist.test.labels})) 56 | #print softmax_prob_average 57 | #print softmax_prob 58 | def main(argv=None): 59 | evaluate() 60 | 61 | if __name__ == '__main__': 62 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 63 | tf.app.run() 64 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/evaluate_mc_dropout_py3.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | #function for MC dropout evaluation 8 | def mc_dropout_logits(x,keep_prob,T): 9 | model= Model() 10 | prediction_list=[] 11 | for i in xrange(T): 12 | prediction=model.inference(x,keep_prob) 13 | prediction_list=[prediction_list,prediction] 14 | print (prediction) 15 | #logits=tf.reduce_mean(prediction_list,0) 16 | #return logits 17 | #print prediction_list 18 | 19 | def evaluate(): 20 | with tf.Graph().as_default(): 21 | #images, labels = mnist.load_test_data(FLAGS.test_data) 22 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 23 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 24 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 25 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 26 | 27 | model = Model() 28 | softmax_prob_layer=[] 29 | #logits with drop out 30 | logits = model.inference(x, keep_prob=0.5) 31 | #normalized probabilities 32 | softmax_prob = tf.nn.softmax(logits) 33 | 34 | #accuracy = model.accuracy(logits, y) 35 | saver = tf.train.Saver() 36 | with tf.Session() as sess: 37 | tf.global_variables_initializer().run() 38 | 39 | #print(sess.run([logits_layer],feed_dict={x_: mnist.test.images, y: mnist.test.labels})) 40 | #total_accuracy = sess.run([accuracy], 41 | #feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 42 | 43 | saver.restore(sess, FLAGS.checkpoint_file_path) 44 | #Monte Carlo stocastic forward pass: 45 | for i in range(50): # change hard-coded T = forward passes 46 | softmax_prob_layer.append(sess.run([softmax_prob],feed_dict={x_: mnist.test.images, y: mnist.test.labels})) 47 | 48 | softmax_prob_sum = tf.convert_to_tensor(np.array(softmax_prob_layer)) 49 | softmax_prob_average = tf.reduce_mean(softmax_prob_sum,0) 50 | softmax_prob_average = tf.squeeze(softmax_prob_average) 51 | #obtain the test accuracy: 52 | accuracy = model.accuracy(softmax_prob_average, y) 53 | total_accuracy = sess.run([accuracy],feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 54 | print('Test accuracy: {}'.format(total_accuracy)) 55 | f = open('trainingAccuracies.log','a+'); 56 | f.write('Test accuracy: {}'.format(total_accuracy)) 57 | #print(sess.run([softmax_prob_average],feed_dict={x_: mnist.test.images, y: mnist.test.labels})) 58 | #print softmax_prob_average 59 | #print softmax_prob 60 | def main(argv=None): 61 | evaluate() 62 | 63 | if __name__ == '__main__': 64 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-5000', 'path to checkpoint file') 65 | tf.app.run() 66 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/evaluation.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | 4 | for n in `seq 0 10000 10000000` 5 | do 6 | check=/home/si318/Desktop/MLSALT4/lenet-all-standard-dropout/checkpoints/model.ckpt-$n 7 | python3 mc_drop.py --checkpoint_file_path $check --T 50 8 | done 9 | 10 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/mc_drop.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | 9 | def evaluate(): 10 | with tf.Graph().as_default(): 11 | 12 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 13 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 14 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 15 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 16 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 17 | 18 | model = Model() 19 | logits = model.inference(x, keep_prob=0.5) 20 | softmax = tf.nn.softmax(logits) 21 | shape = tf.shape(softmax) 22 | saver = tf.train.Saver(max_to_keep = None) 23 | 24 | mean = tf.reduce_mean(softmax_tensors,0) 25 | accuracy = model.accuracy(mean,y) 26 | 27 | 28 | #this is for each forward pass accuracy 29 | softmax_each=tf.placeholder(tf.float32, shape=[10000, 10]) 30 | accuracyi=model.accuracy(softmax_each,y) 31 | 32 | 33 | softmax_list = [] 34 | accuracy_list = [] 35 | with tf.Session() as sess: 36 | tf.global_variables_initializer().run() 37 | saver.restore(sess, FLAGS.checkpoint_file_path) 38 | for i in range(FLAGS.T): 39 | softmaxi = sess.run([softmax], 40 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 41 | softmax_list.append(softmaxi) 42 | 43 | 44 | 45 | #added for accuracy of each forward pass: 46 | print(softmaxi[0]) 47 | accuracyi1=sess.run([accuracyi], 48 | feed_dict={softmax_each:np.squeeze(np.array(softmaxi[0])), y: mnist.test.labels}) 49 | accuracy_list.append(accuracyi1) 50 | 51 | 52 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 53 | 54 | total_accuracy = sess.run([accuracy], 55 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: mnist.test.labels}) 56 | #print (softmax_list[0].shape) 57 | standard_deviation=np.std(np.array(accuracy_list)) 58 | 59 | print('Test accuracy: {}'.format(total_accuracy)) 60 | print('Standard deviation of 10 forward passes:',standard_deviation) 61 | print('Accuracy list is',accuracy_list) 62 | 63 | 64 | 65 | 66 | def main(argv=None): 67 | evaluate() 68 | 69 | if __name__ == '__main__': 70 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-180000', 'path to checkpoint file') 71 | tf.app.flags.DEFINE_integer('T', 10, 'Number of Forward Passes') 72 | tf.app.run() 73 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | #This is a minor modification TensorFlow Code 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | 6 | import math 7 | 8 | from tensorflow.python.framework import constant_op 9 | from tensorflow.python.framework import dtypes 10 | from tensorflow.python.framework import ops 11 | from tensorflow.python.ops import control_flow_ops 12 | from tensorflow.python.ops import math_ops 13 | from tensorflow.python.ops import random_ops 14 | 15 | def poly_inverse_time_decay(learning_rate, 16 | global_step, 17 | decay_steps, 18 | decay_rate, 19 | power, 20 | staircase=False, 21 | name=None): 22 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 23 | When training a model, it is often recommended to lower the learning rate as 24 | the training progresses. This function applies an inverse decay function 25 | to a provided initial learning rate. It requires an `global_step` value to 26 | compute the decayed learning rate. You can just pass a TensorFlow variable 27 | that you increment at each training step. 28 | The function returns the decayed learning rate. It is computed as: 29 | ```python 30 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 31 | decay_step) 32 | ``` 33 | or, if `staircase` is `True`, as: 34 | ```python 35 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 36 | decay_step)) 37 | ``` 38 | Example: decay 1/t with a rate of 0.5: 39 | ```python 40 | ... 41 | global_step = tf.Variable(0, trainable=False) 42 | learning_rate = 0.1 43 | decay_steps = 1.0 44 | decay_rate = 0.5 45 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 46 | decay_steps, decay_rate) 47 | # Passing global_step to minimize() will increment it at each step. 48 | learning_step = ( 49 | tf.train.GradientDescentOptimizer(learning_rate) 50 | .minimize(...my loss..., global_step=global_step) 51 | ) 52 | ``` 53 | Args: 54 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 55 | Python number. The initial learning rate. 56 | global_step: A Python number. 57 | Global step to use for the decay computation. Must not be negative. 58 | decay_steps: How often to apply decay. 59 | decay_rate: A Python number. The decay rate. 60 | staircase: Whether to apply decay in a discrete staircase, as opposed to 61 | continuous, fashion. 62 | name: String. Optional name of the operation. Defaults to 63 | 'InverseTimeDecay'. 64 | Returns: 65 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 66 | learning rate. 67 | Raises: 68 | ValueError: if `global_step` is not supplied. 69 | """ 70 | if global_step is None: 71 | raise ValueError("global_step is required for inverse_time_decay.") 72 | with ops.name_scope(name, "InverseTimeDecay", 73 | [learning_rate, global_step, decay_rate]) as name: 74 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 75 | dtype = learning_rate.dtype 76 | global_step = math_ops.cast(global_step, dtype) 77 | decay_steps = math_ops.cast(decay_steps, dtype) 78 | decay_rate = math_ops.cast(decay_rate, dtype) 79 | power = math_ops.cast(power, dtype) 80 | p = global_step / decay_steps 81 | if staircase: 82 | p = math_ops.floor(p) 83 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 84 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 85 | 86 | return math_ops.div(learning_rate, denom, name=name) 87 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=tf.train.get_global_step()) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | #images, labels = mnist.load_test_data(FLAGS.test_data) 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 13 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 14 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=1.0) 18 | accuracy = model.accuracy(logits, y) 19 | 20 | saver = tf.train.Saver() 21 | 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | 26 | total_accuracy = sess.run([accuracy], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | print('Test accuracy: {}'.format(total_accuracy)) 29 | 30 | 31 | def main(argv=None): 32 | evaluate() 33 | 34 | if __name__ == '__main__': 35 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 36 | tf.app.run() -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/mc_drop.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | 9 | def evaluate(): 10 | with tf.Graph().as_default(): 11 | 12 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 13 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 14 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 15 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 16 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 17 | 18 | model = Model() 19 | logits = model.inference(x, keep_prob=0.5) 20 | softmax = tf.nn.softmax(logits) 21 | shape = tf.shape(softmax) 22 | saver = tf.train.Saver(max_to_keep = None) 23 | 24 | mean = tf.reduce_mean(softmax_tensors,0) 25 | accuracy = model.accuracy(mean,y) 26 | 27 | softmax_list = [] 28 | with tf.Session() as sess: 29 | tf.global_variables_initializer().run() 30 | saver.restore(sess, FLAGS.checkpoint_file_path) 31 | for i in range(FLAGS.T): 32 | softmaxi = sess.run([softmax], 33 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 34 | softmax_list.append(softmaxi) 35 | 36 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 37 | 38 | total_accuracy = sess.run([accuracy], 39 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: mnist.test.labels}) 40 | print('Test accuracy: {}'.format(total_accuracy)) 41 | 42 | 43 | 44 | 45 | 46 | def main(argv=None): 47 | evaluate() 48 | 49 | if __name__ == '__main__': 50 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000000', 'path to checkpoint file') 51 | tf.app.flags.DEFINE_integer('T', 10, 'Number of Forward Passes') 52 | tf.app.run() 53 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/train.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | from Model import Model 5 | import math 6 | import numpy as np 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | def train(): 11 | model = Model() 12 | 13 | with tf.Graph().as_default(): 14 | # Load training data and use test as evaluation set in one hot format 15 | mnist_ = input_data.read_data_sets("MNIST_data/", one_hot=True) 16 | perm =np.random.permutation(1875) 17 | images = mnist_.train.images[perm] 18 | labels = mnist_.train.labels[perm] 19 | print (images.shape) #[1875,28,28,1] np array 20 | 21 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 22 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 23 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 24 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 25 | global_step = tf.contrib.framework.get_or_create_global_step() 26 | 27 | logits = model.inference(x, keep_prob=keep_prob) 28 | loss = model.loss(logits=logits, labels=y) 29 | 30 | accuracy = model.accuracy(logits, y) 31 | summary_op = tf.summary.merge_all() 32 | train_op = model.train(loss, global_step=global_step) 33 | 34 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 35 | input_size = 1875 36 | porp = int(math.ceil(input_size/batch_size)) 37 | 38 | init = tf.global_variables_initializer() 39 | saver = tf.train.Saver(max_to_keep = 100000) 40 | 41 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 42 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 43 | sess.run(init) 44 | for i in range(FLAGS.num_iter): 45 | if i%(porp) == 0: 46 | permutation=np.random.permutation(input_size) #create a list with random indexes 47 | increment = 0 #restart the increment variable 48 | 49 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 50 | increment += 1 51 | image_batch=images[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 52 | label_batch=labels[batch_idx] 53 | 54 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 55 | feed_dict={x_: image_batch, y: label_batch, keep_prob: 0.5}) 56 | 57 | writer.add_summary(summary, i) 58 | if i % 5000 == 0: 59 | f = open('trainingStdDrop.log', 'a+') 60 | validation_accuracy = accuracy.eval(feed_dict={x_: mnist_.test.images, y: mnist_.test.labels, keep_prob: 1.0}) 61 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 62 | f.close() 63 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 64 | 65 | def main(argv=None): 66 | train() 67 | 68 | 69 | if __name__ == '__main__': 70 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 71 | tf.app.flags.DEFINE_integer('num_iter', 1000000, 'number of training iterations') 72 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 73 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 74 | 75 | tf.app.run() 76 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-32/trainingStdDrop.log: -------------------------------------------------------------------------------- 1 | 0, 85.74201202392578, 0.10100000351667404 2 | 1000, 5.491013050079346, 0.7785000205039978 3 | 2000, 4.203330993652344, 0.9085000157356262 4 | 0, 123.6547622680664, 0.10320000350475311 5 | 1000, 217.3306121826172, 0.10100000351667404 6 | 2000, 198.7882843017578, 0.10100000351667404 7 | 3000, 182.8346710205078, 0.10100000351667404 8 | 4000, 168.98521423339844, 0.10100000351667404 9 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=tf.train.get_global_step()) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | #images, labels = mnist.load_test_data(FLAGS.test_data) 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 13 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 14 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=1.0) 18 | accuracy = model.accuracy(logits, y) 19 | 20 | saver = tf.train.Saver() 21 | 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | 26 | total_accuracy = sess.run([accuracy], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | print('Test accuracy: {}'.format(total_accuracy)) 29 | 30 | 31 | def main(argv=None): 32 | evaluate() 33 | 34 | if __name__ == '__main__': 35 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 36 | tf.app.run() -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/evaluation.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | 4 | for n in `seq 0 10000 10000000` 5 | do 6 | check=/home/si318/Desktop/MLSALT4/lenet-all-standard-dropout/checkpoints/model.ckpt-$n 7 | python3 mc_drop.py --checkpoint_file_path $check --T 50 8 | done 9 | 10 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/mc_drop.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | 9 | def evaluate(): 10 | with tf.Graph().as_default(): 11 | 12 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 13 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 14 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 15 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 16 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 17 | 18 | model = Model() 19 | logits = model.inference(x, keep_prob=0.5) 20 | softmax = tf.nn.softmax(logits) 21 | shape = tf.shape(softmax) 22 | saver = tf.train.Saver(max_to_keep = None) 23 | 24 | mean = tf.reduce_mean(softmax_tensors,0) 25 | accuracy = model.accuracy(mean,y) 26 | 27 | softmax_list = [] 28 | with tf.Session() as sess: 29 | tf.global_variables_initializer().run() 30 | saver.restore(sess, FLAGS.checkpoint_file_path) 31 | for i in range(FLAGS.T): 32 | softmaxi = sess.run([softmax], 33 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 34 | softmax_list.append(softmaxi) 35 | 36 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 37 | 38 | total_accuracy = sess.run([accuracy], 39 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: mnist.test.labels}) 40 | print('Test accuracy: {}'.format(total_accuracy)) 41 | 42 | 43 | 44 | 45 | 46 | def main(argv=None): 47 | evaluate() 48 | 49 | if __name__ == '__main__': 50 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000000', 'path to checkpoint file') 51 | tf.app.flags.DEFINE_integer('T', 10, 'Number of Forward Passes') 52 | tf.app.run() 53 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) -------------------------------------------------------------------------------- /lenet-all-standard-dropout/subsets/1-4/train.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | from Model import Model 5 | import math 6 | import numpy as np 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | def train(): 11 | model = Model() 12 | 13 | with tf.Graph().as_default(): 14 | # Load training data and use test as evaluation set in one hot format 15 | mnist_ = input_data.read_data_sets("MNIST_data/", one_hot=True) 16 | perm =np.random.permutation(13750) 17 | images = mnist_.train.images[perm] 18 | labels = mnist_.train.labels[perm] 19 | print (images.shape) #[13750,28,28,1] np array 20 | 21 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 22 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 23 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 24 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 25 | global_step = tf.contrib.framework.get_or_create_global_step() 26 | 27 | logits = model.inference(x, keep_prob=keep_prob) 28 | loss = model.loss(logits=logits, labels=y) 29 | 30 | accuracy = model.accuracy(logits, y) 31 | summary_op = tf.summary.merge_all() 32 | train_op = model.train(loss, global_step=global_step) 33 | 34 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 35 | input_size = 13750 36 | porp = int(math.ceil(input_size/batch_size)) 37 | 38 | init = tf.global_variables_initializer() 39 | saver = tf.train.Saver(max_to_keep = 100000) 40 | 41 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 42 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 43 | sess.run(init) 44 | for i in range(FLAGS.num_iter): 45 | if i%(porp) == 0: 46 | permutation=np.random.permutation(input_size) #create a list with random indexes 47 | increment = 0 #restart the increment variable 48 | 49 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 50 | increment += 1 51 | image_batch=images[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 52 | label_batch=labels[batch_idx] 53 | 54 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 55 | feed_dict={x_: image_batch, y: label_batch, keep_prob: 0.5}) 56 | 57 | writer.add_summary(summary, i) 58 | if i % 5000 == 0: 59 | f = open('trainingStdDrop.log', 'a+') 60 | validation_accuracy = accuracy.eval(feed_dict={x_: mnist_.test.images, y: mnist_.test.labels, keep_prob: 1.0}) 61 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 62 | f.close() 63 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 64 | 65 | def main(argv=None): 66 | train() 67 | 68 | 69 | if __name__ == '__main__': 70 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 71 | tf.app.flags.DEFINE_integer('num_iter', 1000000, 'number of training iterations') 72 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 73 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 74 | 75 | tf.app.run() 76 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/train.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | from Model import Model 5 | 6 | 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | def train(): 11 | model = Model() 12 | 13 | with tf.Graph().as_default(): 14 | # Load training data and use test as evaluation set in one hot format 15 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 16 | # images = mnist.train.images # Returns np.array 17 | # labels = np.asarray(mnist.train.labels, dtype=np.int32) 18 | # val_images = mnist.test.images # Returns np.array 19 | # val_labels = np.asarray(mnist.test.labels, dtype=np.int32) 20 | 21 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 22 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 23 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 24 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 25 | global_step = tf.contrib.framework.get_or_create_global_step() 26 | 27 | logits = model.inference(x, keep_prob=keep_prob) 28 | loss = model.loss(logits=logits, labels=y) 29 | 30 | accuracy = model.accuracy(logits, y) 31 | summary_op = tf.summary.merge_all() 32 | train_op = model.train(loss, global_step=global_step) 33 | 34 | init = tf.global_variables_initializer() 35 | saver = tf.train.Saver(max_to_keep = 100000) 36 | 37 | with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess: 38 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 39 | sess.run(init) 40 | for i in range(FLAGS.num_iter): 41 | batch = mnist.train.next_batch(FLAGS.batch_size) 42 | 43 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 44 | feed_dict={x_: batch[0], y: batch[1], keep_prob: 0.5}) 45 | writer.add_summary(summary, i) 46 | if i % 10000 == 0: 47 | f = open('trainingStdDrop.log', 'a+') 48 | validation_accuracy = accuracy.eval(feed_dict={x_: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0}) 49 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 50 | f.close() 51 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 52 | 53 | def main(argv=None): 54 | train() 55 | 56 | 57 | if __name__ == '__main__': 58 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 59 | tf.app.flags.DEFINE_integer('num_iter', 10000000, 'number of training iterations') 60 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 61 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 62 | 63 | tf.app.run() 64 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=tf.train.get_global_step()) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | #images, labels = mnist.load_test_data(FLAGS.test_data) 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 13 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 14 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=1.0) 18 | accuracy = model.accuracy(logits, y) 19 | 20 | saver = tf.train.Saver() 21 | 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | 26 | total_accuracy = sess.run([accuracy], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | print('Test accuracy: {}'.format(total_accuracy)) 29 | 30 | 31 | def main(argv=None): 32 | evaluate() 33 | 34 | if __name__ == '__main__': 35 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 36 | tf.app.run() -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/mc_drop_eval.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ["CUDA_VISIBLE_DEVICES"]="-1" # force CPU 3 | import tensorflow as tf 4 | from tensorflow.examples.tutorials.mnist import input_data 5 | from Model import Model 6 | import numpy as np 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | 11 | def evaluate(): 12 | with tf.Graph().as_default(): 13 | 14 | mnist = input_data.read_data_sets("/home/rns38/Documents/MLSALT4/lenet-all-standard-dropout/MNIST_data/", one_hot=True) 15 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 16 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 17 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 18 | softmax_tensors = tf.placeholder(tf.float32, shape=[None, 10000, 10]) 19 | 20 | model = Model() 21 | logits = model.inference(x, keep_prob=FLAGS.prob) 22 | softmax = tf.nn.softmax(logits) 23 | shape = tf.shape(softmax) 24 | saver = tf.train.Saver(max_to_keep = None) 25 | 26 | mean = tf.reduce_mean(softmax_tensors,0) 27 | 28 | accuracy = model.accuracy(mean,y) 29 | 30 | softmax_list = [] 31 | with tf.Session() as sess: 32 | tf.global_variables_initializer().run() 33 | saver.restore(sess, FLAGS.checkpoint_file_path) 34 | for i in range(FLAGS.T): 35 | softmaxi = sess.run([softmax], 36 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 37 | softmax_list.append(softmaxi) 38 | 39 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 40 | #print (np.squeeze(np.array(softmax_list)).shape) 41 | for i in range(FLAGS.T): 42 | if i>1: 43 | arr=np.squeeze(np.array(softmax_list)[:i,:,:]) 44 | else : 45 | arr=np.array(softmax_list)[0,:,:] 46 | 47 | total_accuracy, soft = sess.run([accuracy,mean], 48 | feed_dict={softmax_tensors: arr, y: mnist.test.labels}) 49 | if i==FLAGS.T-1: 50 | 51 | f=open('mc_drop_eval.log','a') 52 | split_path=FLAGS.checkpoint_file_path.split() 53 | iteration_number=split_path[0][-6:] 54 | #print (iteration_number) 55 | f.write(str(total_accuracy)+'\n') 56 | f.close() 57 | #np.max(soft.flatten())) 58 | #L=np.array([soft.flatten(), mnist.test.labels.flatten()]) 59 | #L=np.transpose(L) 60 | #L=L[L[:,0].argsort()] 61 | #np.savetxt('uncertainty2.out', L, delimiter=',') 62 | 63 | 64 | def main(argv=None): 65 | evaluate() 66 | 67 | if __name__ == '__main__': 68 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000000', 'path to checkpoint file') 69 | tf.app.flags.DEFINE_integer('T', 50, 'Number of Forward Passes') 70 | tf.app.flags.DEFINE_float('prob', 0.5, 'probability of dropout') 71 | tf.app.run() 72 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/mc_eval.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for n in `seq 0 5000 245000` 4 | do 5 | check=/home/si318/Desktop/MLSALT4/lenet-all-standard-dropout/trained_onnoise_02/checkpoints/model.ckpt-$n 6 | python mc_drop_eval.py --checkpoint_file_path $check --T 50 7 | done 8 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/noisy.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | import numpy as np 5 | import tensorflow as tf 6 | 7 | noiseLevel= [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1] 8 | 9 | tf.logging.set_verbosity(tf.logging.INFO) 10 | 11 | mnist = tf.contrib.learn.datasets.load_dataset("mnist") 12 | # train_data = mnist.train.images # np.array 13 | # train_labels = np.asarray(mnist.train.labels, dtype=np.int32) 14 | eval_data = mnist.train.images # np.array 15 | eval_labels = np.asarray(mnist.train.labels, dtype=np.int32) 16 | eval_data = eval_data.reshape(55000,28,28) 17 | 18 | 19 | for i in range(len(noiseLevel)): 20 | dataset = np.zeros([55000,28,28]) 21 | count2 = 0 22 | for image in eval_data: 23 | image = image.reshape(28,28) 24 | noisePlaces=np.random.choice([0, 1], size=(28,28), p=[1-noiseLevel[i], noiseLevel[i]]) 25 | noiseScale=np.random.rand(28,28) 26 | noise=noiseScale*noisePlaces 27 | noisy_image= np.mod(image + noise, 1) 28 | #print (noisy_image.shape) 29 | dataset[count2,:,:] = noisy_image 30 | count2 += 1 31 | np.save(str(noiseLevel[i]), dataset) 32 | 33 | 34 | 35 | 36 | 37 | # plt.gray() # use this line if you don't want to see it in color 38 | # plt.imshow(eval_data[9999]) 39 | # plt.show() 40 | # plt.gray() # use this line if you don't want to see it in color 41 | # plt.imshow(dataset[9999]) 42 | # plt.show() 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/std_drop_eval.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ["CUDA_VISIBLE_DEVICES"]="-1" # force CPU 3 | import tensorflow as tf 4 | from tensorflow.examples.tutorials.mnist import input_data 5 | from Model import Model 6 | 7 | FLAGS = tf.app.flags.FLAGS 8 | 9 | 10 | def evaluate(): 11 | with tf.Graph().as_default(): 12 | #images, labels = mnist.load_test_data(FLAGS.test_data) 13 | mnist = input_data.read_data_sets("/home/rns38/Documents/MLSALT4/lenet-all-standard-dropout/MNIST_data/", one_hot=True) 14 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 15 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 16 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 17 | 18 | model = Model() 19 | logits = model.inference(x, keep_prob=1.0) 20 | accuracy = model.accuracy(logits, y) 21 | 22 | saver = tf.train.Saver() 23 | 24 | with tf.Session() as sess: 25 | tf.global_variables_initializer().run() 26 | saver.restore(sess, FLAGS.checkpoint_file_path) 27 | 28 | total_accuracy = sess.run([accuracy], 29 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 30 | print('Test accuracy: {}'.format(total_accuracy)) 31 | #below are added by ms for output into a txt file 32 | output=total_accuracy[0] 33 | f=open('std_drop_eval.log','a') 34 | #f.write(str(total_accuracy)+'\n') 35 | f.write(str(output)+'\n') 36 | f.close() 37 | 38 | 39 | def main(argv=None): 40 | evaluate() 41 | 42 | if __name__ == '__main__': 43 | #tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 44 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000000', 'path to checkpoint file') 45 | tf.app.run() 46 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/std_eval.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for n in `seq 0 5000 245000` 4 | do 5 | check=/home/si318/Desktop/MLSALT4/lenet-all-standard-dropout/trained_onnoise_02/checkpoints/model.ckpt-$n 6 | python std_drop_eval.py --checkpoint_file_path $check 7 | done 8 | 9 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_02/train.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | from Model import Model 5 | import math 6 | import numpy as np 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | def train(): 11 | model = Model() 12 | 13 | with tf.Graph().as_default(): 14 | 15 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 16 | labels = mnist.train.labels 17 | images = np.load('0.2.npy') 18 | images = images.reshape([55000,28,28,1]) 19 | eval_data = mnist.test.images 20 | eval_data = eval_data.reshape([10000,28,28,1]) 21 | 22 | x = tf.placeholder(tf.float32, shape=[None, 28,28,1]) #data gets loaded as a 28x8 vector 23 | #x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 24 | 25 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 26 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 27 | global_step = tf.contrib.framework.get_or_create_global_step() 28 | 29 | logits = model.inference(x, keep_prob=keep_prob) 30 | loss = model.loss(logits=logits, labels=y) 31 | 32 | accuracy = model.accuracy(logits, y) 33 | summary_op = tf.summary.merge_all() 34 | train_op = model.train(loss, global_step=global_step) 35 | 36 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 37 | input_size = 55000 38 | porp = int(math.ceil(input_size/batch_size)) 39 | 40 | init = tf.global_variables_initializer() 41 | saver = tf.train.Saver(max_to_keep = 100000) 42 | 43 | 44 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 45 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 46 | sess.run(init) 47 | for i in range(FLAGS.num_iter): 48 | if i%(porp) == 0: 49 | permutation=np.random.permutation(input_size) #create a list with random indexes 50 | increment = 0 #restart the increment variable 51 | 52 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 53 | increment += 1 54 | image_batch=images[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 55 | label_batch=labels[batch_idx] 56 | 57 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 58 | feed_dict={x: image_batch, y: label_batch, keep_prob: 0.5}) 59 | 60 | writer.add_summary(summary, i) 61 | if i % 5000 == 0: 62 | f = open('trainingStdDrop.log', 'a+') 63 | validation_accuracy = accuracy.eval(feed_dict={x: eval_data, y: mnist.test.labels, keep_prob: 1.0}) 64 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 65 | f.close() 66 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 67 | 68 | def main(argv=None): 69 | train() 70 | 71 | 72 | if __name__ == '__main__': 73 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 74 | tf.app.flags.DEFINE_integer('num_iter', 250000, 'number of training iterations') 75 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 76 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 77 | 78 | tf.app.run() 79 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | dropout1 = tf.nn.dropout(conv1, keep_prob) #do dropout 21 | h_pool1 = self._create_max_pool_2x2(dropout1) #do max pooling. output size [batch_size, 14,14,20] 22 | 23 | with tf.variable_scope('conv2') as scope: 24 | kernel = self._create_weights([5, 5, 20, 50]) 25 | conv = self._create_conv2d(h_pool1, kernel) 26 | bias = self._create_bias([50]) 27 | preactivation = tf.nn.bias_add(conv, bias) 28 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 29 | self._activation_summary(conv2) 30 | 31 | 32 | dropout2 = tf.nn.dropout(conv2, keep_prob) 33 | h_pool2 = self._create_max_pool_2x2(dropout2) #output size [batch_size, 7, 7, 50] 34 | 35 | with tf.variable_scope('dense') as scope: 36 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 37 | W_dense = self._create_weights([7 * 7 * 50, 500]) 38 | b_dense = self._create_bias([500]) 39 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 40 | self._activation_summary(dense) 41 | 42 | with tf.variable_scope('logit') as scope: 43 | W_logit = self._create_weights([500, self._num_labels]) 44 | b_logit = self._create_bias([self._num_labels]) 45 | dense_drop = tf.nn.dropout(dense, keep_prob) 46 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 47 | self._activation_summary(logit) 48 | return logit 49 | 50 | def train(self, loss, global_step): 51 | #should probably make these variables arguements but cba 52 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 53 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 54 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 55 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 56 | train_op = optimizer.minimize( 57 | loss=loss, 58 | global_step=tf.train.get_global_step()) 59 | return train_op 60 | 61 | def loss(self, logits, labels): 62 | with tf.variable_scope('loss') as scope: 63 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 64 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 65 | tf.add_to_collection('losses', cross_entropy_mean) 66 | 67 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 68 | 69 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 70 | 71 | def accuracy(self, logits, y): 72 | with tf.variable_scope('accuracy') as scope: 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 74 | name=scope.name) #calculates how much % of our predictions is correct 75 | tf.summary.scalar('accuracy', accuracy) 76 | return accuracy 77 | 78 | def _create_conv2d(self, x, W): 79 | return tf.nn.conv2d(input=x, 80 | filter=W, #4d tensor that includes the size of the filter and number of filters 81 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 82 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 83 | 84 | def _create_max_pool_2x2(self, input): 85 | return tf.nn.max_pool(value=input, 86 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 87 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 88 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 89 | 90 | def _create_weights(self, shape): 91 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 92 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 93 | tf.add_to_collection('losses', weight_decay) 94 | return var 95 | 96 | def _create_bias(self, shape): 97 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 98 | 99 | def _activation_summary(self, x): 100 | #This is simply to catch information for visualization using tensorboard 101 | tensor_name = x.op.name 102 | tf.summary.histogram(tensor_name + '/activations', x) 103 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 104 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/empty: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | #images, labels = mnist.load_test_data(FLAGS.test_data) 11 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 12 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 13 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 14 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=1.0) 18 | accuracy = model.accuracy(logits, y) 19 | 20 | saver = tf.train.Saver() 21 | 22 | with tf.Session() as sess: 23 | tf.global_variables_initializer().run() 24 | saver.restore(sess, FLAGS.checkpoint_file_path) 25 | 26 | total_accuracy = sess.run([accuracy], 27 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 28 | print('Test accuracy: {}'.format(total_accuracy)) 29 | 30 | 31 | def main(argv=None): 32 | evaluate() 33 | 34 | if __name__ == '__main__': 35 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000-10000', 'path to checkpoint file') 36 | tf.app.run() -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/mc_drop.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import numpy as np 5 | 6 | FLAGS = tf.app.flags.FLAGS 7 | 8 | 9 | def evaluate(): 10 | with tf.Graph().as_default(): 11 | 12 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 13 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 14 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 15 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 16 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 17 | 18 | model = Model() 19 | logits = model.inference(x, keep_prob=0.5) 20 | softmax = tf.nn.softmax(logits) 21 | shape = tf.shape(softmax) 22 | saver = tf.train.Saver(max_to_keep = None) 23 | 24 | mean = tf.reduce_mean(softmax_tensors,0) 25 | accuracy = model.accuracy(mean,y) 26 | 27 | softmax_list = [] 28 | with tf.Session() as sess: 29 | tf.global_variables_initializer().run() 30 | saver.restore(sess, FLAGS.checkpoint_file_path) 31 | for i in range(FLAGS.T): 32 | softmaxi = sess.run([softmax], 33 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 34 | softmax_list.append(softmaxi) 35 | 36 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 37 | 38 | total_accuracy = sess.run([accuracy], 39 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: mnist.test.labels}) 40 | print('Test accuracy: {}'.format(total_accuracy)) 41 | 42 | 43 | 44 | 45 | 46 | def main(argv=None): 47 | evaluate() 48 | 49 | if __name__ == '__main__': 50 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000000', 'path to checkpoint file') 51 | tf.app.flags.DEFINE_integer('T', 10, 'Number of Forward Passes') 52 | tf.app.run() 53 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/noisy.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | import numpy as np 5 | import tensorflow as tf 6 | 7 | noiseLevel= [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1] 8 | 9 | tf.logging.set_verbosity(tf.logging.INFO) 10 | 11 | mnist = tf.contrib.learn.datasets.load_dataset("mnist") 12 | # train_data = mnist.train.images # np.array 13 | # train_labels = np.asarray(mnist.train.labels, dtype=np.int32) 14 | eval_data = mnist.train.images # np.array 15 | eval_labels = np.asarray(mnist.train.labels, dtype=np.int32) 16 | eval_data = eval_data.reshape(55000,28,28) 17 | 18 | 19 | for i in range(len(noiseLevel)): 20 | dataset = np.zeros([55000,28,28]) 21 | count2 = 0 22 | for image in eval_data: 23 | image = image.reshape(28,28) 24 | noisePlaces=np.random.choice([0, 1], size=(28,28), p=[1-noiseLevel[i], noiseLevel[i]]) 25 | noiseScale=np.random.rand(28,28) 26 | noise=noiseScale*noisePlaces 27 | noisy_image= np.mod(image + noise, 1) 28 | #print (noisy_image.shape) 29 | dataset[count2,:,:] = noisy_image 30 | count2 += 1 31 | np.save(str(noiseLevel[i]), dataset) 32 | 33 | 34 | 35 | 36 | 37 | # plt.gray() # use this line if you don't want to see it in color 38 | # plt.imshow(eval_data[9999]) 39 | # plt.show() 40 | # plt.gray() # use this line if you don't want to see it in color 41 | # plt.imshow(dataset[9999]) 42 | # plt.show() 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) -------------------------------------------------------------------------------- /lenet-all-standard-dropout/trainednoise_04/train.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | from Model import Model 5 | import math 6 | import numpy as np 7 | 8 | FLAGS = tf.app.flags.FLAGS 9 | 10 | def train(): 11 | model = Model() 12 | 13 | with tf.Graph().as_default(): 14 | 15 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 16 | labels = mnist.train.labels 17 | images = np.load('0.4.npy') 18 | images = images.reshape([55000,28,28,1]) 19 | eval_data = mnist.test.images 20 | eval_data = eval_data.reshape([10000,28,28,1]) 21 | 22 | x = tf.placeholder(tf.float32, shape=[None, 28,28,1]) #data gets loaded as a 28x8 vector 23 | #x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 24 | 25 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 26 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 27 | global_step = tf.contrib.framework.get_or_create_global_step() 28 | 29 | logits = model.inference(x, keep_prob=keep_prob) 30 | loss = model.loss(logits=logits, labels=y) 31 | 32 | accuracy = model.accuracy(logits, y) 33 | summary_op = tf.summary.merge_all() 34 | train_op = model.train(loss, global_step=global_step) 35 | 36 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 37 | input_size = 55000 38 | porp = int(math.ceil(input_size/batch_size)) 39 | 40 | init = tf.global_variables_initializer() 41 | saver = tf.train.Saver(max_to_keep = 100000) 42 | 43 | 44 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 45 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 46 | sess.run(init) 47 | for i in range(FLAGS.num_iter): 48 | if i%(porp) == 0: 49 | permutation=np.random.permutation(input_size) #create a list with random indexes 50 | increment = 0 #restart the increment variable 51 | 52 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 53 | increment += 1 54 | image_batch=images[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 55 | label_batch=labels[batch_idx] 56 | 57 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 58 | feed_dict={x: image_batch, y: label_batch, keep_prob: 0.5}) 59 | 60 | writer.add_summary(summary, i) 61 | if i % 5000 == 0: 62 | f = open('trainingStdDrop.log', 'a+') 63 | validation_accuracy = accuracy.eval(feed_dict={x: eval_data, y: mnist.test.labels, keep_prob: 1.0}) 64 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 65 | f.close() 66 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 67 | 68 | def main(argv=None): 69 | train() 70 | 71 | 72 | if __name__ == '__main__': 73 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 74 | tf.app.flags.DEFINE_integer('num_iter', 250000, 'number of training iterations') 75 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 76 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 77 | 78 | tf.app.run() 79 | -------------------------------------------------------------------------------- /lenet-ip/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | h_pool1 = self._create_max_pool_2x2(conv1) #do max pooling. output size [batch_size, 14,14,20] 21 | 22 | with tf.variable_scope('conv2') as scope: 23 | kernel = self._create_weights([5, 5, 20, 50]) 24 | conv = self._create_conv2d(h_pool1, kernel) 25 | bias = self._create_bias([50]) 26 | preactivation = tf.nn.bias_add(conv, bias) 27 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 28 | self._activation_summary(conv2) 29 | 30 | h_pool2 = self._create_max_pool_2x2(conv2) #output size [batch_size, 7, 7, 50] 31 | 32 | with tf.variable_scope('dense') as scope: 33 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 34 | W_dense = self._create_weights([7 * 7 * 50, 500]) 35 | b_dense = self._create_bias([500]) 36 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 37 | self._activation_summary(dense) 38 | 39 | with tf.variable_scope('logit') as scope: 40 | W_logit = self._create_weights([500, self._num_labels]) 41 | b_logit = self._create_bias([self._num_labels]) 42 | dense_drop = tf.nn.dropout(dense, keep_prob) 43 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 44 | self._activation_summary(logit) 45 | return logit 46 | 47 | def train(self, loss, global_step): 48 | #should probably make these variables arguements but cba 49 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 50 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 51 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 52 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 53 | train_op = optimizer.minimize( 54 | loss=loss, 55 | global_step=global_step) 56 | return train_op 57 | 58 | def loss(self, logits, labels): 59 | with tf.variable_scope('loss') as scope: 60 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 61 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 62 | tf.add_to_collection('losses', cross_entropy_mean) 63 | 64 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 65 | 66 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 67 | 68 | def accuracy(self, logits, y): 69 | with tf.variable_scope('accuracy') as scope: 70 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 71 | name=scope.name) #calculates how much % of our predictions is correct 72 | tf.summary.scalar('accuracy', accuracy) 73 | return accuracy 74 | 75 | def _create_conv2d(self, x, W): 76 | return tf.nn.conv2d(input=x, 77 | filter=W, #4d tensor that includes the size of the filter and number of filters 78 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 79 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 80 | 81 | def _create_max_pool_2x2(self, input): 82 | return tf.nn.max_pool(value=input, 83 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 84 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 85 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 86 | 87 | def _create_weights(self, shape): 88 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 89 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 90 | tf.add_to_collection('losses', weight_decay) 91 | return var 92 | 93 | def _create_bias(self, shape): 94 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 95 | 96 | def _activation_summary(self, x): 97 | #This is simply to catch information for visualization using tensorboard 98 | tensor_name = x.op.name 99 | tf.summary.histogram(tensor_name + '/activations', x) 100 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 101 | -------------------------------------------------------------------------------- /lenet-ip/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | #This is a minor modification TensorFlow Code 2 | from __future__ import absolute_import 3 | from __future__ import division 4 | from __future__ import print_function 5 | 6 | import math 7 | 8 | from tensorflow.python.framework import constant_op 9 | from tensorflow.python.framework import dtypes 10 | from tensorflow.python.framework import ops 11 | from tensorflow.python.ops import control_flow_ops 12 | from tensorflow.python.ops import math_ops 13 | from tensorflow.python.ops import random_ops 14 | 15 | def poly_inverse_time_decay(learning_rate, 16 | global_step, 17 | decay_steps, 18 | decay_rate, 19 | power, 20 | staircase=False, 21 | name=None): 22 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 23 | When training a model, it is often recommended to lower the learning rate as 24 | the training progresses. This function applies an inverse decay function 25 | to a provided initial learning rate. It requires an `global_step` value to 26 | compute the decayed learning rate. You can just pass a TensorFlow variable 27 | that you increment at each training step. 28 | The function returns the decayed learning rate. It is computed as: 29 | ```python 30 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 31 | decay_step) 32 | ``` 33 | or, if `staircase` is `True`, as: 34 | ```python 35 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 36 | decay_step)) 37 | ``` 38 | Example: decay 1/t with a rate of 0.5: 39 | ```python 40 | ... 41 | global_step = tf.Variable(0, trainable=False) 42 | learning_rate = 0.1 43 | decay_steps = 1.0 44 | decay_rate = 0.5 45 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 46 | decay_steps, decay_rate) 47 | # Passing global_step to minimize() will increment it at each step. 48 | learning_step = ( 49 | tf.train.GradientDescentOptimizer(learning_rate) 50 | .minimize(...my loss..., global_step=global_step) 51 | ) 52 | ``` 53 | Args: 54 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 55 | Python number. The initial learning rate. 56 | global_step: A Python number. 57 | Global step to use for the decay computation. Must not be negative. 58 | decay_steps: How often to apply decay. 59 | decay_rate: A Python number. The decay rate. 60 | staircase: Whether to apply decay in a discrete staircase, as opposed to 61 | continuous, fashion. 62 | name: String. Optional name of the operation. Defaults to 63 | 'InverseTimeDecay'. 64 | Returns: 65 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 66 | learning rate. 67 | Raises: 68 | ValueError: if `global_step` is not supplied. 69 | """ 70 | if global_step is None: 71 | raise ValueError("global_step is required for inverse_time_decay.") 72 | with ops.name_scope(name, "InverseTimeDecay", 73 | [learning_rate, global_step, decay_rate]) as name: 74 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 75 | dtype = learning_rate.dtype 76 | global_step = math_ops.cast(global_step, dtype) 77 | decay_steps = math_ops.cast(decay_steps, dtype) 78 | decay_rate = math_ops.cast(decay_rate, dtype) 79 | power = math_ops.cast(power, dtype) 80 | p = global_step / decay_steps 81 | if staircase: 82 | p = math_ops.floor(p) 83 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 84 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 85 | 86 | return math_ops.div(learning_rate, denom, name=name) 87 | 88 | -------------------------------------------------------------------------------- /lenet-ip/subsets/1-32/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | h_pool1 = self._create_max_pool_2x2(conv1) #do max pooling. output size [batch_size, 14,14,20] 21 | 22 | with tf.variable_scope('conv2') as scope: 23 | kernel = self._create_weights([5, 5, 20, 50]) 24 | conv = self._create_conv2d(h_pool1, kernel) 25 | bias = self._create_bias([50]) 26 | preactivation = tf.nn.bias_add(conv, bias) 27 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 28 | self._activation_summary(conv2) 29 | 30 | h_pool2 = self._create_max_pool_2x2(conv2) #output size [batch_size, 7, 7, 50] 31 | 32 | with tf.variable_scope('dense') as scope: 33 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 34 | W_dense = self._create_weights([7 * 7 * 50, 500]) 35 | b_dense = self._create_bias([500]) 36 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 37 | self._activation_summary(dense) 38 | 39 | with tf.variable_scope('logit') as scope: 40 | W_logit = self._create_weights([500, self._num_labels]) 41 | b_logit = self._create_bias([self._num_labels]) 42 | dense_drop = tf.nn.dropout(dense, keep_prob) 43 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 44 | self._activation_summary(logit) 45 | return logit 46 | 47 | def train(self, loss, global_step): 48 | #should probably make these variables arguements but cba 49 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 50 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 51 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 52 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 53 | train_op = optimizer.minimize( 54 | loss=loss, 55 | global_step=global_step) 56 | return train_op 57 | 58 | def loss(self, logits, labels): 59 | with tf.variable_scope('loss') as scope: 60 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 61 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 62 | tf.add_to_collection('losses', cross_entropy_mean) 63 | 64 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 65 | 66 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 67 | 68 | def accuracy(self, logits, y): 69 | with tf.variable_scope('accuracy') as scope: 70 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 71 | name=scope.name) #calculates how much % of our predictions is correct 72 | tf.summary.scalar('accuracy', accuracy) 73 | return accuracy 74 | 75 | def _create_conv2d(self, x, W): 76 | return tf.nn.conv2d(input=x, 77 | filter=W, #4d tensor that includes the size of the filter and number of filters 78 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 79 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 80 | 81 | def _create_max_pool_2x2(self, input): 82 | return tf.nn.max_pool(value=input, 83 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 84 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 85 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 86 | 87 | def _create_weights(self, shape): 88 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 89 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 90 | tf.add_to_collection('losses', weight_decay) 91 | return var 92 | 93 | def _create_bias(self, shape): 94 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 95 | 96 | def _activation_summary(self, x): 97 | #This is simply to catch information for visualization using tensorboard 98 | tensor_name = x.op.name 99 | tf.summary.histogram(tensor_name + '/activations', x) 100 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 101 | -------------------------------------------------------------------------------- /lenet-ip/subsets/1-32/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) 86 | 87 | -------------------------------------------------------------------------------- /lenet-ip/subsets/1-32/train.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import math 5 | import numpy as np 6 | 7 | FLAGS = tf.app.flags.FLAGS 8 | 9 | def train(): 10 | model = Model() 11 | 12 | with tf.Graph().as_default(): 13 | # Load training data and use test as evaluation set in one hot format 14 | mnist_ = input_data.read_data_sets("MNIST_data/", one_hot=True) 15 | perm =np.random.permutation(1875) 16 | images = mnist_.train.images[perm] 17 | labels = mnist_.train.labels[perm] 18 | print (images.shape) #[1875,28,28,1] np array 19 | 20 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 21 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 22 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 23 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 24 | global_step = tf.contrib.framework.get_or_create_global_step() 25 | 26 | logits = model.inference(x, keep_prob=keep_prob) 27 | loss = model.loss(logits=logits, labels=y) 28 | 29 | accuracy = model.accuracy(logits, y) 30 | summary_op = tf.summary.merge_all() 31 | train_op = model.train(loss, global_step=global_step) 32 | 33 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 34 | input_size = 1875 35 | porp = int(math.ceil(input_size/batch_size)) 36 | 37 | init = tf.global_variables_initializer() 38 | saver = tf.train.Saver(max_to_keep = 100000) 39 | 40 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 41 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 42 | sess.run(init) 43 | for i in range(FLAGS.num_iter): 44 | if i%(porp) == 0: 45 | permutation=np.random.permutation(input_size) #create a list with random indexes 46 | increment = 0 #restart the increment variable 47 | 48 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 49 | increment += 1 50 | image_batch=images[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 51 | label_batch=labels[batch_idx] 52 | 53 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 54 | feed_dict={x_: image_batch, y: label_batch, keep_prob: 0.5}) 55 | 56 | writer.add_summary(summary, i) 57 | if i % 5000 == 0: 58 | f = open('trainingStdDrop.log', 'a+') 59 | validation_accuracy = accuracy.eval(feed_dict={x_: mnist_.test.images, y: mnist_.test.labels, keep_prob: 1.0}) 60 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 61 | f.close() 62 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 63 | 64 | def main(argv=None): 65 | train() 66 | 67 | 68 | if __name__ == '__main__': 69 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 70 | tf.app.flags.DEFINE_integer('num_iter', 1000000, 'number of training iterations') 71 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 72 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 73 | 74 | tf.app.run() 75 | -------------------------------------------------------------------------------- /lenet-ip/subsets/1-4/Model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import poly_inverse_time_decay as td 3 | 4 | 5 | class Model(object): 6 | def __init__(self, batch_size=64, learning_rate=0.01, num_labels=10): 7 | self._batch_size = batch_size 8 | self._learning_rate = learning_rate 9 | self._num_labels = num_labels 10 | 11 | def inference(self, images, keep_prob): 12 | with tf.variable_scope('conv1') as scope: 13 | kernel = self._create_weights([5, 5, 1, 20]) #[filter_height, filter_width, in_channels, out_channels] 14 | conv = self._create_conv2d(images, kernel) #create the conv layer 15 | bias = self._create_bias([20]) #create bias variable 16 | preactivation = tf.nn.bias_add(conv, bias) #add bias 17 | conv1 = tf.nn.relu(preactivation, name=scope.name) #put through RELU activation function. #output size [batch_size, 28,28,20] 18 | self._activation_summary(conv1) #this is for TensorBoard visualization 19 | 20 | h_pool1 = self._create_max_pool_2x2(conv1) #do max pooling. output size [batch_size, 14,14,20] 21 | 22 | with tf.variable_scope('conv2') as scope: 23 | kernel = self._create_weights([5, 5, 20, 50]) 24 | conv = self._create_conv2d(h_pool1, kernel) 25 | bias = self._create_bias([50]) 26 | preactivation = tf.nn.bias_add(conv, bias) 27 | conv2 = tf.nn.relu(preactivation, name=scope.name) #outputsize [batch_size, 14,14,50] 28 | self._activation_summary(conv2) 29 | 30 | h_pool2 = self._create_max_pool_2x2(conv2) #output size [batch_size, 7, 7, 50] 31 | 32 | with tf.variable_scope('dense') as scope: 33 | reshape = tf.reshape(h_pool2, [-1, 7 * 7 * 50]) 34 | W_dense = self._create_weights([7 * 7 * 50, 500]) 35 | b_dense = self._create_bias([500]) 36 | dense = tf.nn.relu(tf.matmul(reshape, W_dense) + b_dense, name=scope.name) 37 | self._activation_summary(dense) 38 | 39 | with tf.variable_scope('logit') as scope: 40 | W_logit = self._create_weights([500, self._num_labels]) 41 | b_logit = self._create_bias([self._num_labels]) 42 | dense_drop = tf.nn.dropout(dense, keep_prob) 43 | logit = tf.nn.bias_add(tf.matmul(dense_drop, W_logit), b_logit, name=scope.name) 44 | self._activation_summary(logit) 45 | return logit 46 | 47 | def train(self, loss, global_step): 48 | #should probably make these variables arguements but cba 49 | #learning_rate = tf.train.inverse_time_decay(self._learning_rate, global_step, decay_step, decay_rate) 50 | learning_rate = td.poly_inverse_time_decay(self._learning_rate, global_step, decay_steps = 1, decay_rate = 0.0001, power = 0.75) 51 | #optimizer = tf.train.GradientDescentOptimizer(learning_rate) 52 | optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = 0.9, use_nesterov=True) 53 | train_op = optimizer.minimize( 54 | loss=loss, 55 | global_step=global_step) 56 | return train_op 57 | 58 | def loss(self, logits, labels): 59 | with tf.variable_scope('loss') as scope: 60 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) 61 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name=scope.name) #computes the mean loss = cost. Minimizing mean loss. 62 | tf.add_to_collection('losses', cross_entropy_mean) 63 | 64 | tf.summary.scalar('cost', tf.add_n(tf.get_collection('losses'))) 65 | 66 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 67 | 68 | def accuracy(self, logits, y): 69 | with tf.variable_scope('accuracy') as scope: 70 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), dtype=tf.float32), 71 | name=scope.name) #calculates how much % of our predictions is correct 72 | tf.summary.scalar('accuracy', accuracy) 73 | return accuracy 74 | 75 | def _create_conv2d(self, x, W): 76 | return tf.nn.conv2d(input=x, 77 | filter=W, #4d tensor that includes the size of the filter and number of filters 78 | strides=[1, 1, 1, 1], #The stride of the sliding window for each dimension of the input tensor. 79 | padding='SAME') #padding set so the output feature maps are the same size as the input feature maps. 80 | 81 | def _create_max_pool_2x2(self, input): 82 | return tf.nn.max_pool(value=input, 83 | ksize=[1, 2, 2, 1], #The size of the window for each dimension of the input tensor. 84 | strides=[1, 2, 2, 1], #The stride of the sliding window for each dimension of the input tensor. 85 | padding='SAME') # padding set so the output feature maps are the same size as the input feature maps. 86 | 87 | def _create_weights(self, shape): 88 | var = tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1, dtype=tf.float32)) 89 | weight_decay = tf.multiply(tf.nn.l2_loss(var), 0.0005, name='weight_loss') 90 | tf.add_to_collection('losses', weight_decay) 91 | return var 92 | 93 | def _create_bias(self, shape): 94 | return tf.Variable(tf.constant(1., shape=shape, dtype=tf.float32)) 95 | 96 | def _activation_summary(self, x): 97 | #This is simply to catch information for visualization using tensorboard 98 | tensor_name = x.op.name 99 | tf.summary.histogram(tensor_name + '/activations', x) 100 | tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x)) 101 | -------------------------------------------------------------------------------- /lenet-ip/subsets/1-4/poly_inverse_time_decay.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import math 6 | 7 | from tensorflow.python.framework import constant_op 8 | from tensorflow.python.framework import dtypes 9 | from tensorflow.python.framework import ops 10 | from tensorflow.python.ops import control_flow_ops 11 | from tensorflow.python.ops import math_ops 12 | from tensorflow.python.ops import random_ops 13 | 14 | def poly_inverse_time_decay(learning_rate, 15 | global_step, 16 | decay_steps, 17 | decay_rate, 18 | power, 19 | staircase=False, 20 | name=None): 21 | """Applies inverse time decay, with an additional polynomial argument p to the initial learning rate. 22 | When training a model, it is often recommended to lower the learning rate as 23 | the training progresses. This function applies an inverse decay function 24 | to a provided initial learning rate. It requires an `global_step` value to 25 | compute the decayed learning rate. You can just pass a TensorFlow variable 26 | that you increment at each training step. 27 | The function returns the decayed learning rate. It is computed as: 28 | ```python 29 | decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / 30 | decay_step) 31 | ``` 32 | or, if `staircase` is `True`, as: 33 | ```python 34 | decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / 35 | decay_step)) 36 | ``` 37 | Example: decay 1/t with a rate of 0.5: 38 | ```python 39 | ... 40 | global_step = tf.Variable(0, trainable=False) 41 | learning_rate = 0.1 42 | decay_steps = 1.0 43 | decay_rate = 0.5 44 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, 45 | decay_steps, decay_rate) 46 | # Passing global_step to minimize() will increment it at each step. 47 | learning_step = ( 48 | tf.train.GradientDescentOptimizer(learning_rate) 49 | .minimize(...my loss..., global_step=global_step) 50 | ) 51 | ``` 52 | Args: 53 | learning_rate: A scalar `float32` or `float64` `Tensor` or a 54 | Python number. The initial learning rate. 55 | global_step: A Python number. 56 | Global step to use for the decay computation. Must not be negative. 57 | decay_steps: How often to apply decay. 58 | decay_rate: A Python number. The decay rate. 59 | staircase: Whether to apply decay in a discrete staircase, as opposed to 60 | continuous, fashion. 61 | name: String. Optional name of the operation. Defaults to 62 | 'InverseTimeDecay'. 63 | Returns: 64 | A scalar `Tensor` of the same type as `learning_rate`. The decayed 65 | learning rate. 66 | Raises: 67 | ValueError: if `global_step` is not supplied. 68 | """ 69 | if global_step is None: 70 | raise ValueError("global_step is required for inverse_time_decay.") 71 | with ops.name_scope(name, "InverseTimeDecay", 72 | [learning_rate, global_step, decay_rate]) as name: 73 | learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") 74 | dtype = learning_rate.dtype 75 | global_step = math_ops.cast(global_step, dtype) 76 | decay_steps = math_ops.cast(decay_steps, dtype) 77 | decay_rate = math_ops.cast(decay_rate, dtype) 78 | power = math_ops.cast(power, dtype) 79 | p = global_step / decay_steps 80 | if staircase: 81 | p = math_ops.floor(p) 82 | const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) 83 | denom = math_ops.pow(math_ops.add(const, math_ops.multiply(decay_rate, p)),power) 84 | 85 | return math_ops.div(learning_rate, denom, name=name) 86 | 87 | -------------------------------------------------------------------------------- /lenet-ip/subsets/1-4/train.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | import math 5 | import numpy as np 6 | 7 | FLAGS = tf.app.flags.FLAGS 8 | 9 | def train(): 10 | model = Model() 11 | 12 | with tf.Graph().as_default(): 13 | # Load training data and use test as evaluation set in one hot format 14 | mnist_ = input_data.read_data_sets("MNIST_data/", one_hot=True) 15 | perm =np.random.permutation(15000) 16 | images = mnist_.train.images[perm] 17 | labels = mnist_.train.labels[perm] 18 | print (images.shape) #[13750,28,28,1] np array 19 | 20 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 21 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 22 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 23 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 24 | global_step = tf.contrib.framework.get_or_create_global_step() 25 | 26 | logits = model.inference(x, keep_prob=keep_prob) 27 | loss = model.loss(logits=logits, labels=y) 28 | 29 | accuracy = model.accuracy(logits, y) 30 | summary_op = tf.summary.merge_all() 31 | train_op = model.train(loss, global_step=global_step) 32 | 33 | batch_size = FLAGS.batch_size #batch size, this might not be correct size 34 | input_size = 15000 35 | porp = int(math.ceil(input_size/batch_size)) 36 | 37 | init = tf.global_variables_initializer() 38 | saver = tf.train.Saver(max_to_keep = 100000) 39 | 40 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 41 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 42 | sess.run(init) 43 | for i in range(FLAGS.num_iter): 44 | if i%(porp) == 0: 45 | permutation=np.random.permutation(input_size) #create a list with random indexes 46 | increment = 0 #restart the increment variable 47 | 48 | batch_idx = permutation[increment*batch_size:(increment+1)*batch_size] 49 | increment += 1 50 | image_batch=images[batch_idx] #this is a list with batch size number of elements. Each element is a (32,32,3) array (images) 51 | label_batch=labels[batch_idx] 52 | 53 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 54 | feed_dict={x_: image_batch, y: label_batch, keep_prob: 0.5}) 55 | 56 | writer.add_summary(summary, i) 57 | if i % 5000 == 0: 58 | f = open('trainingStdDrop.log', 'a+') 59 | validation_accuracy = accuracy.eval(feed_dict={x_: mnist_.test.images, y: mnist_.test.labels, keep_prob: 1.0}) 60 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 61 | f.close() 62 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 63 | 64 | def main(argv=None): 65 | train() 66 | 67 | 68 | if __name__ == '__main__': 69 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 70 | tf.app.flags.DEFINE_integer('num_iter', 1000000, 'number of training iterations') 71 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 72 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 73 | 74 | tf.app.run() 75 | -------------------------------------------------------------------------------- /lenet-ip/train.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | from Model import Model 4 | 5 | FLAGS = tf.app.flags.FLAGS 6 | 7 | def train(): 8 | model = Model() 9 | 10 | with tf.Graph().as_default(): 11 | # Load training data and use test as evaluation set in one hot format 12 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 13 | # images = mnist.train.images # Returns np.array 14 | # labels = np.asarray(mnist.train.labels, dtype=np.int32) 15 | # val_images = mnist.test.images # Returns np.array 16 | # val_labels = np.asarray(mnist.test.labels, dtype=np.int32) 17 | 18 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 19 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 20 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 21 | keep_prob = tf.placeholder(tf.float32, name='dropout_prob') 22 | global_step = tf.contrib.framework.get_or_create_global_step() 23 | 24 | logits = model.inference(x, keep_prob=keep_prob) 25 | loss = model.loss(logits=logits, labels=y) 26 | 27 | accuracy = model.accuracy(logits, y) 28 | summary_op = tf.summary.merge_all() 29 | train_op = model.train(loss, global_step=global_step) 30 | 31 | init = tf.global_variables_initializer() 32 | saver = tf.train.Saver(max_to_keep = None) 33 | 34 | with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: 35 | writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph) 36 | sess.run(init) 37 | for i in range(FLAGS.num_iter): 38 | batch = mnist.train.next_batch(FLAGS.batch_size) 39 | 40 | _, cur_loss, summary = sess.run([train_op, loss, summary_op], 41 | feed_dict={x_: batch[0], y: batch[1], keep_prob: 0.5}) 42 | writer.add_summary(summary, i) 43 | if i % 10000 == 0: 44 | f = open('trainingStdDrop.log', 'a+') 45 | validation_accuracy = accuracy.eval(feed_dict={x_: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0}) 46 | f.write('{}, {}, {} \n'.format(i, cur_loss, validation_accuracy)) 47 | f.close() 48 | saver.save(sess, FLAGS.checkpoint_file_path+"-"+str(i)) 49 | 50 | def main(argv=None): 51 | train() 52 | 53 | 54 | if __name__ == '__main__': 55 | tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches') 56 | tf.app.flags.DEFINE_integer('num_iter', 1000000, 'number of training iterations') 57 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt', 'path to checkpoint file') 58 | tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries') 59 | 60 | tf.app.run() 61 | -------------------------------------------------------------------------------- /lenet-ip/uncertainty.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ["CUDA_VISIBLE_DEVICES"]="-1" # force to CPU-only 3 | import tensorflow as tf 4 | from tensorflow.examples.tutorials.mnist import input_data 5 | from Model import Model 6 | import numpy as np 7 | FLAGS = tf.app.flags.FLAGS 8 | def evaluate(): 9 | with tf.Graph().as_default(): 10 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 11 | x_ = tf.placeholder(tf.float32, shape=[None, 784]) #data gets loaded as a 28x8 vector 12 | x = tf.reshape(x_,[-1,28,28,1],name = 'x') #mnist dataset is shape 28,28,1 13 | y = tf.placeholder(shape=[None, 10], dtype=tf.float32, name='y') #10 labels 14 | softmax_tensors = tf.placeholder(tf.float32, shape=[FLAGS.T, 10000, 10]) 15 | 16 | model = Model() 17 | logits = model.inference(x, keep_prob=FLAGS.prob) 18 | softmax = tf.nn.softmax(logits) 19 | shape = tf.shape(softmax) 20 | saver = tf.train.Saver(max_to_keep = None) 21 | mean = tf.reduce_mean(softmax_tensors,0) 22 | accuracy = model.accuracy(mean,y) 23 | softmax_list = [] 24 | with tf.Session() as sess: 25 | tf.global_variables_initializer().run() 26 | saver.restore(sess, FLAGS.checkpoint_file_path) 27 | for i in range(FLAGS.T): 28 | softmaxi = sess.run([softmax], 29 | feed_dict={x_: mnist.test.images, y: mnist.test.labels}) 30 | softmax_list.append(softmaxi) 31 | #mean_prob = sess.run([mean], feed_dict = {softmax_tensors: np.squeeze(np.array(softmax_list))}) 32 | total_accuracy, soft = sess.run([accuracy,mean], 33 | feed_dict={softmax_tensors: np.squeeze(np.array(softmax_list)), y: mnist.test.labels}) 34 | print('Test accuracy: {}'.format(total_accuracy)) 35 | #np.max(soft.flatten())) 36 | L=np.array([soft.flatten(), mnist.test.labels.flatten()]) 37 | L=np.transpose(L) 38 | #L=L[L[:,0].argsort()] 39 | np.savetxt('lenet-ip_mcdrop_uncertainty.log', L, delimiter=',') 40 | 41 | 42 | 43 | def main(argv=None): 44 | evaluate() 45 | if __name__ == '__main__': 46 | tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-990000', 'path to checkpoint file') 47 | tf.app.flags.DEFINE_integer('T', 50, 'Number of Forward Passes') 48 | tf.app.flags.DEFINE_float('prob', 0.5, 'probability of dropout') 49 | tf.app.run() 50 | -------------------------------------------------------------------------------- /mnist_LeNet.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | tf.logging.set_verbosity(tf.logging.INFO) 9 | 10 | def LeNet_all(features, labels, mode): 11 | #This function defines our CNN structure. 12 | 13 | # Input Layer 14 | input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) #batchsize, width, height, channels. 15 | #reshaping the data to expected shape. -1 just means that we define batchsize as how many inputs we have. (So not defined) 16 | 17 | # Convolutional Layer #1 18 | conv1 = tf.layers.conv2d( 19 | inputs=input_layer, #define the input 20 | filters=20, #number of filters 21 | kernel_size=[5, 5], #size of the filter/kernel 22 | padding="same", #padding set so the output feature maps are the same size as the input feature maps. 23 | activation=tf.nn.relu) #Relu activation function 24 | #output size will be [batch_size, 28,28,20] (since we had 20 filter 25 | dropout1 = tf.layers.dropout( 26 | inputs=conv1, rate=0.5, training=mode == tf.estimator.ModeKeys.TRAIN) #dropout layer 27 | pool1 = tf.layers.max_pooling2d(inputs=dropout1, pool_size=[2, 2], strides=2) #maxpool layer 28 | #output size now [batch size, 14,14,20] after pooling 29 | 30 | # Convolutional Layer #2 and Pooling Layer #2 31 | conv2 = tf.layers.conv2d( 32 | inputs=pool1, 33 | filters=50, 34 | kernel_size=[5, 5], 35 | padding="same", 36 | activation=tf.nn.relu) 37 | #output size: [batch size, 14,14,50] 38 | dropout2 = tf.layers.dropout( 39 | inputs=conv2, rate=0.5, training=mode == tf.estimator.ModeKeys.TRAIN) 40 | pool2 = tf.layers.max_pooling2d(inputs=dropout2, pool_size=[2, 2], strides=2) 41 | #output size [7,7,50] 42 | 43 | # Dense Layer 44 | pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 50]) 45 | dense = tf.layers.dense(inputs=pool2_flat, units=500, activation=tf.nn.relu) #this is our fully connected layer 46 | dropout3 = tf.layers.dropout( 47 | inputs=dense, rate=0.5, training=mode == tf.estimator.ModeKeys.TRAIN) #droput. rate = dropout rate. 48 | #training is a boolean to specify whether we are training or not. Dropout is only performed when training = True. 49 | 50 | # Logits Layer 51 | logits = tf.layers.dense(inputs=dropout3, units=10) #This is our output layer/unscaled probabilities. takes in dropout ([batchsize, 500]) and outputs 52 | # [batch_size, 10]. 53 | 54 | 55 | #create dictionary with our predictions, and then return it as an EstimatorSpec with appropriate information 56 | predictions = { 57 | "classes": tf.argmax(input=logits, axis=1), #class with highest value gets picked. 58 | "probabilities": tf.nn.softmax(logits, name="softmax_tensor") #probability of each class, using softmax 59 | } 60 | if mode == tf.estimator.ModeKeys.PREDICT: 61 | return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) 62 | 63 | #define our loss function. Cross entropy in this case. 64 | loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) 65 | 66 | # Configure the Training Op (for TRAIN mode) 67 | if mode == tf.estimator.ModeKeys.TRAIN: 68 | global_step = tf.Variable(0, trainable=False) #Passing global_step to minimize() will increment it at each step 69 | learning_rate = 0.01 70 | decay_step = 1 71 | decay_rate = 0.0001 #decay rate 72 | learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, decay_step, decay_rate) 73 | optimizer = tf.train.GradientDescentOptimizer(learning_rate) 74 | train_op = optimizer.minimize( 75 | loss=loss, 76 | global_step=tf.train.get_global_step()) 77 | return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) 78 | 79 | # Add evaluation metrics (for EVAL mode) 80 | eval_metric_ops = { 81 | "accuracy": tf.metrics.accuracy( 82 | labels=labels, predictions=predictions["classes"])} 83 | return tf.estimator.EstimatorSpec( 84 | mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) 85 | 86 | #Now having defined the model 87 | 88 | def main(unused_argv): 89 | # Load training and eval data 90 | #-------------------------------------------------------- 91 | mnist = tf.contrib.learn.datasets.load_dataset("mnist") 92 | train_data = mnist.train.images # np.array 93 | train_labels = np.asarray(mnist.train.labels, dtype=np.int32) 94 | eval_data = mnist.test.images # np.array 95 | eval_labels = np.asarray(mnist.test.labels, dtype=np.int32) 96 | #-------------------------------------------------------- 97 | 98 | #Create estimator: 99 | mnist_classifier = tf.estimator.Estimator( 100 | model_fn=LeNet_all, model_dir="mnist_convnet_model/") 101 | # Set up logging for predictions 102 | tensors_to_log = {"probabilities": "softmax_tensor"} 103 | logging_hook = tf.train.LoggingTensorHook( 104 | tensors=tensors_to_log, every_n_iter=50) 105 | #train the model 106 | train_input_fn = tf.estimator.inputs.numpy_input_fn( 107 | x={"x": train_data}, #training features 108 | y=train_labels, #training labels 109 | batch_size=64, 110 | num_epochs=None, #model will train until the specified number of steps is reached. 111 | shuffle=True) #shuffle the data 112 | mnist_classifier.train( 113 | input_fn=train_input_fn, 114 | steps=1000000, #model will train for 20000 steps 115 | hooks=[logging_hook]) #specify the logging hook. 116 | # Evaluate the model and print results 117 | eval_input_fn = tf.estimator.inputs.numpy_input_fn( 118 | x={"x": eval_data}, 119 | y=eval_labels, 120 | num_epochs=1, #since we only do 1 forward run. Change this is MC dropout I think. 121 | shuffle=False) 122 | eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) 123 | print(eval_results) 124 | 125 | if __name__ == "__main__": 126 | tf.app.run() 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | --------------------------------------------------------------------------------