├── .gitignore ├── LICENSE.txt ├── NumNum ├── load_data.py ├── model_trial_1.ckpt ├── model_trial_1.py ├── model_trial_1_command_results.txt ├── model_trial_2.py ├── model_trial_2_command_results.txt ├── report │ ├── report.pdf │ └── report_raw_do_not_open.zip └── ritchieng_report.docx └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /NumNum/logs 3 | /NumNum/SVHN.pickle 4 | /NumNum/train.tar.gz 5 | /NumNum/test.tar.gz 6 | /NumNum/extra.tar.gz 7 | /NumNum/train 8 | /NumNum/test 9 | /NumNum/extra 10 | /NumNum/log_trial_1 11 | /NumNum/log_trial_2 12 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Ritchie Ng and Contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /NumNum/load_data.py: -------------------------------------------------------------------------------- 1 | # Import Modules 2 | from __future__ import print_function 3 | from six.moves import cPickle as pickle 4 | from six.moves import range 5 | from six.moves.urllib.request import urlretrieve 6 | from scipy import ndimage 7 | from PIL import Image 8 | import numpy as np 9 | import os 10 | import sys 11 | import tarfile 12 | import h5py 13 | from numpy import random 14 | 15 | # Download data 16 | print('Downloading data...') 17 | 18 | url = 'http://ufldl.stanford.edu/housenumbers/' 19 | 20 | def maybe_download(filename, force=False): 21 | """Download a file if not present, and make sure it's the right size.""" 22 | if force or not os.path.exists(filename): 23 | print('Attempting to download:', filename) 24 | filename, _ = urlretrieve(url + filename, filename) 25 | print('Download Complete!') 26 | statinfo = os.stat(filename) 27 | return filename 28 | 29 | train_filename = maybe_download('train.tar.gz') 30 | test_filename = maybe_download('test.tar.gz') 31 | extra_filename = maybe_download('extra.tar.gz') 32 | 33 | print('Successfully downloaded data!') 34 | 35 | 36 | # Unzip Data 37 | print('Unzipping data...') 38 | np.random.seed(8) 39 | 40 | def maybe_extract(filename, force=False): 41 | # Remove .tar.gz 42 | root = os.path.splitext(os.path.splitext(filename)[0])[0] 43 | if os.path.isdir(root) and not force: 44 | # You may override by setting force=True. 45 | print('%s already present - Skipping extraction of %s.' % (root, filename)) 46 | else: 47 | print('Extracting data for %s. This may take a while. Please wait.' % root) 48 | tar = tarfile.open(filename) 49 | sys.stdout.flush() 50 | tar.extractall() 51 | tar.close() 52 | data_folders = root 53 | print(data_folders) 54 | return data_folders 55 | 56 | train_folders = maybe_extract(train_filename) 57 | test_folders = maybe_extract(test_filename) 58 | extra_folders = maybe_extract(extra_filename) 59 | 60 | print('Successfully unzipped data!') 61 | 62 | # Create dictionary for bounding boxes 63 | print('Creating dictionary of bounding boxes...') 64 | class DigitStructFile: 65 | def __init__(self, inf): 66 | self.inf = h5py.File(inf, 'r') 67 | self.digitStructName = self.inf['digitStruct']['name'] 68 | self.digitStructBbox = self.inf['digitStruct']['bbox'] 69 | 70 | def getName(self,n): 71 | return ''.join([chr(c[0]) for c in self.inf[self.digitStructName[n][0]].value]) 72 | 73 | def bboxHelper(self,attr): 74 | if (len(attr) > 1): 75 | attr = [self.inf[attr.value[j].item()].value[0][0] for j in range(len(attr))] 76 | else: 77 | attr = [attr.value[0][0]] 78 | return attr 79 | 80 | def getBbox(self,n): 81 | bbox = {} 82 | bb = self.digitStructBbox[n].item() 83 | bbox['height'] = self.bboxHelper(self.inf[bb]["height"]) 84 | bbox['label'] = self.bboxHelper(self.inf[bb]["label"]) 85 | bbox['left'] = self.bboxHelper(self.inf[bb]["left"]) 86 | bbox['top'] = self.bboxHelper(self.inf[bb]["top"]) 87 | bbox['width'] = self.bboxHelper(self.inf[bb]["width"]) 88 | return bbox 89 | 90 | def getDigitStructure(self,n): 91 | s = self.getBbox(n) 92 | s['name']=self.getName(n) 93 | return s 94 | 95 | def getAllDigitStructure(self): 96 | return [self.getDigitStructure(i) for i in range(len(self.digitStructName))] 97 | 98 | def getAllDigitStructure_ByDigit(self): 99 | pictDat = self.getAllDigitStructure() 100 | result = [] 101 | structCnt = 1 102 | for i in range(len(pictDat)): 103 | item = { 'filename' : pictDat[i]["name"] } 104 | figures = [] 105 | for j in range(len(pictDat[i]['height'])): 106 | figure = {} 107 | figure['height'] = pictDat[i]['height'][j] 108 | figure['label'] = pictDat[i]['label'][j] 109 | figure['left'] = pictDat[i]['left'][j] 110 | figure['top'] = pictDat[i]['top'][j] 111 | figure['width'] = pictDat[i]['width'][j] 112 | figures.append(figure) 113 | structCnt = structCnt + 1 114 | item['boxes'] = figures 115 | result.append(item) 116 | return result 117 | 118 | print("Successfully created dictionary of bounding boxes!") 119 | 120 | 121 | # Get Digit Structure 122 | print('Getting digit structure for training data...') 123 | digitFileTrain=DigitStructFile(os.path.join('train','digitStruct.mat')) 124 | train_data=digitFileTrain.getAllDigitStructure_ByDigit() 125 | print('Success!') 126 | 127 | print('Getting digit structure for test data...') 128 | digitFileTest=DigitStructFile(os.path.join('test','digitStruct.mat')) 129 | test_data=digitFileTest.getAllDigitStructure_ByDigit() 130 | print('Success!') 131 | 132 | 133 | print('Getting digit structure for extra data...') 134 | digitFileExtra=DigitStructFile(os.path.join('extra','digitStruct.mat')) 135 | extra_data=digitFileExtra.getAllDigitStructure_ByDigit() 136 | print('Success!') 137 | 138 | # Crop Training Images 139 | print('Cropping training images...') 140 | train_imsize = np.ndarray([len(train_data),2]) 141 | for i in np.arange(len(train_data)): 142 | filename = train_data[i]['filename'] 143 | fullname = os.path.join(train_folders, filename) 144 | im = Image.open(fullname) 145 | train_imsize[i, :] = im.size[:] 146 | 147 | print('Success!') 148 | 149 | # Crop Test Images 150 | print('Cropping test images...') 151 | test_imsize = np.ndarray([len(test_data),2]) 152 | for i in np.arange(len(test_data)): 153 | filename = test_data[i]['filename'] 154 | fullname = os.path.join(test_folders, filename) 155 | im = Image.open(fullname) 156 | test_imsize[i, :] = im.size[:] 157 | 158 | print('Success!') 159 | 160 | # Crop Extra Images 161 | print('Cropping extra images...') 162 | extra_imsize = np.ndarray([len(extra_data),2]) 163 | for i in np.arange(len(extra_data)): 164 | filename = extra_data[i]['filename'] 165 | fullname = os.path.join(extra_folders, filename) 166 | im = Image.open(fullname) 167 | extra_imsize[i, :] = im.size[:] 168 | 169 | print('Success!') 170 | 171 | # Use extra data 172 | def generate_dataset(data, folder): 173 | 174 | dataset = np.ndarray([len(data),32,32,1], dtype='float32') 175 | labels = np.ones([len(data),6], dtype=int) * 10 176 | for i in np.arange(len(data)): 177 | filename = data[i]['filename'] 178 | fullname = os.path.join(folder, filename) 179 | im = Image.open(fullname) 180 | boxes = data[i]['boxes'] 181 | num_digit = len(boxes) 182 | labels[i,0] = num_digit 183 | top = np.ndarray([num_digit], dtype='float32') 184 | left = np.ndarray([num_digit], dtype='float32') 185 | height = np.ndarray([num_digit], dtype='float32') 186 | width = np.ndarray([num_digit], dtype='float32') 187 | for j in np.arange(num_digit): 188 | if j < 5: 189 | labels[i,j+1] = boxes[j]['label'] 190 | if boxes[j]['label'] == 10: labels[i,j+1] = 0 191 | else: print('#',i,'image has more than 5 digits.') 192 | top[j] = boxes[j]['top'] 193 | left[j] = boxes[j]['left'] 194 | height[j] = boxes[j]['height'] 195 | width[j] = boxes[j]['width'] 196 | 197 | im_top = np.amin(top) 198 | im_left = np.amin(left) 199 | im_height = np.amax(top) + height[np.argmax(top)] - im_top 200 | im_width = np.amax(left) + width[np.argmax(left)] - im_left 201 | 202 | im_top = np.floor(im_top - 0.1 * im_height) 203 | im_left = np.floor(im_left - 0.1 * im_width) 204 | im_bottom = np.amin([np.ceil(im_top + 1.2 * im_height), im.size[1]]) 205 | im_right = np.amin([np.ceil(im_left + 1.2 * im_width), im.size[0]]) 206 | 207 | im = im.crop((im_left, im_top, im_right, im_bottom)).resize([32,32], Image.ANTIALIAS) 208 | im = np.dot(np.array(im, dtype='float32'), [[0.2989],[0.5870],[0.1140]]) 209 | mean = np.mean(im, dtype='float32') 210 | std = np.std(im, dtype='float32', ddof=1) 211 | if std < 1e-4: std = 1. 212 | im = (im - mean) / std 213 | dataset[i,:,:,:] = im[:,:,:] 214 | 215 | return dataset, labels 216 | 217 | print('Generating training dataset and labels...') 218 | train_dataset, train_labels = generate_dataset(train_data, train_folders) 219 | print('Success! \n Training set: {} \n Training labels: {}'.format(train_dataset.shape, train_labels.shape)) 220 | 221 | 222 | print('Generating testing dataset and labels...') 223 | test_dataset, test_labels = generate_dataset(test_data, test_folders) 224 | print('Success! \n Testing set: {} \n Testing labels: {}'.format(test_dataset.shape, test_labels.shape)) 225 | 226 | print('Generating extra dataset and labels...') 227 | extra_dataset, extra_labels = generate_dataset(extra_data, extra_folders) 228 | print('Success! \n Testing set: {} \n Testing labels: {}'.format(extra_dataset.shape, extra_labels.shape)) 229 | 230 | 231 | # Clean up data by deleting digits more than 5 (very few) 232 | print('Cleaning up training data...') 233 | train_dataset = np.delete(train_dataset, 29929, axis=0) 234 | train_labels = np.delete(train_labels, 29929, axis=0) 235 | print('Success!') 236 | 237 | # Expand Training Data 238 | print('Expanding training data randomly...') 239 | 240 | random.seed(8) 241 | 242 | n_labels = 10 243 | valid_index = [] 244 | valid_index2 = [] 245 | train_index = [] 246 | train_index2 = [] 247 | for i in np.arange(n_labels): 248 | valid_index.extend(np.where(train_labels[:,1] == (i))[0][:400].tolist()) 249 | train_index.extend(np.where(train_labels[:,1] == (i))[0][400:].tolist()) 250 | valid_index2.extend(np.where(extra_labels[:,1] == (i))[0][:200].tolist()) 251 | train_index2.extend(np.where(extra_labels[:,1] == (i))[0][200:].tolist()) 252 | 253 | random.shuffle(valid_index) 254 | random.shuffle(train_index) 255 | random.shuffle(valid_index2) 256 | random.shuffle(train_index2) 257 | 258 | valid_dataset = np.concatenate((extra_dataset[valid_index2,:,:,:], train_dataset[valid_index,:,:,:]), axis=0) 259 | valid_labels = np.concatenate((extra_labels[valid_index2,:], train_labels[valid_index,:]), axis=0) 260 | train_dataset_new = np.concatenate((extra_dataset[train_index2,:,:,:], train_dataset[train_index,:,:,:]), axis=0) 261 | train_labels_new = np.concatenate((extra_labels[train_index2,:], train_labels[train_index,:]), axis=0) 262 | 263 | print('Success! \n Training set: {} \n Training labels: {}'.format(train_dataset_new.shape, train_labels_new.shape)) 264 | print('Success! \n Validation set: {} \n Validation labels: {}'.format(valid_dataset.shape, valid_labels.shape)) 265 | print('Success! \n Testing set: {} \n Testing labels: {}'.format(test_dataset.shape, test_labels.shape)) 266 | 267 | 268 | # Create Pickling File 269 | print('Pickling data...') 270 | pickle_file = 'SVHN.pickle' 271 | 272 | try: 273 | f = open(pickle_file, 'wb') 274 | save = { 275 | 'train_dataset': train_dataset_new, 276 | 'train_labels': train_labels_new, 277 | 'valid_dataset': valid_dataset, 278 | 'valid_labels': valid_labels, 279 | 'test_dataset': test_dataset, 280 | 'test_labels': test_labels, 281 | } 282 | pickle.dump(save, f, pickle.HIGHEST_PROTOCOL) 283 | f.close() 284 | except Exception as e: 285 | print('Unable to save data to {}: {}'.format(pickle_file, e)) 286 | raise 287 | 288 | statinfo = os.stat(pickle_file) 289 | print('Success!') 290 | print('Compressed pickle size: {}'.format(statinfo.st_size)) 291 | -------------------------------------------------------------------------------- /NumNum/model_trial_1.ckpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ritchieng/NumNum/97851ccb74610c414bf474016915173b8abfe913/NumNum/model_trial_1.ckpt -------------------------------------------------------------------------------- /NumNum/model_trial_1.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tarfile 4 | import pickle 5 | import matplotlib.pyplot as plt 6 | import numpy as np, h5py 7 | import pandas as pd 8 | from PIL import Image 9 | import tensorflow as tf 10 | 11 | 12 | print('Loading pickled data...') 13 | 14 | pickle_file = 'SVHN.pickle' 15 | 16 | with open(pickle_file, 'rb') as f: 17 | save = pickle.load(f) 18 | X_train = save['train_dataset'] 19 | y_train = save['train_labels'] 20 | X_val = save['valid_dataset'] 21 | y_val = save['valid_labels'] 22 | X_test = save['test_dataset'] 23 | y_test = save['test_labels'] 24 | del save 25 | print('Training data shape:', X_train.shape) 26 | print('Training label shape:',y_train.shape) 27 | print('Validation data shape:', X_val.shape) 28 | print('Validation label shape:', y_val.shape) 29 | print('Test data shape:', X_test.shape) 30 | print('Test label shape:', y_test.shape) 31 | 32 | print('Data successfully loaded!') 33 | 34 | 35 | 36 | print('Defining accuracy function...') 37 | def accuracy(predictions, labels): 38 | return (100.0 * np.sum(np.argmax(predictions, 2).T == labels) 39 | / predictions.shape[1] / predictions.shape[0]) 40 | print('Accuracy function defined!') 41 | 42 | # CNN Model 43 | print('Loading data and building computation graph...') 44 | 45 | '''Basic information''' 46 | # We processed image size to be 32 47 | image_size = 32 48 | # Number of channels: 1 because greyscale 49 | num_channels = 1 50 | # Mini-batch size 51 | batch_size = 16 52 | # Number of output labels 53 | num_labels = 11 54 | 55 | '''Filters''' 56 | # depth: number of filters (output channels) - should be increasing 57 | # num_channels: number of input channels set at 1 previously 58 | patch_size = 5 59 | depth_1 = 16 60 | depth_2 = depth_1 * 2 61 | depth_3 = depth_2 * 3 62 | 63 | # Number of hidden nodes in fully connected layer 1 64 | num_hidden = 64 65 | shape = [batch_size, image_size, image_size, num_channels] 66 | 67 | graph = tf.Graph() 68 | 69 | with graph.as_default(): 70 | 71 | '''Input Data''' 72 | # X_train: (223965, 32, 32, 1) 73 | tf_train_dataset = tf.placeholder( 74 | tf.float32, shape=(batch_size, image_size, image_size, num_channels)) 75 | 76 | # y_train: (223965, 7) 77 | tf_train_labels = tf.placeholder( 78 | tf.int32, shape=(batch_size, 6)) 79 | 80 | # X_val: (11788, 32, 32, 1) 81 | tf_valid_dataset = tf.constant(X_val) 82 | 83 | # X_test: (13067, 32, 32, 1) 84 | tf_test_dataset = tf.constant(X_test) 85 | 86 | '''Variables''' 87 | 88 | # Create Variables Function 89 | def init_weights(shape, name): 90 | return tf.Variable( 91 | tf.random_normal(shape=shape, stddev=0.01), 92 | name=name) 93 | 94 | def init_biases(shape, name): 95 | return tf.Variable( 96 | tf.constant(1.0, shape=shape), 97 | name=name 98 | ) 99 | 100 | # Create Function for Image Size: Pooling 101 | # 3 Convolutions 102 | # 2 Max Pooling 103 | def output_size_pool(input_size, conv_filter_size, pool_filter_size, 104 | padding, conv_stride, pool_stride): 105 | if padding == 'same': 106 | padding = -1.00 107 | elif padding == 'valid': 108 | padding = 0.00 109 | else: 110 | return None 111 | # After convolution 1 112 | output_1 = ( 113 | ((input_size - conv_filter_size - 2 * padding) / conv_stride) + 1.00) 114 | # After pool 1 115 | output_2 = ( 116 | ((output_1 - pool_filter_size - 2 * padding) / pool_stride) + 1.00) 117 | # After convolution 2 118 | output_3 = ( 119 | ((output_2 - conv_filter_size - 2 * padding) / conv_stride) + 1.00) 120 | # After pool 2 121 | output_4 = ( 122 | ((output_3 - pool_filter_size - 2 * padding) / pool_stride) + 1.00) 123 | # After convolution 2 124 | output_5 = ( 125 | ((output_4 - conv_filter_size - 2 * padding) / conv_stride) + 1.00) 126 | # After pool 2 127 | # output_6 = ( 128 | # ((output_5 - pool_filter_size - 2 * padding) / pool_stride) + 1.00) 129 | return int(output_5) 130 | 131 | # Convolution 1 132 | # Input channels: num_channels = 1 133 | # Output channels: depth = depth_1 134 | w_c1 = init_weights([patch_size, patch_size, num_channels, depth_1], 'w_c1') 135 | b_c1 = init_biases([depth_1], 'b_c1') 136 | 137 | # Convolution 2 138 | # Input channels: num_channels = depth_1 139 | # Output channels: depth = depth_2 140 | w_c2 = init_weights([patch_size, patch_size, depth_1, depth_2], 'w_c2') 141 | b_c2 = init_biases([depth_2], 'b_c2') 142 | 143 | # Convolution 3 144 | # Input channels: num_channels = depth_2 145 | # Output channels: depth = depth_3 146 | w_c3 = init_weights([patch_size, patch_size, depth_2, depth_3], 'w_c3') 147 | b_c3 = init_biases([depth_3], 'b_c3') 148 | 149 | # Fully Connect Layer 1 150 | final_image_size = output_size_pool(input_size=image_size, 151 | conv_filter_size=5, pool_filter_size=2, 152 | padding='valid', conv_stride=1, 153 | pool_stride=2) 154 | print('Final image size after convolutions {}'.format(final_image_size)) 155 | w_fc1 = init_weights([final_image_size*final_image_size*depth_3, num_hidden], 'w_fc1') 156 | b_fc1 = init_biases([num_hidden], 'b_fc1') 157 | 158 | # Softmax 1 159 | w_s1 = init_weights([num_hidden, num_labels], 'w_s1') 160 | b_s1 = init_biases([num_labels], 'b_s1') 161 | 162 | # Softmax 2 163 | w_s2 = init_weights([num_hidden, num_labels], 'w_s2') 164 | b_s2 = init_biases([num_labels], 'b_s2') 165 | 166 | # Softmax 3 167 | w_s3 = init_weights([num_hidden, num_labels], 'w_s3') 168 | b_s3 = init_biases([num_labels], 'b_s3') 169 | 170 | # Softmax 4 171 | w_s4 = init_weights([num_hidden, num_labels], 'w_s4') 172 | b_s4 = init_biases([num_labels], 'b_s4') 173 | 174 | # Softmax 5 175 | w_s5 = init_weights([num_hidden, num_labels], 'w_s5') 176 | b_s5 = init_biases([num_labels], 'b_s5') 177 | 178 | def model(data, keep_prob, shape): 179 | with tf.name_scope("conv_layer_1"): 180 | conv_1 = tf.nn.conv2d( 181 | data, w_c1, strides=[1, 1, 1, 1], padding='VALID') 182 | hidden_conv_1 = tf.nn.relu(conv_1 + b_c1) 183 | pool_1 = tf.nn.max_pool( 184 | hidden_conv_1, [1, 2, 2, 1], [1, 2, 2, 1], padding='VALID') 185 | with tf.name_scope("conv_layer_2"): 186 | conv_2 = tf.nn.conv2d( 187 | pool_1, w_c2, strides=[1, 1, 1, 1], padding='VALID') 188 | hidden_conv_2 = tf.nn.relu(conv_2 + b_c2) 189 | pool_2 = tf.nn.max_pool( 190 | hidden_conv_2, [1, 2, 2, 1], [1, 2, 2, 1], padding='VALID') 191 | with tf.name_scope("conv_layer_3"): 192 | conv_3 = tf.nn.conv2d( 193 | pool_2, w_c3, strides=[1, 1, 1, 1], padding='VALID') 194 | hidden_conv_3 = tf.nn.relu(conv_3 + b_c3) 195 | with tf.name_scope("fc_layer_1"): 196 | hidden_drop = tf.nn.dropout(hidden_conv_3, keep_prob) 197 | shape = hidden_drop.get_shape().as_list() 198 | reshape = tf.reshape( 199 | hidden_drop, [shape[0], shape[1] * shape[2] * shape[3]]) 200 | hidden_fc = tf.nn.relu( 201 | tf.matmul(reshape, w_fc1) + b_fc1) 202 | with tf.name_scope("softmax_1"): 203 | logits_1 = tf.matmul(hidden_fc, w_s1) + b_s1 204 | with tf.name_scope("softmax_2"): 205 | logits_2 = tf.matmul(hidden_fc, w_s2) + b_s2 206 | with tf.name_scope("softmax_3"): 207 | logits_3 = tf.matmul(hidden_fc, w_s3) + b_s3 208 | with tf.name_scope("softmax_4"): 209 | logits_4 = tf.matmul(hidden_fc, w_s4) + b_s4 210 | with tf.name_scope("softmax_5"): 211 | logits_5 = tf.matmul(hidden_fc, w_s5) + b_s5 212 | return [logits_1, logits_2, logits_3, logits_4, logits_5] 213 | 214 | '''Training Computation''' 215 | [logits_1, logits_2, logits_3, logits_4, logits_5] = model( 216 | tf_train_dataset, 0.5, shape) 217 | 218 | '''Loss Function''' 219 | with tf.name_scope("loss"): 220 | loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 221 | logits_1, tf_train_labels[:, 1])) + \ 222 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 223 | logits_2, tf_train_labels[:, 2])) + \ 224 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 225 | logits_3, tf_train_labels[:, 3])) + \ 226 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 227 | logits_4, tf_train_labels[:, 4])) + \ 228 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 229 | logits_5, tf_train_labels[:, 5])) 230 | # Add scalar summary for cost 231 | tf.scalar_summary("loss", loss) 232 | 233 | '''Optimizer''' 234 | # Decaying learning rate 235 | # count the number of steps taken 236 | global_step = tf.Variable(0) 237 | start_learning_rate = 0.05 238 | learning_rate = tf.train.exponential_decay( 239 | start_learning_rate, global_step, 100000, 0.96, staircase=True) 240 | 241 | optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize( 242 | loss, global_step=global_step) 243 | 244 | '''Predictions''' 245 | def softmax_combine(dataset, shape): 246 | train_prediction = tf.pack([ 247 | tf.nn.softmax(model(dataset, 1.0, shape)[0]), 248 | tf.nn.softmax(model(dataset, 1.0, shape)[1]), 249 | tf.nn.softmax(model(dataset, 1.0, shape)[2]), 250 | tf.nn.softmax(model(dataset, 1.0, shape)[3]), 251 | tf.nn.softmax(model(dataset, 1.0, shape)[4])]) 252 | return train_prediction 253 | 254 | train_prediction = softmax_combine(tf_train_dataset, shape) 255 | valid_prediction = softmax_combine(tf_valid_dataset, shape) 256 | test_prediction = softmax_combine(tf_test_dataset, shape) 257 | 258 | '''Save Model (will be initiated later)''' 259 | saver = tf.train.Saver() 260 | 261 | '''Histogram for Weights''' 262 | # Add histogram summaries for weights 263 | tf.histogram_summary("w_c1_summ", w_c1) 264 | tf.histogram_summary("b_c1_summ", b_c1) 265 | 266 | tf.histogram_summary("w_c2_summ", w_c2) 267 | tf.histogram_summary("b_c2_summ", b_c2) 268 | 269 | tf.histogram_summary("w_c3_summ", w_c3) 270 | tf.histogram_summary("b_c3_summ", b_c3) 271 | 272 | tf.histogram_summary("w_fc1_summ", w_fc1) 273 | tf.histogram_summary("b_fc1_summ", b_fc1) 274 | 275 | tf.histogram_summary("w_s1_summ", w_s1) 276 | tf.histogram_summary("b_s1_summ", b_s1) 277 | 278 | tf.histogram_summary("w_s2_summ", w_s2) 279 | tf.histogram_summary("b_s2_summ", b_s2) 280 | 281 | tf.histogram_summary("w_s3_summ", w_s3) 282 | tf.histogram_summary("b_s3_summ", b_s3) 283 | 284 | tf.histogram_summary("w_s4_summ", w_s4) 285 | tf.histogram_summary("b_s4_summ", b_s4) 286 | 287 | tf.histogram_summary("w_s5_summ", w_s5) 288 | tf.histogram_summary("b_s5_summ", b_s5) 289 | 290 | print('Data loaded and computation graph built!') 291 | 292 | num_steps = 60000 293 | 294 | print('Running computation and iteration...') 295 | print('If you are unable to save the summary, please change the path to where you want it to write.') 296 | 297 | with tf.Session(graph=graph) as session: 298 | writer = tf.train.SummaryWriter("/log_trial_1", session.graph) # for 0.8 299 | merged = tf.merge_all_summaries() 300 | 301 | '''If you want to restore model''' 302 | # saver.restore(session, "model_trial_1.ckpt") 303 | # print("Model restored!") 304 | 305 | tf.initialize_all_variables().run() 306 | print('Initialized') 307 | for step in range(num_steps): 308 | offset = (step * batch_size) % (y_train.shape[0] - batch_size) 309 | batch_data = X_train[offset:(offset + batch_size), :, :, :] 310 | batch_labels = y_train[offset:(offset + batch_size), :] 311 | feed_dict = {tf_train_dataset: batch_data, 312 | tf_train_labels: batch_labels} 313 | _, l, predictions, summary = session.run([optimizer, loss, train_prediction, merged], 314 | feed_dict=feed_dict) 315 | writer.add_summary(summary) 316 | if (step % 500 == 0): 317 | print(('Minibatch loss at step {}: {}').format(step, l)) 318 | print( 319 | ('Minibatch accuracy: {}%'.format(accuracy(predictions, batch_labels[:,1:6])))) 320 | print( 321 | ('Validation accuracy: {}%'.format(accuracy(valid_prediction.eval(), 322 | y_val[:,1:6])))) 323 | print( 324 | ('Test accuracy: {}%'.format(accuracy(test_prediction.eval(), y_test[:,1:6])))) 325 | 326 | save_path = saver.save(session, "model_trial_1.ckpt") 327 | print('Model saved in file: {}'.format(save_path)) 328 | 329 | 330 | print('Successfully completed computation and iterations!') 331 | 332 | print('To view Tensorboard\'s visualizations, please run \ 333 | \'tensorboard --logdir=log_trial_1\' in your terminal') 334 | 335 | 336 | -------------------------------------------------------------------------------- /NumNum/model_trial_1_command_results.txt: -------------------------------------------------------------------------------- 1 | ubuntu@ip-172-31-54-80:/mnt/deep_learning$ python model_trial_1.py 2 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcublas.so locally 3 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcudnn.so locally 4 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcufft.so locally 5 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcuda.so locally 6 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcurand.so locally 7 | /home/ubuntu/miniconda/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20. 8 | "This module will be removed in 0.20.", DeprecationWarning) 9 | Loading pickled data... 10 | ('Training data shape:', (230070, 32, 32, 1)) 11 | ('Training label shape:', (230070, 6)) 12 | ('Validation data shape:', (5684, 32, 32, 1)) 13 | ('Validation label shape:', (5684, 6)) 14 | ('Test data shape:', (13068, 32, 32, 1)) 15 | ('Test label shape:', (13068, 6)) 16 | Data successfully loaded! 17 | Defining accuracy function... 18 | Accuracy function defined! 19 | Loading data and building computation graph... 20 | Final image size after convolutions 1 21 | Data loaded and computation graph built! 22 | Running computation and iteration... 23 | I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:925] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 24 | I tensorflow/core/common_runtime/gpu/gpu_init.cc:102] Found device 0 with properties: 25 | name: GRID K520 26 | major: 3 minor: 0 memoryClockRate (GHz) 0.797 27 | pciBusID 0000:00:03.0 28 | Total memory: 4.00GiB 29 | Free memory: 3.95GiB 30 | I tensorflow/core/common_runtime/gpu/gpu_init.cc:126] DMA: 0 31 | I tensorflow/core/common_runtime/gpu/gpu_init.cc:136] 0: Y 32 | I tensorflow/core/common_runtime/gpu/gpu_device.cc:838] Creating TensorFlow device (/gpu:0) -> (device: 0, name: GRID K520, pci bus id: 0000:00:03.0) 33 | Initialized 34 | Minibatch loss at step 0: 12.0505943298 35 | Minibatch accuracy: 1.25% 36 | Validation accuracy: 57.248416608% 37 | Minibatch loss at step 500: 6.88450098038 38 | Minibatch accuracy: 56.25% 39 | Validation accuracy: 57.4700914849% 40 | Minibatch loss at step 1000: 6.68981122971 41 | Minibatch accuracy: 53.75% 42 | Validation accuracy: 57.4700914849% 43 | Minibatch loss at step 1500: 5.83104801178 44 | Minibatch accuracy: 65.0% 45 | Validation accuracy: 56.3828289937% 46 | Minibatch loss at step 2000: 5.84136390686 47 | Minibatch accuracy: 61.25% 48 | Validation accuracy: 60.5700211119% 49 | Minibatch loss at step 2500: 6.04740285873 50 | Minibatch accuracy: 57.5% 51 | Validation accuracy: 62.2097114708% 52 | Minibatch loss at step 3000: 5.89772129059 53 | Minibatch accuracy: 63.75% 54 | Validation accuracy: 64.8733286418% 55 | Minibatch loss at step 3500: 5.3234629631 56 | Minibatch accuracy: 68.75% 57 | Validation accuracy: 69.3314567206% 58 | Minibatch loss at step 4000: 5.03387022018 59 | Minibatch accuracy: 67.5% 60 | Validation accuracy: 71.2948627727% 61 | Minibatch loss at step 4500: 3.92141985893 62 | Minibatch accuracy: 72.5% 63 | Validation accuracy: 74.9366643209% 64 | Minibatch loss at step 5000: 4.70345878601 65 | Minibatch accuracy: 76.25% 66 | Validation accuracy: 76.5904292752% 67 | Minibatch loss at step 5500: 4.18564510345 68 | Minibatch accuracy: 77.5% 69 | Validation accuracy: 78.8705137227% 70 | Minibatch loss at step 6000: 2.89691853523 71 | Minibatch accuracy: 80.0% 72 | Validation accuracy: 76.9528501056% 73 | Minibatch loss at step 6500: 4.4373884201 74 | Minibatch accuracy: 77.5% 75 | Validation accuracy: 79.6094299789% 76 | Minibatch loss at step 7000: 4.27867364883 77 | Minibatch accuracy: 76.25% 78 | Validation accuracy: 80.8268824771% 79 | Minibatch loss at step 7500: 3.14099359512 80 | Minibatch accuracy: 86.25% 81 | Validation accuracy: 80.9254046446% 82 | Minibatch loss at step 8000: 2.99073219299 83 | Minibatch accuracy: 85.0% 84 | Validation accuracy: 82.3117522871% 85 | Minibatch loss at step 8500: 3.87283110619 86 | Minibatch accuracy: 83.75% 87 | Validation accuracy: 82.902885292% 88 | Minibatch loss at step 9000: 2.27327084541 89 | Minibatch accuracy: 88.75% 90 | Validation accuracy: 84.4229415904% 91 | Minibatch loss at step 9500: 3.23587727547 92 | Minibatch accuracy: 86.25% 93 | Validation accuracy: 83.645320197% 94 | Minibatch loss at step 10000: 2.83368802071 95 | Minibatch accuracy: 83.75% 96 | Validation accuracy: 83.2547501759% 97 | Minibatch loss at step 10500: 2.37021636963 98 | Minibatch accuracy: 87.5% 99 | Validation accuracy: 83.6945812808% 100 | Minibatch loss at step 11000: 2.77257108688 101 | Minibatch accuracy: 87.5% 102 | Validation accuracy: 83.2688247713% 103 | Minibatch loss at step 11500: 3.16635322571 104 | Minibatch accuracy: 78.75% 105 | Validation accuracy: 84.3033075299% 106 | Minibatch loss at step 12000: 2.74341273308 107 | Minibatch accuracy: 81.25% 108 | Validation accuracy: 85.4714989444% 109 | Minibatch loss at step 12500: 2.43959283829 110 | Minibatch accuracy: 88.75% 111 | Validation accuracy: 84.5531315975% 112 | Minibatch loss at step 13000: 3.70430445671 113 | Minibatch accuracy: 76.25% 114 | Validation accuracy: 84.8346235046% 115 | Minibatch loss at step 13500: 2.03159546852 116 | Minibatch accuracy: 87.5% 117 | Validation accuracy: 85.1724137931% 118 | Minibatch loss at step 14000: 4.17940807343 119 | Minibatch accuracy: 76.25% 120 | Validation accuracy: 85.8479943702% 121 | Minibatch loss at step 14500: 4.05599737167 122 | Minibatch accuracy: 82.5% 123 | Validation accuracy: 83.8142153413% 124 | Minibatch loss at step 15000: 2.02430272102 125 | Minibatch accuracy: 90.0% 126 | Validation accuracy: 86.1893033075% 127 | Minibatch loss at step 15500: 2.76390194893 128 | Minibatch accuracy: 88.75% 129 | Validation accuracy: 85.6755805771% 130 | Minibatch loss at step 16000: 2.317735672 131 | Minibatch accuracy: 95.0% 132 | Validation accuracy: 86.6080225194% 133 | Minibatch loss at step 16500: 3.12809824944 134 | Minibatch accuracy: 81.25% 135 | Validation accuracy: 82.227304715% 136 | Minibatch loss at step 17000: 2.13054251671 137 | Minibatch accuracy: 91.25% 138 | Validation accuracy: 86.4180154821% 139 | Minibatch loss at step 17500: 3.64493107796 140 | Minibatch accuracy: 77.5% 141 | Validation accuracy: 85.3870513723% 142 | Minibatch loss at step 18000: 3.49307441711 143 | Minibatch accuracy: 85.0% 144 | Validation accuracy: 86.4461646728% 145 | Minibatch loss at step 18500: 2.58382773399 146 | Minibatch accuracy: 92.5% 147 | Validation accuracy: 86.2983814215% 148 | Minibatch loss at step 19000: 2.25456190109 149 | Minibatch accuracy: 92.5% 150 | Validation accuracy: 85.0562983814% 151 | Minibatch loss at step 19500: 3.96861839294 152 | Minibatch accuracy: 85.0% 153 | Validation accuracy: 86.0063335679% 154 | Minibatch loss at step 20000: 3.81250452995 155 | Minibatch accuracy: 86.25% 156 | Validation accuracy: 85.5137227305% 157 | Minibatch loss at step 20500: 3.51889204979 158 | Minibatch accuracy: 86.25% 159 | Validation accuracy: 85.5559465165% 160 | Minibatch loss at step 21000: 3.24685740471 161 | Minibatch accuracy: 90.0% 162 | Validation accuracy: 85.6720619282% 163 | Minibatch loss at step 21500: 2.68059849739 164 | Minibatch accuracy: 91.25% 165 | Validation accuracy: 86.2209711471% 166 | Minibatch loss at step 22000: 3.06141424179 167 | Minibatch accuracy: 87.5% 168 | Validation accuracy: 85.9007741027% 169 | Minibatch loss at step 22500: 2.09622073174 170 | Minibatch accuracy: 92.5% 171 | Validation accuracy: 85.4222378607% 172 | Minibatch loss at step 23000: 3.05345273018 173 | Minibatch accuracy: 88.75% 174 | Validation accuracy: 85.348346235% 175 | Minibatch loss at step 23500: 2.67105126381 176 | Minibatch accuracy: 85.0% 177 | Validation accuracy: 85.4081632653% 178 | Minibatch loss at step 24000: 3.59496998787 179 | Minibatch accuracy: 81.25% 180 | Validation accuracy: 86.1224489796% 181 | Minibatch loss at step 24500: 4.03045558929 182 | Minibatch accuracy: 87.5% 183 | Validation accuracy: 85.3800140746% 184 | Minibatch loss at step 25000: 1.18831312656 185 | Minibatch accuracy: 96.25% 186 | Validation accuracy: 86.2631949331% 187 | Minibatch loss at step 25500: 3.19347548485 188 | Minibatch accuracy: 87.5% 189 | Validation accuracy: 87.0478536242% 190 | Minibatch loss at step 26000: 3.03034424782 191 | Minibatch accuracy: 88.75% 192 | Validation accuracy: 86.2139338494% 193 | Minibatch loss at step 26500: 2.87255167961 194 | Minibatch accuracy: 82.5% 195 | Validation accuracy: 85.3905700211% 196 | Minibatch loss at step 27000: 2.26518821716 197 | Minibatch accuracy: 85.0% 198 | Validation accuracy: 86.2913441239% 199 | Minibatch loss at step 27500: 2.07746839523 200 | Minibatch accuracy: 90.0% 201 | Validation accuracy: 86.4004222379% 202 | Minibatch loss at step 28000: 2.49602365494 203 | Minibatch accuracy: 88.75% 204 | Validation accuracy: 86.703026038% 205 | Minibatch loss at step 28500: 3.42557883263 206 | Minibatch accuracy: 83.75% 207 | Validation accuracy: 86.1118930331% 208 | Minibatch loss at step 29000: 2.44106936455 209 | Minibatch accuracy: 90.0% 210 | Validation accuracy: 86.7522871217% 211 | Minibatch loss at step 29500: 1.90538573265 212 | Minibatch accuracy: 96.25% 213 | Validation accuracy: 85.6368754398% 214 | Minibatch loss at step 30000: 3.83414435387 215 | Minibatch accuracy: 86.25% 216 | Validation accuracy: 85.8761435609% 217 | Minibatch loss at step 30500: 2.91315960884 218 | Minibatch accuracy: 88.75% 219 | Validation accuracy: 86.6854327938% 220 | Minibatch loss at step 31000: 1.25981771946 221 | Minibatch accuracy: 93.75% 222 | Validation accuracy: 84.2505277973% 223 | Minibatch loss at step 31500: 2.96369576454 224 | Minibatch accuracy: 91.25% 225 | Validation accuracy: 86.5587614356% 226 | Minibatch loss at step 32000: 2.88761568069 227 | Minibatch accuracy: 83.75% 228 | Validation accuracy: 85.1935256861% 229 | Minibatch loss at step 32500: 2.89916682243 230 | Minibatch accuracy: 88.75% 231 | Validation accuracy: 86.8156228008% 232 | Minibatch loss at step 33000: 2.74593281746 233 | Minibatch accuracy: 87.5% 234 | Validation accuracy: 85.6579873329% 235 | Minibatch loss at step 33500: 2.50029277802 236 | Minibatch accuracy: 91.25% 237 | Validation accuracy: 86.7839549613% 238 | Minibatch loss at step 34000: 4.32548141479 239 | Minibatch accuracy: 76.25% 240 | Validation accuracy: 86.0063335679% 241 | Minibatch loss at step 34500: 2.3687171936 242 | Minibatch accuracy: 88.75% 243 | Validation accuracy: 85.9429978888% 244 | Minibatch loss at step 35000: 3.10691404343 245 | Minibatch accuracy: 78.75% 246 | Validation accuracy: 86.375791696% 247 | Minibatch loss at step 35500: 3.01476550102 248 | Minibatch accuracy: 90.0% 249 | Validation accuracy: 84.0218156228% 250 | Minibatch loss at step 36000: 2.51575422287 251 | Minibatch accuracy: 88.75% 252 | Validation accuracy: 86.2385643913% 253 | Minibatch loss at step 36500: 3.00953578949 254 | Minibatch accuracy: 86.25% 255 | Validation accuracy: 85.5313159747% 256 | Minibatch loss at step 37000: 3.69128155708 257 | Minibatch accuracy: 87.5% 258 | Validation accuracy: 85.4468684025% 259 | Minibatch loss at step 37500: 1.98807919025 260 | Minibatch accuracy: 93.75% 261 | Validation accuracy: 86.6854327938% 262 | Minibatch loss at step 38000: 3.44806814194 263 | Minibatch accuracy: 78.75% 264 | Validation accuracy: 85.8163265306% 265 | Minibatch loss at step 38500: 2.91740989685 266 | Minibatch accuracy: 81.25% 267 | Validation accuracy: 86.8402533427% 268 | Minibatch loss at step 39000: 1.97872400284 269 | Minibatch accuracy: 88.75% 270 | Validation accuracy: 86.1681914145% 271 | Minibatch loss at step 39500: 4.24053287506 272 | Minibatch accuracy: 82.5% 273 | Validation accuracy: 84.9472202674% 274 | Minibatch loss at step 40000: 4.04936361313 275 | Minibatch accuracy: 92.5% 276 | Validation accuracy: 86.3300492611% 277 | Minibatch loss at step 40500: 3.3367960453 278 | Minibatch accuracy: 85.0% 279 | Validation accuracy: 86.5200562984% 280 | Minibatch loss at step 41000: 1.9304972887 281 | Minibatch accuracy: 92.5% 282 | Validation accuracy: 86.6537649543% 283 | Minibatch loss at step 41500: 2.612023592 284 | Minibatch accuracy: 88.75% 285 | Validation accuracy: 86.6924700915% 286 | Minibatch loss at step 42000: 3.31084656715 287 | Minibatch accuracy: 88.75% 288 | Validation accuracy: 85.8479943702% 289 | Minibatch loss at step 42500: 2.24249958992 290 | Minibatch accuracy: 85.0% 291 | Validation accuracy: 86.2737508797% 292 | Minibatch loss at step 43000: 3.01506495476 293 | Minibatch accuracy: 86.25% 294 | Validation accuracy: 86.48486981% 295 | Minibatch loss at step 43500: 2.33385705948 296 | Minibatch accuracy: 91.25% 297 | Validation accuracy: 85.3553835327% 298 | Minibatch loss at step 44000: 1.93855512142 299 | Minibatch accuracy: 92.5% 300 | Validation accuracy: 86.9880365939% 301 | Minibatch loss at step 44500: 3.41089677811 302 | Minibatch accuracy: 87.5% 303 | Validation accuracy: 87.2026741731% 304 | Minibatch loss at step 45000: 2.48114562035 305 | Minibatch accuracy: 88.75% 306 | Validation accuracy: 87.0760028149% 307 | Minibatch loss at step 45500: 4.25272083282 308 | Minibatch accuracy: 83.75% 309 | Validation accuracy: 85.981703026% 310 | Minibatch loss at step 46000: 3.34993004799 311 | Minibatch accuracy: 85.0% 312 | Validation accuracy: 85.8937368051% 313 | Minibatch loss at step 46500: 2.80544042587 314 | Minibatch accuracy: 86.25% 315 | Validation accuracy: 86.3863476425% 316 | Minibatch loss at step 47000: 2.22520899773 317 | Minibatch accuracy: 86.25% 318 | Validation accuracy: 86.3722730471% 319 | Minibatch loss at step 47500: 2.26676368713 320 | Minibatch accuracy: 90.0% 321 | Validation accuracy: 86.2033779029% 322 | Minibatch loss at step 48000: 3.06219935417 323 | Minibatch accuracy: 87.5% 324 | Validation accuracy: 86.3089373681% 325 | Minibatch loss at step 48500: 2.88023138046 326 | Minibatch accuracy: 88.75% 327 | Validation accuracy: 87.1780436312% 328 | Minibatch loss at step 49000: 1.71527051926 329 | Minibatch accuracy: 93.75% 330 | Validation accuracy: 87.1252638987% 331 | Minibatch loss at step 49500: 3.56804656982 332 | Minibatch accuracy: 78.75% 333 | Validation accuracy: 86.6467276566% 334 | Minibatch loss at step 50000: 3.69273471832 335 | Minibatch accuracy: 85.0% 336 | Validation accuracy: 86.1963406052% 337 | Minibatch loss at step 50500: 3.43105697632 338 | Minibatch accuracy: 86.25% 339 | Validation accuracy: 87.0443349754% 340 | Minibatch loss at step 51000: 1.89972150326 341 | Minibatch accuracy: 91.25% 342 | Validation accuracy: 86.6396903589% 343 | Minibatch loss at step 51500: 3.00917696953 344 | Minibatch accuracy: 86.25% 345 | Validation accuracy: 86.7874736101% 346 | Minibatch loss at step 52000: 3.74160337448 347 | Minibatch accuracy: 80.0% 348 | Validation accuracy: 85.8620689655% 349 | Minibatch loss at step 52500: 2.00872707367 350 | Minibatch accuracy: 91.25% 351 | Validation accuracy: 86.9493314567% 352 | Minibatch loss at step 53000: 1.81841909885 353 | Minibatch accuracy: 92.5% 354 | Validation accuracy: 86.6608022519% 355 | Minibatch loss at step 53500: 2.21344065666 356 | Minibatch accuracy: 90.0% 357 | Validation accuracy: 87.1358198452% 358 | Minibatch loss at step 54000: 1.09682476521 359 | Minibatch accuracy: 93.75% 360 | Validation accuracy: 86.0028149191% 361 | Minibatch loss at step 54500: 1.67963933945 362 | Minibatch accuracy: 93.75% 363 | Validation accuracy: 86.593947924% 364 | Minibatch loss at step 55000: 3.47032308578 365 | Minibatch accuracy: 83.75% 366 | Validation accuracy: 83.3990147783% 367 | Minibatch loss at step 55500: 2.76714706421 368 | Minibatch accuracy: 86.25% 369 | Validation accuracy: 85.4890921886% 370 | Minibatch loss at step 56000: 4.06651353836 371 | Minibatch accuracy: 80.0% 372 | Validation accuracy: 86.5974665728% 373 | Minibatch loss at step 56500: 1.95235359669 374 | Minibatch accuracy: 92.5% 375 | Validation accuracy: 86.6045038705% 376 | Minibatch loss at step 57000: 2.05255413055 377 | Minibatch accuracy: 91.25% 378 | Validation accuracy: 86.6713581985% 379 | Minibatch loss at step 57500: 3.5107550621 380 | Minibatch accuracy: 82.5% 381 | Validation accuracy: 85.4890921886% 382 | Minibatch loss at step 58000: 4.08419799805 383 | Minibatch accuracy: 76.25% 384 | Validation accuracy: 85.8691062632% 385 | Minibatch loss at step 58500: 3.22356629372 386 | Minibatch accuracy: 85.0% 387 | Validation accuracy: 87.1358198452% 388 | Minibatch loss at step 59000: 3.6902692318 389 | Minibatch accuracy: 78.75% 390 | Validation accuracy: 85.9570724842% 391 | Minibatch loss at step 59500: 2.60494422913 392 | Minibatch accuracy: 90.0% 393 | Validation accuracy: 87.2202674173% 394 | W tensorflow/core/common_runtime/bfc_allocator.cc:213] Ran out of memory trying to allocate 6.79GiB. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory is available. 395 | Test accuracy: 87.6400367309% 396 | Model saved in file: model_trial_1.ckpt 397 | Successfully completed computation and iterations! 398 | To view Tensorboard's visualizations, please run 'tensorboard --logdir=logs' in your terminal 399 | ubuntu@ip-172-31-54-80:/mnt/deep_learning$ -------------------------------------------------------------------------------- /NumNum/model_trial_2.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tarfile 4 | import pickle 5 | import matplotlib.pyplot as plt 6 | import numpy as np, h5py 7 | import pandas as pd 8 | from PIL import Image 9 | import tensorflow as tf 10 | from sklearn.cross_validation import train_test_split 11 | 12 | 13 | print('Loading pickled data...') 14 | 15 | pickle_file = 'SVHN.pickle' 16 | 17 | with open(pickle_file, 'rb') as f: 18 | save = pickle.load(f) 19 | X_train = save['train_dataset'] 20 | y_train = save['train_labels'] 21 | X_val = save['valid_dataset'] 22 | y_val = save['valid_labels'] 23 | X_test = save['test_dataset'] 24 | y_test = save['test_labels'] 25 | del save 26 | print('Training data shape:', X_train.shape) 27 | print('Training label shape:',y_train.shape) 28 | print('Validation data shape:', X_val.shape) 29 | print('Validation label shape:', y_val.shape) 30 | print('Test data shape:', X_test.shape) 31 | print('Test label shape:', y_test.shape) 32 | 33 | print('Data successfully loaded!') 34 | 35 | 36 | 37 | print('Defining accuracy function...') 38 | def accuracy(predictions, labels): 39 | return (100.0 * np.sum(np.argmax(predictions, 2).T == labels) 40 | / predictions.shape[1] / predictions.shape[0]) 41 | print('Accuracy function defined!') 42 | 43 | # CNN Model 44 | print('Loading data and building computation graph...') 45 | 46 | '''Basic information''' 47 | # We processed image size to be 32 48 | image_size = 32 49 | # Number of channels: 1 because greyscale 50 | num_channels = 1 51 | # Mini-batch size 52 | batch_size = 16 53 | # Number of output labels 54 | num_labels = 11 55 | 56 | '''Filters''' 57 | # depth: number of filters (output channels) - should be increasing 58 | # num_channels: number of input channels set at 1 previously 59 | patch_size = 5 60 | depth_1 = 16 61 | depth_2 = depth_1 * 2 62 | depth_3 = depth_2 * 3 63 | 64 | # Number of hidden nodes in fully connected layer 1 65 | num_hidden = 64 66 | shape = [batch_size, image_size, image_size, num_channels] 67 | 68 | graph = tf.Graph() 69 | 70 | with graph.as_default(): 71 | 72 | '''Input Data''' 73 | # X_train: (223965, 32, 32, 1) 74 | tf_train_dataset = tf.placeholder( 75 | tf.float32, shape=(batch_size, image_size, image_size, num_channels)) 76 | 77 | # y_train: (223965, 7) 78 | tf_train_labels = tf.placeholder( 79 | tf.int32, shape=(batch_size, 6)) 80 | 81 | # X_val: (11788, 32, 32, 1) 82 | tf_valid_dataset = tf.constant(X_val) 83 | 84 | # X_test: (13067, 32, 32, 1) 85 | tf_test_dataset = tf.constant(X_test) 86 | 87 | '''Variables''' 88 | 89 | # Create Variables Function 90 | def init_weights_conv(shape, name): 91 | return tf.get_variable(shape=shape, name=name, 92 | initializer=tf.contrib.layers.xavier_initializer_conv2d()) 93 | def init_weights_fc(shape, name): 94 | return tf.get_variable(shape=shape, name=name, 95 | initializer=tf.contrib.layers.xavier_initializer()) 96 | 97 | def init_biases(shape, name): 98 | return tf.Variable( 99 | tf.constant(1.0, shape=shape), 100 | name=name 101 | ) 102 | 103 | # Create Function for Image Size: Pooling 104 | # 3 Convolutions 105 | # 2 Max Pooling 106 | def output_size_pool(input_size, conv_filter_size, pool_filter_size, 107 | padding, conv_stride, pool_stride): 108 | if padding == 'same': 109 | padding = -1.00 110 | elif padding == 'valid': 111 | padding = 0.00 112 | else: 113 | return None 114 | # After convolution 1 115 | output_1 = ( 116 | ((input_size - conv_filter_size - 2 * padding) / conv_stride) + 1.00) 117 | # After pool 1 118 | output_2 = ( 119 | ((output_1 - pool_filter_size - 2 * padding) / pool_stride) + 1.00) 120 | # After convolution 2 121 | output_3 = ( 122 | ((output_2 - conv_filter_size - 2 * padding) / conv_stride) + 1.00) 123 | # After pool 2 124 | output_4 = ( 125 | ((output_3 - pool_filter_size - 2 * padding) / pool_stride) + 1.00) 126 | # After convolution 2 127 | output_5 = ( 128 | ((output_4 - conv_filter_size - 2 * padding) / conv_stride) + 1.00) 129 | # After pool 2 130 | # output_6 = ( 131 | # ((output_5 - pool_filter_size - 2 * padding) / pool_stride) + 1.00) 132 | return int(output_5) 133 | 134 | # Convolution 1 135 | # Input channels: num_channels = 1 136 | # Output channels: depth = depth_1 137 | w_c1 = init_weights_conv([patch_size, patch_size, num_channels, depth_1], 'w_c1') 138 | b_c1 = init_biases([depth_1], 'b_c1') 139 | 140 | # Convolution 2 141 | # Input channels: num_channels = depth_1 142 | # Output channels: depth = depth_2 143 | w_c2 = init_weights_conv([patch_size, patch_size, depth_1, depth_2], 'w_c2') 144 | b_c2 = init_biases([depth_2], 'b_c2') 145 | 146 | # Convolution 3 147 | # Input channels: num_channels = depth_2 148 | # Output channels: depth = depth_3 149 | w_c3 = init_weights_conv([patch_size, patch_size, depth_2, depth_3], 'w_c3') 150 | b_c3 = init_biases([depth_3], 'b_c3') 151 | 152 | # Fully Connect Layer 1 153 | final_image_size = output_size_pool(input_size=image_size, 154 | conv_filter_size=5, pool_filter_size=2, 155 | padding='valid', conv_stride=1, 156 | pool_stride=2) 157 | print('Final image size after convolutions {}'.format(final_image_size)) 158 | w_fc1 = init_weights_fc([final_image_size*final_image_size*depth_3, num_hidden], 'w_fc1') 159 | b_fc1 = init_biases([num_hidden], 'b_fc1') 160 | 161 | # Softmax 1 162 | w_s1 = init_weights_fc([num_hidden, num_labels], 'w_s1') 163 | b_s1 = init_biases([num_labels], 'b_s1') 164 | 165 | # Softmax 2 166 | w_s2 = init_weights_fc([num_hidden, num_labels], 'w_s2') 167 | b_s2 = init_biases([num_labels], 'b_s2') 168 | 169 | # Softmax 3 170 | w_s3 = init_weights_fc([num_hidden, num_labels], 'w_s3') 171 | b_s3 = init_biases([num_labels], 'b_s3') 172 | 173 | # Softmax 4 174 | w_s4 = init_weights_fc([num_hidden, num_labels], 'w_s4') 175 | b_s4 = init_biases([num_labels], 'b_s4') 176 | 177 | # Softmax 5 178 | w_s5 = init_weights_fc([num_hidden, num_labels], 'w_s5') 179 | b_s5 = init_biases([num_labels], 'b_s5') 180 | 181 | def model(data, keep_prob, shape): 182 | with tf.name_scope("conv_layer_1"): 183 | conv_1 = tf.nn.conv2d( 184 | data, w_c1, strides=[1, 1, 1, 1], padding='VALID') 185 | hidden_conv_1 = tf.nn.relu(conv_1 + b_c1) 186 | pool_1 = tf.nn.max_pool( 187 | hidden_conv_1, [1, 2, 2, 1], [1, 2, 2, 1], padding='VALID') 188 | with tf.name_scope("conv_layer_2"): 189 | conv_2 = tf.nn.conv2d( 190 | pool_1, w_c2, strides=[1, 1, 1, 1], padding='VALID') 191 | hidden_conv_2 = tf.nn.relu(conv_2 + b_c2) 192 | pool_2 = tf.nn.max_pool( 193 | hidden_conv_2, [1, 2, 2, 1], [1, 2, 2, 1], padding='VALID') 194 | with tf.name_scope("conv_layer_3"): 195 | conv_3 = tf.nn.conv2d( 196 | pool_2, w_c3, strides=[1, 1, 1, 1], padding='VALID') 197 | hidden_conv_3 = tf.nn.relu(conv_3 + b_c3) 198 | with tf.name_scope("fc_layer_1"): 199 | hidden_drop = tf.nn.dropout(hidden_conv_3, keep_prob) 200 | shape = hidden_drop.get_shape().as_list() 201 | reshape = tf.reshape( 202 | hidden_drop, [shape[0], shape[1] * shape[2] * shape[3]]) 203 | hidden_fc = tf.nn.relu( 204 | tf.matmul(reshape, w_fc1) + b_fc1) 205 | with tf.name_scope("softmax_1"): 206 | logits_1 = tf.matmul(hidden_fc, w_s1) + b_s1 207 | with tf.name_scope("softmax_2"): 208 | logits_2 = tf.matmul(hidden_fc, w_s2) + b_s2 209 | with tf.name_scope("softmax_3"): 210 | logits_3 = tf.matmul(hidden_fc, w_s3) + b_s3 211 | with tf.name_scope("softmax_4"): 212 | logits_4 = tf.matmul(hidden_fc, w_s4) + b_s4 213 | with tf.name_scope("softmax_5"): 214 | logits_5 = tf.matmul(hidden_fc, w_s5) + b_s5 215 | return [logits_1, logits_2, logits_3, logits_4, logits_5] 216 | 217 | '''Training Computation''' 218 | [logits_1, logits_2, logits_3, logits_4, logits_5] = model( 219 | tf_train_dataset, 0.9, shape) 220 | 221 | '''Loss Function''' 222 | with tf.name_scope("loss"): 223 | loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 224 | logits_1, tf_train_labels[:, 1])) + \ 225 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 226 | logits_2, tf_train_labels[:, 2])) + \ 227 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 228 | logits_3, tf_train_labels[:, 3])) + \ 229 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 230 | logits_4, tf_train_labels[:, 4])) + \ 231 | tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( 232 | logits_5, tf_train_labels[:, 5])) 233 | # Add scalar summary for cost 234 | tf.scalar_summary("loss", loss) 235 | 236 | '''Optimizer''' 237 | # Decaying learning rate 238 | # count the number of steps taken 239 | global_step = tf.Variable(0) 240 | start_learning_rate = 0.05 241 | learning_rate = tf.train.exponential_decay( 242 | start_learning_rate, global_step, 10000, 0.96) 243 | 244 | optimizer = tf.train.AdagradOptimizer(learning_rate).minimize( 245 | loss, global_step=global_step) 246 | 247 | '''Predictions''' 248 | def softmax_combine(dataset, shape): 249 | train_prediction = tf.pack([ 250 | tf.nn.softmax(model(dataset, 1.0, shape)[0]), 251 | tf.nn.softmax(model(dataset, 1.0, shape)[1]), 252 | tf.nn.softmax(model(dataset, 1.0, shape)[2]), 253 | tf.nn.softmax(model(dataset, 1.0, shape)[3]), 254 | tf.nn.softmax(model(dataset, 1.0, shape)[4])]) 255 | return train_prediction 256 | 257 | train_prediction = softmax_combine(tf_train_dataset, shape) 258 | valid_prediction = softmax_combine(tf_valid_dataset, shape) 259 | test_prediction = softmax_combine(tf_test_dataset, shape) 260 | 261 | '''Save Model (will be initiated later)''' 262 | saver = tf.train.Saver() 263 | 264 | '''Histogram for Weights''' 265 | # Add histogram summaries for weights 266 | tf.histogram_summary("w_c1_summ", w_c1) 267 | tf.histogram_summary("b_c1_summ", b_c1) 268 | 269 | tf.histogram_summary("w_c2_summ", w_c2) 270 | tf.histogram_summary("b_c2_summ", b_c2) 271 | 272 | tf.histogram_summary("w_c3_summ", w_c3) 273 | tf.histogram_summary("b_c3_summ", b_c3) 274 | 275 | tf.histogram_summary("w_fc1_summ", w_fc1) 276 | tf.histogram_summary("b_fc1_summ", b_fc1) 277 | 278 | tf.histogram_summary("w_s1_summ", w_s1) 279 | tf.histogram_summary("b_s1_summ", b_s1) 280 | 281 | tf.histogram_summary("w_s2_summ", w_s2) 282 | tf.histogram_summary("b_s2_summ", b_s2) 283 | 284 | tf.histogram_summary("w_s3_summ", w_s3) 285 | tf.histogram_summary("b_s3_summ", b_s3) 286 | 287 | tf.histogram_summary("w_s4_summ", w_s4) 288 | tf.histogram_summary("b_s4_summ", b_s4) 289 | 290 | tf.histogram_summary("w_s5_summ", w_s5) 291 | tf.histogram_summary("b_s5_summ", b_s5) 292 | 293 | print('Data loaded and computation graph built!') 294 | 295 | num_steps = 60000 296 | 297 | print('Running computation and iteration...') 298 | print('If you are unable to save the summary, please change the path to where you want it to write.') 299 | 300 | with tf.Session(graph=graph) as session: 301 | writer = tf.train.SummaryWriter("log_trial_2", session.graph) # for 0.8 302 | merged = tf.merge_all_summaries() 303 | 304 | '''If you want to restore model''' 305 | # saver.restore(session, "model_trial_1.ckpt") 306 | # print("Model restored!") 307 | 308 | tf.initialize_all_variables().run() 309 | print('Initialized') 310 | for step in range(num_steps): 311 | offset = (step * batch_size) % (y_train.shape[0] - batch_size) 312 | batch_data = X_train[offset:(offset + batch_size), :, :, :] 313 | batch_labels = y_train[offset:(offset + batch_size), :] 314 | feed_dict = {tf_train_dataset: batch_data, 315 | tf_train_labels: batch_labels} 316 | _, l, predictions, summary = session.run([optimizer, loss, train_prediction, merged], 317 | feed_dict=feed_dict) 318 | writer.add_summary(summary) 319 | if (step % 500 == 0): 320 | print(('Minibatch loss at step {}: {}').format(step, l)) 321 | print( 322 | ('Minibatch accuracy: {}%'.format(accuracy(predictions, batch_labels[:,1:6])))) 323 | print( 324 | ('Validation accuracy: {}%'.format(accuracy(valid_prediction.eval(), 325 | y_val[:,1:6])))) 326 | print( 327 | ('Test accuracy: {}%'.format(accuracy(test_prediction.eval(), y_test[:,1:6])))) 328 | 329 | save_path = saver.save(session, "model_trial_2.ckpt") 330 | print('Model saved in file: {}'.format(save_path)) 331 | 332 | 333 | print('Successfully completed computation and iterations!') 334 | 335 | print('To view Tensorboard\'s visualizations, please run \ 336 | \'tensorboard --logdir=log_trial_2\' in your terminal') 337 | 338 | 339 | -------------------------------------------------------------------------------- /NumNum/model_trial_2_command_results.txt: -------------------------------------------------------------------------------- 1 | ubuntu@ip-172-31-54-80:/mnt/deep_learning$ python model_trial_2.py 2 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcublas.so locally 3 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcudnn.so locally 4 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcufft.so locally 5 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcuda.so locally 6 | I tensorflow/stream_executor/dso_loader.cc:108] successfully opened CUDA library libcurand.so locally 7 | /home/ubuntu/miniconda/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20. 8 | "This module will be removed in 0.20.", DeprecationWarning) 9 | Loading pickled data... 10 | ('Training data shape:', (230070, 32, 32, 1)) 11 | ('Training label shape:', (230070, 6)) 12 | ('Validation data shape:', (5684, 32, 32, 1)) 13 | ('Validation label shape:', (5684, 6)) 14 | ('Test data shape:', (13068, 32, 32, 1)) 15 | ('Test label shape:', (13068, 6)) 16 | Data successfully loaded! 17 | Defining accuracy function... 18 | Accuracy function defined! 19 | Loading data and building computation graph... 20 | Final image size after convolutions 1 21 | Data loaded and computation graph built! 22 | Running computation and iteration... 23 | If you are unable to save the summary, please change the path to where you want it to write. 24 | I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:925] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 25 | I tensorflow/core/common_runtime/gpu/gpu_init.cc:102] Found device 0 with properties: 26 | name: GRID K520 27 | major: 3 minor: 0 memoryClockRate (GHz) 0.797 28 | pciBusID 0000:00:03.0 29 | Total memory: 4.00GiB 30 | Free memory: 3.95GiB 31 | I tensorflow/core/common_runtime/gpu/gpu_init.cc:126] DMA: 0 32 | I tensorflow/core/common_runtime/gpu/gpu_init.cc:136] 0: Y 33 | I tensorflow/core/common_runtime/gpu/gpu_device.cc:838] Creating TensorFlow device (/gpu:0) -> (device: 0, name: GRID K520, pci bus id: 0000:00:03.0) 34 | Initialized 35 | Minibatch loss at step 0: 24.6852779388 36 | Minibatch accuracy: 8.75% 37 | Validation accuracy: 57.2660098522% 38 | Minibatch loss at step 500: 6.7720451355 39 | Minibatch accuracy: 56.25% 40 | Validation accuracy: 57.4771287825% 41 | Minibatch loss at step 1000: 6.77860546112 42 | Minibatch accuracy: 55.0% 43 | Validation accuracy: 58.3532723434% 44 | Minibatch loss at step 1500: 5.84234142303 45 | Minibatch accuracy: 66.25% 46 | Validation accuracy: 61.4039408867% 47 | Minibatch loss at step 2000: 4.92063093185 48 | Minibatch accuracy: 66.25% 49 | Validation accuracy: 65.0457424349% 50 | Minibatch loss at step 2500: 4.94031000137 51 | Minibatch accuracy: 70.0% 52 | Validation accuracy: 66.7452498241% 53 | Minibatch loss at step 3000: 4.0554189682 54 | Minibatch accuracy: 75.0% 55 | Validation accuracy: 70.3553835327% 56 | Minibatch loss at step 3500: 4.09882259369 57 | Minibatch accuracy: 72.5% 58 | Validation accuracy: 73.0647431386% 59 | Minibatch loss at step 4000: 4.00511074066 60 | Minibatch accuracy: 73.75% 61 | Validation accuracy: 75.8022519353% 62 | Minibatch loss at step 4500: 3.58182144165 63 | Minibatch accuracy: 77.5% 64 | Validation accuracy: 77.3750879662% 65 | Minibatch loss at step 5000: 3.26156830788 66 | Minibatch accuracy: 80.0% 67 | Validation accuracy: 77.6460239268% 68 | Minibatch loss at step 5500: 4.20952224731 69 | Minibatch accuracy: 75.0% 70 | Validation accuracy: 78.9232934553% 71 | Minibatch loss at step 6000: 3.16578555107 72 | Minibatch accuracy: 75.0% 73 | Validation accuracy: 79.8944405348% 74 | Minibatch loss at step 6500: 3.47639656067 75 | Minibatch accuracy: 78.75% 76 | Validation accuracy: 80.3729767769% 77 | Minibatch loss at step 7000: 3.56078243256 78 | Minibatch accuracy: 77.5% 79 | Validation accuracy: 81.3652357495% 80 | Minibatch loss at step 7500: 2.14208745956 81 | Minibatch accuracy: 90.0% 82 | Validation accuracy: 81.9634060521% 83 | Minibatch loss at step 8000: 2.94690990448 84 | Minibatch accuracy: 82.5% 85 | Validation accuracy: 82.7973258269% 86 | Minibatch loss at step 8500: 2.45778536797 87 | Minibatch accuracy: 87.5% 88 | Validation accuracy: 83.2653061224% 89 | Minibatch loss at step 9000: 1.68624770641 90 | Minibatch accuracy: 86.25% 91 | Validation accuracy: 83.5714285714% 92 | Minibatch loss at step 9500: 3.2386059761 93 | Minibatch accuracy: 75.0% 94 | Validation accuracy: 83.7579169599% 95 | Minibatch loss at step 10000: 1.50795674324 96 | Minibatch accuracy: 92.5% 97 | Validation accuracy: 83.7368050669% 98 | Minibatch loss at step 10500: 1.37326681614 99 | Minibatch accuracy: 93.75% 100 | Validation accuracy: 83.7368050669% 101 | Minibatch loss at step 11000: 1.34895992279 102 | Minibatch accuracy: 91.25% 103 | Validation accuracy: 84.3736805067% 104 | Minibatch loss at step 11500: 2.7125878334 105 | Minibatch accuracy: 82.5% 106 | Validation accuracy: 84.6833216045% 107 | Minibatch loss at step 12000: 2.50231361389 108 | Minibatch accuracy: 83.75% 109 | Validation accuracy: 84.7959183673% 110 | Minibatch loss at step 12500: 1.29541063309 111 | Minibatch accuracy: 95.0% 112 | Validation accuracy: 85.3870513723% 113 | Minibatch loss at step 13000: 3.05928611755 114 | Minibatch accuracy: 82.5% 115 | Validation accuracy: 84.9014778325% 116 | Minibatch loss at step 13500: 2.12882089615 117 | Minibatch accuracy: 88.75% 118 | Validation accuracy: 85.1513019001% 119 | Minibatch loss at step 14000: 3.68164110184 120 | Minibatch accuracy: 83.75% 121 | Validation accuracy: 86.0591133005% 122 | Minibatch loss at step 14500: 3.61965465546 123 | Minibatch accuracy: 82.5% 124 | Validation accuracy: 85.8831808586% 125 | Minibatch loss at step 15000: 2.51434469223 126 | Minibatch accuracy: 83.75% 127 | Validation accuracy: 86.5869106263% 128 | Minibatch loss at step 15500: 2.1530623436 129 | Minibatch accuracy: 88.75% 130 | Validation accuracy: 86.3335679099% 131 | Minibatch loss at step 16000: 1.31691002846 132 | Minibatch accuracy: 93.75% 133 | Validation accuracy: 86.5411681914% 134 | Minibatch loss at step 16500: 1.91876518726 135 | Minibatch accuracy: 86.25% 136 | Validation accuracy: 86.9915552428% 137 | Minibatch loss at step 17000: 1.79984605312 138 | Minibatch accuracy: 90.0% 139 | Validation accuracy: 86.7945109078% 140 | Minibatch loss at step 17500: 2.83466601372 141 | Minibatch accuracy: 81.25% 142 | Validation accuracy: 86.9458128079% 143 | Minibatch loss at step 18000: 2.08052873611 144 | Minibatch accuracy: 88.75% 145 | Validation accuracy: 86.5095003519% 146 | Minibatch loss at step 18500: 1.69891023636 147 | Minibatch accuracy: 93.75% 148 | Validation accuracy: 87.3187895848% 149 | Minibatch loss at step 19000: 1.72224640846 150 | Minibatch accuracy: 93.75% 151 | Validation accuracy: 87.1956368754% 152 | Minibatch loss at step 19500: 2.70904374123 153 | Minibatch accuracy: 85.0% 154 | Validation accuracy: 87.4208304011% 155 | Minibatch loss at step 20000: 2.09364247322 156 | Minibatch accuracy: 86.25% 157 | Validation accuracy: 87.0337790289% 158 | Minibatch loss at step 20500: 3.19808340073 159 | Minibatch accuracy: 82.5% 160 | Validation accuracy: 85.9042927516% 161 | Minibatch loss at step 21000: 3.17224526405 162 | Minibatch accuracy: 82.5% 163 | Validation accuracy: 87.1604503871% 164 | Minibatch loss at step 21500: 1.7398866415 165 | Minibatch accuracy: 90.0% 166 | Validation accuracy: 87.9767769177% 167 | Minibatch loss at step 22000: 2.44544625282 168 | Minibatch accuracy: 86.25% 169 | Validation accuracy: 87.4032371569% 170 | Minibatch loss at step 22500: 1.59228682518 171 | Minibatch accuracy: 95.0% 172 | Validation accuracy: 87.5404644616% 173 | Minibatch loss at step 23000: 1.96063005924 174 | Minibatch accuracy: 87.5% 175 | Validation accuracy: 88.0119634061% 176 | Minibatch loss at step 23500: 2.02179741859 177 | Minibatch accuracy: 92.5% 178 | Validation accuracy: 87.9239971851% 179 | Minibatch loss at step 24000: 3.01138615608 180 | Minibatch accuracy: 78.75% 181 | Validation accuracy: 87.4630541872% 182 | Minibatch loss at step 24500: 1.9803122282 183 | Minibatch accuracy: 88.75% 184 | Validation accuracy: 88.5045742435% 185 | Minibatch loss at step 25000: 1.09110844135 186 | Minibatch accuracy: 95.0% 187 | Validation accuracy: 87.7691766362% 188 | Minibatch loss at step 25500: 1.50219655037 189 | Minibatch accuracy: 95.0% 190 | Validation accuracy: 88.1527093596% 191 | Minibatch loss at step 26000: 2.07877397537 192 | Minibatch accuracy: 88.75% 193 | Validation accuracy: 88.5925404645% 194 | Minibatch loss at step 26500: 1.87961292267 195 | Minibatch accuracy: 83.75% 196 | Validation accuracy: 88.4975369458% 197 | Minibatch loss at step 27000: 1.80905413628 198 | Minibatch accuracy: 93.75% 199 | Validation accuracy: 88.0365939479% 200 | Minibatch loss at step 27500: 1.37362003326 201 | Minibatch accuracy: 96.25% 202 | Validation accuracy: 88.3673469388% 203 | Minibatch loss at step 28000: 1.19686949253 204 | Minibatch accuracy: 91.25% 205 | Validation accuracy: 88.5045742435% 206 | Minibatch loss at step 28500: 1.96773004532 207 | Minibatch accuracy: 87.5% 208 | Validation accuracy: 88.7966220971% 209 | Minibatch loss at step 29000: 1.05563390255 210 | Minibatch accuracy: 95.0% 211 | Validation accuracy: 88.5995777621% 212 | Minibatch loss at step 29500: 0.892932772636 213 | Minibatch accuracy: 95.0% 214 | Validation accuracy: 89.0816326531% 215 | Minibatch loss at step 30000: 1.48054420948 216 | Minibatch accuracy: 91.25% 217 | Validation accuracy: 89.1027445461% 218 | Minibatch loss at step 30500: 1.96582722664 219 | Minibatch accuracy: 90.0% 220 | Validation accuracy: 89.2505277973% 221 | Minibatch loss at step 31000: 0.767149329185 222 | Minibatch accuracy: 96.25% 223 | Validation accuracy: 88.90921886% 224 | Minibatch loss at step 31500: 1.637622118 225 | Minibatch accuracy: 90.0% 226 | Validation accuracy: 88.9655172414% 227 | Minibatch loss at step 32000: 2.63760662079 228 | Minibatch accuracy: 88.75% 229 | Validation accuracy: 88.4904996481% 230 | Minibatch loss at step 32500: 1.93419110775 231 | Minibatch accuracy: 86.25% 232 | Validation accuracy: 89.2223786066% 233 | Minibatch loss at step 33000: 1.40185725689 234 | Minibatch accuracy: 86.25% 235 | Validation accuracy: 89.0570021112% 236 | Minibatch loss at step 33500: 1.40551245213 237 | Minibatch accuracy: 92.5% 238 | Validation accuracy: 89.3771991555% 239 | Minibatch loss at step 34000: 3.09178757668 240 | Minibatch accuracy: 82.5% 241 | Validation accuracy: 89.1238564391% 242 | Minibatch loss at step 34500: 0.807315051556 243 | Minibatch accuracy: 95.0% 244 | Validation accuracy: 89.4053483462% 245 | Minibatch loss at step 35000: 1.85568845272 246 | Minibatch accuracy: 87.5% 247 | Validation accuracy: 89.6305418719% 248 | Minibatch loss at step 35500: 1.86417269707 249 | Minibatch accuracy: 90.0% 250 | Validation accuracy: 88.8529204785% 251 | Minibatch loss at step 36000: 1.68539631367 252 | Minibatch accuracy: 91.25% 253 | Validation accuracy: 89.6375791696% 254 | Minibatch loss at step 36500: 1.9363039732 255 | Minibatch accuracy: 87.5% 256 | Validation accuracy: 89.1379310345% 257 | Minibatch loss at step 37000: 2.64721107483 258 | Minibatch accuracy: 91.25% 259 | Validation accuracy: 88.9408866995% 260 | Minibatch loss at step 37500: 1.50412797928 261 | Minibatch accuracy: 91.25% 262 | Validation accuracy: 89.345531316% 263 | Minibatch loss at step 38000: 3.39422678947 264 | Minibatch accuracy: 76.25% 265 | Validation accuracy: 89.3983110486% 266 | Minibatch loss at step 38500: 2.67686247826 267 | Minibatch accuracy: 87.5% 268 | Validation accuracy: 89.4581280788% 269 | Minibatch loss at step 39000: 0.910995483398 270 | Minibatch accuracy: 95.0% 271 | Validation accuracy: 89.1942294159% 272 | Minibatch loss at step 39500: 3.38206410408 273 | Minibatch accuracy: 81.25% 274 | Validation accuracy: 89.0534834624% 275 | Minibatch loss at step 40000: 1.04500234127 276 | Minibatch accuracy: 97.5% 277 | Validation accuracy: 89.4264602393% 278 | Minibatch loss at step 40500: 1.80597698689 279 | Minibatch accuracy: 90.0% 280 | Validation accuracy: 89.4510907811% 281 | Minibatch loss at step 41000: 1.5854074955 282 | Minibatch accuracy: 92.5% 283 | Validation accuracy: 89.1449683322% 284 | Minibatch loss at step 41500: 2.18178343773 285 | Minibatch accuracy: 88.75% 286 | Validation accuracy: 89.3596059113% 287 | Minibatch loss at step 42000: 2.79684853554 288 | Minibatch accuracy: 86.25% 289 | Validation accuracy: 89.4862772695% 290 | Minibatch loss at step 42500: 1.04474651814 291 | Minibatch accuracy: 93.75% 292 | Validation accuracy: 89.5953553835% 293 | Minibatch loss at step 43000: 2.03591012955 294 | Minibatch accuracy: 91.25% 295 | Validation accuracy: 89.672765658% 296 | Minibatch loss at step 43500: 0.884207248688 297 | Minibatch accuracy: 96.25% 298 | Validation accuracy: 89.6833216045% 299 | Minibatch loss at step 44000: 1.15575289726 300 | Minibatch accuracy: 95.0% 301 | Validation accuracy: 90.1125967628% 302 | Minibatch loss at step 44500: 2.58864116669 303 | Minibatch accuracy: 88.75% 304 | Validation accuracy: 90.2850105559% 305 | Minibatch loss at step 45000: 2.20746231079 306 | Minibatch accuracy: 91.25% 307 | Validation accuracy: 90.0703729768% 308 | Minibatch loss at step 45500: 2.37037158012 309 | Minibatch accuracy: 85.0% 310 | Validation accuracy: 89.6798029557% 311 | Minibatch loss at step 46000: 1.71285831928 312 | Minibatch accuracy: 90.0% 313 | Validation accuracy: 90.021111893% 314 | Minibatch loss at step 46500: 1.46003592014 315 | Minibatch accuracy: 88.75% 316 | Validation accuracy: 89.8135116115% 317 | Minibatch loss at step 47000: 1.59427475929 318 | Minibatch accuracy: 96.25% 319 | Validation accuracy: 90.1759324419% 320 | Minibatch loss at step 47500: 1.13918030262 321 | Minibatch accuracy: 95.0% 322 | Validation accuracy: 89.9683321605% 323 | Minibatch loss at step 48000: 1.9422557354 324 | Minibatch accuracy: 91.25% 325 | Validation accuracy: 90.1337086559% 326 | Minibatch loss at step 48500: 1.5261310339 327 | Minibatch accuracy: 95.0% 328 | Validation accuracy: 90.3307529909% 329 | Minibatch loss at step 49000: 1.27112996578 330 | Minibatch accuracy: 95.0% 331 | Validation accuracy: 89.7114707952% 332 | Minibatch loss at step 49500: 1.94590842724 333 | Minibatch accuracy: 87.5% 334 | Validation accuracy: 89.7677691766% 335 | Minibatch loss at step 50000: 1.57406949997 336 | Minibatch accuracy: 90.0% 337 | Validation accuracy: 90.3237156932% 338 | Minibatch loss at step 50500: 2.35499668121 339 | Minibatch accuracy: 90.0% 340 | Validation accuracy: 89.869809993% 341 | Minibatch loss at step 51000: 1.19329309464 342 | Minibatch accuracy: 95.0% 343 | Validation accuracy: 90.1372273047% 344 | Minibatch loss at step 51500: 1.20900642872 345 | Minibatch accuracy: 92.5% 346 | Validation accuracy: 89.9155524279% 347 | Minibatch loss at step 52000: 1.1290293932 348 | Minibatch accuracy: 93.75% 349 | Validation accuracy: 90.4503870514% 350 | Minibatch loss at step 52500: 1.26053905487 351 | Minibatch accuracy: 91.25% 352 | Validation accuracy: 90.2463054187% 353 | Minibatch loss at step 53000: 0.929253041744 354 | Minibatch accuracy: 97.5% 355 | Validation accuracy: 90.3624208304% 356 | Minibatch loss at step 53500: 2.21037530899 357 | Minibatch accuracy: 88.75% 358 | Validation accuracy: 90.3800140746% 359 | Minibatch loss at step 54000: 1.17362296581 360 | Minibatch accuracy: 91.25% 361 | Validation accuracy: 90.2005629838% 362 | Minibatch loss at step 54500: 0.996218085289 363 | Minibatch accuracy: 93.75% 364 | Validation accuracy: 90.3448275862% 365 | Minibatch loss at step 55000: 2.77048206329 366 | Minibatch accuracy: 83.75% 367 | Validation accuracy: 89.584799437% 368 | Minibatch loss at step 55500: 2.86125254631 369 | Minibatch accuracy: 92.5% 370 | Validation accuracy: 90.3659394792% 371 | Minibatch loss at step 56000: 2.17242717743 372 | Minibatch accuracy: 85.0% 373 | Validation accuracy: 90.087966221% 374 | Minibatch loss at step 56500: 0.638386487961 375 | Minibatch accuracy: 98.75% 376 | Validation accuracy: 89.8275862069% 377 | Minibatch loss at step 57000: 1.77463257313 378 | Minibatch accuracy: 88.75% 379 | Validation accuracy: 90.1020408163% 380 | Minibatch loss at step 57500: 2.48961424828 381 | Minibatch accuracy: 88.75% 382 | Validation accuracy: 90.0527797326% 383 | Minibatch loss at step 58000: 3.00290465355 384 | Minibatch accuracy: 82.5% 385 | Validation accuracy: 90.39408867% 386 | Minibatch loss at step 58500: 1.90595173836 387 | Minibatch accuracy: 92.5% 388 | Validation accuracy: 90.5137227305% 389 | Minibatch loss at step 59000: 2.11096811295 390 | Minibatch accuracy: 86.25% 391 | Validation accuracy: 90.6896551724% 392 | Minibatch loss at step 59500: 0.900547921658 393 | Minibatch accuracy: 96.25% 394 | Validation accuracy: 90.5594651654% 395 | W tensorflow/core/common_runtime/bfc_allocator.cc:213] Ran out of memory trying to allocate 6.79GiB. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory is available. 396 | Test accuracy: 91.6176920722% 397 | Model saved in file: model_trial_2.ckpt 398 | Successfully completed computation and iterations! 399 | To view Tensorboard's visualizations, please run 'tensorboard --logdir=log_trial_2' in your terminal 400 | ubuntu@ip-172-31-54-80:/mnt/deep_learning$ ls 401 | checkpoint model_trial_1.ckpt model_trial_2.py train 402 | extra model_trial_1.ckpt.meta model_trial_3.py train.tar.gz 403 | extra.tar.gz model_trial_1.py SVHN.pickle 404 | load_data.py model_trial_2.ckpt test 405 | logs model_trial_2.ckpt.meta test.tar.gz 406 | ubuntu@ip-172-31-54-80:/mnt/deep_learning$ -------------------------------------------------------------------------------- /NumNum/report/report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ritchieng/NumNum/97851ccb74610c414bf474016915173b8abfe913/NumNum/report/report.pdf -------------------------------------------------------------------------------- /NumNum/report/report_raw_do_not_open.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ritchieng/NumNum/97851ccb74610c414bf474016915173b8abfe913/NumNum/report/report_raw_do_not_open.zip -------------------------------------------------------------------------------- /NumNum/ritchieng_report.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ritchieng/NumNum/97851ccb74610c414bf474016915173b8abfe913/NumNum/ritchieng_report.docx -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CNN for Multi-Digit Classification 2 | 3 | This project explores how Convolutional Neural Networks (CNNs) can be used to effectively identify a series of digits from real-world images that are obtained from “The Street View House Numbers (SVHN) Dataset”. CNNs have evolved dramatically every year since the inception of the ImageNet Challenge in 2010. 4 | 5 | ## Problem Statement 6 | I am attempting to predict a series of numbers given an image of house numbers from the SVHN dataset. An important thing to take note is that instead of the standard identification of numbers, as with the MNIST dataset, I now need to correctly detect the numbers and the sequence of numbers. 7 | 8 | ## Programming Language 9 | I used Python and Tensorflow to build the model. This implementation also uses TensorBoard extensively for visualizations. 10 | 11 | ## Problems running Tensorflow? Use TFAMI. 12 | I recommend starting a GPU instance using Amazon's AWS. I have created an image and replicated it across all regions. You can easily run this set of code on the GPU instance within a few minutes. Simply search for `TFAMI` under `community AMIs` when you are launching your instance. More information on the specific IDs can be obtained from the following [Github repository](https://github.com/ritchieng/tensorflow-aws-ami). 13 | 14 | ## How to use this code base 15 | 1. Create the relevant folders with the commands 16 | - ```mkdir log_trial_1``` 17 | - ```mkdir log_trial_2``` 18 | 2. You can load the data and pre-process all the images with one single command ```python load_data.py``` 19 | 3. Load the first model using the command ```python model_trial_1.py``` 20 | - The output should resemble something similar to this [output](https://github.com/ritchieng/NumNum/blob/master/NumNum/model_trial_1_command_results.txt). 21 | 4. You can view Tensorboard's visualizations using the command ```tensorboard --logdir=log_trial_1``` 22 | 5. Load the second model using the commmand ```python model_trial_2.py``` 23 | - The output should resemble something similar to this [output](https://github.com/ritchieng/NumNum/blob/master/NumNum/model_trial_1_command_results.txt). 24 | 6. You can view Tensorboard's visualizations using the command ```tensorboard --logdir=log_trial_2``` 25 | - You may encounter an issue whereby it says ```Port is in use: 6006``` if you run tensorboard twice on different trials. 26 | - Simply run the command ```lsof -i:6006``` or whatever the port number is. 27 | - Then run the command ```kill -9 ``` where the PID is the number you can find when you run the command above. 28 | - Simply run the command to launch Tensorboard again ```tensorboard --logdir=log_trial_2``` 29 | 30 | 31 | ## Detailed Report 32 | To guide you through, I have made a detailed report. You can refer to the report [here](https://github.com/ritchieng/NumNum/blob/master/NumNum/report/report.pdf). 33 | 34 | ## Academic Journals and Resources 35 | 1. [Multi-digit recognition](https://arxiv.org/abs/1312.6082) 36 | 2. [The Street View House Numbers (SVHN) Dataset](http://ufldl.stanford.edu/housenumbers/) 37 | 38 | ## Licensing 39 | This is an open source project governed by the license in this repository. --------------------------------------------------------------------------------