├── README.md ├── concept.png ├── fine_tuning.py ├── hdf5_with_preprocess.py ├── plot_train_test_error.py ├── result_analysis.py └── train_test_data.py /README.md: -------------------------------------------------------------------------------- 1 | # Poshmark-Deep-Learning 2 | Developed a procedure to automatically predict the aesthetic quality of fashion item photos applying CNN. 3 | ![](concept.png) 4 | -------------------------------------------------------------------------------- /concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CorrineTan/Poshmark-Deep-Learning/ab875a4ef0ffc5e258a52312267dbb25ce1d020b/concept.png -------------------------------------------------------------------------------- /fine_tuning.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | # os.chdir('..') 4 | import sys 5 | # sys.path.insert(0, './python') 6 | 7 | import caffe 8 | import numpy as np 9 | from pylab import * 10 | import pickle 11 | 12 | def fine_tune(solver_path, trained_model_path, scratch_solver_path, 13 | result_suffix): 14 | # get_ipython().magic(u'matplotlib inline') 15 | # get_ipython().system(u'diff models/bvlc_reference_caffenet/train_val.prototxt models/poshmark_734/train_val.prototxt') 16 | niter = 100000 17 | test_interval = 200 18 | 19 | calc_avg_error_interval = 10000 20 | 21 | big_plot_interval = niter 22 | zoomed_in_plot_interval = niter/10 23 | train_error_show_interval = 200 24 | 25 | # solver_path = 'models/poshmark_734_no_oversample/solver.prototxt' 26 | # trained_model_path = 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel' 27 | # scratch_solver_path = 'models/poshmark_734_no_oversample/solver.prototxt' 28 | # result_suffix = '_no_oversample_20160210' 29 | 30 | result_directory = 'result/result' + result_suffix 31 | 32 | if not os.path.exists(result_directory): 33 | os.makedirs(result_directory) 34 | 35 | # losses will also be stored in the log 36 | train_loss = np.zeros(niter) 37 | test_loss = np.zeros(niter/test_interval + 1) 38 | scratch_train_loss = np.zeros(niter) 39 | scratch_test_loss = np.zeros(niter/test_interval + 1) 40 | 41 | caffe.set_device(0) 42 | caffe.set_mode_gpu() 43 | # We create a solver that fine-tunes from a previously trained network. 44 | solver = caffe.SGDSolver(solver_path) 45 | solver.net.copy_from(trained_model_path) 46 | # For reference, we also create a solver that does no finetuning. 47 | scratch_solver = caffe.SGDSolver(scratch_solver_path) 48 | 49 | # We run the solver for niter times, and record the training loss. 50 | for it in range(niter): 51 | solver.step(1) # SGD by Caffe 52 | scratch_solver.step(1) 53 | # store the train loss 54 | train_loss[it] = solver.net.blobs['loss'].data 55 | scratch_train_loss[it] = scratch_solver.net.blobs['loss'].data 56 | if it % 10 == 0: 57 | print 'iter %d, finetune_loss=%f, scratch_loss=%f' % (it, train_loss[it], scratch_train_loss[it]) 58 | if it % test_interval == 0: 59 | solver.test_nets[0].forward() 60 | test_loss[it/test_interval] = solver.test_nets[0].blobs['loss'].data 61 | scratch_solver.test_nets[0].forward() 62 | scratch_test_loss[it/test_interval] = scratch_solver.test_nets[0].blobs['loss'].data 63 | print 'iter %d, test_finetune_loss=%f, test_scratch_loss=%f' % (it, test_loss[it/test_interval], scratch_test_loss[it/test_interval]) 64 | print 'done' 65 | 66 | # calculate the average rmse after certain numbers of iterations 67 | avg_train_rmse = np.empty((niter/calc_avg_error_interval, 1), dtype=np.float64) 68 | avg_test_rmse = np.empty((niter/calc_avg_error_interval, 1), dtype=np.float64) 69 | avg_scratch_train_rmse = np.empty((niter/calc_avg_error_interval, 1), dtype=np.float64) 70 | avg_scratch_test_rmse = np.empty((niter/calc_avg_error_interval, 1), dtype=np.float64) 71 | for idx, i in enumerate(range(0, niter, calc_avg_error_interval)): 72 | avg_train_rmse[idx] = np.mean((train_loss[i:(i + calc_avg_error_interval)]*2)**0.5, axis = 0) 73 | avg_test_rmse[idx] = np.mean((test_loss[(i/test_interval):\ 74 | ((i+calc_avg_error_interval)/test_interval)]*2)**0.5, axis = 0) 75 | avg_scratch_train_rmse[idx] = np.mean((scratch_train_loss[i:(i + calc_avg_error_interval)]*2)**0.5, axis = 0) 76 | avg_scratch_test_rmse[idx] = np.mean((scratch_test_loss[(i/test_interval):\ 77 | ((i+calc_avg_error_interval)/test_interval)]*2)**0.5, axis = 0) 78 | print(avg_train_rmse) 79 | print(avg_test_rmse) 80 | print(avg_scratch_train_rmse) 81 | print(avg_scratch_test_rmse) 82 | 83 | with open(result_directory + 84 | '/train_test_errors' + result_suffix + '.pickle', 'w') as f: 85 | pickle.dump([train_loss, scratch_train_loss, test_loss, scratch_test_loss, 86 | avg_train_rmse, avg_test_rmse, avg_scratch_train_rmse, 87 | avg_scratch_test_rmse], f) 88 | 89 | solver.net.save(result_directory + 90 | '/fine_tune' + result_suffix + '.caffemodel') 91 | scratch_solver.net.save(result_directory + 92 | '/build_from_scratch' + result_suffix + '.caffemodel') 93 | 94 | # get_ipython().magic(u'matplotlib qt') 95 | plt.figure(0) 96 | x_range = range(0, zoomed_in_plot_interval, train_error_show_interval) 97 | fine_tune_train_plot, = plt.plot(x_range, (train_loss[x_range]*2)**0.5, label='Fine tune train loss') 98 | scratch_train_plot, = plt.plot(x_range, (scratch_train_loss[x_range]*2)**0.5, label='Build from scratch train loss') 99 | plt.legend(handles=[fine_tune_train_plot, 100 | scratch_train_plot]) 101 | plt.grid() 102 | plt.xlabel('Number of iterations') 103 | plt.ylabel('RMSE') 104 | plt.show(block=False) 105 | plt.savefig(result_directory + 106 | '/train_errors_zoomed_in' + result_suffix + '.png') 107 | 108 | # get_ipython().magic(u'matplotlib qt') 109 | plt.figure(1) 110 | x_range = range(0, big_plot_interval, train_error_show_interval) 111 | fine_tune_train_plot, = plt.plot(x_range, (train_loss[x_range]*2)**0.5, label='Fine tune train loss') 112 | scratch_train_plot, = plt.plot(x_range, (scratch_train_loss[x_range]*2)**0.5, label='Build from scratch train loss') 113 | plt.legend(handles=[fine_tune_train_plot, scratch_train_plot]) 114 | plt.grid() 115 | plt.xlabel('Number of iterations') 116 | plt.ylabel('RMSE') 117 | plt.show(block=False) 118 | plt.savefig(result_directory + 119 | '/train_errors' + result_suffix + '.png') 120 | 121 | # get_ipython().magic(u'matplotlib qt') 122 | plt.figure(2) 123 | start_ite = 0 124 | end_ite = zoomed_in_plot_interval 125 | iterations = range(start_ite, end_ite, test_interval) 126 | x_range = range(start_ite/test_interval, end_ite/test_interval) 127 | 128 | fine_tune_test_plot, = plt.plot(iterations, (test_loss[x_range]*2)**0.5, label='Fine tune test loss') 129 | scratch_test_plot, = plt.plot(iterations, (scratch_test_loss[x_range]*2)**0.5, label='Build from scratch test loss') 130 | plt.legend(handles=[fine_tune_test_plot, scratch_test_plot]) 131 | plt.grid() 132 | plt.xlabel('Number of iterations') 133 | plt.ylabel('RMSE') 134 | plt.show(block=False) 135 | plt.savefig(result_directory + 136 | '/test_errors_zoomed_in' + result_suffix + '.png') 137 | 138 | # get_ipython().magic(u'matplotlib qt') 139 | plt.figure(3) 140 | start_ite = 0 141 | end_ite = big_plot_interval 142 | iterations = range(start_ite, end_ite, test_interval) 143 | x_range = range(start_ite/test_interval, end_ite/test_interval) 144 | 145 | fine_tune_test_plot, = plt.plot(iterations, (test_loss[x_range]*2)**0.5, label='Fine tune test loss') 146 | scratch_test_plot, = plt.plot(iterations, (scratch_test_loss[x_range]*2)**0.5, label='Build from scratch test loss') 147 | plt.legend(handles=[fine_tune_test_plot, scratch_test_plot]) 148 | plt.grid() 149 | plt.xlabel('Number of iterations') 150 | plt.ylabel('RMSE') 151 | plt.show(block=False) 152 | plt.savefig(result_directory + 153 | '/test_errors' + result_suffix + '.png') 154 | 155 | def cross_validation(model_dir, trained_model_path, result_suffix, numFold): 156 | for i in range(numFold): 157 | solver_path = model_dir + '/solver_' + str(i) + '.prototxt' 158 | scratch_solver_path = solver_path 159 | iter_result_suffix = result_suffix + '_fold' + str(i) 160 | fine_tune(solver_path, trained_model_path, scratch_solver_path, 161 | result_suffix) 162 | 163 | def main(): 164 | # for one fine tune task 165 | # solver_path = 'models/poshmark_734_no_oversample/solver.prototxt' 166 | # trained_model_path = 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel' 167 | # scratch_solver_path = solver_path 168 | # result_suffix = '_no_oversample_20160210' 169 | # fine_tune(solver_path, trained_model_path, scratch_solver_path, 170 | # result_suffix) 171 | # for cross validation 172 | model_dir = 'models/poshmark_734_oversample_cross_validation' 173 | trained_model_path = \ 174 | 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel' 175 | result_suffix = 'oversample_cv_20160211' 176 | cross_validation(model_dir, trained_model_path, result_suffix, 5) 177 | 178 | 179 | if __name__ == '__main__': 180 | main() -------------------------------------------------------------------------------- /hdf5_with_preprocess.py: -------------------------------------------------------------------------------- 1 | import h5py, os 2 | import caffe 3 | import numpy as np 4 | import scipy.ndimage as ndimage 5 | import scipy.misc 6 | import matplotlib.pyplot as plt 7 | import skimage.transform 8 | 9 | # ?????????????????????????????? 224 or 227? 10 | IMG_RESHAPE = 224 11 | # IMG_RESHAPE = 227 12 | IMG_UNCROPPED = 256 13 | 14 | def resize_convert(img_names, scores_raw, path = None, if_oversample = False): 15 | ''' 16 | Load images, set to BGR mode and transpose to CxHxW 17 | and subtract the Imagenet mean. If if_oversample is True, 18 | perform data augmentation. 19 | 20 | Parameters: 21 | --------- 22 | img_names (list): list of image names to be processed. 23 | path (string): path to images. 24 | if_oversample (bool): if True then data augmentation is performed 25 | on each image, and 10 crops of size 224x224 are produced 26 | from each image. If False, then a single 224x224 is produced. 27 | ''' 28 | 29 | path = path if path is not None else '' 30 | if if_oversample == False: 31 | all_imgs = np.empty((len(img_names), 3, IMG_RESHAPE, IMG_RESHAPE), dtype='float32') 32 | else: 33 | all_imgs = np.empty((len(img_names), 3, IMG_UNCROPPED, IMG_UNCROPPED), dtype='float32') 34 | 35 | #load the imagenet mean 36 | mean_val = np.load('../ilsvrc12_data/ilsvrc_2012_mean.npy') 37 | 38 | for i, img_name in enumerate(img_names): 39 | # Read as HxWxC, RGB, range 0-255 40 | # Caffe needs CxHxW, BGR, range 0-255 41 | img = ndimage.imread(path+img_name, mode='RGB') 42 | 43 | #subtract the mean of Imagenet 44 | #First, resize to 256 so we can subtract the mean of dims 256x256 45 | img = img[...,::-1] #Convert RGB TO BGR 46 | 47 | # img = caffe.io.resize_image(img, (IMG_UNCROPPED, IMG_UNCROPPED, 3), interp_order=1) 48 | # img = scipy.misc.imresize(img, (IMG_UNCROPPED, IMG_UNCROPPED, 3), interp='bilinear') 49 | img = skimage.transform.resize(img, (IMG_UNCROPPED, IMG_UNCROPPED, 3), 50 | order=1, preserve_range=True) 51 | # print(img.dtype) 52 | # print('Range of images: max: ' + str(np.amax(img[:,:,0])) + ' ; min: ' + str(np.amin(img[:,:,0])) ) 53 | # print('Range of images: max: ' + str(np.amax(img[:,:,1])) + ' ; min: ' + str(np.amin(img[:,:,1])) ) 54 | # print('Range of images: max: ' + str(np.amax(img[:,:,2])) + ' ; min: ' + str(np.amin(img[:,:,2])) ) 55 | 56 | # print('Reshaped shape of img is:' + str(img.shape)) 57 | img = np.transpose(img, (2, 0, 1)) #HxWxC => CxHxW 58 | 59 | # Since mean is given in Caffe channel order: 3xWxH 60 | # shape of mean_val is:(3, 256, 256) 61 | # Assume it also is given in BGR order 62 | img = img.astype('float32') 63 | img = img - mean_val 64 | 65 | #set to 0-1 range => I don't think googleNet requires this 66 | #I tried both and it didn't make a difference 67 | #img = img/255 68 | 69 | #resize images down since GoogleNet accepts 224x224 crops 70 | if if_oversample == False: 71 | img = np.transpose(img, (1,2,0)) # CxHxW => HxWxC 72 | # img = caffe.io.resiz_eimage(img, (IMG_RESHAPE, IMG_RESHAPE), interp_order=1) 73 | # img = scipy.misc.imresize(img, (IMG_RESHAPE, IMG_RESHAPE, 3), interp='bilinear') 74 | img = skimage.transform.resize(img, (IMG_RESHAPE, IMG_RESHAPE, 3), 75 | order=1, preserve_range=True) 76 | #skimage.transform.resize changes the dtype to float64 77 | img = img.astype('float32') 78 | img = np.transpose(img, (2,0,1)) #convert to CxHxW for Caffe 79 | 80 | all_imgs[i, :, :, :] = img 81 | # plt.imshow(np.transpose(all_imgs[0,:,:,:] + mean_val, (1,2,0))/255) 82 | # plt.show() 83 | 84 | # oversampling requires HxWxC order (from CxHxW) 85 | if if_oversample: 86 | all_imgs = np.transpose(all_imgs, (0, 2, 3, 1)) 87 | # caffe.io.oversample get 4 corners, 1 center, and their mirrors 88 | # https://github.com/BVLC/caffe/blob/master/python/caffe/io.py 89 | # all_imgs = caffe.io.oversample(all_imgs, (IMG_RESHAPE, IMG_RESHAPE)) 90 | all_imgs = oversample(all_imgs, (IMG_RESHAPE, IMG_RESHAPE)) 91 | all_imgs = np.transpose(all_imgs, (0,3,1,2)) #convert to CxHxW for Caffe 92 | # repeat a column vector 93 | scores = np.repeat(scores_raw, 10, axis = 0) 94 | 95 | else: 96 | scores = scores_raw 97 | 98 | return [all_imgs, scores] 99 | 100 | def creat_hdf5(file_name, suffix = '', if_oversample = False): 101 | with open('./output/' + file_name + '.txt', 'r') as data_txt: 102 | lines = data_txt.readlines() 103 | # If you do not have enough memory split data into 104 | # multiple batches and generate multiple separate h5 files 105 | img_names = [None] * len(lines) 106 | scores_raw = np.zeros((len(lines), 1), dtype='f4') 107 | for i,l in enumerate(lines): 108 | sp = l.split(' ') 109 | img_names[i] = sp[0] 110 | scores_raw[i, 0] = float(sp[1]) 111 | 112 | # Switch if_oversample to True if want data augmentation! 113 | # [images, scores] = resize_convert(img_names = img_names, 114 | # scores_raw = scores_raw, if_oversample) 115 | [images, scores] = resize_convert(img_names = img_names, 116 | scores_raw = scores_raw, 117 | if_oversample = if_oversample) 118 | 119 | with h5py.File('../poshmark_734_data/hdf5_patch_224/' + file_name + suffix \ 120 | + '.h5','w') as h5_file: 121 | h5_file.create_dataset('images', data = images) # note the name given to the dataset! 122 | h5_file.create_dataset('scores', data = scores) # note the name given to the dataset! 123 | with open('../poshmark_734_data/hdf5_patch_224/' + file_name + suffix \ 124 | + '_h5_list.txt','w') as h5_list: 125 | h5_list.write('./poshmark_734_data/hdf5_patch_224/' + file_name \ 126 | + suffix + '.h5') # list all h5 files you are going to use 127 | 128 | def oversample(images, crop_dims): 129 | """ 130 | Crop images into the four corners, center, and their mirrored versions. 131 | 132 | Parameters 133 | ---------- 134 | image : iterable of (H x W x K) ndarrays 135 | crop_dims : (height, width) tuple for the crops. 136 | 137 | Returns 138 | ------- 139 | crops : (10*N x H x W x K) ndarray of crops for number of inputs N. 140 | """ 141 | # Dimensions and center. 142 | im_shape = np.array(images[0].shape) 143 | crop_dims = np.array(crop_dims) 144 | im_center = im_shape[:2] / 2.0 145 | 146 | # Make crop coordinates 147 | h_indices = (0, im_shape[0] - crop_dims[0]) 148 | w_indices = (0, im_shape[1] - crop_dims[1]) 149 | crops_ix = np.empty((5, 4), dtype=int) 150 | curr = 0 151 | for i in h_indices: 152 | for j in w_indices: 153 | crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) 154 | curr += 1 155 | crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate([ 156 | -crop_dims / 2.0, 157 | crop_dims / 2.0 158 | ]) 159 | # to make crops_ix a 10x4 array 160 | crops_ix = np.tile(crops_ix, (2, 1)) 161 | 162 | # Extract crops 163 | print ('len(images) = ' + str(len(images))) 164 | crops = np.empty((10 * len(images), crop_dims[0], crop_dims[1], 165 | im_shape[-1]), dtype=np.float32) 166 | ix = 0 167 | for i, im in enumerate(images): 168 | for crop in crops_ix: 169 | crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] 170 | ix += 1 171 | print('image idx = ' + str(i)) 172 | print('ix = ' + str(ix)) 173 | 174 | crops[ix-5:ix] = crops[ix-5:ix, :, ::-1, :] # flip for mirrors 175 | return crops 176 | 177 | def main(): 178 | # # the input file name matches the output file name 179 | # creat_hdf5(file_name = 'train', suffix = '_no_oversample') 180 | # creat_hdf5(file_name = 'test', suffix = '_no_oversample') 181 | 182 | for i in range(5): 183 | creat_hdf5(file_name = 'train_' + str(i), 184 | suffix = '_mirror_oversample_224', 185 | if_oversample = True) 186 | creat_hdf5(file_name = 'test_' + str(i), 187 | suffix = '_mirror_oversample_224', 188 | if_oversample = True) 189 | 190 | if __name__ == '__main__': 191 | main() -------------------------------------------------------------------------------- /plot_train_test_error.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | # os.chdir('..') 4 | import sys 5 | # sys.path.insert(0, './python') 6 | 7 | import caffe 8 | import numpy as np 9 | from pylab import * 10 | import pickle 11 | 12 | # get_ipython().magic(u'matplotlib inline') 13 | # get_ipython().system(u'diff models/bvlc_reference_caffenet/train_val.prototxt models/poshmark_734/train_val.prototxt') 14 | niter = 100000 15 | test_interval = 200 16 | 17 | calc_avg_error_interval = 10000 18 | 19 | big_plot_interval = niter 20 | zoomed_in_plot_interval = niter/10 21 | train_error_show_interval = 200 22 | 23 | result_suffix = '_oversample_cv_20160219_fold1_test' 24 | 25 | with open('result/result' + result_suffix + \ 26 | '/train_test_errors' + result_suffix + '.pickle', 'r') as f: 27 | [train_loss, scratch_train_loss, test_loss, scratch_test_loss, 28 | avg_train_rmse, avg_test_rmse, avg_scratch_train_rmse, 29 | avg_scratch_test_rmse] = pickle.load(f) 30 | 31 | # get_ipython().magic(u'matplotlib qt') 32 | plt.figure(0) 33 | x_range = range(0, zoomed_in_plot_interval, train_error_show_interval) 34 | fine_tune_train_plot, = plt.plot(x_range, (train_loss[x_range]*2)**0.5, label='Fine tune train loss') 35 | scratch_train_plot, = plt.plot(x_range, (scratch_train_loss[x_range]*2)**0.5, label='Build from scratch train loss') 36 | # plt.legend(handles=[fine_tune_train_plot, scratch_train_plot]) 37 | plt.grid() 38 | plt.xlabel('Number of iterations') 39 | plt.ylabel('RMSE') 40 | plt.show(block=False) 41 | plt.savefig('result/result' + result_suffix + \ 42 | '/train_errors_zoomed_in' + result_suffix + '.png') 43 | 44 | # get_ipython().magic(u'matplotlib qt') 45 | plt.figure(1) 46 | x_range = range(0, big_plot_interval, train_error_show_interval) 47 | fine_tune_train_plot, = plt.plot(x_range, (train_loss[x_range]*2)**0.5, label='Fine tune train loss') 48 | scratch_train_plot, = plt.plot(x_range, (scratch_train_loss[x_range]*2)**0.5, label='Build from scratch train loss') 49 | # plt.legend(handles=[fine_tune_train_plot, scratch_train_plot]) 50 | plt.grid() 51 | plt.xlabel('Number of iterations') 52 | plt.ylabel('RMSE') 53 | plt.show(block=False) 54 | plt.savefig('result/result' + result_suffix + \ 55 | '/train_errors' + result_suffix + '.png') 56 | 57 | # get_ipython().magic(u'matplotlib qt') 58 | plt.figure(2) 59 | start_ite = 0 60 | end_ite = zoomed_in_plot_interval 61 | iterations = range(start_ite, end_ite, test_interval) 62 | x_range = range(start_ite/test_interval, end_ite/test_interval) 63 | 64 | fine_tune_test_plot, = plt.plot(iterations, (test_loss[x_range]*2)**0.5, label='Fine tune test loss') 65 | scratch_test_plot, = plt.plot(iterations, (scratch_test_loss[x_range]*2)**0.5, label='Build from scratch test loss') 66 | # plt.legend(handles=[fine_tune_test_plot, scratch_test_plot]) 67 | plt.grid() 68 | plt.xlabel('Number of iterations') 69 | plt.ylabel('RMSE') 70 | plt.show(block=False) 71 | plt.savefig('result/result' + result_suffix + \ 72 | '/test_errors_zoomed_in' + result_suffix + '.png') 73 | 74 | # get_ipython().magic(u'matplotlib qt') 75 | plt.figure(3) 76 | start_ite = 0 77 | end_ite = big_plot_interval 78 | iterations = range(start_ite, end_ite, test_interval) 79 | x_range = range(start_ite/test_interval, end_ite/test_interval) 80 | 81 | fine_tune_test_plot, = plt.plot(iterations, (test_loss[x_range]*2)**0.5, label='Fine tune test loss') 82 | scratch_test_plot, = plt.plot(iterations, (scratch_test_loss[x_range]*2)**0.5, label='Build from scratch test loss') 83 | # plt.legend(handles=[fine_tune_test_plot, scratch_test_plot]) 84 | plt.grid() 85 | plt.xlabel('Number of iterations') 86 | plt.ylabel('RMSE') 87 | plt.show(block=False) 88 | plt.savefig('result/result' + result_suffix + \ 89 | '/test_errors' + result_suffix + '.png') -------------------------------------------------------------------------------- /result_analysis.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | import sys 4 | 5 | # import caffe 6 | import numpy as np 7 | from pylab import * 8 | import pickle 9 | 10 | def read_pickle_data(pickle_path): 11 | pickle_file = open(pickle_path, 'r') 12 | [train_loss, scratch_train_loss, test_loss, scratch_test_loss, 13 | avg_train_rmse, avg_test_rmse, avg_scratch_train_rmse, 14 | avg_scratch_test_rmse] = pickle.load(pickle_file) 15 | return [train_loss, scratch_train_loss, test_loss, scratch_test_loss, 16 | avg_train_rmse, avg_test_rmse, avg_scratch_train_rmse, 17 | avg_scratch_test_rmse] 18 | 19 | def calc_avg_error_after_iter(avg_train_rmse, avg_test_rmse, 20 | avg_scratch_train_rmse, avg_scratch_test_rmse, iteration): 21 | start_idx = iteration/10000 - 1 22 | avg_train_rmse_after_iter = np.mean(avg_train_rmse[start_idx:]) 23 | avg_test_rmse_after_iter = np.mean(avg_test_rmse[start_idx:]) 24 | avg_scratch_train_rmse_after_iter = \ 25 | np.mean(avg_scratch_train_rmse[start_idx:]) 26 | avg_scratch_test_rmse_after_iter = \ 27 | np.mean(avg_scratch_test_rmse[start_idx:]) 28 | 29 | print('avg_train_rmse_after_iter = ' + str(avg_train_rmse_after_iter)) 30 | print('avg_test_rmse_after_iter = ' + str(avg_test_rmse_after_iter)) 31 | print('avg_scratch_train_rmse_after_iter = ' + \ 32 | str(avg_scratch_train_rmse_after_iter)) 33 | print('avg_scratch_test_rmse_after_iter = ' + \ 34 | str(avg_scratch_test_rmse_after_iter)) 35 | 36 | return [avg_train_rmse_after_iter, avg_test_rmse_after_iter, 37 | avg_scratch_train_rmse_after_iter, 38 | avg_scratch_test_rmse_after_iter] 39 | 40 | def print_avg_loss_table_latex(avg_train_rmse, avg_test_rmse, 41 | avg_scratch_train_rmse, avg_scratch_test_rmse): 42 | for i in range(len(avg_train_rmse)): 43 | print(str(i * 10000) + '-' + str(i * 10000 + 10000) + ' & ' + '{0:.3f}'.format(avg_train_rmse[i][0]) + ' & ' + '{0:.3f}'.format(avg_scratch_train_rmse[i][0])+ ' & ' + '{0:.3f}'.format(avg_test_rmse[i][0]) + ' & ' + '{0:.3f}'.format(avg_scratch_test_rmse[i][0]) + ' \\\\ \\hline') 44 | 45 | def print_avg_loss_table_latex_no_scratch(avg_train_rmse, avg_test_rmse): 46 | for i in range(len(avg_train_rmse)): 47 | print(str(i * 10000) + '-' + str(i * 10000 + 10000) + ' & ' + '{0:.3f}'.format(avg_train_rmse[i][0]) + ' & ' + '{0:.3f}'.format(avg_test_rmse[i][0]) + ' \\\\ \\hline') 48 | 49 | 50 | 51 | 52 | def main(): 53 | # with scratch net, one test run 54 | # pickle_path_prefix = './result/result_oversample_cv_20160219_fold' 55 | # pickle_path_suffix = 'train_test_errors_oversample_cv_20160219_fold' 56 | 57 | # no scratch net, multiple test runs 58 | pickle_path_prefix = './result/result_cv_multi_test_runs_20160301_fold' 59 | pickle_path_suffix = 'train_test_errors_cv_multi_test_runs_20160301_fold' 60 | num_fold = 5 61 | total_train_error = 0 62 | total_test_error = 0 63 | total_scratch_train_error = 0 64 | total_scratch_test_error = 0 65 | 66 | for i in range(num_fold): 67 | print('Fold ' + str(i) + 68 | ' ====================================') 69 | 70 | pickle_path = pickle_path_prefix + str(i) + '/' + \ 71 | pickle_path_suffix + str(i) + '.pickle' 72 | [train_loss, scratch_train_loss, test_loss, scratch_test_loss, 73 | avg_train_rmse, avg_test_rmse, avg_scratch_train_rmse, 74 | avg_scratch_test_rmse] = read_pickle_data(pickle_path) 75 | 76 | [avg_train_rmse_after_iter, avg_test_rmse_after_iter, 77 | avg_scratch_train_rmse_after_iter, 78 | avg_scratch_test_rmse_after_iter] = \ 79 | calc_avg_error_after_iter( 80 | avg_train_rmse, avg_test_rmse, 81 | avg_scratch_train_rmse, avg_scratch_test_rmse, 40000) 82 | 83 | total_train_error += avg_train_rmse_after_iter 84 | total_test_error += avg_test_rmse_after_iter 85 | total_scratch_train_error += avg_scratch_train_rmse_after_iter 86 | total_scratch_test_error += avg_scratch_test_rmse_after_iter 87 | 88 | # print_avg_loss_table_latex(avg_train_rmse, avg_test_rmse, 89 | # avg_scratch_train_rmse, avg_scratch_test_rmse) 90 | print_avg_loss_table_latex_no_scratch(avg_train_rmse, avg_test_rmse) 91 | 92 | print('===============================================================') 93 | print('Cross-validation average: \n' 94 | + 'train = ' + str(total_train_error/num_fold) + '\n' 95 | + 'test = ' + str(total_test_error/num_fold) + '\n' 96 | + 'scratch_train = ' + str(total_scratch_train_error/num_fold) + '\n' 97 | + 'scratch_test = ' + str(total_scratch_test_error/num_fold) + '\n') 98 | 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /train_test_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 22 15:55:16 2015 4 | 5 | @author: JianyuWang 6 | """ 7 | import sys, getopt 8 | import random 9 | 10 | def parseArgs(argv): 11 | csvFile = '' 12 | imageDir = '' 13 | outputDir = '' 14 | try: 15 | opts, args = getopt.getopt(argv,"hc:i:o:f:", 16 | ["cFile=", "iDir=", "oDir=", "numFold="]) 17 | except getopt.GetoptError: 18 | print('train_test_data.py -c -i ' + \ 19 | '-o -n ') 20 | sys.exit(2) 21 | for opt, arg in opts: 22 | if opt == '-h': 23 | print('train_test_data.py -c -i ' + \ 24 | '-o -n ') 25 | sys.exit() 26 | elif opt in ("-c", "--cFile"): 27 | csvFile = arg 28 | elif opt in ("-i", "--iDir"): 29 | imageDir = arg 30 | elif opt in ("-o", "--oDir"): 31 | outputDir = arg 32 | elif opt in ("-f", "--numFold"): 33 | numFold = arg 34 | print('Csv file is: ' + csvFile) 35 | print('Image directory is: ' + imageDir) 36 | print('Output direcotry is: ' + outputDir) 37 | print('Number of fold is: ' + numFold) 38 | return [csvFile, imageDir, outputDir, int(numFold)] 39 | 40 | def parseScoreCSV(csvFile): 41 | dataList = [] 42 | 43 | csvFile = open(csvFile, 'r') 44 | # skip the header row 45 | csvFile.readline() 46 | for line in csvFile: 47 | lineList = line.split(',') 48 | # do not include the \n 49 | dataList.append([lineList[0], float(lineList[-1][0:-1])]) 50 | return dataList 51 | 52 | 53 | def separateTrainTest(dataList, numFold): 54 | trainList = [] 55 | testList = [] 56 | rSeed = 1 57 | # rSeed = 2 58 | 59 | # get random separation fold 60 | random.seed(rSeed) 61 | testIdx = random.sample(xrange(len(dataList)), 62 | int(len(dataList)/numFold)) 63 | for i in range(len(dataList)): 64 | if i in testIdx: 65 | testList.append(dataList[i]) 66 | else: 67 | trainList.append(dataList[i]) 68 | 69 | return [trainList, testList] 70 | 71 | def makeTrainTestTxt(trainList, testList, imageDir, outputDir, 72 | outputSuffix = None): 73 | trainFile = open(outputDir + '/train' + outputSuffix + '.txt', 'w') 74 | testFile = open(outputDir + '/test' + outputSuffix + '.txt', 'w') 75 | for img in trainList: 76 | trainFile.write(imageDir + '/' + img[0] + ' ' + str(img[1]) + '\n') 77 | for img in testList: 78 | testFile.write(imageDir + '/' + img[0] + ' ' + str(img[1]) + '\n') 79 | trainFile.close() 80 | testFile.close() 81 | 82 | def makeCrossValidation(dataList, numFold, imageDir, outputDir): 83 | oldTestList = [] 84 | leftoverDataList = dataList 85 | for i in range(numFold): 86 | [leftoverDataList, testList] = \ 87 | separateTrainTest(leftoverDataList, numFold - i) 88 | # trainList is always the previous testList and the leftover 89 | makeTrainTestTxt(oldTestList + leftoverDataList, testList, imageDir, 90 | outputDir, '_' + str(i)) 91 | oldTestList = oldTestList + testList 92 | 93 | 94 | if __name__ == "__main__": 95 | [csvFile, imageDir, outputDir, numFold] = parseArgs(sys.argv[1:]) 96 | dataList = parseScoreCSV(csvFile) 97 | # To make just one test set 98 | # [trainList, testList] = separateTrainTest(dataList, numFold) 99 | # makeTrainTestTxt(trainList, testList, imageDir, outputDir) 100 | 101 | # To make cross-validation 102 | makeCrossValidation(dataList, numFold, imageDir, outputDir) 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | --------------------------------------------------------------------------------