Try your data instead of MNIST data in CNN tutorials.
5 | 6 | Before run the source code 7 | ------------------------- 8 | You must prepare the dataset like following. 9 | 10 | The root directory of your own dataset. 11 | - Class 1 12 | + image 1-1 13 | + image 1-2 14 | ... 15 | + image 1-n 16 | - Class 2 17 | + image 2-1 18 | + image 2-2 19 | ... 20 | + image 2-n 21 | - Class 3 22 | ... 23 | - Class N 24 | 25 | How to run? 26 | ----------- 27 | Enter the commend on your terminal like following. 28 | ``` 29 | $ python run.py 30 | 31 | ... 32 | 33 | Enter the path where the images are located. 34 | >> {Enter the path where root directory of your own dataset.} 35 | ``` 36 | 37 | And following commend will help you when you run the source code. 38 | ``` 39 | $ python run.py -h 40 | ``` 41 | 42 | 43 | 44 | 45 |46 | First commit: May.24.2017 47 | Developed by YeongHyeon Park (Korea) 48 |
49 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | import source.developed as dev 2 | dev.print_stamp() 3 | 4 | print("\n***** Load modules *****") 5 | import os, argparse 6 | import tensorflow as tf 7 | 8 | # custom module 9 | import source.constructor as cntrt 10 | import source.model as model 11 | import source.sub_procedure as subp 12 | print(" Load module complete") 13 | 14 | def main(): 15 | usr_path = "" 16 | print("Enter the path where the images are located.") 17 | usr_path = input(">> ") 18 | 19 | if(os.path.exists(usr_path)): 20 | dataset, classes, min_b = cntrt.load_dataset(path=usr_path, img_h=FLAGS.height, img_w=FLAGS.width) 21 | 22 | # Separate composition and execute 23 | sess = tf.InteractiveSession() 24 | 25 | # Initialize placeholdersshape[0] 26 | # x is image, y_ is label 27 | #x = tf.placeholder(tf.float32, shape=[None, img_length]) 28 | #y_ = tf.placeholder(tf.float32, shape=[None, classes]) 29 | height, width, dimension = dataset.train.shape 30 | x = tf.placeholder(tf.float32, shape=[None, height, width, dimension]) 31 | y_ = tf.placeholder(tf.float32, shape=[None, classes]) 32 | 33 | keep_prob, train_step, accuracy = model.conv_neural_network(x, y_, height=height, width=width, dimension=dimension, classes=classes) 34 | 35 | print("\n***** Training with CNN *****") 36 | sess.run(tf.global_variables_initializer()) 37 | print(" Initialized all variables") 38 | 39 | #Create a saver object which will save all the variables 40 | saver = tf.train.Saver() 41 | 42 | batch_size = FLAGS.batch 43 | if(batch_size > min_b): 44 | batch_size = min_b 45 | subp.training(steps=FLAGS.steps, batch_size=batch_size, dataset=dataset, sess=sess, keep_prob=keep_prob, train_step=train_step, accuracy=accuracy, x=x, y_=y_, saver=saver) 46 | 47 | 48 | else: 49 | print("Invalid path.") 50 | 51 | if __name__ == "__main__": 52 | parser = argparse.ArgumentParser() 53 | parser.add_argument('--steps', type=int, default=1000, help='Default: 1000') 54 | parser.add_argument('--batch', type=int, default=10, help='Default: 10. Batches per iteration, the number of data to be training and testing.') 55 | parser.add_argument('--height', type=float, default=1, help='Default: 28. Enter the size to resize images what you want') 56 | parser.add_argument('--width', type=float, default=1, help='Default: 28. Enter the size to resize images what you want') 57 | FLAGS, unparsed = parser.parse_known_args() 58 | 59 | main() 60 | -------------------------------------------------------------------------------- /source/__pycache__/constructor.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YeongHyeon/CNN_Own_Dataset/97e66745b8d75b45479e24e33fd327978e0f0809/source/__pycache__/constructor.cpython-35.pyc -------------------------------------------------------------------------------- /source/__pycache__/developed.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YeongHyeon/CNN_Own_Dataset/97e66745b8d75b45479e24e33fd327978e0f0809/source/__pycache__/developed.cpython-35.pyc -------------------------------------------------------------------------------- /source/__pycache__/model.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YeongHyeon/CNN_Own_Dataset/97e66745b8d75b45479e24e33fd327978e0f0809/source/__pycache__/model.cpython-35.pyc -------------------------------------------------------------------------------- /source/__pycache__/sub_procedure.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YeongHyeon/CNN_Own_Dataset/97e66745b8d75b45479e24e33fd327978e0f0809/source/__pycache__/sub_procedure.cpython-35.pyc -------------------------------------------------------------------------------- /source/__pycache__/utility.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YeongHyeon/CNN_Own_Dataset/97e66745b8d75b45479e24e33fd327978e0f0809/source/__pycache__/utility.cpython-35.pyc -------------------------------------------------------------------------------- /source/constructor.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import os, sys, glob, shutil, random, inspect 4 | import scipy.misc 5 | 6 | import source.utility as util 7 | 8 | from tensorflow.contrib.learn.python.learn.datasets import base 9 | from tensorflow.python.framework import dtypes 10 | 11 | PACK_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))+"/.." 12 | 13 | class DataSet(object): 14 | 15 | def __init__(self, who_am_i, datas, labels, class_len, height=0, width=0): 16 | self._who_am_i = who_am_i 17 | 18 | combined = list(zip(datas, labels)) 19 | random.shuffle(combined) 20 | self._datas, self._labels = zip(*combined) 21 | 22 | self._class_len = class_len 23 | 24 | form = scipy.misc.imread(self._datas[0]) 25 | dimension = 0 26 | if(len(form.shape) < 3): 27 | dimension = 1 28 | else: 29 | dimension = form.shape[2] 30 | 31 | 32 | if((height == 0) or (width == 0)): 33 | self._height = form.shape[0] 34 | self._width = form.shape[1] 35 | else: 36 | self._height = height 37 | self._width = width 38 | self._dimension = dimension 39 | 40 | @property 41 | def amount(self): 42 | return int(len(self._datas)) 43 | 44 | @property 45 | def shape(self): 46 | return self._height, self._width, self._dimension 47 | 48 | def next_batch(self, batch_size=10): 49 | 50 | datas = np.empty((0, self._height, self._width, self._dimension), int) 51 | labels = np.empty((0, self._class_len), int) 52 | 53 | 54 | for idx in range(batch_size): 55 | random.randint(0, len(self._datas)-1) 56 | tmp_img = scipy.misc.imread(self._datas[idx]) 57 | tmp_img = scipy.misc.imresize(tmp_img, (self._height, self._width)) 58 | tmp_img = tmp_img.reshape(1, self._height, self._width, self._dimension) 59 | 60 | datas = np.append(datas, tmp_img, axis=0) 61 | labels = np.append(labels, np.eye(self._class_len)[int(np.asfarray(self._labels[idx]))].reshape(1, self._class_len), axis=0) 62 | 63 | 64 | return datas, labels 65 | 66 | def split_data(path=None): 67 | 68 | util.directory_check(PACK_PATH+"/train") 69 | util.directory_check(PACK_PATH+"/test") 70 | util.directory_check(PACK_PATH+"/valid") 71 | 72 | directories = [] 73 | for dirname in os.listdir(path): 74 | directories.append(dirname) 75 | 76 | for di in directories: 77 | if(not(os.path.exists(PACK_PATH+"/train/"+di))): 78 | os.mkdir("./train/"+di) 79 | if(not(os.path.exists(PACK_PATH+"/test/"+di))): 80 | os.mkdir(PACK_PATH+"/test/"+di) 81 | if(not(os.path.exists(PACK_PATH+"/valid/"+di))): 82 | os.mkdir(PACK_PATH+"/valid/"+di) 83 | 84 | extensions = [".jpg",".JPG",".jpeg",".JPEG"] 85 | for di in directories: 86 | files = [] 87 | for ex in extensions: 88 | for filename in glob.glob(path+"/"+di+"/*"+ex): 89 | files.append(filename) 90 | random.shuffle(files) 91 | 92 | tr_point = int(len(files)*0.8) 93 | te_point = int(len(files)*0.9) 94 | va_point = int(len(files)*1.0) 95 | 96 | train = files[:tr_point] 97 | test = files[tr_point:te_point] 98 | valid = files[te_point:va_point] 99 | 100 | util.copy_file_as_image(train, PACK_PATH+"/train/"+di) 101 | util.copy_file_as_image(test, PACK_PATH+"/test/"+di) 102 | util.copy_file_as_image(valid, PACK_PATH+"/valid/"+di) 103 | 104 | 105 | def path_to_dirlist(path=None): 106 | 107 | directories = [] 108 | for dirname in os.listdir(path): 109 | directories.append(dirname) 110 | 111 | return directories 112 | 113 | def dirlist_to_dataset(path=None, dirlist=None): 114 | 115 | extensions = [".jpg",".JPG",".jpeg",".JPEG"] 116 | 117 | height = 0 118 | width = 0 119 | dimension = 0 120 | classes = len(dirlist) 121 | 122 | for di in dirlist: 123 | for ex in extensions: 124 | for fi in glob.glob(path+"/"+di+"/*"+ex): 125 | sample = scipy.misc.imread(fi) 126 | if(len(sample.shape) < 3): 127 | dimension = 1 128 | else: 129 | dimension = sample.shape[2] 130 | height = sample.shape[0] 131 | width = sample.shape[1] 132 | break 133 | 134 | data_list = [] 135 | label_list = [] 136 | 137 | label_num = 0 138 | for di in dirlist: 139 | for ex in extensions: 140 | for fi in glob.glob(path+"/"+di+"/*"+ex): 141 | data_list.append(fi) 142 | label_list.append(label_num) 143 | label_num = label_num + 1 144 | 145 | return data_list, label_list, classes 146 | 147 | def load_dataset(path=None, img_h=28, img_w=28): 148 | 149 | print("\n***** Load dataset *****") 150 | 151 | split_data(path=path) 152 | 153 | dirlist = path_to_dirlist(path=PACK_PATH+"/train") 154 | if(len(dirlist) > 0): 155 | train_datas, train_labels, classes = dirlist_to_dataset(path="./train", dirlist=dirlist) 156 | 157 | dirlist = path_to_dirlist(path=PACK_PATH+"/test") 158 | if(len(dirlist) > 0): 159 | test_datas, test_labels, classes = dirlist_to_dataset(path="./test", dirlist=dirlist) 160 | 161 | dirlist = path_to_dirlist(path=PACK_PATH+"/valid") 162 | if(len(dirlist) > 0): 163 | valid_datas, valid_labels, classes = dirlist_to_dataset(path="./valid", dirlist=dirlist) 164 | 165 | train = DataSet(who_am_i="train", datas=train_datas, labels=train_labels, class_len=classes, height=img_h, width=img_w) 166 | test = DataSet(who_am_i="test", datas=test_datas, labels=test_labels, class_len=classes, height=img_h, width=img_w) 167 | validation = DataSet(who_am_i="valid", datas=valid_datas, labels=valid_labels, class_len=classes, height=img_h, width=img_w) 168 | 169 | num_train = train.amount 170 | num_test = test.amount 171 | print(" Num of Train images : "+str(num_train)) 172 | print(" Num of Test images : "+str(num_test)) 173 | 174 | return base.Datasets(train=train, test=test, validation=validation), classes, min(num_train, num_test) 175 | -------------------------------------------------------------------------------- /source/developed.py: -------------------------------------------------------------------------------- 1 | stamp=""" 2 | Developed by... 3 | _ _ __ __ _ _ ____ _ _ _ _ __ __ _ _ 4 | | | | _| \ \ | __| | | | | | _| \ \ | 5 | | / _| ()| | |_ | | | / _| ()| | _ 6 | |__/____|__/_|_|____| |_|_/_/____|__/_|_| (_) 7 | """ 8 | 9 | def print_stamp(): 10 | print(stamp) 11 | print("https://github.com/YeongHyeon/CNN_Own_Dataset") 12 | -------------------------------------------------------------------------------- /source/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['TF_CPP_MIN_LOG_LEVEL']='2' 3 | 4 | import tensorflow as tf 5 | 6 | def weight_variable(shape): 7 | # standard deviation = 0.1 8 | initial = tf.truncated_normal(shape, stddev=0.1) 9 | return tf.Variable(initial) 10 | 11 | def bias_variable(shape): 12 | initial = tf.constant(0.1, shape=shape) 13 | return tf.Variable(initial) 14 | 15 | def conv2d(x, W): 16 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 17 | 18 | def max_pool_2x2(x): 19 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], 20 | strides=[1, 2, 2, 1], padding='SAME') 21 | 22 | def conv_neural_network(x, y_, height=28, width=28, dimension=1, classes=None): 23 | 24 | print("\n***** Initialize CNN Layers *****") 25 | 26 | print("\n* Layer 1 Init") 27 | # 5, 5: window size 28 | # 1: number of input channel 29 | # 32: number of output channel 30 | W_conv1 = weight_variable([5, 5, dimension, 32]) 31 | b_conv1 = bias_variable([32]) 32 | 33 | # Convolusion x(input data) and W(weight) and add b(bias) 34 | # And apply relu function 35 | h_conv1 = tf.nn.relu(conv2d(x, W_conv1) + b_conv1) 36 | # Apply max pooling on (h = x conv W + b) 37 | h_pool1 = max_pool_2x2(h_conv1) 38 | print(" "+str(h_pool1.shape)) 39 | 40 | print("\n* Layer 2 Init") 41 | W_conv2 = weight_variable([5, 5, 32, 64]) 42 | b_conv2 = bias_variable([64]) 43 | 44 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) 45 | h_pool2 = max_pool_2x2(h_conv2) 46 | print(" "+str(h_pool2.shape)) 47 | 48 | """ 49 | One benefit of replacing a fully connected layer with a convolutional layer is that the number of parameters to adjust are reduced due to the fact that the weights are shared in a convolutional layer. 50 | This means faster and more robust learning. Additionally max pooling can be used just after a convolutional layer to reduce the dimensionality of the layer. 51 | This means improved robustness to distortions in input stimuli and a better overall performance. 52 | reference: https://www.quora.com/What-are-the-benefits-of-converting-a-fully-connected-layer-in-a-deep-neural-network-to-an-equivalent-convolutional-layer 53 | """ 54 | print("\n* Fully connected Layer Init") 55 | # 7*7: frame size : 28*28 -> 14*14 -> 7*7 (caused by max pool) 56 | # 64: number of output channel of Layer 2 57 | full_flat = int(h_pool2.shape[1] * h_pool2.shape[2] * h_pool2.shape[3]) 58 | full_con = 1024 59 | W_fc1 = weight_variable([full_flat, full_con]) 60 | b_fc1 = bias_variable([full_con]) 61 | 62 | h_pool2_flat = tf.reshape(h_pool2, [-1, full_flat]) 63 | h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) 64 | print(" "+str(h_fc1.shape)) 65 | 66 | print("\n* Dropout Layer Init") 67 | # Prevention overfitting 68 | keep_prob = tf.placeholder(tf.float32) 69 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 70 | print(" "+str(h_fc1_drop.shape)) 71 | 72 | print("\n* Softmax Layer Init") 73 | W_fc2 = weight_variable([full_con, classes]) 74 | b_fc2 = bias_variable([classes]) 75 | 76 | y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2) 77 | print(" "+str(y_conv.shape)) 78 | 79 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1])) 80 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # return 81 | 82 | correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1)) 83 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # return 84 | 85 | return keep_prob, train_step, accuracy 86 | -------------------------------------------------------------------------------- /source/sub_procedure.py: -------------------------------------------------------------------------------- 1 | import os, inspect 2 | import source.utility as util 3 | 4 | PACK_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))+"/.." 5 | 6 | def training(steps=None, batch_size=None, dataset=None, sess=None, keep_prob=None, train_step=None, accuracy=None, x=None, y_=None, saver=None): 7 | 8 | print_step = 10 9 | if(steps <= 10): 10 | print_step = 1 11 | 12 | train_acc_list = [] 13 | test_acc_list = [] 14 | print("\n Training... steps: %d, batch size: %d\n" %(steps, batch_size)) 15 | 16 | for i in range(steps): 17 | # dataset.train.next_batch returns images, labels 18 | train = dataset.train.next_batch(batch_size) 19 | if i%print_step == 0: 20 | test = dataset.test.next_batch(batch_size) 21 | 22 | train_accuracy = accuracy.eval(feed_dict={x:train[0], y_:train[1], keep_prob: 1.0}) 23 | test_accuracy = accuracy.eval(feed_dict={x:test[0], y_:test[1], keep_prob: 1.0}) 24 | train_acc_list.append(train_accuracy) 25 | test_acc_list.append(test_accuracy) 26 | print(" Step %d\n\ttrain acc, test acc | %.5f, %.5f\r" %(i, train_accuracy, test_accuracy)) 27 | train_step.run(feed_dict={x:train[0], y_:train[1], keep_prob: 0.5}) # dropout 50% 28 | 29 | test = dataset.test.next_batch(batch_size) 30 | print("\n Final Test accuracy %g"%accuracy.eval(feed_dict={x:test[0], y_:test[1], keep_prob: 1.0})) 31 | 32 | if(not(os.path.exists("./checkpoint"))): 33 | os.mkdir("./checkpoint") 34 | else: 35 | pass 36 | saver.save(sess, "./checkpoint/checkpoint",global_step=1000) 37 | 38 | util.show_graph(train_acc_list, test_acc_list) 39 | util.save_graph_as_image(train_acc_list, test_acc_list) 40 | -------------------------------------------------------------------------------- /source/utility.py: -------------------------------------------------------------------------------- 1 | # import modules 2 | import os 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import psutil, shutil 6 | import scipy.misc 7 | 8 | # from modules 9 | from datetime import datetime 10 | 11 | def directory_check(dir_name): 12 | if(os.path.exists(dir_name)): 13 | shutil.rmtree(dir_name) 14 | os.mkdir(dir_name) 15 | else: 16 | os.mkdir(dir_name) 17 | 18 | def copy_file_as_image(origin, copy): 19 | count = 0 20 | for ori in origin: 21 | shutil.copy(ori, copy+"/"+str(count)+".jpg") 22 | count = count + 1 23 | 24 | def memory_check(): 25 | pid = os.getpid() 26 | proc = psutil.Process(pid) 27 | used_mem = proc.memory_info()[0] 28 | 29 | print(" Memory Used: %.2f GB\t( %.2f MB )" %(used_mem/(2**30), used_mem/(2**20))) 30 | 31 | return used_mem 32 | 33 | def show_graph(train_acc_list, test_acc_list): 34 | # draw graph 35 | markers = {'train': 'o', 'test': 's'} 36 | x = np.arange(len(train_acc_list)) 37 | plt.plot(x, train_acc_list, label='train acc') 38 | plt.plot(x, test_acc_list, label='test acc', linestyle='--') 39 | plt.xlabel("epochs") 40 | plt.ylabel("accuracy") 41 | plt.ylim(0, 1.1) 42 | plt.legend(loc='lower right') 43 | plt.show() 44 | 45 | def save_graph_as_image(train_list, test_list, ylabel=""): 46 | 47 | x = np.arange(len(train_list)) 48 | plt.clf() 49 | plt.plot(x, train_list, label="train "+ylabel) 50 | plt.plot(x, test_list, label="test "+ylabel, linestyle='--') 51 | plt.xlabel("step") 52 | plt.ylabel(ylabel) 53 | plt.ylim(-0.1, max([1, max(train_list), max(test_list)])*1.1) 54 | plt.legend(loc='lower right') 55 | 56 | if(not(os.path.exists("./graph"))): 57 | os.mkdir("./graph") 58 | else: 59 | pass 60 | now = datetime.now() 61 | 62 | print(" Save "+ylabel+" graph in ./graph") 63 | plt.savefig("./graph/"+ylabel+"_"+now.strftime('%Y%m%d_%H%M%S%f')+".png") 64 | 65 | def save_matrix_as_image(filename, matrix): 66 | matrix = matrix.real 67 | 68 | min_idx = matrix.argmin() 69 | bias = matrix[int(min_idx/matrix.shape[0])][int(min_idx%matrix.shape[0])] 70 | matrix = matrix + abs(bias) 71 | 72 | scipy.misc.imsave(filename+".jpg", matrix) 73 | --------------------------------------------------------------------------------