├── Intro ├── __init__.py ├── hello_world.py └── math_ops.py ├── Kaggle ├── README.md ├── __init__.py ├── classif_fish.py ├── classif_fish_exo.py ├── extract_deepFeatures_fish.py ├── inception_utils.py └── inception_v3.py ├── MNIST ├── __init__.py ├── autoencoder.py ├── autoencoder_exo.py ├── conv_ae.py ├── conv_ae_exo.py ├── gan_from_web.py ├── gan_mlp.py ├── lenet.py ├── lenet_exo.py ├── mlp.py ├── mlp_exo.py ├── one_conv.py ├── one_conv_exo.py ├── rnn.py ├── rnn_exo.py ├── softmax.py └── softmax_exo.py ├── README.md ├── RL ├── q_learning_neural_net.py ├── q_learning_neural_net_exo.py └── random_walk.py └── SGD ├── __init__.py ├── binary_classif.py ├── binary_classif_exo.py ├── linear_regression.py └── linear_regression_exo.py /Intro/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fabienbaradel/Tensorflow-tutorials/3a3c6a233156d0463e0d874e97941c3331ced6ee/Intro/__init__.py -------------------------------------------------------------------------------- /Intro/hello_world.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | """ 4 | First steps with Tensorflow: 5 | - introduction to constant op 6 | - introduction to variable 7 | - first approach of feeding data 8 | """ 9 | 10 | 11 | ###################################### 12 | ### CREATE A CONSTANT AND PRINT IT ### 13 | ###################################### 14 | 15 | # Create a constant 16 | hello = tf.constant('Welcome to TensorFlow ') 17 | # Info about the constant 18 | print(hello) 19 | 20 | # Intro tf session 21 | with tf.Session() as sess: 22 | # Run the op 23 | print(sess.run(hello)) 24 | 25 | 26 | ####################################### 27 | ### FEEDING STRING TO A PLACEHOLDER ### 28 | ####################################### 29 | 30 | # 1) Create a placeholder 31 | my_placeholder = tf.placeholder(dtype=tf.string) 32 | 33 | # 2) Create my operation 34 | my_string = my_placeholder 35 | 36 | # 3) Intro tf session 37 | with tf.Session() as sess: 38 | # Run the op 39 | print(sess.run(my_string, feed_dict={my_placeholder: "hello"})) 40 | 41 | 42 | ################ 43 | ### EXERCISE ### 44 | ################ 45 | """ 46 | Print the concatenation of the hello constant and your name: 47 | => you only have to create an operation between 'hello' and 'placeholder' 48 | """ 49 | 50 | # Placeholder and the constant 51 | my_placeholder = tf.placeholder(dtype=tf.string) 52 | hello = tf.constant('Welcome to TensorFlow ') 53 | 54 | # Create your own operation (only one line...) 55 | welcome = tf.add(my_placeholder, hello) 56 | 57 | # Intro tf session 58 | with tf.Session() as sess: 59 | # Run the op 60 | print(sess.run(welcome, feed_dict={my_placeholder: "Fabien"})) 61 | print(sess.run(welcome, feed_dict={my_placeholder: "Jean"})) 62 | print(sess.run(welcome, feed_dict={my_placeholder: "Romaric"})) 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /Intro/math_ops.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | """ 4 | Same as with string, we now discover basic operations in Tensorflow for interger/float 5 | """ 6 | 7 | ############################ 8 | ## DEALING WITH CONSTANT ### 9 | ############################ 10 | 11 | a = tf.constant(2) 12 | b = tf.constant(3) 13 | a 14 | 15 | # Define some operations 16 | add = tf.add(a, b) 17 | mul = tf.mul(a, b) 18 | 19 | 20 | # Launch the default graph. 21 | with tf.Session() as sess: 22 | # Print the TF constants 23 | print("The constant a: " + str(a) ) # a is not equal to 2 but the constant a is assign to 2 during the session 24 | print("The constant b: " + str(b)) 25 | 26 | # Get the values of each constants 27 | (a_value, b_value) = sess.run([a, b]) # a_value and b_value are not anymore Tensorflow operation 28 | print("a=%i, b=%i" % (a_value, b_value)) 29 | 30 | # Basic math operation on constant 31 | print("Addition with constants: %i" % sess.run(add)) 32 | print("Multiplication with constants: %i" % sess.run(mul)) 33 | 34 | 35 | ####################################### 36 | ### WITH PLACEHOLDERS AND VARIABLES ### 37 | ####################################### 38 | 39 | a = tf.placeholder(tf.int16) 40 | b = tf.placeholder(tf.int16) 41 | 42 | # Define some operations 43 | add = tf.add(a, b) 44 | mul = tf.mul(a, b) 45 | 46 | # Launch the default graph. 47 | with tf.Session() as sess: 48 | # Run every operation with variable input 49 | print("Addition with variables: %i" % sess.run(add, feed_dict={a: 2, b: 3})) 50 | print("Multiplication with variables: %i" % sess.run(mul, feed_dict={a: 2, b: 3})) 51 | 52 | 53 | ################## 54 | ### EXERCISE 1 ### 55 | ################## 56 | """ 57 | Dealing with matrices 58 | """ 59 | import numpy as np 60 | 61 | # My two placeholders for the matrices 62 | A = tf.placeholder(tf.int32, shape=[2,3]) 63 | A.get_shape() 64 | B = tf.placeholder(tf.int32, shape=[2,3]) 65 | B.get_shape() 66 | 67 | # Compute = A'.B 68 | mult = tf.matmul(A,B, transpose_a=True) 69 | 70 | # Launch the default graph. 71 | with tf.Session() as sess: 72 | # Create two numpy matrices 73 | A_value = np.array([1, 2, 3, 4, 5, 6]).reshape((2,3)) 74 | print("A = ") 75 | print(A_value) 76 | print("") 77 | B_value = np.array([1, 0, 2, 0, 0, -1]).reshape((2, 3)) 78 | print("B = ") 79 | print(B_value) 80 | print("") 81 | 82 | 83 | # Get value for the mult operation 84 | mult_value = sess.run(mult, feed_dict={A: A_value, B: B_value}) 85 | print("mult A' by B = ") 86 | print(mult_value) 87 | print("") 88 | 89 | 90 | ################## 91 | ### EXERCISE 2 ### 92 | ################## 93 | """ 94 | Dealing with matrices again 95 | """ 96 | 97 | # My two placeholders for the matrices 98 | A = tf.placeholder(tf.int32, shape=[2,3]) 99 | A.get_shape() 100 | B = tf.placeholder(tf.int32, shape=[2,3]) 101 | B.get_shape() 102 | 103 | # Pairwise multiplication and sum over the matrix 104 | mult_element_by_element = tf.mul(A,B) #????? # see tf.mul 105 | sum = tf.reduce_sum(mult_element_by_element) #????? # see tf.reduce_sum 106 | 107 | # Launch the default graph. 108 | with tf.Session() as sess: 109 | # Create two numpy matrices 110 | A_value = np.array([1, 2, 3, 4, 5, 6]).reshape((2,3)) 111 | print("A = ") 112 | print(A_value) 113 | print("") 114 | B_value = np.array([1, 0, 2, 0, 0, -1]).reshape((2, 3)) 115 | print("B = ") 116 | print(B_value) 117 | print("") 118 | 119 | # Get value for the element by element multiplication operation 120 | mult_element_by_element_value = sess.run(mult_element_by_element, feed_dict={A: A_value, B: B_value}) 121 | print("A by B element by element = ") 122 | print(mult_element_by_element_value) 123 | print("") 124 | 125 | # Get value for sum operation 126 | sum_value = sess.run(sum, feed_dict={A: A_value, B: B_value}) 127 | print("A by B element by element then sum = ") 128 | print(sum_value) 129 | print("") 130 | 131 | 132 | 133 | 134 | 135 | 136 | -------------------------------------------------------------------------------- /Kaggle/README.md: -------------------------------------------------------------------------------- 1 | # README 2 | 3 | How to make a first (honorable) submission to [The Natue Conservancy Fisheries Monitoring](https://www.kaggle.com/c/the-nature-conservancy-fisheries-monitoring) challenge. 4 | 5 | We use a deep learning pretrained models (already trained on Imagenet) and extract deep Features from it. 6 | Each images is now summarize by a vector of size 2048. This part is done by [here](https://github.com/fabienbaradel/Tensorflow-tutorials/blob/master/Kaggle/extract_deepFeatures_fish.py). 7 | You have to download the image from the Kaggle website to do it (Don't do it by yourself it takes a while...) 8 | The pretrained inception v3 checkpoint can be download [here](https://drive.google.com/file/d/0B3K4bVd6ydRwVzRqLUJvVTJTSTA/view?usp=sharing). 9 | 10 | In your virtual machine you have 4 .txt in this repository with the labels/features/image_name of the Fish dataset: 11 | - fish_features.txt: inception_v3 features of the training set 12 | - fish_labels.txt: labels of the training set 13 | - fish_features_test.txt: inception_v3 features of the test set 14 | - pic_names_test.py: name of the pictures of the test set 15 | The .txt files can be download [here](https://drive.google.com/file/d/0B3K4bVd6ydRwR1VrVkwtblJnNHc/view?usp=sharing) if you are not working with the VM. Don't forget to add them in this directory on your laptop. 16 | 17 | Complete the [classif_fish_exo.py](https://github.com/fabienbaradel/Tensorflow-tutorials/blob/master/Kaggle/classif_fish.py). 18 | You will create a classification from the dee features. Feel free to build the network you want. 19 | Running this code will create a pred.csv file in this directory which corresponds to the submission file required by Kaggle. 20 | 21 | A simple softmax regression with 10 epochs leads to a score of ~1.14 (rank: 300/1100). 22 | Improve it and submit the pred.csv in Kaggle! ;) -------------------------------------------------------------------------------- /Kaggle/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fabienbaradel/Tensorflow-tutorials/3a3c6a233156d0463e0d874e97941c3331ced6ee/Kaggle/__init__.py -------------------------------------------------------------------------------- /Kaggle/classif_fish.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import os 6 | from numpy import random 7 | import csv 8 | 9 | 10 | class Fish_classif(object): 11 | """ 12 | Fish classif from Inception_v3 features (2048) 13 | """ 14 | 15 | def __init__(self, learning_rate, batch_size, num_epoch): 16 | """ 17 | Init the class with some parameters 18 | :param learning_rate: 19 | :param batch_size: 20 | """ 21 | # Parameters 22 | self.learning_rate = learning_rate 23 | self.batch_size = batch_size 24 | self.num_epochs = num_epoch 25 | self.num_classes = 8 26 | self.input_size = 2048 27 | self.training_size = 3720 28 | self.batch_per_epoch = int(self.training_size/self.batch_size) 29 | self.display_step = 1 30 | 31 | # Placeholders 32 | self.X = tf.placeholder(tf.float32, [None, 2048]) 33 | self.Y = tf.placeholder(tf.float32, [None]) 34 | 35 | # one hot encoding 36 | self.Y_one_hot = tf.one_hot(tf.cast(self.Y, tf.int32), self.num_classes) 37 | 38 | 39 | # Load dataset 40 | print("Dataset loading...") 41 | self.data_X = np.loadtxt('fish_features.txt') 42 | self.data_Y = np.loadtxt('fish_labels.txt') 43 | print(str(self.data_X.shape[0])+" training data.") 44 | 45 | # Split train / validation 46 | ind = range(self.data_X.shape[0]) 47 | random.shuffle(ind) 48 | ind_train, ind_val = ind[:self.training_size], ind[self.training_size:] 49 | self.train_X, self.train_Y = self.data_X[ind_train, :], self.data_Y[ind_train] 50 | self.val_X, self.val_Y = self.data_X[ind_val, :], self.data_Y[ind_val] 51 | 52 | # Test 53 | self.test_X = np.loadtxt('fish_features_test.txt') 54 | 55 | print("Done!") 56 | 57 | 58 | 59 | def inference(self): 60 | """ 61 | Softmax regression 62 | :return: 63 | """ 64 | 65 | # Set model weights 66 | self.W = tf.Variable(tf.zeros([2048, 8])) 67 | self.b = tf.Variable(tf.zeros([8])) 68 | 69 | # Construct the inference 70 | self.logits = tf.add(tf.matmul(self.X, self.W), self.b) # WX+b : the logits 71 | self.Y_hat = tf.nn.softmax(self.logits) # softmax 72 | 73 | def losses(self): 74 | """ 75 | Compute the cross entropy loss 76 | :return: 77 | """ 78 | # cross entropy loss 79 | self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.logits, self.Y_one_hot)) 80 | 81 | 82 | 83 | def optimizer(self): 84 | """ 85 | Create a optimizer and therefore a training operation 86 | :return: 87 | """ 88 | # The optimizer 89 | self.opt = tf.train.AdagradOptimizer(self.learning_rate) 90 | 91 | # Training operation to run later 92 | self.train_op = self.opt.minimize(self.loss) 93 | 94 | 95 | def metrics(self): 96 | """ 97 | Compute the accuracy 98 | :return: 99 | """ 100 | # Label prediction of the model (the highest one) 101 | self.predicted_label = tf.argmax(self.Y_hat, 1) 102 | # Real class: 103 | self.real_label = tf.argmax(self.Y_one_hot, 1) 104 | # Number of correct prediction 105 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 106 | # Calculate accuracy 107 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 108 | self.accuracy = tf.mul(100.0, self.accuracy) 109 | 110 | 111 | def train(self): 112 | """ 113 | Train the model 114 | :return: 115 | """ 116 | # Initializing the variables 117 | init = tf.global_variables_initializer() 118 | 119 | # Launch the graph 120 | with tf.Session() as sess: 121 | sess.run(init) 122 | 123 | # Training cycle 124 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 125 | for s in range(self.batch_per_epoch): 126 | # Get batch over the dataset 127 | location = np.random.choice(range(self.training_size), self.batch_size) 128 | batch_xs, batch_ys = self.train_X[location], self.train_Y[location] 129 | 130 | # Apply the training op 131 | (_, 132 | loss_train, 133 | accuracy_train) = sess.run([self.train_op, 134 | self.loss, 135 | self.accuracy], 136 | feed_dict={self.X: batch_xs, 137 | self.Y: batch_ys}) 138 | # Model on validation set 139 | (loss_val, 140 | accuracy_val) = sess.run([self.loss, 141 | self.accuracy], 142 | feed_dict={self.X: self.val_X, 143 | self.Y: self.val_Y}) 144 | 145 | # Print loss and accuracy on the batch 146 | if s % 200 == 0: 147 | print("\033[1;37;40mStep: %04d , " 148 | "TRAIN: loss = %.4f - accuracy = %.2f | " 149 | "VALIDATION: loss = %.4f - accuracy = %.2f" 150 | % ((epoch*self.batch_per_epoch + s), 151 | loss_train, accuracy_train, 152 | loss_val, accuracy_val) ) 153 | 154 | 155 | # Do prediction for the test set and write a p.csv file 156 | Y_hat = sess.run(self.Y_hat, 157 | feed_dict={self.X: self.test_X}) 158 | 159 | # Write prediction into a file 160 | csv_pred_file = open('./pred.csv', 'w') 161 | img_name_file = open('pic_names_test.txt', 'r') 162 | wr = csv.writer(csv_pred_file) 163 | wr.writerow(['image', 'ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT']) 164 | for i in range(Y_hat.shape[0]): 165 | img_name = img_name_file.readline() 166 | img_name = img_name.rstrip('\n') 167 | row = [img_name]+list(Y_hat[i,:]) 168 | wr.writerow(row) 169 | 170 | 171 | 172 | def main(_): 173 | """ 174 | Main function 175 | :param _: 176 | :return: 177 | """ 178 | 179 | # Instanciate a Fish classif 180 | model = Fish_classif(learning_rate=0.01, 181 | batch_size=64, 182 | num_epoch=10) 183 | # Setup the graph 184 | model.inference() 185 | 186 | # Compute loss and metrics 187 | model.losses() 188 | model.metrics() 189 | 190 | # Create an optimzer 191 | model.optimizer() 192 | 193 | # And finally train your model! 194 | model.train() 195 | 196 | 197 | 198 | # To start the app for tensorflow 199 | if __name__ == '__main__': 200 | tf.app.run() 201 | 202 | -------------------------------------------------------------------------------- /Kaggle/classif_fish_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import os 6 | from numpy import random 7 | import csv 8 | 9 | 10 | class Fish_classif(object): 11 | """ 12 | Fish classif from Inception_v3 features (2048) 13 | """ 14 | 15 | def __init__(self, learning_rate, batch_size, num_epoch): 16 | """ 17 | Init the class with some parameters 18 | :param learning_rate: 19 | :param batch_size: 20 | """ 21 | # Parameters 22 | self.learning_rate = learning_rate 23 | self.batch_size = batch_size 24 | self.num_epochs = num_epoch 25 | self.num_classes = 8 26 | self.input_size = 2048 27 | self.training_size = 3720 28 | self.batch_per_epoch = int(self.training_size/self.batch_size) 29 | self.display_step = 1 30 | 31 | # Placeholders 32 | self.X = tf.placeholder(tf.float32, [None, 2048]) 33 | self.Y = tf.placeholder(tf.float32, [None]) 34 | 35 | # one hot encoding 36 | self.Y_one_hot = tf.one_hot(tf.cast(self.Y, tf.int32), self.num_classes) 37 | 38 | 39 | # Load dataset 40 | print("Dataset loading...") 41 | self.data_X = np.loadtxt('fish_features.txt') 42 | self.data_Y = np.loadtxt('fish_labels.txt') 43 | print(str(self.data_X.shape[0])+" training data.") 44 | 45 | # Split train / validation 46 | ind = range(self.data_X.shape[0]) 47 | random.shuffle(ind) 48 | ind_train, ind_val = ind[:self.training_size], ind[self.training_size:] 49 | self.train_X, self.train_Y = self.data_X[ind_train, :], self.data_Y[ind_train] 50 | self.val_X, self.val_Y = self.data_X[ind_val, :], self.data_Y[ind_val] 51 | 52 | # Test 53 | self.test_X = np.loadtxt('fish_features_test.txt') 54 | 55 | print("Done!") 56 | 57 | 58 | 59 | def inference(self): 60 | """ 61 | Softmax regression 62 | :return: 63 | """ 64 | 65 | # Construct the inference 66 | self.logits = ??????? 67 | self.Y_hat = ??????? 68 | 69 | def losses(self): 70 | """ 71 | Compute the cross entropy loss 72 | :return: 73 | """ 74 | # cross entropy loss 75 | self.loss = ?????? 76 | 77 | 78 | 79 | def optimizer(self): 80 | """ 81 | Create a optimizer and therefore a training operation 82 | :return: 83 | """ 84 | ??????? 85 | 86 | 87 | def metrics(self): 88 | """ 89 | Compute the accuracy 90 | :return: 91 | """ 92 | # Label prediction of the model (the highest one) 93 | self.predicted_label = tf.argmax(self.Y_hat, 1) 94 | # Real class: 95 | self.real_label = tf.argmax(self.Y_one_hot, 1) 96 | # Number of correct prediction 97 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 98 | # Calculate accuracy 99 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 100 | self.accuracy = tf.mul(100.0, self.accuracy) 101 | 102 | 103 | def train(self): 104 | """ 105 | Train the model 106 | :return: 107 | """ 108 | # Initializing the variables 109 | init = tf.global_variables_initializer() 110 | 111 | # Launch the graph 112 | with tf.Session() as sess: 113 | sess.run(init) 114 | 115 | # Training cycle 116 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 117 | for s in range(self.batch_per_epoch): 118 | # Get batch over the dataset 119 | location = np.random.choice(range(self.training_size), self.batch_size) 120 | batch_xs, batch_ys = self.train_X[location], self.train_Y[location] 121 | 122 | # Apply the training op 123 | (_, 124 | loss_train, 125 | accuracy_train) = sess.run([self.train_op, 126 | self.loss, 127 | self.accuracy], 128 | feed_dict={self.X: batch_xs, 129 | self.Y: batch_ys}) 130 | # Model on validation set 131 | (loss_val, 132 | accuracy_val) = sess.run([self.loss, 133 | self.accuracy], 134 | feed_dict={self.X: self.val_X, 135 | self.Y: self.val_Y}) 136 | 137 | # Print loss and accuracy on the batch 138 | if s % 200 == 0: 139 | print("\033[1;37;40mStep: %04d , " 140 | "TRAIN: loss = %.4f - accuracy = %.2f | " 141 | "VALIDATION: loss = %.4f - accuracy = %.2f" 142 | % ((epoch*self.batch_per_epoch + s), 143 | loss_train, accuracy_train, 144 | loss_val, accuracy_val) ) 145 | 146 | 147 | # Do prediction for the test set and write a p.csv file 148 | Y_hat = sess.run(self.Y_hat, 149 | feed_dict={self.X: self.test_X}) 150 | 151 | # Write prediction into a file 152 | csv_pred_file = open('./pred.csv', 'w') 153 | img_name_file = open('pic_names_test.txt', 'r') 154 | wr = csv.writer(csv_pred_file) 155 | wr.writerow(['image', 'ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT']) 156 | for i in range(Y_hat.shape[0]): 157 | img_name = img_name_file.readline() 158 | img_name = img_name.rstrip('\n') 159 | row = [img_name]+list(Y_hat[i,:]) 160 | wr.writerow(row) 161 | 162 | 163 | 164 | def main(_): 165 | """ 166 | Main function 167 | :param _: 168 | :return: 169 | """ 170 | 171 | # Instanciate a Fish classif 172 | model = Fish_classif(learning_rate=0.01, 173 | batch_size=64, 174 | num_epoch=10) 175 | # Setup the graph 176 | model.inference() 177 | 178 | # Compute loss and metrics 179 | model.losses() 180 | model.metrics() 181 | 182 | # Create an optimzer 183 | model.optimizer() 184 | 185 | # And finally train your model! 186 | model.train() 187 | 188 | 189 | 190 | # To start the app for tensorflow 191 | if __name__ == '__main__': 192 | tf.app.run() 193 | 194 | -------------------------------------------------------------------------------- /Kaggle/extract_deepFeatures_fish.py: -------------------------------------------------------------------------------- 1 | """ 2 | Extraction of features from Inception v3 model (pretrained on Imagenet) 3 | """ 4 | 5 | 6 | import os.path 7 | import re 8 | import sys 9 | import tarfile 10 | import time 11 | 12 | import numpy as np 13 | from six.moves import urllib 14 | import tensorflow as tf 15 | from PIL import Image 16 | import tensorflow as tf 17 | 18 | import inception_v3 as inception 19 | from scipy import misc 20 | import re 21 | 22 | slim = tf.contrib.slim 23 | 24 | FLAGS = tf.app.flags.FLAGS 25 | 26 | 27 | tf.app.flags.DEFINE_string('ckpt_path', './inception_v3.ckpt', 28 | 'The directory where the model was written to or an absolute path to a ' 29 | 'checkpoint file.') 30 | tf.app.flags.DEFINE_string('rgb_dir', '/Users/fabien/Documents/Kaggle/TheNatureConservancyFisheriesMonitoring/Data', 31 | """Absolute path to image files.""") 32 | tf.app.flags.DEFINE_string('inception_bottlenecks_dir', '/Users/fabien/Documents/Kaggle/TheNatureConservancyFisheriesMonitoring/Data/inceptionv3_features', 33 | """Where to write the extracted bottlenecks.""") 34 | 35 | 36 | 37 | def main(_): 38 | # Placeholder to feed the image 39 | images = tf.placeholder(tf.float32, [None, None, None, 3]) 40 | 41 | # get the logist and bottlenecks for each images 42 | with slim.arg_scope(inception.inception_v3_arg_scope()): 43 | logits, end_points = inception.inception_v3(images, num_classes=1001, is_training=False) 44 | bottlenecks = end_points['PreLogits'] 45 | bottlenecks = tf.squeeze(bottlenecks) 46 | 47 | # List of the classes 48 | list_classes = os.listdir(os.path.join(FLAGS.rgb_dir, 'train')) 49 | list_classes = [x for x in list_classes if 'DS' not in x ] 50 | 51 | # Create the directories of the features for each classes 52 | for c in list_classes: 53 | if not os.path.exists(os.path.join(FLAGS.inception_bottlenecks_dir, c)): 54 | os.makedirs(os.path.join(FLAGS.inception_bottlenecks_dir, c)) 55 | 56 | print("All the classes: "+str(list_classes)) 57 | 58 | # Saver to restore the Inception v3 pretrained model on Imagenet 59 | saver = tf.train.Saver(max_to_keep=None) 60 | 61 | # # Start the session 62 | with tf.Session() as sess: 63 | # Restore the pretrained model 64 | print("Restoring inception v3 model from: " + FLAGS.ckpt_path) 65 | saver.restore(sess, FLAGS.ckpt_path) 66 | print("Restored!") 67 | 68 | # Loop over all the directories (video) 69 | for label in list_classes: 70 | list_img = os.listdir(os.path.join(FLAGS.rgb_dir, 'train', label)) 71 | list_img = [x for x in list_img if 'DS' not in x] 72 | print("Class: " + label + " has " + str(len(list_img)) + " images.") 73 | for img in list_img: 74 | print(img) 75 | # Catch file from directory one by one and convert them to np array 76 | img_filename = os.path.join(FLAGS.rgb_dir, 'train', label, img) 77 | image = Image.open(img_filename) 78 | image = image.resize((299, 299)) 79 | image_data = np.array(image) 80 | # Inception v3 preprocessing 81 | image_data = image_data / 255.0 82 | image_data = image_data - 0.5 83 | image_data = image_data * 2 84 | image_data = image_data.reshape((1, 299, 299, 3)) 85 | 86 | # Catch the features 87 | bottlenecks_v = sess.run(bottlenecks, {images: image_data}) 88 | 89 | # Save bottlenecks 90 | txtfile_bottlenecks = os.path.join(FLAGS.inception_bottlenecks_dir, 'train', label, img + '.txt') 91 | np.savetxt(txtfile_bottlenecks, bottlenecks_v) 92 | 93 | # Finally do the same for test images 94 | list_test_img = os.listdir(os.path.join(FLAGS.rgb_dir, 'test_stg1')) 95 | list_test_img = [x for x in list_test_img if 'DS' not in x] 96 | for img in list_test_img: 97 | print(img) 98 | # Catch file from directory one by one and convert them to np array 99 | img_filename = os.path.join(FLAGS.rgb_dir, 'test_stg1', img) 100 | image = Image.open(img_filename) 101 | image = image.resize((299, 299)) 102 | image_data = np.array(image) 103 | image_data = image_data / 255.0 104 | image_data = image_data - 0.5 105 | image_data = image_data * 2 106 | image_data = image_data.reshape((1, 299, 299, 3)) 107 | 108 | # Catch the logits 109 | bottlenecks_v = sess.run(bottlenecks, {images: image_data}) 110 | 111 | # Save bottlenecks 112 | txtfile_bottlenecks = os.path.join(FLAGS.inception_bottlenecks_dir, 'test_stg1', img + '.txt') 113 | np.savetxt(txtfile_bottlenecks, bottlenecks_v) 114 | 115 | 116 | ################################################ 117 | ## Merge features/labels to one big .txt file ## 118 | ################################################ 119 | # Training set 120 | print("Creating training set") 121 | list_features = [] 122 | list_labels = [] 123 | for c in list_classes: 124 | print(c) 125 | list_img_features = os.listdir(os.path.join(FLAGS.inception_bottlenecks_dir,'train', c)) 126 | for img_features in list_img_features: 127 | features = np.loadtxt(os.path.join(FLAGS.inception_bottlenecks_dir,'train', c, img_features)) 128 | list_features.append(features) 129 | list_labels.append(list_classes.index(c)) 130 | array_features = np.asarray(list_features) 131 | array_labels = np.asarray(list_labels) 132 | # Save to txt files 133 | np.savetxt('./fish_features.txt', array_features) 134 | np.savetxt('./fish_labels.txt', array_labels) 135 | 136 | # Test set 137 | print("Creating test set") 138 | list_features = [] 139 | list_img_name = [] 140 | list_img_features = os.listdir(os.path.join(FLAGS.inception_bottlenecks_dir,'test_stg1')) 141 | for img_features in list_img_features: 142 | features = np.loadtxt(os.path.join(FLAGS.inception_bottlenecks_dir,'test_stg1', img_features)) 143 | list_features.append(features) 144 | list_labels.append(list_classes.index(c)) 145 | list_img_name.append(img_features.split('.')[0] + '.jpg') 146 | array_features = np.asarray(list_features) 147 | np.savetxt('./fish_features_test.txt', array_features) 148 | 149 | # Write name of each pictures for submission 150 | with open('./pic_names_test.txt', 'w') as thefile: 151 | for item in list_img_name: 152 | thefile.write("%s\n" % item) 153 | 154 | 155 | if __name__ == '__main__': 156 | tf.app.run() 157 | -------------------------------------------------------------------------------- /Kaggle/inception_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Contains common code shared by all inception models. 16 | 17 | Usage of arg scope: 18 | with slim.arg_scope(inception_arg_scope()): 19 | logits, end_points = inception.inception_v3(images, num_classes, 20 | is_training=is_training) 21 | 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | import tensorflow as tf 28 | 29 | slim = tf.contrib.slim 30 | 31 | 32 | def inception_arg_scope(weight_decay=0.00004, 33 | use_batch_norm=True, 34 | batch_norm_decay=0.9997, 35 | batch_norm_epsilon=0.001): 36 | """Defines the default arg scope for inception models. 37 | 38 | Args: 39 | weight_decay: The weight decay to use for regularizing the model. 40 | use_batch_norm: "If `True`, batch_norm is applied after each convolution. 41 | batch_norm_decay: Decay for batch norm moving average. 42 | batch_norm_epsilon: Small float added to variance to avoid dividing by zero 43 | in batch norm. 44 | 45 | Returns: 46 | An `arg_scope` to use for the inception models. 47 | """ 48 | batch_norm_params = { 49 | # Decay for the moving averages. 50 | 'decay': batch_norm_decay, 51 | # epsilon to prevent 0s in variance. 52 | 'epsilon': batch_norm_epsilon, 53 | # collection containing update_ops. 54 | 'updates_collections': tf.GraphKeys.UPDATE_OPS, 55 | } 56 | if use_batch_norm: 57 | normalizer_fn = slim.batch_norm 58 | normalizer_params = batch_norm_params 59 | else: 60 | normalizer_fn = None 61 | normalizer_params = {} 62 | # Set weight_decay for weights in Conv and FC layers. 63 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 64 | weights_regularizer=slim.l2_regularizer(weight_decay)): 65 | with slim.arg_scope( 66 | [slim.conv2d], 67 | weights_initializer=slim.variance_scaling_initializer(), 68 | activation_fn=tf.nn.relu, 69 | normalizer_fn=normalizer_fn, 70 | normalizer_params=normalizer_params) as sc: 71 | return sc 72 | -------------------------------------------------------------------------------- /Kaggle/inception_v3.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Contains the definition for inception v3 classification network.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import tensorflow as tf 22 | 23 | import inception_utils 24 | 25 | slim = tf.contrib.slim 26 | trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) 27 | 28 | 29 | def inception_v3_base(inputs, 30 | final_endpoint='Mixed_7c', 31 | min_depth=16, 32 | depth_multiplier=1.0, 33 | scope=None): 34 | """Inception model from http://arxiv.org/abs/1512.00567. 35 | 36 | Constructs an Inception v3 network from inputs to the given final endpoint. 37 | This method can construct the network up to the final inception block 38 | Mixed_7c. 39 | 40 | Note that the names of the layers in the paper do not correspond to the names 41 | of the endpoints registered by this function although they build the same 42 | network. 43 | 44 | Here is a mapping from the old_names to the new names: 45 | Old name | New name 46 | ======================================= 47 | conv0 | Conv2d_1a_3x3 48 | conv1 | Conv2d_2a_3x3 49 | conv2 | Conv2d_2b_3x3 50 | pool1 | MaxPool_3a_3x3 51 | conv3 | Conv2d_3b_1x1 52 | conv4 | Conv2d_4a_3x3 53 | pool2 | MaxPool_5a_3x3 54 | mixed_35x35x256a | Mixed_5b 55 | mixed_35x35x288a | Mixed_5c 56 | mixed_35x35x288b | Mixed_5d 57 | mixed_17x17x768a | Mixed_6a 58 | mixed_17x17x768b | Mixed_6b 59 | mixed_17x17x768c | Mixed_6c 60 | mixed_17x17x768d | Mixed_6d 61 | mixed_17x17x768e | Mixed_6e 62 | mixed_8x8x1280a | Mixed_7a 63 | mixed_8x8x2048a | Mixed_7b 64 | mixed_8x8x2048b | Mixed_7c 65 | 66 | Args: 67 | inputs: a tensor of size [batch_size, height, width, channels]. 68 | final_endpoint: specifies the endpoint to construct the network up to. It 69 | can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 70 | 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 71 | 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 72 | 'Mixed_6d', 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c']. 73 | min_depth: Minimum depth value (number of channels) for all convolution ops. 74 | Enforced when depth_multiplier < 1, and not an active constraint when 75 | depth_multiplier >= 1. 76 | depth_multiplier: Float multiplier for the depth (number of channels) 77 | for all convolution ops. The value must be greater than zero. Typical 78 | usage will be to set this value in (0, 1) to reduce the number of 79 | parameters or computation cost of the model. 80 | scope: Optional variable_scope. 81 | 82 | Returns: 83 | tensor_out: output tensor corresponding to the final_endpoint. 84 | end_points: a set of activations for external use, for example summaries or 85 | losses. 86 | 87 | Raises: 88 | ValueError: if final_endpoint is not set to one of the predefined values, 89 | or depth_multiplier <= 0 90 | """ 91 | # end_points will collect relevant activations for external use, for example 92 | # summaries or losses. 93 | end_points = {} 94 | 95 | if depth_multiplier <= 0: 96 | raise ValueError('depth_multiplier is not greater than zero.') 97 | depth = lambda d: max(int(d * depth_multiplier), min_depth) 98 | 99 | with tf.variable_scope(scope, 'InceptionV3', [inputs]): 100 | with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], 101 | stride=1, padding='VALID'): 102 | # 299 x 299 x 3 103 | end_point = 'Conv2d_1a_3x3' 104 | net = slim.conv2d(inputs, depth(32), [3, 3], stride=2, scope=end_point) 105 | end_points[end_point] = net 106 | if end_point == final_endpoint: return net, end_points 107 | # 149 x 149 x 32 108 | end_point = 'Conv2d_2a_3x3' 109 | net = slim.conv2d(net, depth(32), [3, 3], scope=end_point) 110 | end_points[end_point] = net 111 | if end_point == final_endpoint: return net, end_points 112 | # 147 x 147 x 32 113 | end_point = 'Conv2d_2b_3x3' 114 | net = slim.conv2d(net, depth(64), [3, 3], padding='SAME', scope=end_point) 115 | end_points[end_point] = net 116 | if end_point == final_endpoint: return net, end_points 117 | # 147 x 147 x 64 118 | end_point = 'MaxPool_3a_3x3' 119 | net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) 120 | end_points[end_point] = net 121 | if end_point == final_endpoint: return net, end_points 122 | # 73 x 73 x 64 123 | end_point = 'Conv2d_3b_1x1' 124 | net = slim.conv2d(net, depth(80), [1, 1], scope=end_point) 125 | end_points[end_point] = net 126 | if end_point == final_endpoint: return net, end_points 127 | # 73 x 73 x 80. 128 | end_point = 'Conv2d_4a_3x3' 129 | net = slim.conv2d(net, depth(192), [3, 3], scope=end_point) 130 | end_points[end_point] = net 131 | if end_point == final_endpoint: return net, end_points 132 | # 71 x 71 x 192. 133 | end_point = 'MaxPool_5a_3x3' 134 | net = slim.max_pool2d(net, [3, 3], stride=2, scope=end_point) 135 | end_points[end_point] = net 136 | if end_point == final_endpoint: return net, end_points 137 | # 35 x 35 x 192. 138 | 139 | # Inception blocks 140 | with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], 141 | stride=1, padding='SAME'): 142 | # mixed: 35 x 35 x 256. 143 | end_point = 'Mixed_5b' 144 | with tf.variable_scope(end_point): 145 | with tf.variable_scope('Branch_0'): 146 | branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') 147 | with tf.variable_scope('Branch_1'): 148 | branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0a_1x1') 149 | branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], 150 | scope='Conv2d_0b_5x5') 151 | with tf.variable_scope('Branch_2'): 152 | branch_2 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') 153 | branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], 154 | scope='Conv2d_0b_3x3') 155 | branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], 156 | scope='Conv2d_0c_3x3') 157 | with tf.variable_scope('Branch_3'): 158 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 159 | branch_3 = slim.conv2d(branch_3, depth(32), [1, 1], 160 | scope='Conv2d_0b_1x1') 161 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 162 | end_points[end_point] = net 163 | if end_point == final_endpoint: return net, end_points 164 | 165 | # mixed_1: 35 x 35 x 288. 166 | end_point = 'Mixed_5c' 167 | with tf.variable_scope(end_point): 168 | with tf.variable_scope('Branch_0'): 169 | branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') 170 | with tf.variable_scope('Branch_1'): 171 | branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0b_1x1') 172 | branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], 173 | scope='Conv_1_0c_5x5') 174 | with tf.variable_scope('Branch_2'): 175 | branch_2 = slim.conv2d(net, depth(64), [1, 1], 176 | scope='Conv2d_0a_1x1') 177 | branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], 178 | scope='Conv2d_0b_3x3') 179 | branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], 180 | scope='Conv2d_0c_3x3') 181 | with tf.variable_scope('Branch_3'): 182 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 183 | branch_3 = slim.conv2d(branch_3, depth(64), [1, 1], 184 | scope='Conv2d_0b_1x1') 185 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 186 | end_points[end_point] = net 187 | if end_point == final_endpoint: return net, end_points 188 | 189 | # mixed_2: 35 x 35 x 288. 190 | end_point = 'Mixed_5d' 191 | with tf.variable_scope(end_point): 192 | with tf.variable_scope('Branch_0'): 193 | branch_0 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') 194 | with tf.variable_scope('Branch_1'): 195 | branch_1 = slim.conv2d(net, depth(48), [1, 1], scope='Conv2d_0a_1x1') 196 | branch_1 = slim.conv2d(branch_1, depth(64), [5, 5], 197 | scope='Conv2d_0b_5x5') 198 | with tf.variable_scope('Branch_2'): 199 | branch_2 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') 200 | branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], 201 | scope='Conv2d_0b_3x3') 202 | branch_2 = slim.conv2d(branch_2, depth(96), [3, 3], 203 | scope='Conv2d_0c_3x3') 204 | with tf.variable_scope('Branch_3'): 205 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 206 | branch_3 = slim.conv2d(branch_3, depth(64), [1, 1], 207 | scope='Conv2d_0b_1x1') 208 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 209 | end_points[end_point] = net 210 | if end_point == final_endpoint: return net, end_points 211 | 212 | # mixed_3: 17 x 17 x 768. 213 | end_point = 'Mixed_6a' 214 | with tf.variable_scope(end_point): 215 | with tf.variable_scope('Branch_0'): 216 | branch_0 = slim.conv2d(net, depth(384), [3, 3], stride=2, 217 | padding='VALID', scope='Conv2d_1a_1x1') 218 | with tf.variable_scope('Branch_1'): 219 | branch_1 = slim.conv2d(net, depth(64), [1, 1], scope='Conv2d_0a_1x1') 220 | branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], 221 | scope='Conv2d_0b_3x3') 222 | branch_1 = slim.conv2d(branch_1, depth(96), [3, 3], stride=2, 223 | padding='VALID', scope='Conv2d_1a_1x1') 224 | with tf.variable_scope('Branch_2'): 225 | branch_2 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', 226 | scope='MaxPool_1a_3x3') 227 | net = tf.concat(3, [branch_0, branch_1, branch_2]) 228 | end_points[end_point] = net 229 | if end_point == final_endpoint: return net, end_points 230 | 231 | # mixed4: 17 x 17 x 768. 232 | end_point = 'Mixed_6b' 233 | with tf.variable_scope(end_point): 234 | with tf.variable_scope('Branch_0'): 235 | branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 236 | with tf.variable_scope('Branch_1'): 237 | branch_1 = slim.conv2d(net, depth(128), [1, 1], scope='Conv2d_0a_1x1') 238 | branch_1 = slim.conv2d(branch_1, depth(128), [1, 7], 239 | scope='Conv2d_0b_1x7') 240 | branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], 241 | scope='Conv2d_0c_7x1') 242 | with tf.variable_scope('Branch_2'): 243 | branch_2 = slim.conv2d(net, depth(128), [1, 1], scope='Conv2d_0a_1x1') 244 | branch_2 = slim.conv2d(branch_2, depth(128), [7, 1], 245 | scope='Conv2d_0b_7x1') 246 | branch_2 = slim.conv2d(branch_2, depth(128), [1, 7], 247 | scope='Conv2d_0c_1x7') 248 | branch_2 = slim.conv2d(branch_2, depth(128), [7, 1], 249 | scope='Conv2d_0d_7x1') 250 | branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], 251 | scope='Conv2d_0e_1x7') 252 | with tf.variable_scope('Branch_3'): 253 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 254 | branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], 255 | scope='Conv2d_0b_1x1') 256 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 257 | end_points[end_point] = net 258 | if end_point == final_endpoint: return net, end_points 259 | 260 | # mixed_5: 17 x 17 x 768. 261 | end_point = 'Mixed_6c' 262 | with tf.variable_scope(end_point): 263 | with tf.variable_scope('Branch_0'): 264 | branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 265 | with tf.variable_scope('Branch_1'): 266 | branch_1 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') 267 | branch_1 = slim.conv2d(branch_1, depth(160), [1, 7], 268 | scope='Conv2d_0b_1x7') 269 | branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], 270 | scope='Conv2d_0c_7x1') 271 | with tf.variable_scope('Branch_2'): 272 | branch_2 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') 273 | branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], 274 | scope='Conv2d_0b_7x1') 275 | branch_2 = slim.conv2d(branch_2, depth(160), [1, 7], 276 | scope='Conv2d_0c_1x7') 277 | branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], 278 | scope='Conv2d_0d_7x1') 279 | branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], 280 | scope='Conv2d_0e_1x7') 281 | with tf.variable_scope('Branch_3'): 282 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 283 | branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], 284 | scope='Conv2d_0b_1x1') 285 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 286 | end_points[end_point] = net 287 | if end_point == final_endpoint: return net, end_points 288 | # mixed_6: 17 x 17 x 768. 289 | end_point = 'Mixed_6d' 290 | with tf.variable_scope(end_point): 291 | with tf.variable_scope('Branch_0'): 292 | branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 293 | with tf.variable_scope('Branch_1'): 294 | branch_1 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') 295 | branch_1 = slim.conv2d(branch_1, depth(160), [1, 7], 296 | scope='Conv2d_0b_1x7') 297 | branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], 298 | scope='Conv2d_0c_7x1') 299 | with tf.variable_scope('Branch_2'): 300 | branch_2 = slim.conv2d(net, depth(160), [1, 1], scope='Conv2d_0a_1x1') 301 | branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], 302 | scope='Conv2d_0b_7x1') 303 | branch_2 = slim.conv2d(branch_2, depth(160), [1, 7], 304 | scope='Conv2d_0c_1x7') 305 | branch_2 = slim.conv2d(branch_2, depth(160), [7, 1], 306 | scope='Conv2d_0d_7x1') 307 | branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], 308 | scope='Conv2d_0e_1x7') 309 | with tf.variable_scope('Branch_3'): 310 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 311 | branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], 312 | scope='Conv2d_0b_1x1') 313 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 314 | end_points[end_point] = net 315 | if end_point == final_endpoint: return net, end_points 316 | 317 | # mixed_7: 17 x 17 x 768. 318 | end_point = 'Mixed_6e' 319 | with tf.variable_scope(end_point): 320 | with tf.variable_scope('Branch_0'): 321 | branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 322 | with tf.variable_scope('Branch_1'): 323 | branch_1 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 324 | branch_1 = slim.conv2d(branch_1, depth(192), [1, 7], 325 | scope='Conv2d_0b_1x7') 326 | branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], 327 | scope='Conv2d_0c_7x1') 328 | with tf.variable_scope('Branch_2'): 329 | branch_2 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 330 | branch_2 = slim.conv2d(branch_2, depth(192), [7, 1], 331 | scope='Conv2d_0b_7x1') 332 | branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], 333 | scope='Conv2d_0c_1x7') 334 | branch_2 = slim.conv2d(branch_2, depth(192), [7, 1], 335 | scope='Conv2d_0d_7x1') 336 | branch_2 = slim.conv2d(branch_2, depth(192), [1, 7], 337 | scope='Conv2d_0e_1x7') 338 | with tf.variable_scope('Branch_3'): 339 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 340 | branch_3 = slim.conv2d(branch_3, depth(192), [1, 1], 341 | scope='Conv2d_0b_1x1') 342 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 343 | end_points[end_point] = net 344 | if end_point == final_endpoint: return net, end_points 345 | 346 | # mixed_8: 8 x 8 x 1280. 347 | end_point = 'Mixed_7a' 348 | with tf.variable_scope(end_point): 349 | with tf.variable_scope('Branch_0'): 350 | branch_0 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 351 | branch_0 = slim.conv2d(branch_0, depth(320), [3, 3], stride=2, 352 | padding='VALID', scope='Conv2d_1a_3x3') 353 | with tf.variable_scope('Branch_1'): 354 | branch_1 = slim.conv2d(net, depth(192), [1, 1], scope='Conv2d_0a_1x1') 355 | branch_1 = slim.conv2d(branch_1, depth(192), [1, 7], 356 | scope='Conv2d_0b_1x7') 357 | branch_1 = slim.conv2d(branch_1, depth(192), [7, 1], 358 | scope='Conv2d_0c_7x1') 359 | branch_1 = slim.conv2d(branch_1, depth(192), [3, 3], stride=2, 360 | padding='VALID', scope='Conv2d_1a_3x3') 361 | with tf.variable_scope('Branch_2'): 362 | branch_2 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', 363 | scope='MaxPool_1a_3x3') 364 | net = tf.concat(3, [branch_0, branch_1, branch_2]) 365 | end_points[end_point] = net 366 | if end_point == final_endpoint: return net, end_points 367 | # mixed_9: 8 x 8 x 2048. 368 | end_point = 'Mixed_7b' 369 | with tf.variable_scope(end_point): 370 | with tf.variable_scope('Branch_0'): 371 | branch_0 = slim.conv2d(net, depth(320), [1, 1], scope='Conv2d_0a_1x1') 372 | with tf.variable_scope('Branch_1'): 373 | branch_1 = slim.conv2d(net, depth(384), [1, 1], scope='Conv2d_0a_1x1') 374 | branch_1 = tf.concat(3, [ 375 | slim.conv2d(branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), 376 | slim.conv2d(branch_1, depth(384), [3, 1], scope='Conv2d_0b_3x1')]) 377 | with tf.variable_scope('Branch_2'): 378 | branch_2 = slim.conv2d(net, depth(448), [1, 1], scope='Conv2d_0a_1x1') 379 | branch_2 = slim.conv2d( 380 | branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') 381 | branch_2 = tf.concat(3, [ 382 | slim.conv2d(branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), 383 | slim.conv2d(branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1')]) 384 | with tf.variable_scope('Branch_3'): 385 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 386 | branch_3 = slim.conv2d( 387 | branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') 388 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 389 | end_points[end_point] = net 390 | if end_point == final_endpoint: return net, end_points 391 | 392 | # mixed_10: 8 x 8 x 2048. 393 | end_point = 'Mixed_7c' 394 | with tf.variable_scope(end_point): 395 | with tf.variable_scope('Branch_0'): 396 | branch_0 = slim.conv2d(net, depth(320), [1, 1], scope='Conv2d_0a_1x1') 397 | with tf.variable_scope('Branch_1'): 398 | branch_1 = slim.conv2d(net, depth(384), [1, 1], scope='Conv2d_0a_1x1') 399 | branch_1 = tf.concat(3, [ 400 | slim.conv2d(branch_1, depth(384), [1, 3], scope='Conv2d_0b_1x3'), 401 | slim.conv2d(branch_1, depth(384), [3, 1], scope='Conv2d_0c_3x1')]) 402 | with tf.variable_scope('Branch_2'): 403 | branch_2 = slim.conv2d(net, depth(448), [1, 1], scope='Conv2d_0a_1x1') 404 | branch_2 = slim.conv2d( 405 | branch_2, depth(384), [3, 3], scope='Conv2d_0b_3x3') 406 | branch_2 = tf.concat(3, [ 407 | slim.conv2d(branch_2, depth(384), [1, 3], scope='Conv2d_0c_1x3'), 408 | slim.conv2d(branch_2, depth(384), [3, 1], scope='Conv2d_0d_3x1')]) 409 | with tf.variable_scope('Branch_3'): 410 | branch_3 = slim.avg_pool2d(net, [3, 3], scope='AvgPool_0a_3x3') 411 | branch_3 = slim.conv2d( 412 | branch_3, depth(192), [1, 1], scope='Conv2d_0b_1x1') 413 | net = tf.concat(3, [branch_0, branch_1, branch_2, branch_3]) 414 | end_points[end_point] = net 415 | if end_point == final_endpoint: return net, end_points 416 | raise ValueError('Unknown final endpoint %s' % final_endpoint) 417 | 418 | 419 | def inception_v3(inputs, 420 | num_classes=1000, 421 | is_training=True, 422 | dropout_keep_prob=0.8, 423 | min_depth=16, 424 | depth_multiplier=1.0, 425 | prediction_fn=slim.softmax, 426 | spatial_squeeze=True, 427 | reuse=None, 428 | scope='InceptionV3'): 429 | """Inception model from http://arxiv.org/abs/1512.00567. 430 | 431 | "Rethinking the Inception Architecture for Computer Vision" 432 | 433 | Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, 434 | Zbigniew Wojna. 435 | 436 | With the default arguments this method constructs the exact model defined in 437 | the paper. However, one can experiment with variations of the inception_v3 438 | network by changing arguments dropout_keep_prob, min_depth and 439 | depth_multiplier. 440 | 441 | The default image size used to train this network is 299x299. 442 | 443 | Args: 444 | inputs: a tensor of size [batch_size, height, width, channels]. 445 | num_classes: number of predicted classes. 446 | is_training: whether is training or not. 447 | dropout_keep_prob: the percentage of activation values that are retained. 448 | min_depth: Minimum depth value (number of channels) for all convolution ops. 449 | Enforced when depth_multiplier < 1, and not an active constraint when 450 | depth_multiplier >= 1. 451 | depth_multiplier: Float multiplier for the depth (number of channels) 452 | for all convolution ops. The value must be greater than zero. Typical 453 | usage will be to set this value in (0, 1) to reduce the number of 454 | parameters or computation cost of the model. 455 | prediction_fn: a function to get predictions out of logits. 456 | spatial_squeeze: if True, logits is of shape is [B, C], if false logits is 457 | of shape [B, 1, 1, C], where B is batch_size and C is number of classes. 458 | reuse: whether or not the network and its variables should be reused. To be 459 | able to reuse 'scope' must be given. 460 | scope: Optional variable_scope. 461 | 462 | Returns: 463 | logits: the pre-softmax activations, a tensor of size 464 | [batch_size, num_classes] 465 | end_points: a dictionary from components of the network to the corresponding 466 | activation. 467 | 468 | Raises: 469 | ValueError: if 'depth_multiplier' is less than or equal to zero. 470 | """ 471 | if depth_multiplier <= 0: 472 | raise ValueError('depth_multiplier is not greater than zero.') 473 | depth = lambda d: max(int(d * depth_multiplier), min_depth) 474 | 475 | with tf.variable_scope(scope, 'InceptionV3', [inputs, num_classes], 476 | reuse=reuse) as scope: 477 | with slim.arg_scope([slim.batch_norm, slim.dropout], 478 | is_training=is_training): 479 | net, end_points = inception_v3_base( 480 | inputs, scope=scope, min_depth=min_depth, 481 | depth_multiplier=depth_multiplier) 482 | 483 | # Auxiliary Head logits 484 | with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], 485 | stride=1, padding='SAME'): 486 | aux_logits = end_points['Mixed_6e'] 487 | with tf.variable_scope('AuxLogits'): 488 | aux_logits = slim.avg_pool2d( 489 | aux_logits, [5, 5], stride=3, padding='VALID', 490 | scope='AvgPool_1a_5x5') 491 | aux_logits = slim.conv2d(aux_logits, depth(128), [1, 1], 492 | scope='Conv2d_1b_1x1') 493 | 494 | # Shape of feature map before the final layer. 495 | kernel_size = _reduced_kernel_size_for_small_input( 496 | aux_logits, [5, 5]) 497 | aux_logits = slim.conv2d( 498 | aux_logits, depth(768), kernel_size, 499 | weights_initializer=trunc_normal(0.01), 500 | padding='VALID', scope='Conv2d_2a_{}x{}'.format(*kernel_size)) 501 | aux_logits = slim.conv2d( 502 | aux_logits, num_classes, [1, 1], activation_fn=None, 503 | normalizer_fn=None, weights_initializer=trunc_normal(0.001), 504 | scope='Conv2d_2b_1x1') 505 | if spatial_squeeze: 506 | aux_logits = tf.squeeze(aux_logits, [1, 2], name='SpatialSqueeze') 507 | end_points['AuxLogits'] = aux_logits 508 | 509 | # Final pooling and prediction 510 | with tf.variable_scope('Logits'): 511 | kernel_size = _reduced_kernel_size_for_small_input(net, [8, 8]) 512 | net = slim.avg_pool2d(net, kernel_size, padding='VALID', 513 | scope='AvgPool_1a_{}x{}'.format(*kernel_size)) 514 | # 1 x 1 x 2048 515 | net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') 516 | end_points['PreLogits'] = net 517 | # 2048 518 | logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, 519 | normalizer_fn=None, scope='Conv2d_1c_1x1') 520 | if spatial_squeeze: 521 | logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') 522 | # 1000 523 | end_points['Logits'] = logits 524 | end_points['Predictions'] = prediction_fn(logits, scope='Predictions') 525 | return logits, end_points 526 | inception_v3.default_image_size = 299 527 | 528 | 529 | def _reduced_kernel_size_for_small_input(input_tensor, kernel_size): 530 | """Define kernel size which is automatically reduced for small input. 531 | 532 | If the shape of the input images is unknown at graph construction time this 533 | function assumes that the input images are is large enough. 534 | 535 | Args: 536 | input_tensor: input tensor of size [batch_size, height, width, channels]. 537 | kernel_size: desired kernel size of length 2: [kernel_height, kernel_width] 538 | 539 | Returns: 540 | a tensor with the kernel size. 541 | 542 | TODO(jrru): Make this function work with unknown shapes. Theoretically, this 543 | can be done with the code below. Problems are two-fold: (1) If the shape was 544 | known, it will be lost. (2) inception.slim.ops._two_element_tuple cannot 545 | handle tensors that define the kernel size. 546 | shape = tf.shape(input_tensor) 547 | return = tf.pack([tf.minimum(shape[1], kernel_size[0]), 548 | tf.minimum(shape[2], kernel_size[1])]) 549 | 550 | """ 551 | shape = input_tensor.get_shape().as_list() 552 | if shape[1] is None or shape[2] is None: 553 | kernel_size_out = kernel_size 554 | else: 555 | kernel_size_out = [min(shape[1], kernel_size[0]), 556 | min(shape[2], kernel_size[1])] 557 | return kernel_size_out 558 | 559 | 560 | inception_v3_arg_scope = inception_utils.inception_arg_scope -------------------------------------------------------------------------------- /MNIST/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fabienbaradel/Tensorflow-tutorials/3a3c6a233156d0463e0d874e97941c3331ced6ee/MNIST/__init__.py -------------------------------------------------------------------------------- /MNIST/autoencoder.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import tensorflow.contrib.slim as slim 7 | 8 | 9 | # Import MNIST data 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | #mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 12 | mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 13 | 14 | 15 | 16 | class Autoencoder(object): 17 | """ 18 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 19 | """ 20 | 21 | def __init__(self, learning_rate, batch_size): 22 | """ 23 | Init the class with some parameters 24 | :param learning_rate: 25 | :param batch_size: 26 | """ 27 | # Parameters 28 | self.learning_rate = learning_rate 29 | self.mnist = mnist 30 | self.batch_size = batch_size 31 | self.num_epochs = 5 32 | self.num_classes = 10 33 | self.input_size = 784 34 | self.input_weight, self.input_height = 28, 28 35 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 36 | self.display_step = 1 37 | 38 | # Placeholders 39 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 40 | 41 | 42 | 43 | def inference(self): 44 | """ 45 | Design the inference model (here a simple neuralnet) 46 | :return: 47 | """ 48 | 49 | 50 | # Building the encoder 51 | def encoder(x): 52 | net = slim.fully_connected(x, 256) 53 | net = slim.fully_connected(net, 128) 54 | return net 55 | 56 | # Building the decoder 57 | def decoder(x): 58 | net = slim.fully_connected(x, 256) 59 | net = slim.fully_connected(net, 784) 60 | return net 61 | 62 | # Construct model 63 | encoder_op = encoder(self.X) 64 | self.X_reconstruct = decoder(encoder_op) 65 | 66 | 67 | def losses(self): 68 | """ 69 | Compute mean square loss 70 | :return: 71 | """ 72 | # cross entropy loss 73 | self.loss = tf.reduce_mean(tf.square(tf.sub(self.X, self.X_reconstruct))) 74 | 75 | 76 | def optimizer(self): 77 | """ 78 | Create a optimizer and therefore a training operation 79 | :return: 80 | """ 81 | # The optimizer 82 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 83 | 84 | # Training operation to run later 85 | self.train_op = self.opt.minimize(self.loss) 86 | 87 | 88 | def train(self): 89 | """ 90 | Train the model on MNIST training set 91 | :return: 92 | """ 93 | # Initializing the variables 94 | init = tf.global_variables_initializer() 95 | 96 | # Launch the graph 97 | with tf.Session() as sess: 98 | sess.run(init) 99 | 100 | # Training cycle 101 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 102 | for s in range(self.batch_per_epoch): 103 | # Get bacth fro MNIST training set 104 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 105 | 106 | # Apply the training op 107 | (_, 108 | loss_train) = sess.run([self.train_op, 109 | self.loss], 110 | feed_dict={self.X: batch_xs}) 111 | # Print loss and accuracy on the batch 112 | if s % 200 == 0: 113 | print("\033[1;37;40mStep: %04d , " 114 | "TRAIN: loss = %.4f" 115 | % ((epoch * self.mnist.train.num_examples + s), 116 | loss_train)) 117 | 118 | 119 | 120 | 121 | # Display logs per epoch step 122 | if (epoch) % self.display_step == 0: 123 | # Compute loss on validation (only 200 random images) 124 | loss_val = sess.run(self.loss, 125 | feed_dict={self.X: mnist.test.images[:200]}) 126 | 127 | # Compute loss on train (only 200 random images) 128 | loss_train = sess.run(self.loss, 129 | feed_dict={self.X: mnist.train.images[:200]}) 130 | 131 | print("\033[1;32;40mEpoch: %04d , " 132 | "TRAIN: loss = %.4f| " 133 | "VALIDATION: loss = %.4f" 134 | % (epoch + 1, 135 | loss_train, 136 | loss_val)) 137 | 138 | 139 | # Plot reconstruted images 140 | X_reconstr = sess.run(self.X_reconstruct, 141 | feed_dict={self.X: mnist.test.images[:10]}) 142 | 143 | # Compare original images with their reconstructions 144 | f, a = plt.subplots(2, 10, figsize=(10, 2)) 145 | for i in range(10): 146 | a[0][i].imshow(np.reshape(mnist.test.images[i], (28, 28))) 147 | a[1][i].imshow(np.reshape(X_reconstr[i], (28, 28))) 148 | f.show() 149 | plt.draw() 150 | plt.waitforbuttonpress() 151 | 152 | 153 | 154 | def main(_): 155 | """ 156 | Main function 157 | :param _: 158 | :return: 159 | """ 160 | 161 | # Instanciate a MNIST class 162 | model = Autoencoder(learning_rate=0.01, 163 | batch_size=100) 164 | # Setup the graph 165 | model.inference() 166 | 167 | # Compute loss and metrics 168 | model.losses() 169 | 170 | # Create an optimzer 171 | model.optimizer() 172 | 173 | # And finally train your model! 174 | model.train() 175 | 176 | 177 | 178 | # To start the app for tensorflow 179 | if __name__ == '__main__': 180 | tf.app.run() 181 | 182 | -------------------------------------------------------------------------------- /MNIST/autoencoder_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import tensorflow.contrib.slim as slim 7 | 8 | # Import MNIST data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | 11 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 12 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 13 | 14 | 15 | class Autoencoder(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 5 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples / self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | 40 | def inference(self): 41 | """ 42 | Design the inference model (here a simple neuralnet) 43 | :return: 44 | """ 45 | 46 | # Building the encoder 47 | def encoder(x): 48 | ????? 49 | 50 | # Building the decoder 51 | def decoder(x): 52 | ????? 53 | 54 | # Construct model 55 | encoder_op = encoder(self.X) 56 | self.X_reconstruct = decoder(encoder_op) 57 | 58 | def losses(self): 59 | """ 60 | Compute mean square loss 61 | :return: 62 | """ 63 | # cross entropy loss 64 | self.loss = ????? 65 | 66 | def optimizer(self): 67 | """ 68 | Create a optimizer and therefore a training operation 69 | :return: 70 | """ 71 | # The optimizer 72 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 73 | 74 | # Training operation to run later 75 | self.train_op = self.opt.minimize(self.loss) 76 | 77 | def train(self): 78 | """ 79 | Train the model on MNIST training set 80 | :return: 81 | """ 82 | # Initializing the variables 83 | init = tf.global_variables_initializer() 84 | 85 | # Launch the graph 86 | with tf.Session() as sess: 87 | sess.run(init) 88 | 89 | # Training cycle 90 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 91 | for s in range(self.batch_per_epoch): 92 | # Get bacth fro MNIST training set 93 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 94 | 95 | # Apply the training op 96 | (_, 97 | loss_train) = sess.run([self.train_op, 98 | self.loss], 99 | feed_dict={self.X: batch_xs}) 100 | # Print loss and accuracy on the batch 101 | if s % 200 == 0: 102 | print("\033[1;37;40mStep: %04d , " 103 | "TRAIN: loss = %.4f" 104 | % ((epoch * self.mnist.train.num_examples + s), 105 | loss_train)) 106 | 107 | # Display logs per epoch step 108 | if (epoch) % self.display_step == 0: 109 | # Compute loss on validation (only 200 random images) 110 | loss_val = sess.run(self.loss, 111 | feed_dict={self.X: mnist.test.images[:200]}) 112 | 113 | # Compute loss on train (only 200 random images) 114 | loss_train = sess.run(self.loss, 115 | feed_dict={self.X: mnist.train.images[:200]}) 116 | 117 | print("\033[1;32;40mEpoch: %04d , " 118 | "TRAIN: loss = %.4f| " 119 | "VALIDATION: loss = %.4f" 120 | % (epoch + 1, 121 | loss_train, 122 | loss_val)) 123 | 124 | # Plot reconstruted images 125 | X_reconstr = sess.run(self.X_reconstruct, 126 | feed_dict={self.X: mnist.test.images[:10]}) 127 | 128 | # Compare original images with their reconstructions 129 | f, a = plt.subplots(2, 10, figsize=(10, 2)) 130 | for i in range(10): 131 | a[0][i].imshow(np.reshape(mnist.test.images[i], (28, 28))) 132 | a[1][i].imshow(np.reshape(X_reconstr[i], (28, 28))) 133 | f.show() 134 | plt.draw() 135 | plt.waitforbuttonpress() 136 | 137 | 138 | def main(_): 139 | """ 140 | Main function 141 | :param _: 142 | :return: 143 | """ 144 | 145 | # Instanciate a MNIST class 146 | model = Autoencoder(learning_rate=0.01, 147 | batch_size=100) 148 | # Setup the graph 149 | model.inference() 150 | 151 | # Compute loss and metrics 152 | model.losses() 153 | 154 | # Create an optimzer 155 | model.optimizer() 156 | 157 | # And finally train your model! 158 | model.train() 159 | 160 | 161 | # To start the app for tensorflow 162 | if __name__ == '__main__': 163 | tf.app.run() 164 | 165 | -------------------------------------------------------------------------------- /MNIST/conv_ae.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import tensorflow.contrib.slim as slim 7 | 8 | # Import MNIST data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | 11 | #mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 12 | mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 13 | 14 | 15 | class Autoencoder(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 2 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples / self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | 40 | def inference(self): 41 | """ 42 | Design the inference model (here a simple neuralnet) 43 | :return: 44 | """ 45 | 46 | # Building the encoder 47 | def encoder(x): 48 | #x = tf.random_normal([784]) 49 | X = tf.reshape(x, [-1, 28, 28, 1]) 50 | net = slim.conv2d(X, 10, [3,3], stride=2, padding="SAME") 51 | net = slim.conv2d(net, 10, [3,3], stride=2, padding="SAME") 52 | net = slim.conv2d(net, 10, [3,3], stride=2, padding="SAME") 53 | return net 54 | 55 | # Building the decoder 56 | def decoder(x): 57 | #x = tf.random_normal([1, 4, 4, 10]) 58 | net = slim.conv2d_transpose(x, 10, [3,3], stride = 2, padding = "SAME") 59 | net = slim.conv2d_transpose(net, 10, [3,3], stride = 2, padding = "SAME") 60 | net = slim.conv2d_transpose(net, 1, [3,3], stride = 2, padding = "SAME") 61 | net = tf.image.resize_nearest_neighbor(net, [28,28]) 62 | net = tf.reshape(net, [-1, 784]) 63 | return net 64 | 65 | 66 | 67 | # Construct model 68 | encoder_op = encoder(self.X) 69 | self.X_reconstruct = decoder(encoder_op) 70 | 71 | def losses(self): 72 | """ 73 | Compute mean square loss 74 | :return: 75 | """ 76 | # cross entropy loss 77 | self.loss = tf.reduce_mean(tf.square(tf.sub(self.X, self.X_reconstruct))) 78 | 79 | def optimizer(self): 80 | """ 81 | Create a optimizer and therefore a training operation 82 | :return: 83 | """ 84 | # The optimizer 85 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 86 | 87 | # Training operation to run later 88 | self.train_op = self.opt.minimize(self.loss) 89 | 90 | def train(self): 91 | """ 92 | Train the model on MNIST training set 93 | :return: 94 | """ 95 | # Initializing the variables 96 | init = tf.global_variables_initializer() 97 | 98 | # Launch the graph 99 | with tf.Session() as sess: 100 | sess.run(init) 101 | 102 | # Training cycle 103 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 104 | for s in range(self.batch_per_epoch): 105 | # Get bacth fro MNIST training set 106 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 107 | 108 | # Apply the training op 109 | (_, 110 | loss_train) = sess.run([self.train_op, 111 | self.loss], 112 | feed_dict={self.X: batch_xs}) 113 | # Print loss and accuracy on the batch 114 | if s % 20 == 0: 115 | print("\033[1;37;40mStep: %04d , " 116 | "TRAIN: loss = %.4f" 117 | % ((epoch * self.mnist.train.num_examples + s), 118 | loss_train)) 119 | 120 | # Display logs per epoch step 121 | if (epoch) % self.display_step == 0: 122 | # Compute loss on validation (only 200 random images) 123 | loss_val = sess.run(self.loss, 124 | feed_dict={self.X: mnist.test.images[:200]}) 125 | 126 | # Compute loss on train (only 200 random images) 127 | loss_train = sess.run(self.loss, 128 | feed_dict={self.X: mnist.train.images[:200]}) 129 | 130 | print("\033[1;32;40mEpoch: %04d , " 131 | "TRAIN: loss = %.4f| " 132 | "VALIDATION: loss = %.4f" 133 | % (epoch + 1, 134 | loss_train, 135 | loss_val)) 136 | 137 | # Plot reconstruted images 138 | X_reconstr = sess.run(self.X_reconstruct, 139 | feed_dict={self.X: mnist.test.images[:10]}) 140 | 141 | # Compare original images with their reconstructions 142 | f, a = plt.subplots(2, 10, figsize=(10, 2)) 143 | for i in range(10): 144 | a[0][i].imshow(np.reshape(mnist.test.images[i], (28, 28))) 145 | a[1][i].imshow(np.reshape(X_reconstr[i], (28, 28))) 146 | f.show() 147 | plt.draw() 148 | plt.waitforbuttonpress() 149 | 150 | 151 | def main(_): 152 | """ 153 | Main function 154 | :param _: 155 | :return: 156 | """ 157 | 158 | # Instanciate a MNIST class 159 | model = Autoencoder(learning_rate=0.01, 160 | batch_size=100) 161 | # Setup the graph 162 | model.inference() 163 | 164 | # Compute loss and metrics 165 | model.losses() 166 | 167 | # Create an optimzer 168 | model.optimizer() 169 | 170 | # And finally train your model! 171 | model.train() 172 | 173 | 174 | # To start the app for tensorflow 175 | if __name__ == '__main__': 176 | tf.app.run() 177 | 178 | -------------------------------------------------------------------------------- /MNIST/conv_ae_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import tensorflow.contrib.slim as slim 7 | 8 | # Import MNIST data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | 11 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 12 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 13 | 14 | 15 | class Autoencoder(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 2 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples / self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | 40 | def inference(self): 41 | """ 42 | Design the inference model (here a simple neuralnet) 43 | :return: 44 | """ 45 | 46 | # Building the encoder 47 | def encoder(x): 48 | #x = tf.random_normal([784]) 49 | ?????? 50 | return ??? 51 | 52 | # Building the decoder 53 | def decoder(x): 54 | #x = tf.random_normal([1, 4, 4, 10]) 55 | ???????? 56 | return ??? 57 | 58 | 59 | 60 | # Construct model 61 | encoder_op = encoder(self.X) 62 | self.X_reconstruct = decoder(encoder_op) 63 | 64 | def losses(self): 65 | """ 66 | Compute mean square loss 67 | :return: 68 | """ 69 | # cross entropy loss 70 | self.loss = ??????? 71 | 72 | def optimizer(self): 73 | """ 74 | Create a optimizer and therefore a training operation 75 | :return: 76 | """ 77 | # The optimizer 78 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 79 | 80 | # Training operation to run later 81 | self.train_op = self.opt.minimize(self.loss) 82 | 83 | def train(self): 84 | """ 85 | Train the model on MNIST training set 86 | :return: 87 | """ 88 | # Initializing the variables 89 | init = tf.global_variables_initializer() 90 | 91 | # Launch the graph 92 | with tf.Session() as sess: 93 | sess.run(init) 94 | 95 | # Training cycle 96 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 97 | for s in range(self.batch_per_epoch): 98 | # Get bacth fro MNIST training set 99 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 100 | 101 | # Apply the training op 102 | (_, 103 | loss_train) = sess.run([self.train_op, 104 | self.loss], 105 | feed_dict={self.X: batch_xs}) 106 | # Print loss and accuracy on the batch 107 | if s % 20 == 0: 108 | print("\033[1;37;40mStep: %04d , " 109 | "TRAIN: loss = %.4f" 110 | % ((epoch * self.mnist.train.num_examples + s), 111 | loss_train)) 112 | 113 | # Display logs per epoch step 114 | if (epoch) % self.display_step == 0: 115 | # Compute loss on validation (only 200 random images) 116 | loss_val = sess.run(self.loss, 117 | feed_dict={self.X: mnist.test.images[:200]}) 118 | 119 | # Compute loss on train (only 200 random images) 120 | loss_train = sess.run(self.loss, 121 | feed_dict={self.X: mnist.train.images[:200]}) 122 | 123 | print("\033[1;32;40mEpoch: %04d , " 124 | "TRAIN: loss = %.4f| " 125 | "VALIDATION: loss = %.4f" 126 | % (epoch + 1, 127 | loss_train, 128 | loss_val)) 129 | 130 | # Plot reconstruted images 131 | X_reconstr = sess.run(self.X_reconstruct, 132 | feed_dict={self.X: mnist.test.images[:10]}) 133 | 134 | # Compare original images with their reconstructions 135 | f, a = plt.subplots(2, 10, figsize=(10, 2)) 136 | for i in range(10): 137 | a[0][i].imshow(np.reshape(mnist.test.images[i], (28, 28))) 138 | a[1][i].imshow(np.reshape(X_reconstr[i], (28, 28))) 139 | f.show() 140 | plt.draw() 141 | plt.waitforbuttonpress() 142 | 143 | 144 | def main(_): 145 | """ 146 | Main function 147 | :param _: 148 | :return: 149 | """ 150 | 151 | # Instanciate a MNIST class 152 | model = Autoencoder(learning_rate=0.01, 153 | batch_size=100) 154 | # Setup the graph 155 | model.inference() 156 | 157 | # Compute loss and metrics 158 | model.losses() 159 | 160 | # Create an optimzer 161 | model.optimizer() 162 | 163 | # And finally train your model! 164 | model.train() 165 | 166 | 167 | # To start the app for tensorflow 168 | if __name__ == '__main__': 169 | tf.app.run() 170 | 171 | -------------------------------------------------------------------------------- /MNIST/gan_from_web.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | def xavier_init(size): 10 | in_dim = size[0] 11 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 12 | return tf.random_normal(shape=size, stddev=xavier_stddev) 13 | 14 | 15 | X = tf.placeholder(tf.float32, shape=[None, 784]) 16 | 17 | D_W1 = tf.Variable(xavier_init([784, 128])) 18 | D_b1 = tf.Variable(tf.zeros(shape=[128])) 19 | 20 | D_W2 = tf.Variable(xavier_init([128, 1])) 21 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 22 | 23 | theta_D = [D_W1, D_W2, D_b1, D_b2] 24 | 25 | 26 | Z = tf.placeholder(tf.float32, shape=[None, 100]) 27 | 28 | G_W1 = tf.Variable(xavier_init([100, 128])) 29 | G_b1 = tf.Variable(tf.zeros(shape=[128])) 30 | 31 | G_W2 = tf.Variable(xavier_init([128, 784])) 32 | G_b2 = tf.Variable(tf.zeros(shape=[784])) 33 | 34 | theta_G = [G_W1, G_W2, G_b1, G_b2] 35 | 36 | 37 | DC_D_W1 = tf.Variable(xavier_init([5, 5, 1, 16])) 38 | DC_D_b1 = tf.Variable(tf.zeros(shape=[16])) 39 | 40 | DC_D_W2 = tf.Variable(xavier_init([3, 3, 16, 32])) 41 | DC_D_b2 = tf.Variable(tf.zeros(shape=[32])) 42 | 43 | DC_D_W3 = tf.Variable(xavier_init([7 * 7 * 32, 128])) 44 | DC_D_b3 = tf.Variable(tf.zeros(shape=[128])) 45 | 46 | DC_D_W4 = tf.Variable(xavier_init([128, 1])) 47 | DC_D_b4 = tf.Variable(tf.zeros(shape=[1])) 48 | 49 | theta_DC_D = [DC_D_W1, DC_D_b1, DC_D_W2, DC_D_b2, DC_D_W3, DC_D_b3, DC_D_W4, DC_D_b4] 50 | 51 | 52 | def sample_Z(m, n): 53 | return np.random.uniform(-1., 1., size=[m, n]) 54 | 55 | 56 | def generator(z): 57 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 58 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 59 | G_prob = tf.nn.sigmoid(G_log_prob) 60 | 61 | return G_prob 62 | 63 | 64 | def discriminator(x): 65 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 66 | D_logit = tf.matmul(D_h1, D_W2) + D_b2 67 | D_prob = tf.nn.sigmoid(D_logit) 68 | 69 | return D_prob, D_logit 70 | 71 | 72 | def dc_generator(z): 73 | pass 74 | 75 | 76 | def dc_discriminator(x): 77 | x = tf.reshape(x, shape=[-1, 28, 28, 1]) 78 | conv1 = tf.nn.relu(tf.nn.conv2d(x, DC_D_W1, strides=[1, 2, 2, 1], padding='SAME') + DC_D_b1) 79 | conv2 = tf.nn.relu(tf.nn.conv2d(conv1, DC_D_W2, strides=[1, 2, 2, 1], padding='SAME') + DC_D_b2) 80 | conv2 = tf.reshape(conv2, shape=[-1, 7 * 7 * 32]) 81 | h = tf.nn.relu(tf.matmul(conv2, DC_D_W3) + DC_D_b3) 82 | logit = tf.matmul(h, DC_D_W4) + DC_D_b4 83 | prob = tf.nn.sigmoid(logit) 84 | 85 | return prob, logit 86 | 87 | 88 | def plot(samples): 89 | fig = plt.figure(figsize=(4, 4)) 90 | gs = gridspec.GridSpec(4, 4) 91 | gs.update(wspace=0.05, hspace=0.05) 92 | 93 | for i, sample in enumerate(samples): 94 | ax = plt.subplot(gs[i]) 95 | plt.axis('off') 96 | ax.set_xticklabels([]) 97 | ax.set_yticklabels([]) 98 | ax.set_aspect('equal') 99 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 100 | 101 | return fig 102 | 103 | 104 | G_sample = generator(Z) 105 | D_real, D_logit_real = discriminator(X)#dc_discriminator(X)# 106 | D_fake, D_logit_fake = discriminator(G_sample)#dc_discriminator(G_sample)# 107 | 108 | # D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake)) 109 | # G_loss = -tf.reduce_mean(tf.log(D_fake)) 110 | 111 | # Alternative losses: 112 | # ------------------- 113 | D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(D_logit_real, tf.ones_like(D_logit_real))) 114 | D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(D_logit_fake, tf.zeros_like(D_logit_fake))) 115 | D_loss = D_loss_real + D_loss_fake 116 | G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(D_logit_fake, tf.ones_like(D_logit_fake))) 117 | 118 | D_solver = tf.train.AdamOptimizer().minimize(D_loss)#, var_list=theta_DC_D) 119 | G_solver = tf.train.AdamOptimizer().minimize(G_loss)##, var_list=theta_G) 120 | 121 | mb_size = 128 122 | Z_dim = 100 123 | 124 | mnist = input_data.read_data_sets('/Users/fabien/Datasets/MNIST', one_hot=True) 125 | 126 | sess = tf.Session() 127 | sess.run(tf.initialize_all_variables()) 128 | 129 | if not os.path.exists('./img_gan/'): 130 | os.makedirs('./img_gan/') 131 | 132 | i = 0 133 | 134 | for it in range(1000000): 135 | if it % 100 == 0: 136 | samples = sess.run(G_sample, feed_dict={Z: sample_Z(16, Z_dim)}) 137 | 138 | fig = plot(samples) 139 | plt.savefig('./img_gan//{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 140 | i += 1 141 | plt.close(fig) 142 | 143 | X_mb, _ = mnist.train.next_batch(mb_size) 144 | 145 | _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(mb_size, Z_dim)}) 146 | _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(mb_size, Z_dim)}) 147 | 148 | if it % 100 == 0: 149 | print('Iter: {}'.format(it)) 150 | print('D loss: {:.4}'. format(D_loss_curr)) 151 | print('G_loss: {:.4}'.format(G_loss_curr)) 152 | print() -------------------------------------------------------------------------------- /MNIST/gan_mlp.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import tensorflow.contrib.slim as slim 7 | 8 | 9 | 10 | # Import MNIST data 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 13 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 14 | 15 | 16 | 17 | 18 | class MNIST_logistic(object): 19 | """ 20 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 21 | """ 22 | 23 | def __init__(self, learning_rate, batch_size): 24 | """ 25 | Init the class with some parameters 26 | :param learning_rate: 27 | :param batch_size: 28 | """ 29 | # Parameters 30 | self.learning_rate = learning_rate 31 | self.mnist = mnist 32 | self.batch_size = batch_size 33 | self.num_epochs = 5 34 | self.num_classes = 10 35 | self.input_size = 784 36 | self.input_weight, self.input_height = 28, 28 37 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 38 | self.display_step = 1 39 | 40 | # Placeholders 41 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 42 | self.Z = tf.placeholder(tf.float32, [None, 100]) # noise for the generator 43 | 44 | # Loss and accuracy tracking 45 | self.list_loss_train = [] 46 | self.list_loss_validation = [] 47 | self.list_accuracy_train = [] 48 | self.list_accuracy_validation = [] 49 | 50 | 51 | 52 | def inference(self): 53 | def generator(z): 54 | net = slim.fully_connected(z, 128) 55 | generated_image = slim.fully_connected(net, 784, activation_fn=None) 56 | return generated_image 57 | 58 | def discriminator(x): 59 | net = slim.fully_connected(x, 128) 60 | is_real = slim.fully_connected(net, 1, activation_fn=None) 61 | return is_real 62 | 63 | self.generated_X = generator(self.Z) 64 | self.D_real = discriminator(self.X) 65 | self.D_fake = discriminator(self.generated_X) 66 | 67 | 68 | 69 | 70 | def losses(self): 71 | """ 72 | GAN losses 73 | :return: 74 | """ 75 | self.D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(self.D_real, tf.ones_like(self.D_real))) 76 | self.D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(self.D_fake, tf.zeros_like(self.D_fake))) 77 | self.D_loss = self.D_loss_real + self.D_loss_fake 78 | self.G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(self.D_fake, tf.ones_like(self.D_fake))) 79 | 80 | 81 | def optimizer(self): 82 | """ 83 | Create a optimizer and therefore a training operation 84 | :return: 85 | """ 86 | # The optimizer for D and G 87 | self.opt_D = tf.train.AdamOptimizer(self.learning_rate) 88 | self.opt_G = tf.train.AdamOptimizer(self.learning_rate) 89 | 90 | # Training operation for D and G solver (minmax problem) 91 | self.train_op_D = self.opt_D.minimize(self.D_loss) 92 | self.train_op_G = self.opt_G.minimize(self.G_loss) 93 | 94 | 95 | 96 | def train(self): 97 | """ 98 | Train the model on MNIST training set 99 | :return: 100 | """ 101 | # Initializing the variables 102 | init = tf.global_variables_initializer() 103 | 104 | def sample_Z(m, n): 105 | '''Uniform prior for G(Z)''' 106 | return np.random.uniform(-1., 1., size=[m, n]) 107 | 108 | # Launch the graph 109 | with tf.Session() as sess: 110 | sess.run(init) 111 | 112 | # Training cycle 113 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 114 | # Averaging loss over the epoch 115 | avg_loss_train = 0.0 116 | for s in range(self.batch_per_epoch): 117 | # Get bacth fro MNIST training set 118 | batch_xs, _ = mnist.train.next_batch(self.batch_size) 119 | 120 | # Apply the training op alternatively 121 | (_, loss_D) = sess.run([self.train_op_D, self.D_loss], 122 | feed_dict={self.X: batch_xs, 123 | self.Z: sample_Z(self.batch_size, 100)}) 124 | (_, loss_G) = sess.run([self.train_op_G, self.G_loss], 125 | feed_dict={self.Z: sample_Z(self.batch_size, 100)}) 126 | 127 | # Print loss and accuracy on the batch 128 | print(loss_D, loss_G) 129 | 130 | 131 | 132 | 133 | def main(_): 134 | """ 135 | Main function 136 | :param _: 137 | :return: 138 | """ 139 | 140 | # Instanciate a MNIST class 141 | model = MNIST_logistic(learning_rate=0.01, 142 | batch_size=100) 143 | # Setup the graph 144 | model.inference() 145 | 146 | # Compute loss and metrics 147 | model.losses() 148 | 149 | # Create an optimzer 150 | model.optimizer() 151 | 152 | # And finally train your model! 153 | model.train() 154 | 155 | 156 | 157 | # To start the app for tensorflow 158 | if __name__ == '__main__': 159 | tf.app.run() 160 | 161 | -------------------------------------------------------------------------------- /MNIST/lenet.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import tensorflow.contrib.slim as slim 6 | 7 | 8 | # Import MNIST data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | #mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 11 | mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 12 | 13 | 14 | 15 | 16 | class MNIST_logistic(object): 17 | """ 18 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 19 | """ 20 | 21 | def __init__(self, learning_rate, batch_size): 22 | """ 23 | Init the class with some parameters 24 | :param learning_rate: 25 | :param batch_size: 26 | """ 27 | # Parameters 28 | self.learning_rate = learning_rate 29 | self.mnist = mnist 30 | self.batch_size = batch_size 31 | self.num_epochs = 50 32 | self.num_classes = 10 33 | self.input_size = 784 34 | self.input_weight, self.input_height = 28, 28 35 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 36 | self.display_step = 1 37 | 38 | # Placeholders 39 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 40 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 41 | 42 | 43 | def inference(self): 44 | """ 45 | LeNet 46 | :return: 47 | """ 48 | 49 | def init_weights(shape): 50 | return tf.Variable(tf.random_normal(shape, stddev=0.01)) 51 | 52 | # input reshaping 53 | #X = tf.random_normal([1, 28, 28,1]) 54 | X = tf.reshape(self.X, [-1, 28, 28, 1]) 55 | 56 | net = slim.conv2d(X, 32, [5, 5], scope='conv1') 57 | net = slim.max_pool2d(net, [2, 2], 2, scope='pool1') 58 | net = slim.conv2d(net, 64, [5, 5], scope='conv2') 59 | net = slim.max_pool2d(net, [2, 2], 2, scope='pool2') 60 | net = slim.flatten(net) 61 | 62 | net = slim.fully_connected(net, 1024, scope='fc3') 63 | self.logits = slim.fully_connected(net, 10, activation_fn=None, 64 | scope='fc4') 65 | 66 | 67 | self.Y_hat = slim.softmax(self.logits, scope='Predictions') # softmax 68 | 69 | def losses(self): 70 | """ 71 | Compute the cross entropy loss 72 | :return: 73 | """ 74 | # cross entropy loss 75 | self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.logits, self.Y)) 76 | 77 | 78 | 79 | def optimizer(self): 80 | """ 81 | Create a optimizer and therefore a training operation 82 | :return: 83 | """ 84 | # The optimizer 85 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 86 | 87 | # Training operation to run later 88 | self.train_op = self.opt.minimize(self.loss) 89 | 90 | 91 | def metrics(self): 92 | """ 93 | Compute the accuracy 94 | :return: 95 | """ 96 | # Label prediction of the model (the highest one) 97 | self.predicted_label = tf.argmax(self.Y_hat, 1) 98 | # Real class: 99 | self.real_label = tf.argmax(self.Y, 1) 100 | # Number of correct prediction 101 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 102 | # Calculate accuracy 103 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 104 | self.accuracy = tf.mul(100.0, self.accuracy) 105 | 106 | 107 | def train(self): 108 | """ 109 | Train the model on MNIST training set 110 | :return: 111 | """ 112 | # Initializing the variables 113 | init = tf.global_variables_initializer() 114 | 115 | # Launch the graph 116 | with tf.Session() as sess: 117 | sess.run(init) 118 | 119 | # Training cycle 120 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 121 | for s in range(self.batch_per_epoch): 122 | # Get bacth fro MNIST training set 123 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 124 | 125 | # Apply the training op 126 | (_, 127 | loss_train, 128 | accuracy_train) = sess.run([self.train_op, 129 | self.loss, 130 | self.accuracy], 131 | feed_dict={self.X: batch_xs, 132 | self.Y: batch_ys}) 133 | 134 | # Print loss and accuracy on the batch 135 | if s % 20 == 0: 136 | print("\033[1;37;40mStep: %04d , " 137 | "TRAIN: loss = %.4f - accuracy = %.2f" 138 | % ((epoch * self.batch_per_epoch + s), 139 | loss_train, accuracy_train)) 140 | 141 | 142 | # Display logs per epoch step 143 | if (epoch) % self.display_step == 0: 144 | # Compute loss on validation set (only 200 random images) 145 | (loss_val, 146 | accuracy_val) = sess.run([self.loss, 147 | self.accuracy], 148 | feed_dict={self.X: mnist.test.images[:1000], 149 | self.Y: mnist.test.labels[:1000]}) 150 | 151 | # Compute loss on training set (only 200 random images) 152 | (loss_train, 153 | accuracy_train) = sess.run([self.loss, 154 | self.accuracy], 155 | feed_dict={self.X: mnist.train.images[:1000], 156 | self.Y: mnist.train.labels[:1000]}) 157 | print("\033[1;32;40mEpoch: %04d , " 158 | "TRAIN: loss = %.4f - accuracy = %.2f | " 159 | "VALIDATION: loss = %.4f - accuracy = %.2f" 160 | % (epoch + 1, 161 | loss_train, accuracy_train, 162 | loss_val, accuracy_val)) 163 | 164 | 165 | def main(_): 166 | """ 167 | Main function 168 | :param _: 169 | :return: 170 | """ 171 | 172 | # Instanciate a MNIST class 173 | model = MNIST_logistic(learning_rate=0.01, 174 | batch_size=128) 175 | # Setup the graph 176 | model.inference() 177 | 178 | # Compute loss and metrics 179 | model.losses() 180 | model.metrics() 181 | 182 | # Create an optimzer 183 | model.optimizer() 184 | 185 | # And finally train your model! 186 | model.train() 187 | 188 | 189 | 190 | # To start the app for tensorflow 191 | if __name__ == '__main__': 192 | tf.app.run() 193 | 194 | -------------------------------------------------------------------------------- /MNIST/lenet_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import tensorflow.contrib.slim as slim 6 | 7 | 8 | # Import MNIST data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 11 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 12 | 13 | 14 | 15 | 16 | class MNIST_lenet(object): 17 | """ 18 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 19 | """ 20 | 21 | def __init__(self, learning_rate, batch_size): 22 | """ 23 | Init the class with some parameters 24 | :param learning_rate: 25 | :param batch_size: 26 | """ 27 | # Parameters 28 | self.learning_rate = learning_rate 29 | self.mnist = mnist 30 | self.batch_size = batch_size 31 | self.num_epochs = 50 32 | self.num_classes = 10 33 | self.input_size = 784 34 | self.input_weight, self.input_height = 28, 28 35 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 36 | self.display_step = 1 37 | 38 | # Placeholders 39 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 40 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 41 | 42 | 43 | def inference(self): 44 | """ 45 | LeNet 46 | :return: 47 | """ 48 | 49 | # Write a lenet using the slim library 50 | # https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim 51 | self.logits = ????? 52 | 53 | 54 | self.Y_hat = ???? 55 | 56 | def losses(self): 57 | """ 58 | Compute the cross entropy loss 59 | :return: 60 | """ 61 | # cross entropy loss 62 | self.loss = ????? 63 | 64 | 65 | 66 | def optimizer(self): 67 | """ 68 | Create a optimizer and therefore a training operation 69 | :return: 70 | """ 71 | # The optimizer 72 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 73 | 74 | # Training operation to run later 75 | self.train_op = self.opt.minimize(self.loss) 76 | 77 | 78 | def metrics(self): 79 | """ 80 | Compute the accuracy 81 | :return: 82 | """ 83 | # Label prediction of the model (the highest one) 84 | self.predicted_label = tf.argmax(self.Y_hat, 1) 85 | # Real class: 86 | self.real_label = tf.argmax(self.Y, 1) 87 | # Number of correct prediction 88 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 89 | # Calculate accuracy 90 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 91 | self.accuracy = tf.mul(100.0, self.accuracy) 92 | 93 | 94 | def train(self): 95 | """ 96 | Train the model on MNIST training set 97 | :return: 98 | """ 99 | # Initializing the variables 100 | init = tf.global_variables_initializer() 101 | 102 | # Launch the graph 103 | with tf.Session() as sess: 104 | sess.run(init) 105 | 106 | # Training cycle 107 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 108 | for s in range(self.batch_per_epoch): 109 | # Get bacth fro MNIST training set 110 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 111 | 112 | # Apply the training op 113 | (_, 114 | loss_train, 115 | accuracy_train) = sess.run([self.train_op, 116 | self.loss, 117 | self.accuracy], 118 | feed_dict={self.X: batch_xs, 119 | self.Y: batch_ys}) 120 | 121 | # Print loss and accuracy on the batch 122 | if s % 20 == 0: 123 | print("\033[1;37;40mStep: %04d , " 124 | "TRAIN: loss = %.4f - accuracy = %.2f" 125 | % ((epoch * self.batch_per_epoch + s), 126 | loss_train, accuracy_train)) 127 | 128 | 129 | # Display logs per epoch step 130 | if (epoch) % self.display_step == 0: 131 | # Compute loss on validation set (only 200 random images) 132 | (loss_val, 133 | accuracy_val) = sess.run([self.loss, 134 | self.accuracy], 135 | feed_dict={self.X: mnist.test.images[:1000], 136 | self.Y: mnist.test.labels[:1000]}) 137 | 138 | # Compute loss on training set (only 200 random images) 139 | (loss_train, 140 | accuracy_train) = sess.run([self.loss, 141 | self.accuracy], 142 | feed_dict={self.X: mnist.train.images[:1000], 143 | self.Y: mnist.train.labels[:1000]}) 144 | print("\033[1;32;40mEpoch: %04d , " 145 | "TRAIN: loss = %.4f - accuracy = %.2f | " 146 | "VALIDATION: loss = %.4f - accuracy = %.2f" 147 | % (epoch + 1, 148 | loss_train, accuracy_train, 149 | loss_val, accuracy_val)) 150 | 151 | 152 | def main(_): 153 | """ 154 | Main function 155 | :param _: 156 | :return: 157 | """ 158 | 159 | # Instanciate a MNIST class 160 | model = MNIST_lenet(learning_rate=0.01, 161 | batch_size=128) 162 | # Setup the graph 163 | model.inference() 164 | 165 | # Compute loss and metrics 166 | model.losses() 167 | model.metrics() 168 | 169 | # Create an optimzer 170 | model.optimizer() 171 | 172 | # And finally train your model! 173 | model.train() 174 | 175 | 176 | 177 | # To start the app for tensorflow 178 | if __name__ == '__main__': 179 | tf.app.run() 180 | 181 | -------------------------------------------------------------------------------- /MNIST/mlp.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | #mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_logistic(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | def inference(self): 43 | """ 44 | Design the inference model (here a simple neuralnet) 45 | :return: 46 | """ 47 | 48 | 49 | # Network Parameters 50 | n_hidden_1 = 256 # 1st layer number of features 51 | n_hidden_2 = 256 # 2nd layer number of features 52 | n_input = 784 # MNIST data input (img shape: 28*28) 53 | n_classes = 10 # MNIST total classes (0-9 digits) 54 | 55 | # Store layers weight & bias 56 | weights = { 57 | 'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])), 58 | 'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])), 59 | 'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes])) 60 | } 61 | biases = { 62 | 'b1': tf.Variable(tf.random_normal([n_hidden_1])), 63 | 'b2': tf.Variable(tf.random_normal([n_hidden_2])), 64 | 'out': tf.Variable(tf.random_normal([n_classes])) 65 | } 66 | 67 | def multilayer_perceptron(x, weights, biases): 68 | # Hidden layer with RELU activation 69 | layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1']) 70 | layer_1 = tf.nn.relu(layer_1) 71 | # Hidden layer with RELU activation 72 | layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2']) 73 | layer_2 = tf.nn.relu(layer_2) 74 | # Output layer with linear activation 75 | out_layer = tf.matmul(layer_2, weights['out']) + biases['out'] 76 | #import ipdb; 77 | #ipdb.set_trace() 78 | return out_layer 79 | 80 | # Construct model 81 | self.logits = multilayer_perceptron(self.X, weights, biases) 82 | self.Y_hat = tf.nn.softmax(self.logits) 83 | 84 | def losses(self): 85 | """ 86 | Compute the cross entropy loss 87 | :return: 88 | """ 89 | # cross entropy loss 90 | self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.logits, self.Y)) 91 | 92 | 93 | def optimizer(self): 94 | """ 95 | Create a optimizer and therefore a training operation 96 | :return: 97 | """ 98 | # The optimizer 99 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 100 | 101 | # Training operation to run later 102 | self.train_op = self.opt.minimize(self.loss) 103 | 104 | 105 | def metrics(self): 106 | """ 107 | Compute the accuracy 108 | :return: 109 | """ 110 | # Label prediction of the model (the highest one) 111 | self.predicted_label = tf.argmax(self.Y_hat, 1) 112 | # Real class: 113 | self.real_label = tf.argmax(self.Y, 1) 114 | # Number of correct prediction 115 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 116 | # Calculate accuracy 117 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 118 | self.accuracy = tf.mul(100.0, self.accuracy) 119 | 120 | 121 | def train(self): 122 | """ 123 | Train the model on MNIST training set 124 | :return: 125 | """ 126 | # Initializing the variables 127 | init = tf.global_variables_initializer() 128 | 129 | # get the current default graph 130 | # G = tf.get_default_graph() 131 | # ops = G.get_operations() 132 | # for op in ops: 133 | # print(op.name) 134 | 135 | # Launch the graph 136 | with tf.Session() as sess: 137 | sess.run(init) 138 | 139 | # Training cycle 140 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 141 | for s in range(self.batch_per_epoch): 142 | # Get bacth fro MNIST training set 143 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 144 | 145 | # Apply the training op 146 | (_, 147 | loss_train, 148 | accuracy_train) = sess.run([self.train_op, 149 | self.loss, 150 | self.accuracy], 151 | feed_dict={self.X: batch_xs, 152 | self.Y: batch_ys}) 153 | # Print loss and accuracy on the batch 154 | if s % 200 == 0: 155 | print("\033[1;37;40mStep: %04d , " 156 | "TRAIN: loss = %.4f - accuracy = %.2f" 157 | % ((epoch * self.batch_per_epoch + s), 158 | loss_train, accuracy_train)) 159 | 160 | 161 | 162 | # Display logs per epoch step 163 | if (epoch) % self.display_step == 0: 164 | # Compute loss on validation set (only 200 random images) 165 | (loss_val, 166 | accuracy_val) = sess.run([self.loss, 167 | self.accuracy], 168 | feed_dict={self.X: mnist.test.images[:1000], 169 | self.Y: mnist.test.labels[:1000]}) 170 | 171 | # Compute loss on training set (only 200 random images) 172 | (loss_train, 173 | accuracy_train) = sess.run([self.loss, 174 | self.accuracy], 175 | feed_dict={self.X: mnist.train.images[:1000], 176 | self.Y: mnist.train.labels[:1000]}) 177 | print("\033[1;32;40mEpoch: %04d , " 178 | "TRAIN: loss = %.4f - accuracy = %.2f | " 179 | "VALIDATION: loss = %.4f - accuracy = %.2f" 180 | % (epoch + 1, 181 | loss_train, accuracy_train, 182 | loss_val, accuracy_val)) 183 | 184 | 185 | def main(_): 186 | """ 187 | Main function 188 | :param _: 189 | :return: 190 | """ 191 | 192 | # Instanciate a MNIST class 193 | model = MNIST_logistic(learning_rate=0.001, 194 | batch_size=100) 195 | # Setup the graph 196 | model.inference() 197 | 198 | # Compute loss and metrics 199 | model.losses() 200 | model.metrics() 201 | 202 | # Create an optimzer 203 | model.optimizer() 204 | 205 | # And finally train your model! 206 | model.train() 207 | 208 | 209 | 210 | # To start the app for tensorflow 211 | if __name__ == '__main__': 212 | tf.app.run() 213 | 214 | -------------------------------------------------------------------------------- /MNIST/mlp_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_mlp(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | def inference(self): 43 | """ 44 | MLP inference model 45 | :return: 46 | """ 47 | 48 | def multilayer_perceptron(x, weights, biases): 49 | ??????? 50 | 51 | # Construct model 52 | self.logits = multilayer_perceptron(self.X) 53 | self.Y_hat = ??????? 54 | 55 | def losses(self): 56 | """ 57 | Compute the cross entropy loss 58 | :return: 59 | """ 60 | # cross entropy loss 61 | self.loss = ????? 62 | 63 | 64 | def optimizer(self): 65 | """ 66 | Create a optimizer and therefore a training operation 67 | :return: 68 | """ 69 | # The optimizer 70 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 71 | 72 | # Training operation to run later 73 | self.train_op = self.opt.minimize(self.loss) 74 | 75 | 76 | def metrics(self): 77 | """ 78 | Compute the accuracy 79 | :return: 80 | """ 81 | # Label prediction of the model (the highest one) 82 | self.predicted_label = tf.argmax(self.Y_hat, 1) 83 | # Real class: 84 | self.real_label = tf.argmax(self.Y, 1) 85 | # Number of correct prediction 86 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 87 | # Calculate accuracy 88 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 89 | self.accuracy = tf.mul(100.0, self.accuracy) 90 | 91 | 92 | def train(self): 93 | """ 94 | Train the model on MNIST training set 95 | :return: 96 | """ 97 | # Initializing the variables 98 | init = tf.global_variables_initializer() 99 | 100 | # Launch the graph 101 | with tf.Session() as sess: 102 | sess.run(init) 103 | 104 | # Training cycle 105 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 106 | for s in range(self.batch_per_epoch): 107 | # Get bacth fro MNIST training set 108 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 109 | 110 | # Apply the training op 111 | (_, 112 | loss_train, 113 | accuracy_train) = sess.run([self.train_op, 114 | self.loss, 115 | self.accuracy], 116 | feed_dict={self.X: batch_xs, 117 | self.Y: batch_ys}) 118 | # Print loss and accuracy on the batch 119 | if s % 200 == 0: 120 | print("\033[1;37;40mStep: %04d , " 121 | "TRAIN: loss = %.4f - accuracy = %.2f" 122 | % ((epoch * self.batch_per_epoch + s), 123 | loss_train, accuracy_train)) 124 | 125 | 126 | 127 | # Display logs per epoch step 128 | if (epoch) % self.display_step == 0: 129 | # Compute loss on validation set (only 200 random images) 130 | (loss_val, 131 | accuracy_val) = sess.run([self.loss, 132 | self.accuracy], 133 | feed_dict={self.X: mnist.test.images[:1000], 134 | self.Y: mnist.test.labels[:1000]}) 135 | 136 | # Compute loss on training set (only 200 random images) 137 | (loss_train, 138 | accuracy_train) = sess.run([self.loss, 139 | self.accuracy], 140 | feed_dict={self.X: mnist.train.images[:1000], 141 | self.Y: mnist.train.labels[:1000]}) 142 | print("\033[1;32;40mEpoch: %04d , " 143 | "TRAIN: loss = %.4f - accuracy = %.2f | " 144 | "VALIDATION: loss = %.4f - accuracy = %.2f" 145 | % (epoch + 1, 146 | loss_train, accuracy_train, 147 | loss_val, accuracy_val)) 148 | 149 | 150 | def main(_): 151 | """ 152 | Main function 153 | :param _: 154 | :return: 155 | """ 156 | 157 | # Instanciate a MNIST class 158 | model = MNIST_mlp(learning_rate=0.001, 159 | batch_size=100) 160 | # Setup the graph 161 | model.inference() 162 | 163 | # Compute loss and metrics 164 | model.losses() 165 | model.metrics() 166 | 167 | # Create an optimzer 168 | model.optimizer() 169 | 170 | # And finally train your model! 171 | model.train() 172 | 173 | 174 | 175 | # To start the app for tensorflow 176 | if __name__ == '__main__': 177 | tf.app.run() 178 | 179 | -------------------------------------------------------------------------------- /MNIST/one_conv.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_logistic(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | 43 | def inference(self): 44 | """ 45 | Design the inference model (here a simple neuralnet) 46 | :return: 47 | """ 48 | 49 | def init_weights(shape): 50 | return tf.Variable(tf.random_normal(shape, stddev=0.01)) 51 | 52 | # input reshaping 53 | X = tf.reshape(self.X, [-1, 28, 28, 1]) 54 | 55 | 56 | # Convolutional layer 57 | nb_filter = 1 58 | self.W_conv1 = init_weights([5, 5, 1, nb_filter]) 59 | self.b_conv1 = init_weights([nb_filter]) 60 | net = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(X, self.W_conv1, 61 | strides=[1, 1, 1, 1], padding='SAME'), self.b_conv1)) 62 | 63 | # Max pooling 64 | net = tf.nn.max_pool(net, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], 65 | padding='SAME') 66 | 67 | # Classifier 68 | net = tf.reshape(net, [-1, 14*14*nb_filter]) 69 | self.W = init_weights([14*14*nb_filter, 10]) 70 | self.b = init_weights([10]) 71 | self.logits = tf.add(tf.matmul(net, self.W), self.b) # WX+b : the logits 72 | 73 | self.Y_hat = tf.nn.softmax(self.logits) # softmax 74 | 75 | def losses(self): 76 | """ 77 | Compute the cross entropy loss 78 | :return: 79 | """ 80 | # cross entropy loss 81 | self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.logits, self.Y)) 82 | 83 | 84 | 85 | def optimizer(self): 86 | """ 87 | Create a optimizer and therefore a training operation 88 | :return: 89 | """ 90 | # The optimizer 91 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 92 | 93 | # Training operation to run later 94 | self.train_op = self.opt.minimize(self.loss) 95 | 96 | 97 | def metrics(self): 98 | """ 99 | Compute the accuracy 100 | :return: 101 | """ 102 | # Label prediction of the model (the highest one) 103 | self.predicted_label = tf.argmax(self.Y_hat, 1) 104 | # Real class: 105 | self.real_label = tf.argmax(self.Y, 1) 106 | # Number of correct prediction 107 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 108 | # Calculate accuracy 109 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 110 | self.accuracy = tf.mul(100.0, self.accuracy) 111 | 112 | 113 | def train(self): 114 | """ 115 | Train the model on MNIST training set 116 | :return: 117 | """ 118 | # Initializing the variables 119 | init = tf.global_variables_initializer() 120 | 121 | # Launch the graph 122 | with tf.Session() as sess: 123 | sess.run(init) 124 | 125 | # Training cycle 126 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 127 | for s in range(self.batch_per_epoch): 128 | # Get bacth fro MNIST training set 129 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 130 | 131 | # Apply the training op 132 | (_, 133 | loss_train, 134 | accuracy_train) = sess.run([self.train_op, 135 | self.loss, 136 | self.accuracy], 137 | feed_dict={self.X: batch_xs, 138 | self.Y: batch_ys}) 139 | 140 | # Print loss and accuracy on the batch 141 | if s % 200 == 0: 142 | print("\033[1;37;40mStep: %04d , " 143 | "TRAIN: loss = %.4f - accuracy = %.2f" 144 | % ((epoch * self.batch_per_epoch + s), 145 | loss_train, accuracy_train)) 146 | 147 | # Display logs per epoch step 148 | if (epoch) % self.display_step == 0: 149 | # Compute loss on validation set (only 200 random images) 150 | (loss_val, 151 | accuracy_val) = sess.run([self.loss, 152 | self.accuracy], 153 | feed_dict={self.X: mnist.test.images[:1000], 154 | self.Y: mnist.test.labels[:1000]}) 155 | 156 | # Compute loss on training set (only 200 random images) 157 | (loss_train, 158 | accuracy_train) = sess.run([self.loss, 159 | self.accuracy], 160 | feed_dict={self.X: mnist.train.images[:1000], 161 | self.Y: mnist.train.labels[:1000]}) 162 | print("\033[1;32;40mEpoch: %04d , " 163 | "TRAIN: loss = %.4f - accuracy = %.2f | " 164 | "VALIDATION: loss = %.4f - accuracy = %.2f" 165 | % (epoch + 1, 166 | loss_train, accuracy_train, 167 | loss_val, accuracy_val)) 168 | 169 | 170 | def main(_): 171 | """ 172 | Main function 173 | :param _: 174 | :return: 175 | """ 176 | 177 | # Instanciate a MNIST class 178 | model = MNIST_logistic(learning_rate=0.01, 179 | batch_size=128) 180 | # Setup the graph 181 | model.inference() 182 | 183 | # Compute loss and metrics 184 | model.losses() 185 | model.metrics() 186 | 187 | # Create an optimzer 188 | model.optimizer() 189 | 190 | # And finally train your model! 191 | model.train() 192 | 193 | 194 | 195 | # To start the app for tensorflow 196 | if __name__ == '__main__': 197 | tf.app.run() 198 | 199 | -------------------------------------------------------------------------------- /MNIST/one_conv_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_conv(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | 43 | def inference(self): 44 | """ 45 | Design the inference model (here a simple neuralnet) 46 | :return: 47 | """ 48 | 49 | def init_weights(shape): 50 | return tf.Variable(tf.random_normal(shape, stddev=0.01)) 51 | 52 | # input reshaping 53 | X = ???? # reshape to [_1, 28, 28, 1] 54 | 55 | 56 | # One convolution, max pooling and fully connected layer 57 | # tf.nn.conv2d() tf.nn.max_pool() 58 | 59 | self.logits = ????? # WX+b : the logits 60 | 61 | self.Y_hat = ?????? # softmax 62 | 63 | def losses(self): 64 | """ 65 | Compute the cross entropy loss 66 | :return: 67 | """ 68 | # cross entropy loss 69 | self.loss = ????? 70 | 71 | 72 | 73 | def optimizer(self): 74 | """ 75 | Create a optimizer and therefore a training operation 76 | :return: 77 | """ 78 | # The optimizer 79 | self.opt = tf.train.AdamOptimizer(self.learning_rate) 80 | 81 | # Training operation to run later 82 | self.train_op = self.opt.minimize(self.loss) 83 | 84 | 85 | def metrics(self): 86 | """ 87 | Compute the accuracy 88 | :return: 89 | """ 90 | # Label prediction of the model (the highest one) 91 | self.predicted_label = tf.argmax(self.Y_hat, 1) 92 | # Real class: 93 | self.real_label = tf.argmax(self.Y, 1) 94 | # Number of correct prediction 95 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 96 | # Calculate accuracy 97 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 98 | self.accuracy = tf.mul(100.0, self.accuracy) 99 | 100 | 101 | def train(self): 102 | """ 103 | Train the model on MNIST training set 104 | :return: 105 | """ 106 | # Initializing the variables 107 | init = tf.global_variables_initializer() 108 | 109 | # Launch the graph 110 | with tf.Session() as sess: 111 | sess.run(init) 112 | 113 | # Training cycle 114 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 115 | for s in range(self.batch_per_epoch): 116 | # Get bacth fro MNIST training set 117 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 118 | 119 | # Apply the training op 120 | (_, 121 | loss_train, 122 | accuracy_train) = sess.run([self.train_op, 123 | self.loss, 124 | self.accuracy], 125 | feed_dict={self.X: batch_xs, 126 | self.Y: batch_ys}) 127 | 128 | # Print loss and accuracy on the batch 129 | if s % 200 == 0: 130 | print("\033[1;37;40mStep: %04d , " 131 | "TRAIN: loss = %.4f - accuracy = %.2f" 132 | % ((epoch * self.batch_per_epoch + s), 133 | loss_train, accuracy_train)) 134 | 135 | # Display logs per epoch step 136 | if (epoch) % self.display_step == 0: 137 | # Compute loss on validation set (only 200 random images) 138 | (loss_val, 139 | accuracy_val) = sess.run([self.loss, 140 | self.accuracy], 141 | feed_dict={self.X: mnist.test.images[:1000], 142 | self.Y: mnist.test.labels[:1000]}) 143 | 144 | # Compute loss on training set (only 200 random images) 145 | (loss_train, 146 | accuracy_train) = sess.run([self.loss, 147 | self.accuracy], 148 | feed_dict={self.X: mnist.train.images[:1000], 149 | self.Y: mnist.train.labels[:1000]}) 150 | print("\033[1;32;40mEpoch: %04d , " 151 | "TRAIN: loss = %.4f - accuracy = %.2f | " 152 | "VALIDATION: loss = %.4f - accuracy = %.2f" 153 | % (epoch + 1, 154 | loss_train, accuracy_train, 155 | loss_val, accuracy_val)) 156 | 157 | 158 | def main(_): 159 | """ 160 | Main function 161 | :param _: 162 | :return: 163 | """ 164 | 165 | # Instanciate a MNIST class 166 | model = MNIST_conv(learning_rate=0.01, 167 | batch_size=128) 168 | # Setup the graph 169 | model.inference() 170 | 171 | # Compute loss and metrics 172 | model.losses() 173 | model.metrics() 174 | 175 | # Create an optimzer 176 | model.optimizer() 177 | 178 | # And finally train your model! 179 | model.train() 180 | 181 | 182 | 183 | # To start the app for tensorflow 184 | if __name__ == '__main__': 185 | tf.app.run() 186 | 187 | -------------------------------------------------------------------------------- /MNIST/rnn.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_logistic(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | 43 | def inference(self): 44 | """ 45 | RNN 46 | :return: 47 | """ 48 | # Reshaping for sequene modelling 49 | X_seq = tf.reshape(self.X, [-1, 28, 28]) #[-1, 28, 28] 50 | 51 | # Define a lstm cell with tensorflow 52 | rnn_cell = tf.nn.rnn_cell.BasicRNNCell(100) 53 | 54 | # Get lstm cell output 55 | outputs, states = tf.nn.dynamic_rnn(cell= rnn_cell, 56 | inputs=X_seq, 57 | dtype=tf.float32) 58 | 59 | # Set model weights 60 | self.W = tf.Variable(tf.zeros([100, 10])) 61 | self.b = tf.Variable(tf.zeros([10])) 62 | 63 | # Classif using rnn inner loop last output 64 | self.logits = tf.add(tf.matmul(outputs[:,-1,:], self.W), self.b) # WX+b : the logits 65 | self.Y_hat = tf.nn.softmax(self.logits) # softmax 66 | 67 | def losses(self): 68 | """ 69 | Compute the cross entropy loss 70 | :return: 71 | """ 72 | # cross entropy loss 73 | self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.logits, self.Y)) 74 | 75 | 76 | 77 | def optimizer(self): 78 | """ 79 | Create a optimizer and therefore a training operation 80 | :return: 81 | """ 82 | # The optimizer 83 | self.opt = tf.train.AdagradOptimizer(self.learning_rate) 84 | 85 | # Training operation to run later 86 | self.train_op = self.opt.minimize(self.loss) 87 | 88 | 89 | def metrics(self): 90 | """ 91 | Compute the accuracy 92 | :return: 93 | """ 94 | # Label prediction of the model (the highest one) 95 | self.predicted_label = tf.argmax(self.Y_hat, 1) 96 | # Real class: 97 | self.real_label = tf.argmax(self.Y, 1) 98 | # Number of correct prediction 99 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 100 | # Calculate accuracy 101 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 102 | self.accuracy = tf.mul(100.0, self.accuracy) 103 | 104 | 105 | def train(self): 106 | """ 107 | Train the model on MNIST training set 108 | :return: 109 | """ 110 | # Initializing the variables 111 | init = tf.global_variables_initializer() 112 | 113 | # Launch the graph 114 | with tf.Session() as sess: 115 | sess.run(init) 116 | 117 | # Training cycle 118 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 119 | for s in range(self.batch_per_epoch): 120 | # Get bacth fro MNIST training set 121 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 122 | 123 | # Apply the training op 124 | (_, 125 | loss_train, 126 | accuracy_train) = sess.run([self.train_op, 127 | self.loss, 128 | self.accuracy], 129 | feed_dict={self.X: batch_xs, 130 | self.Y: batch_ys}) 131 | 132 | # Print loss and accuracy on the batch 133 | if s % 200 == 0: 134 | print("\033[1;37;40mStep: %04d , " 135 | "TRAIN: loss = %.4f - accuracy = %.2f" 136 | % ((epoch*self.batch_per_epoch + s), 137 | loss_train, accuracy_train ) ) 138 | 139 | 140 | 141 | 142 | # Display logs per epoch step 143 | if (epoch) % self.display_step == 0: 144 | # Compute loss on test (only 200 random images) 145 | (loss_test, 146 | accuracy_test) = sess.run([self.loss, 147 | self.accuracy], 148 | feed_dict={self.X: mnist.test.images[:100], 149 | self.Y: mnist.test.labels[:100]}) 150 | # Compute loss on training (only 200 random images) 151 | (loss_train, 152 | accuracy_train) = sess.run([self.loss, 153 | self.accuracy], 154 | feed_dict={self.X: mnist.train.images[:100], 155 | self.Y: mnist.train.labels[:100]}) 156 | print("\033[1;32;40mEpoch: %04d , " 157 | "TRAIN: loss = %.4f - accuracy = %.2f | " 158 | "TEST: loss = %.4f - accuracy = %.2f" 159 | % (epoch + 1, 160 | loss_train, accuracy_train, 161 | loss_test, accuracy_test)) 162 | 163 | 164 | def main(_): 165 | """ 166 | Main function 167 | :param _: 168 | :return: 169 | """ 170 | 171 | # Instanciate a MNIST class 172 | model = MNIST_logistic(learning_rate=0.01, 173 | batch_size=64) 174 | # Setup the graph 175 | model.inference() 176 | 177 | # Compute loss and metrics 178 | model.losses() 179 | model.metrics() 180 | 181 | # Create an optimzer 182 | model.optimizer() 183 | 184 | # And finally train your model! 185 | model.train() 186 | 187 | 188 | 189 | # To start the app for tensorflow 190 | if __name__ == '__main__': 191 | tf.app.run() 192 | 193 | -------------------------------------------------------------------------------- /MNIST/rnn_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_logistic(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | 43 | def inference(self): 44 | """ 45 | RNN 46 | :return: 47 | """ 48 | # Reshaping for sequene modelling 49 | X_seq = ???? # reshape to [-1, 28, 28] 50 | 51 | # Define a lstm cell with tensorflow 52 | rnn_cell = tf.nn.rnn_cell.BasicRNNCell(100) 53 | 54 | # Get lstm cell output 55 | outputs, states = ???? #tf.nn.dynamic_rnn 56 | 57 | # Set model weights 58 | self.W = ???? # variable of size [100, 10] 59 | self.b = ??? # biases for a classif of 10 classes 60 | 61 | # Classif using rnn inner loop last output 62 | self.logits = ?????? # WX+b : the logits using x as the last hidden state 63 | self.Y_hat = ?????? # softmax 64 | 65 | def losses(self): 66 | """ 67 | Compute the cross entropy loss 68 | :return: 69 | """ 70 | # cross entropy loss 71 | self.loss = ?????? 72 | 73 | 74 | 75 | def optimizer(self): 76 | """ 77 | Create a optimizer and therefore a training operation 78 | :return: 79 | """ 80 | # The optimizer 81 | self.opt = tf.train.AdagradOptimizer(self.learning_rate) 82 | 83 | # Training operation to run later 84 | self.train_op = self.opt.minimize(self.loss) 85 | 86 | 87 | def metrics(self): 88 | """ 89 | Compute the accuracy 90 | :return: 91 | """ 92 | # Label prediction of the model (the highest one) 93 | self.predicted_label = tf.argmax(self.Y_hat, 1) 94 | # Real class: 95 | self.real_label = tf.argmax(self.Y, 1) 96 | # Number of correct prediction 97 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 98 | # Calculate accuracy 99 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 100 | self.accuracy = tf.mul(100.0, self.accuracy) 101 | 102 | 103 | def train(self): 104 | """ 105 | Train the model on MNIST training set 106 | :return: 107 | """ 108 | # Initializing the variables 109 | init = tf.global_variables_initializer() 110 | 111 | # Launch the graph 112 | with tf.Session() as sess: 113 | sess.run(init) 114 | 115 | # Training cycle 116 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 117 | for s in range(self.batch_per_epoch): 118 | # Get bacth fro MNIST training set 119 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 120 | 121 | # Apply the training op 122 | (_, 123 | loss_train, 124 | accuracy_train) = sess.run([self.train_op, 125 | self.loss, 126 | self.accuracy], 127 | feed_dict={self.X: batch_xs, 128 | self.Y: batch_ys}) 129 | 130 | # Print loss and accuracy on the batch 131 | if s % 200 == 0: 132 | print("\033[1;37;40mStep: %04d , " 133 | "TRAIN: loss = %.4f - accuracy = %.2f" 134 | % ((epoch*self.batch_per_epoch + s), 135 | loss_train, accuracy_train ) ) 136 | 137 | 138 | 139 | 140 | # Display logs per epoch step 141 | if (epoch) % self.display_step == 0: 142 | # Compute loss on test (only 200 random images) 143 | (loss_test, 144 | accuracy_test) = sess.run([self.loss, 145 | self.accuracy], 146 | feed_dict={self.X: mnist.test.images[:100], 147 | self.Y: mnist.test.labels[:100]}) 148 | # Compute loss on training (only 200 random images) 149 | (loss_train, 150 | accuracy_train) = sess.run([self.loss, 151 | self.accuracy], 152 | feed_dict={self.X: mnist.train.images[:100], 153 | self.Y: mnist.train.labels[:100]}) 154 | print("\033[1;32;40mEpoch: %04d , " 155 | "TRAIN: loss = %.4f - accuracy = %.2f | " 156 | "TEST: loss = %.4f - accuracy = %.2f" 157 | % (epoch + 1, 158 | loss_train, accuracy_train, 159 | loss_test, accuracy_test)) 160 | 161 | 162 | def main(_): 163 | """ 164 | Main function 165 | :param _: 166 | :return: 167 | """ 168 | 169 | # Instanciate a MNIST class 170 | model = MNIST_logistic(learning_rate=0.01, 171 | batch_size=64) 172 | # Setup the graph 173 | model.inference() 174 | 175 | # Compute loss and metrics 176 | model.losses() 177 | model.metrics() 178 | 179 | # Create an optimzer 180 | model.optimizer() 181 | 182 | # And finally train your model! 183 | model.train() 184 | 185 | 186 | 187 | # To start the app for tensorflow 188 | if __name__ == '__main__': 189 | tf.app.run() 190 | 191 | -------------------------------------------------------------------------------- /MNIST/softmax.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | #mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_logistic(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | def inference(self): 43 | """ 44 | Design the inference model (here a simple neuralnet) 45 | :return: 46 | """ 47 | 48 | # Set model weights 49 | self.W = tf.Variable(tf.zeros([784, 10])) 50 | self.b = tf.Variable(tf.zeros([10])) 51 | 52 | # Construct the inference 53 | self.logits = tf.add(tf.matmul(self.X, self.W), self.b) # WX+b : the logits 54 | self.Y_hat = tf.nn.softmax(self.logits) # softmax apply to logits 55 | 56 | def losses(self): 57 | """ 58 | Compute the cross entropy loss 59 | :return: 60 | """ 61 | # cross entropy loss 62 | #self.loss = tf.reduce_mean(-tf.reduce_sum(self.Y * tf.log(self.Y_hat), reduction_indices=1)) 63 | # also equal to: 64 | self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.logits, self.Y)) 65 | 66 | 67 | 68 | def optimizer(self): 69 | """ 70 | Create a optimizer and therefore a training operation 71 | :return: 72 | """ 73 | # The optimizer 74 | self.opt = tf.train.GradientDescentOptimizer(self.learning_rate) 75 | 76 | # Training operation to run later 77 | self.train_op = self.opt.minimize(self.loss) 78 | 79 | 80 | def metrics(self): 81 | """ 82 | Compute the accuracy 83 | :return: 84 | """ 85 | # Label prediction of the model (the highest one) 86 | self.predicted_label = tf.argmax(self.Y_hat, 1) 87 | # Real class: 88 | self.real_label = tf.argmax(self.Y, 1) 89 | # Number of correct prediction 90 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 91 | # Calculate accuracy 92 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 93 | self.accuracy = tf.mul(100.0, self.accuracy) 94 | 95 | 96 | def train(self): 97 | """ 98 | Train the model on MNIST training set 99 | :return: 100 | """ 101 | # Initializing the variables 102 | init = tf.global_variables_initializer() 103 | 104 | # Launch the graph 105 | with tf.Session() as sess: 106 | sess.run(init) 107 | 108 | # Training cycle 109 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 110 | for s in range(self.batch_per_epoch): 111 | # Get bacth fro MNIST training set 112 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 113 | 114 | # Apply the training op 115 | (_, 116 | loss_train, 117 | accuracy_train) = sess.run([self.train_op, 118 | self.loss, 119 | self.accuracy], 120 | feed_dict={self.X: batch_xs, 121 | self.Y: batch_ys}) 122 | 123 | # Print loss and accuracy on the batch 124 | if s % 200 == 0: 125 | print("\033[1;37;40mStep: %04d , " 126 | "TRAIN: loss = %.4f - accuracy = %.2f" 127 | % ((epoch*self.batch_per_epoch + s), 128 | loss_train, accuracy_train ) ) 129 | 130 | 131 | 132 | # Display logs per epoch step 133 | if (epoch) % self.display_step == 0: 134 | # Compute loss on validation set (only 200 random images) 135 | (loss_val, 136 | accuracy_val) = sess.run([self.loss, 137 | self.accuracy], 138 | feed_dict={self.X: mnist.test.images[:1000], 139 | self.Y: mnist.test.labels[:1000]}) 140 | 141 | # Compute loss on training set (only 200 random images) 142 | (loss_train, 143 | accuracy_train) = sess.run([self.loss, 144 | self.accuracy], 145 | feed_dict={self.X: mnist.train.images[:1000], 146 | self.Y: mnist.train.labels[:1000]}) 147 | 148 | print("\033[1;32;40mEpoch: %04d , " 149 | "TRAIN: loss = %.4f - accuracy = %.2f | " 150 | "VALIDATION: loss = %.4f - accuracy = %.2f" 151 | % (epoch + 1, 152 | loss_train, accuracy_train, 153 | loss_val, accuracy_val)) 154 | 155 | 156 | def main(_): 157 | """ 158 | Main function 159 | :param _: 160 | :return: 161 | """ 162 | 163 | # Instanciate a MNIST class 164 | model = MNIST_logistic(learning_rate=0.01, 165 | batch_size=64) 166 | # Setup the graph 167 | model.inference() 168 | 169 | # Compute loss and metrics 170 | model.losses() 171 | model.metrics() 172 | 173 | # Create an optimzer 174 | model.optimizer() 175 | 176 | # And finally train your model! 177 | model.train() 178 | 179 | 180 | 181 | # To start the app for tensorflow 182 | if __name__ == '__main__': 183 | tf.app.run() 184 | 185 | -------------------------------------------------------------------------------- /MNIST/softmax_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Import MNIST data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/home/sid/MNIST", one_hot=True) 10 | #mnist = input_data.read_data_sets("/Users/fabien/Datasets/MNIST", one_hot=True) 11 | 12 | 13 | 14 | 15 | class MNIST_logistic(object): 16 | """ 17 | Class to construct a simple logistic regression on MNIST (i.e a neural net w/o hidden layer) 18 | """ 19 | 20 | def __init__(self, learning_rate, batch_size): 21 | """ 22 | Init the class with some parameters 23 | :param learning_rate: 24 | :param batch_size: 25 | """ 26 | # Parameters 27 | self.learning_rate = learning_rate 28 | self.mnist = mnist 29 | self.batch_size = batch_size 30 | self.num_epochs = 50 31 | self.num_classes = 10 32 | self.input_size = 784 33 | self.input_weight, self.input_height = 28, 28 34 | self.batch_per_epoch = int(self.mnist.train.num_examples/self.batch_size) 35 | self.display_step = 1 36 | 37 | # Placeholders 38 | self.X = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 39 | self.Y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 40 | 41 | 42 | def inference(self): 43 | """ 44 | Design the inference model (here a simple neuralnet) 45 | :return: 46 | """ 47 | 48 | # Set model weights (variable!) 49 | self.W = ????? # input of size 784 and there are 10 classes... 50 | self.b = ????? 51 | 52 | # Construct the inference 53 | self.logits = ????? # WX+b = the logits 54 | self.Y_hat = ????? # softmax apply to logits 55 | 56 | def losses(self): 57 | """ 58 | Compute the cross entropy loss 59 | :return: 60 | """ 61 | # cross entropy loss 62 | self.loss = ??????? 63 | 64 | 65 | 66 | def optimizer(self): 67 | """ 68 | Create a optimizer and therefore a training operation 69 | :return: 70 | """ 71 | # The optimizer 72 | self.opt = tf.train.GradientDescentOptimizer(self.learning_rate) 73 | 74 | # Training operation to run later 75 | self.train_op = self.opt.minimize(self.loss) 76 | 77 | 78 | def metrics(self): 79 | """ 80 | Compute the accuracy 81 | :return: 82 | """ 83 | # Label prediction of the model (the highest one) 84 | self.predicted_label = tf.argmax(self.Y_hat, 1) 85 | # Real class: 86 | self.real_label = tf.argmax(self.Y, 1) 87 | # Number of correct prediction 88 | self.correct_prediction = tf.equal(self.predicted_label, self.real_label) 89 | # Calculate accuracy 90 | self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32)) 91 | self.accuracy = tf.mul(100.0, self.accuracy) 92 | 93 | 94 | def train(self): 95 | """ 96 | Train the model on MNIST training set 97 | :return: 98 | """ 99 | # Initializing the variables 100 | init = tf.global_variables_initializer() 101 | 102 | # Launch the graph 103 | with tf.Session() as sess: 104 | sess.run(init) 105 | 106 | # Training cycle 107 | for epoch in range(self.num_epochs): # 1 epoch = 1 loop over the entire training set 108 | for s in range(self.batch_per_epoch): 109 | # Get bacth fro MNIST training set 110 | batch_xs, batch_ys = mnist.train.next_batch(self.batch_size) 111 | 112 | # Apply the training op 113 | (_, 114 | loss_train, 115 | accuracy_train) = sess.run([self.train_op, 116 | self.loss, 117 | self.accuracy], 118 | feed_dict={self.X: batch_xs, 119 | self.Y: batch_ys}) 120 | 121 | # Print loss and accuracy on the batch 122 | if s % 200 == 0: 123 | print("\033[1;37;40mStep: %04d , " 124 | "TRAIN: loss = %.4f - accuracy = %.2f" 125 | % ((epoch*self.batch_per_epoch + s), 126 | loss_train, accuracy_train ) ) 127 | 128 | 129 | # Display logs per epoch step 130 | if (epoch) % self.display_step == 0: 131 | # Compute loss on validation set (only 200 random images) 132 | (loss_val, 133 | accuracy_val) = sess.run([self.loss, 134 | self.accuracy], 135 | feed_dict={self.X: mnist.test.images[:1000], 136 | self.Y: mnist.test.labels[:1000]}) 137 | 138 | # Compute loss on training set (only 200 random images) 139 | (loss_train, 140 | accuracy_train) = sess.run([self.loss, 141 | self.accuracy], 142 | feed_dict={self.X: mnist.train.images[:1000], 143 | self.Y: mnist.train.labels[:1000]}) 144 | print("\033[1;32;40mEpoch: %04d , " 145 | "TRAIN: loss = %.4f - accuracy = %.2f | " 146 | "VALIDATION: loss = %.4f - accuracy = %.2f" 147 | % (epoch + 1, 148 | loss_train, accuracy_train, 149 | loss_val, accuracy_val)) 150 | 151 | 152 | def main(_): 153 | """ 154 | Main function 155 | :param _: 156 | :return: 157 | """ 158 | 159 | # Instanciate a MNIST class 160 | model = MNIST_logistic(learning_rate=0.01, 161 | batch_size=64) 162 | # Setup the graph 163 | model.inference() 164 | 165 | # Compute loss and metrics 166 | model.losses() 167 | model.metrics() 168 | 169 | # Create an optimzer 170 | model.optimizer() 171 | 172 | # And finally train your model! 173 | model.train() 174 | 175 | 176 | 177 | # To start the app for tensorflow 178 | if __name__ == '__main__': 179 | tf.app.run() 180 | 181 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction to Deep Learning with Tensorflow 2 | 3 | Tutorials with Tensorflow implementations. Initially designed for [ENSAI SID 2017](http://www.ensai.fr/formation/id-3e-annee-ingenieur/filiere-statistique-et-ingenierie-des-donnees.html). 4 | 5 | A .ova file (Ubuntu 16.04 - ~ 3.5 Go) is available on a USB key where Python 2.7 and Tensorflow (0.12.0) are already installed. 6 | Go to VirtualBox and 'import a virtual environment' and select the .ova file. 7 | 8 | Slides available on [my website](https://fabienbaradel.github.io): ["Intro to deep learning with Tensorflow"](https://fabienbaradel.github.io/images/tensorflow_ensai_SID_13_01_17.pdf) 9 | 10 | Feel free to install directly Tensorflow on your laptop [TensorFlow Installation Guide](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/g3doc/get_started/os_setup.md) 11 | 12 | 13 | ## Introduction 14 | - Hello World ([code](Intro/hello_world.py)) 15 | - Basic Maths ([code](Intro/math_ops.py)) 16 | 17 | ## Understanding Stochastic Gradient Descent 18 | - Linear Regression ([exo](SGD/linear_regression_exo.py)) - ([solution](SGD/linear_regression.py)) 19 | - Binary Classification ([exo](SGD/binary_classifcation_exo.py)) - ([solution](SGD/binary_classifcation.py)) 20 | 21 | ## MNIST 22 | - Softmax ([exo](MNIST/softmax_exo.py)) - ([solution](MNIST/softmax.py)) 23 | - Mulilayer Perceptron ([exo](MNIST/mlp_exo.py)) - ([solution](MNIST/mlp.py)) 24 | - One Conv + Max Pool ([exo](MNIST/one_conv_exo.py)) - ([solution](MNIST/one_conv.py)) 25 | - Make your life easier => [SLIM](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim) ;) 26 | - LeNet ([exo](MNIST/lenet_exo.py)) - ([solution](MNIST/lenet.py)) 27 | - Autoencoder ([exo](MNIST/autoencoder_exo.py)) - ([solution](MNIST/autoencoder.py)) 28 | - Conv-Deconv Autoencoder ([exo](MNIST/conv_ae_exo.py)) - ([solution](MNIST/conv_ae.py)) 29 | - RNN ([exo](MNIST/rnn_exo.py)) - ([solution](MNIST/rnn_exo.py)) 30 | - GAN (coming...) 31 | - DCGAN (coming...) 32 | 33 | 34 | ## The Nature Conservancy Fisheries Monitoring ([Kaggle](https://www.kaggle.com/c/the-nature-conservancy-fisheries-monitoring)) 35 | - Have a look at the [README](Kaggle/README.md) of this repository first 36 | - Inception v3 Features Extraction ([code](Kaggle/extract_deepFeatures_fish.py)) 37 | - Classification from DeepFeatures ([exo](Kaggle/classif_fish_exo.py)) - ([softmax regression](kaggle/classif_fish.py)) 38 | 39 | 40 | ## Frozen Lake 41 | - Q-Learning with neural network ([exo](RL/q_learning_neural_net_exo.py)) - ([solution](RL/q_learning_neural_net.py)) 42 | 43 | ## Thanks 44 | Code inspired by other great tutorials from [@aymericdamien](https://github.com/aymericdamien/TensorFlow-Examples) (SGD and MNIST parts), [@awjuliani](https://github.com/awjuliani/DeepRL-Agents) (RL part), [@pkmital](https://github.com/pkmital/tensorflow_tutorials/blob/master/python/09_convolutional_autoencoder.py) (conv-deconv autoencoder). 45 | Here are my favorite deep learning blogs: [colah.github.io](http://colah.github.io/), [karpathy.github.io](http://karpathy.github.io/), [wildml.com](http://www.wildml.com/) 46 | 47 | 48 | 49 | ## Dependencies 50 | ``` 51 | tensorflow (>0.12.1) 52 | numpy 53 | matplotlib 54 | gym (Frozen Lake only) 55 | ``` 56 | 57 | An update one the repo will be done soone (python 3.5 and Tensorflow 1.2) 58 | -------------------------------------------------------------------------------- /RL/q_learning_neural_net.py: -------------------------------------------------------------------------------- 1 | """ 2 | Re-written code from https://github.com/awjuliani/DeepRL-Agents 3 | I only had few omments and a testing part at the end. 4 | """ 5 | import gym 6 | import numpy as np 7 | import random 8 | import tensorflow as tf 9 | import matplotlib.pyplot as plt 10 | import tensorflow.contrib.slim as slim 11 | 12 | 13 | 14 | env = gym.make('FrozenLake-v0') 15 | 16 | np.set_printoptions(suppress=True) 17 | 18 | # Q-network: Q(state) = action where Q is a simple matrix 19 | input_state = tf.placeholder(shape=[1, 16], dtype=tf.float32, name="input_state") 20 | W = tf.Variable(tf.random_uniform([16,4],0,0.01)) 21 | Q_values = tf.matmul(input_state, W) 22 | action = tf.argmax(Q_values, 1) 23 | 24 | # Computing the loss function: sum of squares between Q_target and Q_values 25 | Q_target_values = tf.placeholder(shape=[1, 4], dtype=tf.float32, name="Q_target_values") 26 | diff_tf = Q_target_values - Q_values 27 | loss = tf.reduce_sum(tf.square(Q_target_values - Q_values)) 28 | 29 | 30 | # Define an optimizer to minimize the loss function 31 | opt = tf.train.GradientDescentOptimizer(learning_rate=0.1) 32 | train_op = opt.minimize(loss) 33 | 34 | # Initialize all the variables of the graph 35 | init = tf.global_variables_initializer() 36 | 37 | # Set learning parameters 38 | gamma = .99 # for the Q_target 39 | e = 0.1 # chance of taking a random action at any step 40 | num_episodes = 2000 # total number of episodes during the learning 41 | 42 | 43 | # Create lists to contain total rewards and steps per episode 44 | jList = [] 45 | rList = [] 46 | 47 | with tf.Session() as sess: 48 | sess.run(init) 49 | 50 | ############ 51 | # Training # 52 | ############ 53 | print("\n###########################") 54 | print("## TRAINING YOUR AGENT ####") 55 | print("###########################") 56 | 57 | for i in range(num_episodes): 58 | # Reset environment and get first observation 59 | s = env.reset() 60 | R = 0 # sum of the reward over an episode (1.0 if we reach the goal state, 0 otherwise 61 | j = 0 # number of step in the environment 62 | 63 | # The Q-Network # 64 | # The episode is stopped if we reach a hole, the goal state or after 100 steps 65 | while j < 99 and s not in [5,7,11,12,15]: 66 | j+=1 67 | 68 | # Choose an action by greedily (with e chance of random action) from the Q-network 69 | s_one_hot = np.identity(16)[s:s + 1] # one hot encoding of the state 70 | a, all_Q = sess.run([action, Q_values], feed_dict={input_state: s_one_hot}) 71 | if np.random.rand(1) < e: 72 | a[0] = env.action_space.sample() 73 | 74 | # Get new state and reward from environment 75 | new_s, r, _, _ = env.step(a[0]) 76 | 77 | # Obtain the Q values of the new state by feeding the new state through our network still with the same parameters as for the previous step 78 | new_s_one_hot = np.identity(16)[new_s:new_s + 1] 79 | all_next_Q = sess.run(Q_values, feed_dict={input_state: new_s_one_hot}) 80 | 81 | # Obtain max(all_next_Q) and set our Q_target_values for chosen action. 82 | max_all_next_Q = np.max(all_next_Q) 83 | all_Q_target = all_Q.copy() 84 | all_Q_target[0, a[0]] = r + gamma * max_all_next_Q # Q_target definition 85 | 86 | # Train our network by minimizing SE(all_Q_target - all_Q) 87 | sess.run(train_op, feed_dict={input_state: s_one_hot, 88 | Q_target_values: all_Q_target}) 89 | 90 | 91 | # Update the reward and the current state for the next step 92 | R += r 93 | s = new_s 94 | 95 | # Append final reward value and number of step did during the episode 96 | jList.append(j) 97 | rList.append(R) 98 | 99 | # Print in the log every 100 episodes 100 | if i % 100 == 0 and i != 0: 101 | print("Episodes %04d to %04d: nb of successful episodes = %i/100, nb of steps in avg = %.2f " % (i-100, 102 | i, 103 | int(sum(rList[-100:])), 104 | float(np.mean(jList[-100:])) 105 | )) 106 | # print the W matrix 107 | W_matrix = sess.run(W) 108 | print("\nHere is your the matrix estimated by your network:") 109 | print((" Left - Down - Right - Up")) 110 | print(np.asarray(W_matrix)) 111 | 112 | ############ 113 | # Testing # 114 | ############ 115 | print("\n###########################") 116 | print("## TESTING YOUR AGENT #####") 117 | print("###########################") 118 | # Do 100 episodes with the network whithout random choice to now how good is your agent 119 | nb_successful_episodes = 0 120 | total_nb_of_steps = 0 121 | 122 | for i in range(100): 123 | # Reset environment and get first observation 124 | s = env.reset() 125 | R = 0 # sum of the reward over an episode (1.0 if we reach the goal state, 0 otherwise 126 | j = 0 # number of step in the environment 127 | 128 | while j < 99 and s not in [5,7,11,12,15]: 129 | j+=1 130 | 131 | # Choose an action from the trained Q-network 132 | s_one_hot = np.identity(16)[s:s + 1] # one hot encoding of the state 133 | a, all_Q = sess.run([action, Q_values], feed_dict={input_state: s_one_hot}) 134 | 135 | # Get new state and reward from environment 136 | new_s, r, _, _ = env.step(a[0]) 137 | 138 | # Update the reward and the current state for the next step 139 | R += r 140 | s = new_s 141 | 142 | # Add the final reward after the episode and nb of steps 143 | nb_successful_episodes += R 144 | total_nb_of_steps +=j 145 | 146 | print("Testing on 100 episodes: \nYour agent has finished %i episodes and did %.2f steps in average!" % (int(nb_successful_episodes), 147 | total_nb_of_steps/100.0)) 148 | if nb_successful_episodes < 30: 149 | print("=> Keep coding or maybe just re-run your script...") 150 | else: 151 | print("=> Well done! How to improve your score and make your agent smarter?!\n") 152 | 153 | -------------------------------------------------------------------------------- /RL/q_learning_neural_net_exo.py: -------------------------------------------------------------------------------- 1 | """ 2 | Re-written code from https://github.com/awjuliani/DeepRL-Agents 3 | I only had few omments and a testing part at the end. 4 | """ 5 | 6 | import gym 7 | import numpy as np 8 | import random 9 | import tensorflow as tf 10 | import matplotlib.pyplot as plt 11 | import tensorflow.contrib.slim as slim 12 | 13 | 14 | 15 | env = gym.make('FrozenLake-v0') 16 | 17 | np.set_printoptions(suppress=True) 18 | 19 | # Q-network: Q(state) = action where Q is a simple matrix 20 | input_state = tf.placeholder(shape=[1, 16], dtype=tf.float32, name="input_state") 21 | W = ????? 22 | Q_values = ????? 23 | action = tf.argmax(Q_values, 1) 24 | 25 | # Computing the loss function: sum of squares between Q_target and Q_values 26 | Q_target_values = tf.placeholder(shape=[1, 4], dtype=tf.float32, name="Q_target_values") 27 | diff_tf = ????? 28 | loss = ???? 29 | 30 | 31 | # Define an optimizer to minimize the loss function 32 | opt = ????? 33 | train_op = ????? 34 | 35 | # Initialize all the variables of the graph 36 | init = tf.global_variables_initializer() 37 | 38 | # Set learning parameters 39 | gamma = .99 # for the Q_target 40 | e = 0.1 # chance of taking a random action at any step 41 | num_episodes = 2000 # total number of episodes during the learning 42 | 43 | 44 | # Create lists to contain total rewards and steps per episode 45 | jList = [] 46 | rList = [] 47 | 48 | with tf.Session() as sess: 49 | sess.run(init) 50 | 51 | ############ 52 | # Training # 53 | ############ 54 | print("\n###########################") 55 | print("## TRAINING YOUR AGENT ####") 56 | print("###########################") 57 | 58 | for i in range(num_episodes): 59 | # Reset environment and get first observation 60 | s = env.reset() 61 | R = 0 # sum of the reward over an episode (1.0 if we reach the goal state, 0 otherwise 62 | j = 0 # number of step in the environment 63 | 64 | # The Q-Network # 65 | # The episode is stopped if we reach a hole, the goal state or after 100 steps 66 | while j < 99 and s not in [5,7,11,12,15]: 67 | j+=1 68 | 69 | # Choose an action by greedily (with e chance of random action) from the Q-network 70 | s_one_hot = np.identity(16)[s:s + 1] # one hot encoding of the state 71 | a, all_Q = sess.run([action, Q_values], feed_dict={input_state: ?????}) 72 | if np.random.rand(1) < e: 73 | a[0] = env.action_space.sample() 74 | 75 | # Get new state and reward from environment 76 | new_s, r, _, _ = env.step(a[0]) 77 | 78 | # Obtain the Q values of the new state by feeding the new state through our network still with the same parameters as for the previous step 79 | new_s_one_hot = np.identity(16)[new_s:new_s + 1] 80 | all_next_Q = sess.run(Q_values, feed_dict={input_state: ??????}) 81 | 82 | # Obtain max(all_next_Q) and set our Q_target_values for chosen action. 83 | # Here is a little trick for the backpropagation: do not touch this part for the Q value target 84 | max_all_next_Q = np.max(all_next_Q) 85 | all_Q_target = all_Q.copy() 86 | all_Q_target[0, a[0]] = r + gamma * max_all_next_Q # Q_target definition 87 | 88 | # Train our network by minimizing SE(all_Q_target - all_Q) 89 | sess.run(train_op, feed_dict={input_state: ?????, 90 | Q_target_values: ?????}) 91 | 92 | 93 | # Update the reward and the current state for the next step 94 | R += r 95 | s = new_s 96 | 97 | # Append final reward value and number of step did during the episode 98 | jList.append(j) 99 | rList.append(R) 100 | 101 | # Print in the log every 100 episodes 102 | if i % 100 == 0 and i != 0: 103 | print("Episodes %04d to %04d: nb of successful episodes = %i/100, nb of steps in avg = %.2f " % (i-100, 104 | i, 105 | int(sum(rList[-100:])), 106 | float(np.mean(jList[-100:])) 107 | )) 108 | # print the W matrix 109 | W_matrix = sess.run(W) 110 | print("\nHere is your the matrix estimated by your network:") 111 | print((" Left - Down - Right - Up")) 112 | print(np.asarray(W_matrix)) 113 | 114 | ############ 115 | # Testing # 116 | ############ 117 | print("\n###########################") 118 | print("## TESTING YOUR AGENT #####") 119 | print("###########################") 120 | # Do 100 episodes with the network whithout random choice to now how good is your agent 121 | nb_successful_episodes = 0 122 | total_nb_of_steps = 0 123 | 124 | for i in range(100): 125 | # Reset environment and get first observation 126 | s = env.reset() 127 | R = 0 # sum of the reward over an episode (1.0 if we reach the goal state, 0 otherwise 128 | j = 0 # number of step in the environment 129 | 130 | while j < 99 and s not in [5,7,11,12,15]: 131 | j+=1 132 | 133 | # Choose an action from the trained Q-network 134 | s_one_hot = np.identity(16)[s:s + 1] # one hot encoding of the state 135 | a, all_Q = sess.run([action, Q_values], feed_dict={input_state: ?????}) 136 | 137 | # Get new state and reward from environment 138 | new_s, r, _, _ = env.step(a[0]) 139 | 140 | # Update the reward and the current state for the next step 141 | R += r 142 | s = new_s 143 | 144 | # Add the final reward after the episode and nb of steps 145 | nb_successful_episodes += R 146 | total_nb_of_steps +=j 147 | 148 | print("Testing on 100 episodes: \nYour agent has finished %i episodes and did %.2f steps in average!" % (int(nb_successful_episodes), 149 | total_nb_of_steps/100.0)) 150 | if nb_successful_episodes < 30: 151 | print("=> Keep coding or maybe just re-run your script...") 152 | else: 153 | print("=> Well done! How to improve your score and make your agent smarter?!\n") 154 | 155 | -------------------------------------------------------------------------------- /RL/random_walk.py: -------------------------------------------------------------------------------- 1 | import gym 2 | import tensorflow as tf 3 | import numpy as np 4 | import random 5 | import matplotlib.pyplot as plt 6 | import tensorflow.contrib.slim as slim 7 | 8 | # Take an evironnmenet and have a look at it 9 | env = gym.make('FrozenLake-v0') 10 | #env.render() 11 | 12 | # See the action space: 13 | print(env.action_space) # 0=Left, 1=Down, 2=Right, 3=Up 14 | # State space = range(16) 15 | tuple_actions = ('Left', 'Down', 'Right', 'Up') 16 | 17 | 18 | # Random walk in teh frozen lake 19 | def random_walk(env): 20 | # Reset the environnement 21 | env.reset() 22 | 23 | # Look at the initial environnement 24 | print("Initial state") 25 | env.render() 26 | 27 | # Reward and 'is it done?' of the first state 28 | reward, done = False, 0.0 29 | num_iter = 0 30 | x, y = 0, 0 # initial coordinate 31 | 32 | # Random walk until we win or we fall in a hole 33 | while done == 0.0: 34 | num_iter += 1 35 | # Take a random action and apply it 36 | a = env.action_space.sample() 37 | observation, reward, done, info = env.step(a) 38 | 39 | # Print the environment and your new position 40 | print("\nIter: "+str(num_iter)) 41 | print("Random action: "+str(a)+" = "+tuple_actions[a]) 42 | print("State: "+str(observation)) 43 | 44 | env.render() 45 | 46 | # Check the random walk wins 47 | if reward == 1.0: 48 | print("\nRandom walk reachs the GOAL after %i iterations!" % num_iter) 49 | else: 50 | print("\nYou felt down in a hole after %i iterations..." % num_iter) 51 | 52 | random_walk(env) 53 | 54 | 55 | 56 | #These lines establish the feed-forward part of the network used to choose actions 57 | 58 | -------------------------------------------------------------------------------- /SGD/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fabienbaradel/Tensorflow-tutorials/3a3c6a233156d0463e0d874e97941c3331ced6ee/SGD/__init__.py -------------------------------------------------------------------------------- /SGD/binary_classif.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | # Parameters 8 | BATCH_SIZE = 10 9 | learning_rate = 0.1 10 | training_steps = 100 11 | display_step = 10 12 | 13 | # Training Data 14 | training_size = 100 15 | coord_X = np.random.uniform(0, 1, training_size) 16 | coord_Y = np.random.uniform(0, 1, training_size) 17 | color = np.asarray([0 if x+y>1 else 1 for x,y in zip(coord_X,coord_Y)]) 18 | 19 | coord_XY = np.asarray(zip(coord_X, coord_Y)) 20 | n_samples = len(color) 21 | 22 | 23 | # Create the placeholder for X and Y 24 | X = tf.placeholder(dtype=tf.float32, shape=[None, 2], name = 'X') 25 | Y = tf.placeholder(dtype=tf.float32, shape=[None], name = 'Y') 26 | Y_reshape = tf.reshape(Y, [-1, 1]) 27 | 28 | 29 | # Set model weights 30 | W = tf.Variable(tf.zeros([2, 1])) 31 | b = tf.Variable(tf.zeros([1])) 32 | 33 | # Construct the inference 34 | Y_hat = tf.sigmoid(tf.add(tf.matmul(X, W), b)) # WX+b : the logits 35 | 36 | # Write your loss function: Mean squared error 37 | loss = tf.reduce_sum(tf.abs(tf.sub(Y_reshape, Y_hat)), name='loss') 38 | 39 | # Gradient descent 40 | optimizer = tf.train.GradientDescentOptimizer(learning_rate) 41 | train_op = optimizer.minimize(loss) 42 | 43 | # Initializing the variables 44 | init = tf.global_variables_initializer() 45 | 46 | # Launch the graph 47 | with tf.Session() as sess: 48 | sess.run(init) 49 | 50 | # Fit the data 51 | for step in range(training_steps): 52 | # Uniform batch sampling in the training set 53 | location = np.random.choice(range(n_samples), BATCH_SIZE) 54 | mini_batch_X, mini_batch_Y = coord_XY[location], color[location] 55 | sess.run(train_op, feed_dict={X: mini_batch_X, Y: mini_batch_Y}) 56 | 57 | # Display logs per step 58 | if (step+1) % display_step == 0 or step < 10: 59 | (loss_value, W_value, b_value) = sess.run([loss, W, b], feed_dict={X: coord_XY, Y:color}) 60 | print("Step: %04d , Loss = %.4f , W-1 = (%.2f, %.2f), b-1 = %.2f" 61 | %(step+1, loss_value, W_value[0], W_value[0], b_value)) 62 | 63 | print("Optimization Finished!") 64 | print("") 65 | (loss_value_training, W_value, b_value) = sess.run([loss, W, b], feed_dict={X: coord_XY, Y: color}) 66 | 67 | 68 | # Graphic display and fitted line 69 | # Color for the plot 70 | plot_color = ['r' if c == 0 else 'g' for c in color] 71 | plt.scatter(coord_X, coord_Y, color=plot_color, label='Original data') 72 | xx = np.linspace(0, 1) 73 | yy = (-W_value[0]/W_value[1]) * np.linspace(0, 1) - (b_value) / W_value[1] 74 | plt.plot(xx, yy, 'k-', label='Fitted line - SGD') 75 | plt.legend() 76 | plt.show() 77 | 78 | 79 | 80 | 81 | #from sklearn import linear_model 82 | #logreg = linear_model.LogisticRegression() 83 | #logreg.fit(coord_XY, color) 84 | #logreg.coef_ 85 | #logreg.intercept_ -------------------------------------------------------------------------------- /SGD/binary_classif_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | # Parameters 8 | BATCH_SIZE = 10 9 | learning_rate = 0.1 10 | training_steps = 100 11 | display_step = 10 12 | 13 | # Training Data 14 | training_size = 100 15 | coord_X = np.random.uniform(0, 1, training_size) 16 | coord_Y = np.random.uniform(0, 1, training_size) 17 | color = np.asarray([0 if x+y>1 else 1 for x,y in zip(coord_X,coord_Y)]) 18 | 19 | coord_XY = np.asarray(zip(coord_X, coord_Y)) 20 | n_samples = len(color) 21 | 22 | 23 | # Create the placeholder for X and Y 24 | X = tf.placeholder(dtype=tf.float32, shape=[None, 2], name = 'X') 25 | Y = tf.placeholder(dtype=tf.float32, shape=[None], name = 'Y') 26 | Y_reshape = tf.reshape(Y, [-1, 1]) 27 | 28 | 29 | # Set model weights 30 | W = tf.Variable(tf.zeros([2, 1])) 31 | b = tf.Variable(tf.zeros([1])) 32 | 33 | # Construct the inference 34 | Y_hat = ?????????? 35 | 36 | # Write your loss function: Mean squared error 37 | loss = ?????????? 38 | 39 | # Gradient descent 40 | optimizer = tf.train.GradientDescentOptimizer(learning_rate) 41 | train_op = optimizer.minimize(loss) 42 | 43 | # Initializing the variables 44 | init = tf.global_variables_initializer() 45 | 46 | # Launch the graph 47 | with tf.Session() as sess: 48 | sess.run(init) 49 | 50 | # Fit the data 51 | for step in range(training_steps): 52 | # Uniform batch sampling in the training set 53 | location = np.random.choice(range(n_samples), BATCH_SIZE) 54 | mini_batch_X, mini_batch_Y = ???, ??? # take a sample over your training set (coord_XY and color) 55 | sess.run(train_op, feed_dict={X: ???, Y: ???}) 56 | 57 | # Display logs per step 58 | if (step+1) % display_step == 0 or step < 10: 59 | (loss_value, W_value, b_value) = sess.run([???, ???, ???], feed_dict={X: coord_XY, Y:color}) 60 | print("Step: %04d , Loss = %.4f , W-1 = (%.2f, %.2f), b-1 = %.2f" 61 | %(step+1, loss_value, W_value[0], W_value[0], b_value)) 62 | 63 | print("Optimization Finished!") 64 | print("") 65 | (loss_value_training, W_value, b_value) = sess.run([???, ???, ???], feed_dict={X: coord_XY, Y: color}) 66 | 67 | 68 | # Graphic display and fitted line 69 | # Color for the plot 70 | plot_color = ['r' if c == 0 else 'g' for c in color] 71 | plt.scatter(coord_X, coord_Y, color=plot_color, label='Original data') 72 | xx = np.linspace(0, 1) 73 | yy = (-W_value[0]/W_value[1]) * np.linspace(0, 1) - (b_value) / W_value[1] 74 | plt.plot(xx, yy, 'k-', label='Fitted line - SGD') 75 | plt.legend() 76 | plt.show() 77 | 78 | 79 | 80 | 81 | #from sklearn import linear_model 82 | #logreg = linear_model.LogisticRegression() 83 | #logreg.fit(coord_XY, color) 84 | #logreg.coef_ 85 | #logreg.intercept_ -------------------------------------------------------------------------------- /SGD/linear_regression.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | # Parameters 8 | learning_rate = 0.01 9 | training_steps = 1000 10 | display_step = 25 11 | 12 | # Training Data 13 | train_X = np.asarray([3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167, 14 | 7.042,10.791,5.313,7.997,5.654,9.27,3.1]) 15 | train_Y = np.asarray([1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221, 16 | 2.827,3.465,1.65,2.904,2.42,2.94,1.3]) 17 | n_samples = train_X.shape[0] 18 | BATCH_SIZE = n_samples 19 | 20 | 21 | # Create the placeholder for X and Y 22 | X = tf.placeholder("float", shape=[None]) 23 | Y = tf.placeholder("float", shape=[None]) 24 | 25 | # Set model weights 26 | W = tf.Variable([0.0], name="weight") 27 | b = tf.Variable([0.0], name="bias") 28 | 29 | # Construct a linear model: Y_hat = W*X + b 30 | Y_hat = tf.add(tf.mul(X, W), b) 31 | 32 | # Write your loss function: Mean squared error 33 | loss = tf.reduce_mean(tf.square(Y_hat-Y)) 34 | 35 | # Gradient descent optimizer 36 | optimizer = tf.train.GradientDescentOptimizer(learning_rate) 37 | train_op = optimizer.minimize(loss) 38 | 39 | # Initializing the variables 40 | init = tf.global_variables_initializer() 41 | 42 | 43 | # Launch the graph 44 | with tf.Session() as sess: 45 | sess.run(init) 46 | 47 | # Fit with a mini-batch of data 48 | for step in range(training_steps): 49 | # Uniform batch sampling in the training set 50 | location = np.random.choice(range(n_samples), BATCH_SIZE) 51 | mini_batch_X, mini_batch_Y = train_X[location], train_Y[location] 52 | sess.run(train_op, feed_dict={X: mini_batch_X, Y: mini_batch_Y}) 53 | 54 | # Display logs per step 55 | if (step+1) % display_step == 0 or step == 0: 56 | (loss_value, W_value, b_value) = sess.run([loss, W, b], feed_dict={X: train_X, Y:train_Y}) 57 | print("Step: %04d , Loss = %.4f , W = %.3f , b = %.3f" 58 | %(step+1, loss_value, W_value, b_value)) 59 | #plt.plot(train_X, W_value * train_X + b_value) 60 | 61 | print("Optimization Finished!") 62 | (loss_value_training, W_value, b_value) = sess.run([loss, W, b], feed_dict={X: train_X, Y: train_Y}) 63 | 64 | 65 | # Graphic display 66 | plt.plot(train_X, train_Y, 'ro', label='Original data') 67 | plt.plot(train_X, W_value * train_X + b_value, label='Fitted line - SGD') 68 | plt.show() 69 | 70 | 71 | 72 | # closed form solution for W and B 73 | X = np.vstack([train_X, np.ones(len(train_X))]).T 74 | X.shape 75 | Y = train_Y 76 | Y.shape 77 | W_closed_form, b_closed_form = np.linalg.lstsq(X, Y)[0] 78 | 79 | plt.plot(train_X, W_closed_form * train_X + b_closed_form, label='Fitted line - closed form') 80 | plt.legend(loc=0) 81 | plt.show() 82 | 83 | print("Closed form: W = %.3f , b = %.3f " % ( W_closed_form, b_closed_form)) 84 | print("Gradient descent: W = %.3f , b = %.3f " % (W_value, b_value)) 85 | -------------------------------------------------------------------------------- /SGD/linear_regression_exo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | # Parameters 8 | learning_rate = 0.01 9 | training_steps = 1000 10 | display_step = 25 11 | 12 | # Training Data 13 | train_X = np.asarray([3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167, 14 | 7.042,10.791,5.313,7.997,5.654,9.27,3.1]) 15 | train_Y = np.asarray([1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221, 16 | 2.827,3.465,1.65,2.904,2.42,2.94,1.3]) 17 | n_samples = train_X.shape[0] 18 | BATCH_SIZE = n_samples 19 | 20 | 21 | # Create the placeholder for X and Y 22 | X = tf.placeholder("float", shape=[None]) 23 | Y = tf.placeholder("float", shape=[None]) 24 | 25 | # Set model weights 26 | W = tf.Variable([0.0], name="weight") 27 | b = tf.Variable([0.0], name="bias") 28 | 29 | # Construct a linear model: Y_hat = W*X + b 30 | Y_hat = ???????? 31 | 32 | # Write your loss function: Mean squared error 33 | loss = ???????? 34 | 35 | # Gradient descent optimizer 36 | optimizer = tf.train.GradientDescentOptimizer(learning_rate) 37 | train_op = optimizer.minimize(loss) 38 | 39 | # Initializing the variables 40 | init = tf.global_variables_initializer() 41 | 42 | 43 | # Launch the graph 44 | with tf.Session() as sess: 45 | sess.run(init) 46 | 47 | # Fit with a mini-batch of data 48 | for step in range(training_steps): 49 | # Uniform batch sampling in the training set 50 | location = np.random.choice(range(n_samples), BATCH_SIZE) 51 | mini_batch_X, mini_batch_Y = ????, ??? # take a sample of your data given the index 'location' 52 | sess.run(????, feed_dict={X: ????, Y: ????}) # run the training operation 53 | 54 | # Display logs per step 55 | if (step+1) % display_step == 0 or step == 0: 56 | (loss_value, W_value, b_value) = sess.run([???, ???, ???], feed_dict={X: train_X, Y:train_Y}) # get the loss, W and b values 57 | print("Step: %04d , Loss = %.4f , W = %.3f , b = %.3f" 58 | %(step+1, loss_value, W_value, b_value)) 59 | 60 | print("Optimization Finished!") 61 | (loss_value_training, W_value, b_value) = sess.run([???, ???, ???], feed_dict={X: train_X, Y: train_Y}) # get the loss, W and b values 62 | 63 | 64 | # Graphic display 65 | plt.plot(train_X, train_Y, 'ro', label='Original data') 66 | plt.plot(train_X, W_value * train_X + b_value, label='Fitted line - SGD') 67 | plt.show() 68 | 69 | 70 | 71 | # closed form solution for W and B 72 | X = np.vstack([train_X, np.ones(len(train_X))]).T 73 | X.shape 74 | Y = train_Y 75 | Y.shape 76 | W_closed_form, b_closed_form = np.linalg.lstsq(X, Y)[0] 77 | 78 | plt.plot(train_X, W_closed_form * train_X + b_closed_form, label='Fitted line - closed form') 79 | plt.legend(loc=0) 80 | plt.show() 81 | 82 | print("Closed form: W = %.3f , b = %.3f " % ( W_closed_form, b_closed_form)) 83 | print("Gradient descent: W = %.3f , b = %.3f " % (W_value, b_value)) 84 | --------------------------------------------------------------------------------