├── MNIST_data ├── t10k-images-idx3-ubyte.gz ├── t10k-labels-idx1-ubyte.gz ├── train-images-idx3-ubyte.gz └── train-labels-idx1-ubyte.gz ├── README ├── Sparsity_autoencoder.zip ├── autoencoder.py ├── basic_operations.py ├── bidirectional_rnn.py ├── convolutional_network.py ├── dynamic_rnn.py ├── grbm.py ├── helloworld.py ├── linear_regression.py ├── logistic_regression.py ├── multilayer_perceptron.py ├── nearest_neighbor.py ├── new_filters_at_0.png ├── original_and_reconstruct.png ├── rbm.py ├── rbm.pyc ├── recurrent_network.py ├── tensorboard_basic.py ├── test_filters_at_epoch_0.png ├── utils.py └── utils.pyc /MNIST_data/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/MNIST_data/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /MNIST_data/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/MNIST_data/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /MNIST_data/train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/MNIST_data/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /MNIST_data/train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/MNIST_data/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Deep leaning examples using Tensorflow library: 2 | 1. Linear regression 3 | 2. Logistic regression 4 | 3. Multi layer perceptron 5 | 4. Nearest neighbor 6 | 5. Recurrent neural network 7 | 6. Dynamic recurrent neural network 8 | 7. Bidirectional recurrent neural network 9 | 8. convolutional neural network 10 | 9. Autoencoder 11 | 10. Sparsity Autoencoder 12 | 11. Restrict Boltzmann Machine RBM 13 | 12. Gaussian restrict boltzmann machine GRBM 14 | -------------------------------------------------------------------------------- /Sparsity_autoencoder.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/Sparsity_autoencoder.zip -------------------------------------------------------------------------------- /autoencoder.py: -------------------------------------------------------------------------------- 1 | 2 | """ Auto Encoder Example. 3 | Using an auto encoder on MNIST handwritten digits. 4 | [MNIST Dataset] http://yann.lecun.com/exdb/mnist/ 5 | """ 6 | from __future__ import division, print_function, absolute_import 7 | 8 | import tensorflow as tf 9 | import numpy as np 10 | import matplotlib.pyplot as plt 11 | 12 | #import mnist data 13 | from tensorflow.examples.tutorials.mnist import input_data 14 | mnist = input_data.read_data_sets("MNIST_data", one_hot = True) 15 | 16 | #Parameters 17 | learning_rate = 0.01 18 | training_epochs = 20 19 | batch_size = 256 20 | display_step = 1 21 | examples_to_show = 10 22 | 23 | #network parameters 24 | n_hidden_1 = 256 25 | n_hidden_2 = 128 26 | n_input = 784 27 | 28 | #tf graph input 29 | X = tf.placeholder("float",[None, n_input]) 30 | 31 | weights = { 32 | 'encoder_h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])), 33 | 'encoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])), 34 | 'decoder_h1': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_1])), 35 | 'decoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_input])) 36 | } 37 | 38 | biases = { 39 | 'encoder_h1': tf.Variable(tf.random_normal([n_hidden_1])), 40 | 'encoder_h2': tf.Variable(tf.random_normal([n_hidden_2])), 41 | 'decoder_h1': tf.Variable(tf.random_normal([n_hidden_1])), 42 | 'decoder_h2': tf.Variable(tf.random_normal([n_input])) 43 | } 44 | 45 | # Building the encoder 46 | def encoder(x): 47 | # encoder hidden layer with sigmoid activtion 48 | layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['encoder_h1']), biases['encoder_h1'])) 49 | #encoder hidden layer with sigmoid activation 50 | layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['encoder_h2']), biases['encoder_h2'])) 51 | 52 | return layer_2 53 | #building decoder 54 | def decoder(x): 55 | #decoder hidden layer with sigmoid activation 56 | layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['decoder_h1']), biases['decoder_h1'])) 57 | #decoder hidden layer with sigmoid activation 58 | layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['decoder_h2']), biases['decoder_h2'])) 59 | return layer_2 60 | 61 | #construct the model 62 | encoder_op = encoder(X) 63 | decoder_op = decoder(encoder_op) 64 | 65 | #prediction 66 | y_pred = decoder_op 67 | #targets are inout data 68 | y_true = X 69 | 70 | #define loss and optimizer, minimize the squared error 71 | cost = tf.reduce_mean(tf.pow(y_true - y_pred, 2)) 72 | optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(cost) 73 | 74 | #initializing the variables 75 | init = tf.global_variables_initializer() 76 | 77 | #launch the graph 78 | 79 | with tf.Session() as sess: 80 | sess.run(init) 81 | total_batch = int(mnist.train.num_examples/batch_size) 82 | # training cyccle 83 | for epoch in range(training_epochs): 84 | #loop over all batches 85 | for i in range(total_batch): 86 | batch_xs, batch_ys = mnist.train.next_batch(batch_size) 87 | 88 | #run optimization op (backprop) and cost op (get loss value) 89 | _, c = sess.run([optimizer, cost], feed_dict = {X: batch_xs}) 90 | 91 | #display logs per epoch step 92 | if (epoch +1) % display_step ==0: 93 | print("epoch", '%04d' % (epoch +1), "cost", "{:.9f}".format(c)) 94 | 95 | print("optimization finished") 96 | 97 | #applying encode and decode over test set 98 | encode_decode = sess.run(y_pred, feed_dict = {X : mnist.test.images[:examples_to_show]}) 99 | #compare original images with their reconstructions 100 | f,a = plt.subplots(2, 10, figsize= (10,2)) 101 | for i in range(examples_to_show): 102 | a[0][i].imshow(np.reshape(mnist.test.images[i],(28,28))) 103 | a[1][i].imshow(np.reshape(encode_decode[i], [28,28])) 104 | f.show() 105 | plt.draw() 106 | plt.waitforbuttonpress() -------------------------------------------------------------------------------- /basic_operations.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import tensorflow as tf 4 | 5 | # Basic constant operations 6 | # The value returned by the constructor represents the output 7 | # of the Constant op. 8 | 9 | a = tf.constant(2) 10 | b = tf.constant(3) 11 | 12 | # launch the default graph 13 | with tf.Session() as sess: 14 | print(sess.run(a)) 15 | print("a = 2, b = 3") 16 | print("Addition with constants: %i"%sess.run(a + b)) 17 | print("Multiplication with constants:%i" % sess.run(a*b)) 18 | 19 | # Basic operations with variable as graph input 20 | # The value returned by the constructor represents the output of the variable op. 21 | # tf graph input 22 | a = tf.placeholder(tf.int16) 23 | b = tf.placeholder(tf.int16) 24 | 25 | # Define some operations 26 | add = tf.add(a,b) 27 | mul = tf.multiply(a,b) 28 | 29 | with tf.Session() as sess: 30 | print("Addition with variables: %i" % sess.run(add, feed_dict={a:3, b:4})) 31 | print("Multiplication with variables: %i" %sess.run(mul, feed_dict={a:4, b:5})) 32 | 33 | # ---------------- 34 | # More in details: 35 | # Matrix Multiplication from TensorFlow official tutorial 36 | 37 | # Create a constant op that produces a 1x2 matrix. the op is added as a node to the default graph. 38 | # The value returned by the cnstructor represents the output of the constnt op. 39 | 40 | matrix1 = tf.constant([[3., 3.]]) 41 | 42 | # Create another constant that produces a 2x1 matrix 43 | matrix2 = tf.constant([[2.],[ 2.]]) 44 | 45 | # Create a matmul op that takes matrix1 and matrix 2 as inputs, the returned value product represents the result of the matrix multiplication 46 | 47 | product = tf.matmul(matrix1,matrix2) 48 | 49 | 50 | # To run the matmul op we call the session run() method, passing product which represents the output of the matmul op. This indicates to the call that we want to get the output to the matmul op back 51 | 52 | # all inputs needed by the op are run automatically by the session. ther typically are run in parallel. 53 | 54 | # the call run product thus causes the execution of threes op in the graph: the two constants and matmul 55 | 56 | #the output of the op is returned in result as a numpy ndarray object. 57 | 58 | with tf.Session() as sess: 59 | 60 | result = sess.run(product) 61 | print(result) 62 | -------------------------------------------------------------------------------- /bidirectional_rnn.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A Bidirectional Recurrent Neural Network (LSTM) 3 | ''' 4 | 5 | from __future__ import print_function 6 | 7 | import tensorflow as tf 8 | from tensorflow.contrib import rnn 9 | import numpy as np 10 | 11 | # import mnist data 12 | from tensorflow.examples.tutorials.mnist import input_data 13 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 14 | 15 | ''' 16 | To classify images using a bidirectional recurrent neural network, we consider 17 | every image row as a sequence of pixels. Because MNIST image shape is 28*28px, 18 | we will then handle 28 sequences of 28 steps for every sample. 19 | ''' 20 | 21 | # parameters 22 | learning_rate = 0.001 23 | training_iters = 100000 24 | batch_size = 128 25 | display_step = 10 26 | 27 | #network parameters 28 | n_input = 28 29 | n_steps = 28 30 | n_hidden = 128 31 | n_classes = 10 32 | 33 | #tf graph input 34 | 35 | x = tf.placeholder("float", [None, n_steps, n_input]) 36 | y = tf.placeholder("float", [None, n_classes]) 37 | 38 | #define weights 39 | weights = { 40 | #hidden layer weights 2*n_hidden because of forward + backward cells 41 | 'out' : tf.Variable(tf.random_normal([2*n_hidden, n_classes])) 42 | } 43 | biases = { 44 | 'out' : tf.Variable(tf.random_normal([n_classes])) 45 | } 46 | 47 | def BiRNN(x,weights, biases): 48 | #prepare data shape to match bidirectional rnn function requirements 49 | #current data input shape: (batch_size, n_steps, n_input) 50 | #required shape: n_steps tensors list of shape(batch_size, n_input) 51 | 52 | #unstack to get a list of n_steps tensors of shape (batch_size, n_input) 53 | x = tf.unstack(x, n_steps, 1) 54 | 55 | #define lstm cells with tensorflow 56 | # forward direction cell 57 | lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias = 1.0) 58 | #Backward direction cell 59 | lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias = 1.0) 60 | 61 | #get lstm cell output 62 | try: 63 | output, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32) 64 | 65 | except Exeption: 66 | outputs = rnn.static_bidirectional_rnn(lstm_fw_cell,lstm_bw_cell, x , dtype = tf.float32) 67 | 68 | #linear activation, using rnn inner loop last ouput 69 | return tf.matmul(output[-1], weights['out']) + biases['out'] 70 | 71 | pred = BiRNN(x, weights, biases) 72 | 73 | #define loss and optimizer 74 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)) 75 | optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost) 76 | 77 | #evaluate model 78 | correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) 79 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 80 | 81 | #initializing the variables 82 | init = tf.global_variables_initializer() 83 | 84 | #launch the graph 85 | with tf.Session() as sess: 86 | sess.run(init) 87 | step =1 88 | # keep training until reach the max iterations 89 | while step * batch_size < training_iters: 90 | batch_x, batch_y = mnist.train.next_batch(batch_size) 91 | 92 | #reshape data to get 28 squence of 28 elements 93 | batch_x = batch_x.reshape((batch_size, n_steps, n_input)) 94 | 95 | #run optimzation op (backprop) 96 | sess.run(optimizer, feed_dict = {x : batch_x, y: batch_y}) 97 | if step % display_step == 0: 98 | # calculate batch accuracy 99 | acc = sess.run(accuracy, feed_dict = {x : batch_x, y:batch_y}) 100 | #calculate the loss 101 | loss = sess.run(cost, feed_dict = {x : batch_x, y: batch_y}) 102 | 103 | print("iter", str(step*batch_size), "minibatch loss", "{:.6f}".format(loss),"training accuracy =","{:.5f}".format(acc)) 104 | 105 | step +=1 106 | print("oprimization finished") 107 | 108 | #calculate accuracy for 128 mnist test images 109 | test_len = 128 110 | test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) 111 | test_label = mnist.test.labels[:test_len] 112 | print("testing accuracy:", sess.run(accuracy, feed_dict = {x:test_data, y:test_label})) 113 | -------------------------------------------------------------------------------- /convolutional_network.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A Convolutional Network implementation example using TensorFlow library. 3 | ''' 4 | 5 | from __future__ import print_function 6 | 7 | import tensorflow as tf 8 | #import mnist data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 11 | 12 | # Parameters 13 | learning_rate = 0.001 14 | training_iters = 10000 15 | batch_size = 128 16 | display_step = 10 17 | 18 | # network parameters 19 | n_input = 784 #mnist data input 20 | n_classes = 10 # mnist total classes 21 | dropout = 0.75 # dropout, probability to keep units 22 | 23 | #tf graph input 24 | x = tf.placeholder(tf.float32, [None, n_input]) 25 | y = tf.placeholder(tf.float32, [None, n_classes]) 26 | 27 | keep_prob = tf.placeholder(tf.float32) # dropout 28 | 29 | 30 | # create some wrappers for simplicity 31 | def conv2d(x, W, b, strides = 1): 32 | # conv2d wrapper, with bias and relu activation 33 | x = tf.nn.conv2d(x, W, strides = [1,strides, strides, 1], padding = 'SAME') 34 | x = tf.nn.bias_add(x, b) 35 | return tf.nn.relu(x) 36 | 37 | def maxpool2d(x, k =2): 38 | # maxpool2d wrapper 39 | return tf.nn.max_pool(x, ksize = [1, k , k, 1], strides =[ 1, k, k, 1], padding = 'SAME') 40 | 41 | # create model 42 | def conv_net(x, weights, biases, dropout): 43 | # reshape input picture 44 | x = tf.reshape(x, shape = [-1, 28, 28, 1]) 45 | 46 | #convolution layer 47 | conv1 = conv2d(x , weights['wc1'], biases['bc1']) 48 | #max pooling (down sampling) 49 | conv1 = maxpool2d(conv1, k =2) 50 | 51 | #convolution layer 52 | conv2 = conv2d(conv1, weights['wc2'], biases['bc2']) 53 | #max pooling(down sampling) 54 | conv2 = maxpool2d(conv2, k =2) 55 | 56 | #fully connected layer 57 | #reshape conv2 output to fit fully connected layer input 58 | 59 | fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]]) 60 | fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1']) 61 | fc1 = tf.nn.relu(fc1) 62 | 63 | #apply dropout 64 | fc1 = tf.nn.dropout(fc1, dropout) 65 | 66 | #output, class prediction 67 | out = tf.add(tf.matmul(fc1, weights['out']), biases['out']) 68 | return out 69 | 70 | #store layers weight and bias 71 | 72 | weights = { 73 | #5x5 conv, 1 input, 32 output 74 | 'wc1' : tf.Variable(tf.random_normal([5,5,1,32])), 75 | #5x5 conv, 32 inputs, 64 outputs 76 | 'wc2' : tf.Variable(tf.random_normal([5,5,32,64])), 77 | # fully connected, 7*7*64 inputs, 1024 outputs 78 | 'wd1' : tf.Variable(tf.random_normal([7*7*64,1024])), 79 | # 1024 inputs, 10 outputs 80 | 'out': tf.Variable(tf.random_normal([1024,n_classes])) 81 | } 82 | 83 | biases = { 84 | 'bc1' : tf.Variable(tf.random_normal([32])), 85 | 'bc2' : tf.Variable(tf.random_normal([64])), 86 | 'bd1' : tf.Variable(tf.random_normal([1024])), 87 | 'out' : tf.Variable(tf.random_normal([n_classes])) 88 | } 89 | 90 | #construct model 91 | pred = conv_net(x, weights, biases, keep_prob) 92 | 93 | #define loss and optimizer 94 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)) 95 | optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost) 96 | 97 | #Evaluate model 98 | 99 | correct_pre = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) 100 | accuracy = tf.reduce_mean(tf.cast(correct_pre, tf.float32)) 101 | 102 | #initializing the variables 103 | init = tf.global_variables_initializer() 104 | 105 | #launch the graph 106 | with tf.Session() as sess: 107 | sess.run(init) 108 | step =1 109 | #keep training until reach max iterations 110 | while step * batch_size < training_iters: 111 | batch_x, batch_y = mnist.train.next_batch(batch_size) 112 | 113 | #run optimization op (backprop) 114 | sess.run(optimizer, feed_dict = {x :batch_x, y:batch_y, keep_prob: dropout}) 115 | 116 | if step % display_step == 0: 117 | # calculate batch loss and accuracy 118 | loss, acc = sess.run([cost,accuracy], feed_dict ={x: batch_x, y :batch_y, keep_prob : 1.}) 119 | 120 | print("iter", str(step*batch_size), " minibatch loss =" , "{:.6f}".format(loss) ," Training Accuracy =" ,"{:.5f}".format(acc)) 121 | 122 | step +=1 123 | print("optimization finished") 124 | 125 | #Calculate accuracy for 256 mnist test images 126 | print ("testing accuracy:",sess.run(accuracy, feed_dict = {x: mnist.test.images[:256], y:mnist.test.labels[:256], keep_prob: 1.})) -------------------------------------------------------------------------------- /dynamic_rnn.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | A Dynamic Recurrent Neural Network (LSTM) 4 | ''' 5 | from __future__ import print_function 6 | import tensorflow as tf 7 | import random 8 | 9 | class ToySequenceData(object) : 10 | 11 | """ Generate sequence of data with dynamic length. 12 | This class generate samples for training: 13 | - Class 0: linear sequences (i.e. [0, 1, 2, 3,...]) 14 | - Class 1: random sequences (i.e. [1, 3, 10, 7,...]) 15 | NOTICE: 16 | We have to pad each sequence to reach 'max_seq_len' for TensorFlow 17 | consistency (we cannot feed a numpy array with inconsistent 18 | dimensions). The dynamic calculation will then be perform thanks to 19 | 'seqlen' attribute that records every actual sequence length. 20 | """ 21 | 22 | def __init__(self, n_samples = 1000, max_seq_len = 20, min_seq_len = 3, max_value = 1000): 23 | self.data = [] 24 | self.labels = [] 25 | self.sequen = [] 26 | for i in range(n_samples): 27 | # random sequence length 28 | len = random.randint(min_seq_len, max_seq_len) 29 | #monitor squence length for tensorflow dynamic calculation 30 | self.sequen.append(len) 31 | # add a random or linear int sequence 50% prob 32 | if random.random() < .5: 33 | #generate a linear sequence 34 | rand_start = random.randint(0, max_value - len) 35 | s = [[float(i)/max_value] for i in range(rand_start, rand_start +len)] 36 | 37 | #pad sequence for dimension consistency 38 | s += [[0.] for i in range(max_seq_len - len)] 39 | self.data.append(s) 40 | self.labels.append([1., 0.]) 41 | else: 42 | #gnerate a random sequence 43 | s = [[float(random.randint(0, max_value))/ max_value] for i in range(len)] 44 | #pad sequence for dimension consistency 45 | s += [[0.] for i in range(max_seq_len - len)] 46 | self.data.append(s) 47 | self.labels.append([0., 1.]) 48 | self.batch_id = 0 49 | def next(self, batch_size): 50 | # return a batch of data. when dataset end is reached, start over 51 | if self.batch_id == len(self.data): 52 | self.batch_id = 0 53 | batch_data = (self.data[self.batch_id:min(self.batch_id + batch_size, len(self.data))]) 54 | batch_labels = (self.labels[self.batch_id : min(self.batch_id + batch_size, len(self.data))]) 55 | batch_seqlen = (self.seqlen[self.batch_id: min(self.batch_id + batch_size, len(self.data))]) 56 | 57 | self.batch_id = min(self.batch_id + batch_size, len(self.data)) 58 | return batch_data, batch_labels, batch_seqlen 59 | 60 | 61 | 62 | # ========== 63 | # MODEL 64 | # ========== 65 | 66 | # Parameters 67 | learning_rate = 0.01 68 | training_iters = 1000000 69 | batch_size = 128 70 | display_step = 10 71 | 72 | 73 | # Network Parameters 74 | seq_max_len = 20 # Sequence max length 75 | n_hidden = 64 # hidden layer num of features 76 | n_classes = 2 # linear sequence or not 77 | 78 | trainset = ToySequenceData(n_samples = 1000, max_seq_len = seq_max_len) 79 | testset = ToySequenceData(n_samples = 500, max_seq_len = seq_max_len) 80 | 81 | #tf graph input 82 | x = tf.placeholder("float", [None, seq_max_len, 1]) 83 | y = tf.placeholder("float", [None, n_classes]) 84 | # A placeholder for indicating each sequence length 85 | seqlen = tf.placeholder(tf.int32, [None]) 86 | 87 | 88 | # Define weights 89 | weights = { 90 | 'out': tf.Variable(tf.random_normal([n_hidden, n_classes])) 91 | } 92 | biases = { 93 | 'out': tf.Variable(tf.random_normal([n_classes])) 94 | } 95 | 96 | def dynamicRNN(x, seqlen, weights, biases): 97 | 98 | # Prepare data shape to match `rnn` function requirements 99 | # Current data input shape: (batch_size, n_steps, n_input) 100 | # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) 101 | 102 | # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input) 103 | x = tf.unstack(x, seq_max_len, 1) 104 | 105 | # Define a lstm cell with tensorflow 106 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden) 107 | 108 | # Get lstm cell output, providing 'sequence_length' will perform dynamic 109 | # calculation. 110 | outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x, dtype = tf.float32, sequence_length = seqlen) 111 | 112 | 113 | # When performing dynamic calculation, we must retrieve the last 114 | # dynamically computed output, i.e., if a sequence length is 10, we need 115 | # to retrieve the 10th output. 116 | # However TensorFlow doesn't support advanced indexing yet, so we build 117 | # a custom op that for each sample in batch size, get its length and 118 | # get the corresponding relevant output. 119 | 120 | # 'outputs' is a list of output at every timestep, we pack them in a Tensor 121 | # and change back dimension to [batch_size, n_step, n_input] 122 | 123 | outputs = tf.stack(outputs) 124 | outputs = tf.transpose(outputs, [1,0,2]) 125 | 126 | #Hack to build the indexing and retrieve the right output 127 | batch_size = tf.shape(outputs)[0] 128 | #start indices for each sample 129 | index = tf.range(0, batch_size) * seq_max_len + (seqlen - 1) 130 | #indexing 131 | outputs = tf.gather(tf.reshape(outputs, [-1, n_hidden]), index) 132 | 133 | #linear activation, using outputs conputed above 134 | return tf.matmul(outputs, weight['out']) + biases['out'] 135 | 136 | pred = dynamicRNN(x, seqlen, weights, biases) 137 | 138 | #define loss and optimizer 139 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)) 140 | optimizer = tf.train.GradientDescentOptimizer(learning_rate = learning_rate).minimize(cost) 141 | 142 | #evaluate model 143 | correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y,1)) 144 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 145 | 146 | #initializing the variables 147 | init = tf.global_variables_initializer() 148 | 149 | # Launch the graph 150 | with tf.Session() as sess: 151 | sess.run(init) 152 | step = 1 153 | # Keep training until reach max iterations 154 | while step * batch_size < training_iters: 155 | batch_x, batch_y, batch_seqlen = trainset.next(batch_size) 156 | # Run optimization op (backprop) 157 | sess.run(optimizer, feed_dict={x: batch_x, y: batch_y, seqlen : batch_seqlen}) 158 | 159 | 160 | if step % display_step == 0: 161 | # Calculate batch accuracy 162 | acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y, seqlen: batch_seqlen}) 163 | # Calculate batch loss 164 | loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y, seqlen: batch_seqlen}) 165 | print("Iter ", str(step*batch_size), ", Minibatch Loss= " ,"{:.6f}".format(loss) , ", Training Accuracy= " ,"{:.5f}".format(acc)) 166 | step += 1 167 | print("Optimization Finished!") -------------------------------------------------------------------------------- /grbm.py: -------------------------------------------------------------------------------- 1 | # Gaussian Restrict Boltzmann Machine 2 | 3 | import tensorflow as tf 4 | import math 5 | import timeit 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | from rbm import RBM 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | from utils import tile_raster_images 11 | 12 | class GRBM(RBM): 13 | # Gaussian Restrict Boltzmann Machine 14 | def __init__(self, inp = None, n_visible = 784, n_hidden = 500, W = None, hbias = None, vbias = None, sigma = 1.0): 15 | """ 16 | self.n_visible = n_visible 17 | self.n_hidden = n_hidden 18 | if inp is None: 19 | inp = tf.placeholder(dtype = tf.float32, shape = [None, self.n_visible]) 20 | self.input = inp 21 | if W is None: 22 | low = -4.0 * math.sqrt(6.0 / (n_visible + n_hidden)) 23 | high = 4.0 * math.sqrt(6.0 / (n_visible + n_hidden)) 24 | W = tf.Variable(tf.random_uniform([self.n_visible, self.n_hidden], minval = low, maxval = high, dtype = tf.float32)) 25 | self.W = W 26 | if hbias is None: 27 | hbias = tf.Variable(tf.zeros([n_hidden]), dtype = tf.float32) 28 | self.hbias = hbias 29 | if vbias is None: 30 | vbias = tf.Variable(tf.zeros([n_visible]), dtype = tf.float32) 31 | self.vbias = vbias""" 32 | super(GRBM, self).__init__(inp, n_visible, n_hidden, W, hbias, vbias) 33 | self.sigma = sigma 34 | #self.params = [self.W, self.hbias, self.vbias] 35 | 36 | 37 | def propup(self, visible): 38 | #print(visible.shape) 39 | #print(self.W) 40 | # This function propagates the visible units activation upwards to the hidden unit 41 | return tf.nn.sigmoid(tf.matmul(visible, self.W) / (self.sigma **2) + self.hbias) 42 | 43 | def propdown(self, hidden): 44 | # This function propagates the hidden units activaion downwards to the visible unit 45 | #dist = tf.contrib.distributions.Normal(loc = tf.matmul(hidden, tf.transpose(self.W)) + self.vbias, scale = self.sigma **2) 46 | return tf.matmul(hidden, tf.transpose(self.W)) + self.vbias 47 | #return dist.prob(tf.matmul(hidden, tf.transpose(self.W)) + self.vbias) 48 | 49 | def sample_bernoulli(self, prob): 50 | return tf.nn.relu(tf.sign(prob - tf.random_uniform(tf.shape(prob)))) 51 | 52 | def sample_gaussian(self, x, sigma): 53 | return x + tf.random_normal(tf.shape(x), mean = 0.0, stddev = sigma, dtype=tf.float32) 54 | 55 | 56 | def sample_h_given_v(self, v0_sample): 57 | # This function infers state of hidden units given visible units 58 | # get a sample of the hiddens given their activation 59 | h1_mean = self.propup(v0_sample) 60 | h1_sample = self.sample_bernoulli(h1_mean) 61 | return (h1_mean, h1_sample) 62 | 63 | def sample_v_given_h(self, h0_sample): 64 | # This function infers state of visible units given hidden units 65 | # get a sample of the hiddens given their activation 66 | v1_mean = self.propdown(h0_sample) 67 | v1_sample = self.sample_gaussian(v1_mean, self.sigma) 68 | return (v1_mean, v1_sample) 69 | 70 | '''gibbs_vhv which performs a step of Gibbs sampling starting from the visible units. 71 | As we shall see, this will be useful for sampling from the RBM. 72 | gibbs_hvh which performs a step of Gibbs sampling starting from the hidden units. 73 | This function will be useful for performing CD and PCD updates.''' 74 | 75 | def gibbs_hvh(self, h_sample): 76 | # this function implements one step of Gibbs sampling starting from the hidden state 77 | v1_mean, v1_sample = self.sample_v_given_h(h_sample) 78 | h1_mean, h1_sample = self.sample_h_given_v(v1_sample) 79 | return (v1_mean, v1_sample, h1_mean, h1_sample) 80 | 81 | def gibbs_vhv(self, v_sample): 82 | # this function implements one step of gibbs sampling starting from the visible state 83 | h1_mean, h1_sample = self.sample_h_given_v(v_sample) 84 | v1_mean, v1_sample = self.sample_v_given_h(h1_sample) 85 | return (h1_mean, h1_sample, v1_mean, v1_sample) 86 | 87 | def free_energy(self, v_sample): 88 | #function to compute the free energy which need for computing the gradient 89 | wx_b = tf.matmul(v_sample, self.W) / self.sigma**2 + self.hbias 90 | vbias_term = tf.reduce_sum(0.5*((v_sample - self.vbias)/ self.sigma)**2, axis = 1) 91 | hidden_term = tf.reduce_sum(tf.log(1 + tf.exp(wx_b)),axis =1) 92 | return -hidden_term + vbias_term 93 | 94 | #we then add a train_ops method, whose purpose is to generate the sysbolic gradients from CD-k and PCD-k updates 95 | def train_ops(self, lr = 0.1, persistent = None, k = 1): 96 | ''' This function implements one step of CD-k or PCD-k 97 | :param lr: leaning rate used to train the rbm 98 | :param persistent: none for Cd, for PCD, shared variable containing old state of 99 | Gibbs chain. This must be a shared variable of size(batch_size), number of hidden units) 100 | :param k : number if Gibbs step to do in CD-k, PCD-k 101 | 102 | Return a proxy for the cost and the updates dictionary. 103 | The dictionary contains the updates rules for the weights and biases 104 | but also an update of the shared variable used to store the persistent 105 | chain, if one is used''' 106 | 107 | # compute positive phase 108 | ph_mean, ph_sample = self.sample_h_given_v(self.input) 109 | #decide how to initialize persistent chain: 110 | #for cd, we use the newly generate hidden sample 111 | # for PCD, we initialize from the old state of the chain 112 | 113 | if persistent is None: 114 | chain_start = ph_sample 115 | else: 116 | chain_start = persistent 117 | 118 | #perform actual negative phase 119 | #in order to implement CD-k/ PCd-k we need to scan over the function that implement one gibbs step k times 120 | 121 | cond = lambda i, nv_mean, nv_sample, nh_mean, nh_sample: i < k 122 | body = lambda i, nv_mean, nv_sample, nh_mean, nh_sample: (i+1, ) + self.gibbs_hvh(nh_sample) 123 | 124 | i, nv_mean, nv_sample, nh_mean, nh_sample = tf.while_loop(cond, body, loop_vars = [tf.constant(0), tf.zeros(tf.shape(self.input)), tf.zeros(tf.shape(self.input)), tf.zeros(tf.shape(chain_start)), chain_start]) 125 | # determine gradients on RBM parameters 126 | 127 | # note that we only need the sample at the end of the chain 128 | chain_end = tf.stop_gradient(nv_sample) 129 | 130 | self.cost = tf.reduce_mean(self.free_energy(self.input)) - tf.reduce_mean(self.free_energy(chain_end)) 131 | 132 | #we must not compute the gradient through the gibbs sampling 133 | # compute the gradients 134 | gparams = tf.gradients(ys = [self.cost], xs = self.params) 135 | new_params = [] 136 | for gparam, param in zip(gparams, self.params): 137 | new_params.append(tf.assign(param, param - gparam * lr)) 138 | 139 | if persistent is not None: 140 | new_persistent = [tf.assign(persistent, nh_sample)] 141 | else: 142 | new_persistent = [] 143 | return new_params + new_persistent 144 | 145 | 146 | def get_reconstruct_cost(self): 147 | #compute the cross-entropy of the original inout and the reconstruction 148 | act_h = self.propup(self.input) 149 | act_v = self.propdown(act_h) 150 | #print(act_h.shape) 151 | #print(act_v) 152 | #cross_entropy = -tf.reduce_mean(tf.reduce_sum(self.input * tf.log(act_v) + (1.0 - self.input) * tf.log(1.0 - act_v), axis = 1)) 153 | cross_entropy = tf.reduce_mean(tf.reduce_sum(tf.square(self.input - act_v), axis=1)) 154 | return cross_entropy 155 | ''' 156 | def reconstruction(self, inp): 157 | act_h = self.propup(inp) 158 | return self.propdown(act_h) 159 | ''' 160 | def test_rbm(): 161 | ''' Demonstrate how to train and afterwards sample from 162 | :param leaning_rate: 163 | :param training_ecpochs 164 | :param dataset 165 | :param barch_size 166 | :param n_chains 167 | :param n_samples 168 | ''' 169 | 170 | 171 | #import dataset 172 | mnist = input_data.read_data_sets("MNIST_data", one_hot = True) 173 | # parameters 174 | leaning_rate = 0.01 175 | training_epochs = 1 176 | batch_size = 50 177 | display_step = 1 178 | 179 | 180 | #define input 181 | x = tf.placeholder(tf.float32, [None, 784]) 182 | #network parameters 183 | n_visible = 784 184 | n_hidden = 500 185 | grbm = GRBM(x, n_visible = n_visible, n_hidden = n_hidden) 186 | 187 | cost = grbm.get_reconstruct_cost() 188 | 189 | #create the persistent variable 190 | #persistent_chain = tf.Variable(tf.zeros([batch_size, n_hidden]), dtype = tf.float32) 191 | persistent_chain = None 192 | train = grbm.train_ops(lr = leaning_rate, persistent = persistent_chain, k = 1) 193 | #initializing the variables 194 | init = tf.global_variables_initializer() 195 | 196 | ################# 197 | # training RBM 198 | ################# 199 | 200 | with tf.Session() as sess: 201 | start_time = timeit.default_timer() 202 | sess.run(init) 203 | total_batch = int(mnist.train.num_examples / batch_size) 204 | for epoch in range(training_epochs): 205 | c = 0.0 206 | #print(sess.run(grbm.W)) 207 | # loop over all batches 208 | for i in range(total_batch): 209 | batch_xs, batch_ys = mnist.train.next_batch(batch_size) 210 | 211 | # run optimization op (batchprop) and cost op 212 | _ = sess.run(train, feed_dict = {x : batch_xs}) 213 | c += sess.run(cost, feed_dict = {x : batch_xs})/ total_batch 214 | #display logs per epoch step 215 | if epoch % display_step == 0: 216 | print("epoch", '%04d'%(epoch +1), "cost","{:.4f}".format(c)) 217 | #print(sess.run(grbm.W)) 218 | 219 | #construct image from the weight matrix 220 | 221 | plt.imsave("new_filters_at_{0}.png".format(epoch),tile_raster_images(X = sess.run(tf.transpose(grbm.W)), 222 | img_shape = (28, 28), tile_shape = (10,10), tile_spacing = (1,1)), cmap = 'gray') 223 | plt.show() 224 | end_time = timeit.default_timer() 225 | training_time = end_time - start_time 226 | print("TIME:{0} minutes".format(training_time/ 60,)) 227 | 228 | """ 229 | 230 | ################################# 231 | # Sampling from the RBM # 232 | ################################# 233 | # Reconstruct the image by sampling 234 | print("...Sampling from the RBM") 235 | n_chains = 20 236 | n_batch = 10 237 | n_samples = n_batch *2 238 | number_test_examples = mnist.test.num_examples 239 | #randomly select the n_chains examples 240 | 241 | test_indexs = np.random.randint(number_test_examples - n_chains * n_batch) 242 | test_samples = mnist.test.images[test_indexs:test_indexs + n_chains * n_batch] 243 | #create the persistent variable saving the visiable state 244 | #persistent_v_chain = tf.Variable(tf.to_float(test_samples), dtype = tf.float32) 245 | # the step of gibbs 246 | #step_every = 1000 247 | ''' 248 | # implement the gibbs sampling 249 | cond = lambda j, h_mean, h_sample, v_mean, v_sample: j < step_every 250 | body = lambda j, h_mean, h_sample, v_mean, v_sample: (j+1, ) + grbm.gibbs_vhv(v_sample) 251 | j, h_mean, h_sample, v_mean, v_sample = tf.while_loop(cond, body, loop_vars=[tf.constant(0), tf.zeros([n_chains, n_hidden]), 252 | tf.zeros([n_chains, n_hidden]), tf.zeros(tf.shape(persistent_v_chain)), persistent_v_chain]) 253 | ''' 254 | # Update the persistent_v_chain 255 | #new_persistent_v_chain = tf.assign(persistent_v_chain, v_sample) 256 | # Store the image by sampling 257 | image_data = np.zeros((29*(n_samples+1)+1, 29*(n_chains)-1), 258 | dtype="uint8") 259 | 260 | # Initialize the variable 261 | #sess.run(tf.variables_initializer(var_list=[persistent_v_chain])) 262 | # Do successive sampling 263 | for idx in range(n_batch): 264 | #sample = sess.run(v_mean) 265 | #sess.run(new_persistent_v_chain) 266 | # Add the original images 267 | ''' 268 | image_data[2*idx*29 : 2*idx *29 + 28,:] = tile_raster_images(X=test_samples[idx*n_batch, (idx+1)*n_chains], 269 | img_shape=(28, 28), 270 | tile_shape=(1, n_chains), 271 | tile_spacing=(1, 1)) 272 | ''' 273 | sample = sess.run(grbm.reconstruction, feed_dict = {x : test_samples[idx*n_batch, (idx+1)*n_chains]}) 274 | print("...plotting sample", idx) 275 | image_data[(2*idx +1)*29:(2 * idx +1)*29+28,:] = tile_raster_images(X=sample, 276 | img_shape=(28, 28), 277 | tile_shape=(1, n_chains), 278 | tile_spacing=(1, 1)) 279 | #image = plt.imshow(image_data) 280 | plt.imsave("new_original_and_{0}samples.png".format(n_samples), image_data, cmap = 'gray') 281 | plt.show() 282 | """ 283 | # Randomly select the 'n_chains' examples 284 | 285 | n_chains = 20 286 | n_batch = 10 287 | n_samples = n_batch*2 288 | number_test_examples = mnist.test.num_examples 289 | test_indexs = np.random.randint(number_test_examples - n_chains*n_batch) 290 | test_samples = mnist.test.images[test_indexs:test_indexs+n_chains*n_batch] 291 | image_data = np.zeros((29*(n_samples+1)+1, 29*(n_chains)-1), 292 | dtype="uint8") 293 | # Add the original images 294 | 295 | for i in range(n_batch): 296 | image_data[2*i*29:2*i*29+28,:] = tile_raster_images(X=test_samples[i*n_batch:(i+1)*n_chains], 297 | img_shape=(28, 28), 298 | tile_shape=(1, n_chains), 299 | tile_spacing=(1, 1)) 300 | 301 | samples = sess.run(grbm.reconstruction(x), feed_dict={x:test_samples[i*n_batch:(i+1)*n_chains]}) 302 | image_data[(2*i+1)*29:(2*i+1)*29+28,:] = tile_raster_images(X=samples, 303 | img_shape=(28, 28), 304 | tile_shape=(1, n_chains), 305 | tile_spacing=(1, 1)) 306 | 307 | image = plt.imsave("original_and_reconstruct.png",image_data, cmap = 'gray') 308 | plt.show() 309 | 310 | 311 | if __name__ == '__main__': 312 | test_rbm() 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | -------------------------------------------------------------------------------- /helloworld.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import tensorflow as tf 3 | 4 | hello = tf.constant('Hello World') 5 | # start tf session 6 | sess = tf.Session() 7 | 8 | print(sess.run(hello)) 9 | -------------------------------------------------------------------------------- /linear_regression.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A linear regression learning algorithm example using TensorFlow library. 3 | ''' 4 | 5 | from __future__ import print_function 6 | 7 | import tensorflow as tf 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | rng = np.random 11 | 12 | #parameters 13 | 14 | learning_rate = 0.01 15 | training_epochs = 1000 16 | display_step = 50 17 | 18 | # Training data 19 | train_X = np.asarray([3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167, 7.042, 10.791, 5.313, 7.997, 5.654, 9.27,3.1]) 20 | train_Y = np.asarray([1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221, 2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3]) 21 | n_samples = train_X.shape[0] 22 | #tf graph input 23 | X = tf.placeholder("float") 24 | Y = tf.placeholder("float") 25 | 26 | #Set model weights 27 | W = tf.Variable(rng.randn(), name = "weight") 28 | b = tf.Variable(rng.randn(), name = "bias") 29 | 30 | #Construct a linear model 31 | pred = tf.add(tf.multiply(X, W), b) 32 | 33 | #Mean squared error 34 | cost = tf.reduce_sum(tf.pow(pred - Y, 2))/ (2 * n_samples) 35 | 36 | #Gradient descent 37 | # minimize() know to modify W and b because variable objects are trainable = true by default 38 | 39 | optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 40 | 41 | #Initializing the variables 42 | init = tf.global_variables_initializer() 43 | 44 | #lauch the graph 45 | with tf.Session() as sess: 46 | sess.run(init) 47 | 48 | #Fit all training data 49 | for epoch in range(training_epochs): 50 | for(x, y)in zip(train_X, train_Y): 51 | sess.run(optimizer, feed_dict = {X :x, Y :y}) 52 | 53 | #Display log per epoch step 54 | if (epoch +1) % display_step == 0: 55 | c = sess.run(cost, feed_dict = {X: train_X, Y:train_Y}) 56 | print("Epoch:", '%04d' %(epoch +1), "cost =", "{:.9f}".format(c), "W =", sess.run(W), "b =", sess.run(b)) 57 | 58 | print("Optimization finished") 59 | training_cost = sess.run(cost, feed_dict = {X: train_X, Y:train_Y}) 60 | print("training cost =", training_cost, "W = ",sess.run(W), "b =", sess.run(b) ) 61 | 62 | #graphic display 63 | plt.plot(train_X, train_Y, 'ro', label = 'Original data') 64 | plt.plot(train_X, sess.run(W) * train_X + sess.run(b), label = 'Fitted line') 65 | plt.legend() 66 | plt.show() 67 | 68 | #Testing example 69 | test_X = np.asarray([6.83, 4.668, 8.9, 7.91, 5.7, 8.7, 3.1, 2.1]) 70 | test_Y = np.asarray([1.84, 2.273, 3.2, 2.831, 2.92, 3.24, 1.35, 1.03]) 71 | 72 | print("Testing... (Mean square loss Comparison)") 73 | testing_cost = sess.run(tf.reduce_sum(tf.pow(pred - Y, 2)) / (2 * test_X.shape[0]),feed_dict={X: test_X, Y: test_Y}) 74 | print("testing cost =", testing_cost) 75 | print("absolute mean square loss difference:", abs(training_cost - testing_cost)) 76 | 77 | plt.plot(test_X, test_Y, 'bo', label = 'testing data') 78 | plt.plot(train_X, sess.run(W)* train_X + sess.run(b), label = 'fitted line') 79 | plt.legend() 80 | plt.show() -------------------------------------------------------------------------------- /logistic_regression.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A logistic regression learning algorithm example using TensorFlow library. 3 | ''' 4 | 5 | from __future__ import print_function 6 | import tensorflow as tf 7 | 8 | #import 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 11 | 12 | #parameters 13 | learning_rate = 0.01 14 | training_epochs = 25 15 | batch_size = 100 16 | display_step = 1 17 | 18 | #tf graph input 19 | 20 | x = tf.placeholder(tf.float32, [None, 784]) #mnist data image of shape 784 21 | y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition->10 classes 22 | 23 | #Set model weights 24 | 25 | W = tf.Variable(tf.zeros([784,10])) 26 | b = tf.Variable(tf.zeros([10])) 27 | 28 | #Construct model 29 | pred = tf.nn.softmax(tf.matmul( x, W ) + b) #softmax 30 | 31 | #Minimize error using cross entropy 32 | cost = tf.reduce_mean( - tf.reduce_sum(y * tf.log(pred), reduction_indices = 1)) 33 | 34 | #Gradient descent 35 | optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 36 | 37 | #initializing the variables 38 | init = tf.global_variables_initializer() 39 | 40 | #launch the grap 41 | with tf.Session() as sess: 42 | sess.run(init) 43 | 44 | #Training cycle 45 | for epoch in range(training_epochs): 46 | avg_cost = 0 47 | #print(mnist.train.num_examples) 48 | total_batch = int(mnist.train.num_examples / batch_size) 49 | #lopp over all batchs 50 | for i in range(total_batch): 51 | batch_xs, batch_ys = mnist.train.next_batch(batch_size) 52 | #run optimization op (backprop) and cost op(to get the loss value) 53 | _, c = sess.run([optimizer, cost], feed_dict={x: batch_xs, y: batch_ys}) 54 | #compute average loss 55 | avg_cost += c/total_batch 56 | #Display logs per epoch step 57 | if (epoch +1 ) % display_step ==0: 58 | print("Epoch:", '%04d' % (epoch +1), "cost:", "{:.9f}".format(avg_cost)) 59 | 60 | print("optimization finished") 61 | 62 | #test model 63 | 64 | correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y,1)) 65 | 66 | #calculate accuracy 67 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 68 | print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels})) -------------------------------------------------------------------------------- /multilayer_perceptron.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A Multilayer Perceptron implementation example using TensorFlow library. 3 | ''' 4 | 5 | from __future__ import print_function 6 | 7 | #import Mnist data 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 10 | 11 | import tensorflow as tf 12 | 13 | #Parameters 14 | learning_rate = 0.001 15 | training_epochs = 15 16 | batch_size = 100 17 | display_step = 1 18 | 19 | #Network parameters 20 | n_hidden_1 = 256 # 1st layer number of features 21 | n_hidden_2 = 256 # 2nd layer number of features 22 | n_input = 784 # Mnist data input 23 | n_classes = 10 # the number of classes mnist 24 | 25 | #tf graph input 26 | x = tf.placeholder("float", [None, n_input]) 27 | y = tf.placeholder("float", [None, n_classes]) 28 | 29 | #Create model 30 | def multilayer_perceptron(x, weights, biases): 31 | #Hidden layer with relu activation 32 | layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1']) 33 | layer_1 = tf.nn.relu(layer_1) 34 | #hidden layer with RELU activation 35 | layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2']) 36 | layer_2 = tf.nn.relu(layer_2) 37 | # output layer with linear activation 38 | out_layer = tf.matmul(layer_2, weights['out']) + biases['out'] 39 | return out_layer 40 | 41 | # store layers weights and bias 42 | weights = { 43 | 'h1' : tf.Variable(tf.random_normal([n_input, n_hidden_1])), 44 | 'h2' : tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])), 45 | 'out' : tf.Variable(tf.random_normal([n_hidden_2, n_classes])) 46 | } 47 | 48 | biases = { 49 | 'b1' : tf.Variable(tf.random_normal([n_hidden_1])), 50 | 'b2' : tf.Variable(tf.random_normal([n_hidden_2])), 51 | 'out': tf.Variable(tf.random_normal([n_classes])) 52 | } 53 | 54 | 55 | # construct model 56 | pred = multilayer_perceptron(x, weights, biases) 57 | # define loss and optimizer 58 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)) 59 | optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost) 60 | 61 | #initializing the variables 62 | init = tf.global_variables_initializer() 63 | 64 | #launch the graph 65 | with tf.Session() as sess: 66 | sess.run(init) 67 | 68 | #training cycle 69 | for epoch in range(training_epochs): 70 | avg_cost = 0. 71 | 72 | total_batch = int(mnist.train.num_examples/batch_size) 73 | # Loop over all batches 74 | for i in range(total_batch): 75 | batch_x, batch_y = mnist.train.next_batch(batch_size) 76 | # run optimization op (backprop) and cost op (loss value) 77 | _, c = sess.run([optimizer,cost], feed_dict = {x : batch_x, y : batch_y}) 78 | 79 | #compute average loss 80 | avg_cost += c/total_batch 81 | 82 | #display logs per epoch step 83 | if (epoch +1) % display_step ==0: 84 | print("epoch:", '%04d'% (epoch +1), "cost =", "{:.9f}".format(avg_cost)) 85 | 86 | print("optimization finished") 87 | 88 | #test model 89 | correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y,1)) 90 | 91 | #calculate accuracy 92 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 93 | print("Accuracy", accuracy.eval({x:mnist.test.images, y: mnist.test.labels})) 94 | -------------------------------------------------------------------------------- /nearest_neighbor.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A nearest neighbor learning algorithm example using TensorFlow library. 3 | ''' 4 | 5 | from __future__ import print_function 6 | import numpy as np 7 | import tensorflow as tf 8 | # import mnist data 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 11 | 12 | # In this example, we limit mnist data 13 | 14 | Xtr, Ytr = mnist.train.next_batch(5000) # 5000 for training (nn candidates) 15 | Xte, Yte = mnist.test.next_batch(200) #200 for testing 16 | 17 | # tf graph input 18 | 19 | xtr = tf.placeholder("float", [None, 784]) 20 | xte = tf.placeholder("float", [784]) 21 | 22 | # Nearest neighbor calculation using L1 distance 23 | # calculte L1 distance 24 | 25 | distance = tf.reduce_sum(tf.abs(tf.add(xtr, tf.negative(xte))), reduction_indices = 1) 26 | 27 | # Prediction: get min distance index (nearest neighbor) 28 | pred = tf.arg_min(distance, 0) 29 | 30 | accuracy = 0. 31 | 32 | #Initializing the variables 33 | init = tf.global_variables_initializer() 34 | 35 | with tf.Session() as sess: 36 | sess.run(init) 37 | #loop over test data 38 | for i in range(len(Xte)): 39 | #get nearest neighbor 40 | nn_index = sess.run(pred, feed_dict={xtr: Xtr, xte: Xte[i,:] }) 41 | #get nearest neighbor class label and compare it to its true label 42 | print("Test", i, "Prediction:", np.argmax(Ytr[nn_index]), \ 43 | "true class:", np.argmax(Yte[i])) 44 | # Calculate accuracy 45 | print(Ytr.shape) 46 | print(Yte.shape) 47 | if np.argmax(Ytr[nn_index]) == np.argmax(Yte[i]): 48 | accuracy += 1./len(Xte) 49 | print("Done") 50 | print("Accuracy:", accuracy) 51 | -------------------------------------------------------------------------------- /new_filters_at_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/new_filters_at_0.png -------------------------------------------------------------------------------- /original_and_reconstruct.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/original_and_reconstruct.png -------------------------------------------------------------------------------- /rbm.py: -------------------------------------------------------------------------------- 1 | # Restrict Boltzmann Machine 2 | import tensorflow as tf 3 | import math 4 | import timeit 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | from tensorflow.examples.tutorials.mnist import input_data 9 | from utils import tile_raster_images 10 | 11 | class RBM(object): 12 | """A Restricted Boltzmann Machines class""" 13 | def __init__(self, inp = None, n_visible = 784, n_hidden = 500, W = None, hbias = None, vbias = None): 14 | self.n_visible = n_visible 15 | self.n_hidden = n_hidden 16 | if inp is None: 17 | inp = tf.placeholder(dtype = tf.float32, shape=[None, self.n_visible]) 18 | self.input = inp 19 | if W is None: 20 | low = -4.0 * math.sqrt(6. / (n_visible + n_hidden)) 21 | high = 4.0 * math.sqrt(6. / (n_visible + n_hidden)) 22 | W = tf.Variable(tf.random_uniform([self.n_visible,self.n_hidden], minval = low, maxval = high, dtype = tf.float32)) 23 | self.W = W 24 | if hbias is None: 25 | hbias = tf.Variable(tf.zeros([n_hidden]), dtype = tf.float32) 26 | self.hbias = hbias 27 | if vbias is None: 28 | vbias = tf.Variable(tf.zeros([n_visible]), dtype = tf.float32) 29 | self.vbias = vbias 30 | self.params = [self.W, self.hbias, self.vbias] 31 | 32 | def propup(self, visible): 33 | """This function propagates the visible units activation upwards to the hidden units""" 34 | return tf.nn.sigmoid(tf.matmul(visible,self.W) + self.hbias) 35 | 36 | 37 | def propdown(self, hidden): 38 | """This function propagates the hidden units activation downwards to the visible units""" 39 | return tf.nn.sigmoid(tf.matmul(hidden, tf.transpose(self.W)) + self.vbias) 40 | 41 | def sample_prob(self, prob): 42 | '''Do sampling with the given probability''' 43 | return tf.nn.relu(tf.sign(prob - tf.random_uniform(tf.shape(prob)))) 44 | 45 | def sample_h_given_v(self, v0_sample): 46 | ''' This function infers state of hidden units given visible units''' 47 | # get a sample of the hiddens given their activation 48 | h1_mean = self.propup(v0_sample) 49 | h1_sample = self.sample_prob(h1_mean) 50 | return (h1_mean, h1_sample) 51 | 52 | def sample_v_given_h(self, h0_sample): 53 | ''' This function infers state of visible units given hidden units ''' 54 | # get a sample of the visible given their activation 55 | v1_mean = self.propdown(h0_sample) 56 | v1_sample = self.sample_prob(v1_mean) 57 | return (v1_mean, v1_sample) 58 | 59 | '''gibbs_vhv which performs a step of Gibbs sampling starting from the visible units. 60 | As we shall see, this will be useful for sampling from the RBM. 61 | gibbs_hvh which performs a step of Gibbs sampling starting from the hidden units. 62 | This function will be useful for performing CD and PCD updates.''' 63 | 64 | def gibbs_hvh(self, h_sample): 65 | '''This function implements one step of Gibbs sampling, 66 | starting from the hidden state''' 67 | v1_mean, v1_sample = self.sample_v_given_h(h_sample) 68 | h1_mean, h1_sample = self.sample_h_given_v(v1_sample) 69 | return (v1_mean, v1_sample, h1_mean, h1_sample) 70 | 71 | 72 | def gibbs_vhv(self, v_sample): 73 | ''' This function implements one step of Gibbs sampling, 74 | starting from the visible state''' 75 | h1_mean, h1_sample = self.sample_h_given_v(v_sample) 76 | v1_mean, v1_sample = self.sample_v_given_h(h1_sample) 77 | return (h1_mean, h1_sample, v1_mean, v1_sample) 78 | 79 | 80 | def free_energy(self, v_sample): 81 | '''function to compute the free energy which need for 82 | computing the gradient of the parameters''' 83 | wx_b = tf.matmul(v_sample, self.W) + self.hbias 84 | vbias_term = tf.matmul(v_sample, tf.expand_dims(self.vbias, axis = 1)) 85 | hidden_term = tf.reduce_sum(tf.log(1 + tf.exp(wx_b)), axis = 1) 86 | return -hidden_term - vbias_term 87 | 88 | # we then add a get_cost_updates method, whose purpose is to generate the 89 | # symbolic gradients forn CD-k and PCD-k updates 90 | 91 | def get_cost_updates(self, lr = 0.1, persistent = None, k = 1): 92 | '''This functions implements one step of CD-k or PCD-k 93 | 94 | :param lr: learning rate used to train the RBM 95 | :param persistent: None for CD. For PCD, shared variable containing 96 | old state of Gibbs chain. This must be a shared variable of size(batch 97 | size, number of hidden units) 98 | :param k: number of Gibbs step to do in CD-k/PCD-k 99 | 100 | Return a proxy for the cost and the updates dictionary. 101 | The dictionary contains the update rules for weights and biases 102 | but also an update of the shared variable used to store the persistent 103 | chain, if one is used.''' 104 | 105 | #compute positive phase 106 | 107 | ph_mean, ph_sample = self.sample_h_given_v(self.input) 108 | 109 | #decide how to initialize persistent chain: 110 | # for CD, we use the newly generate hidden sample 111 | #forn PCD, we initialize from the old state of the chain 112 | if persistent is None: 113 | chain_start = ph_sample 114 | else: 115 | chain_start = persistent 116 | # perform actual negative phase 117 | # in order to implement CD-k/ PCD-k we need to scan over the 118 | # function that implements one gibbs step k times 119 | #print( tf.shape(chain_start)) 120 | cond = lambda i, nv_mean, nv_sample, nh_mean, nh_sample: i < k 121 | body = lambda i, nv_mean, nv_sample, nh_mean, nh_sample: (i + 1, ) + self.gibbs_hvh(nh_sample) 122 | i, nv_mean, nv_sample, nh_mean, nh_sample = tf.while_loop(cond, body, loop_vars=[tf.constant(0), tf.zeros(tf.shape(self.input)), tf.zeros(tf.shape(self.input)), tf.zeros(tf.shape(chain_start)), chain_start]) 123 | # determine gradients on RBM parameters 124 | 125 | # note that we only need the sample at the end of the chain 126 | chain_end = tf.stop_gradient(nv_sample) 127 | 128 | self.cost = tf.reduce_mean(self.free_energy(self.input)) - tf.reduce_mean(self.free_energy(chain_end)) 129 | # We must not compute the gradient through the gibbs sampling 130 | #compute the gradients 131 | gparams = tf.gradients(ys = [self.cost], xs = self.params) 132 | new_params = [] 133 | for gparam, param in zip(gparams, self.params): 134 | new_params.append(tf.assign(param, param - gparam * lr)) 135 | 136 | if persistent is not None: 137 | new_persistent = [tf.assign(persistent, nh_sample)] 138 | else: 139 | new_persistent = [] 140 | return new_params + new_persistent 141 | 142 | def get_reconstruction_cost(self): 143 | '''compute the cross-entropy of the original input and the reconstruction''' 144 | act_h = self.propup(self.input) 145 | act_v = self.propdown(act_h) 146 | 147 | cross_entropy = -tf.reduce_mean(tf.reduce_sum(self.input * tf.log(act_v) + (1.0 - self.input)* tf.log(1.0 - act_v), axis = 1)) 148 | return cross_entropy 149 | """ 150 | def get_reconstruction_cost(self): 151 | '''Compute the cross-entropy of the original input and the reconstruction''' 152 | activation_h = self.propup(self.input) 153 | activation_v = self.propdown(activation_h) 154 | # Do this to not get Nan 155 | activation_v_clip = tf.clip_by_value(activation_v, clip_value_min=1e-30, clip_value_max=1.0) 156 | reduce_activation_v_clip = tf.clip_by_value(1.0 - activation_v, clip_value_min=1e-30, clip_value_max=1.0) 157 | cross_entropy = -tf.reduce_mean(tf.reduce_sum(self.input*(tf.log(activation_v_clip)) + 158 | (1.0 - self.input)*(tf.log(reduce_activation_v_clip)), axis=1)) 159 | return cross_entropy 160 | """ 161 | 162 | 163 | def reconstruction(self, v): 164 | h = self.propup(v) 165 | return self.propdown(h) 166 | 167 | 168 | def test_rbm(): 169 | ''' Demonstrate how to train and afterwards sample from it 170 | this is demonstrate on mnist 171 | :param leaning_rate: learning rate used for training the rbm 172 | :param training_epochs: number of epochs used for training 173 | :param dataset: path to the picked dataset 174 | :param batch_size: size of a batch used to train the RBM 175 | :param n_chains: number of parallel GIbbs chains to be used for sampling 176 | :param n_samples: number of samples to plot for each chain 177 | ''' 178 | # import dataset 179 | mnist = input_data.read_data_sets("MNIST_data", one_hot = True) 180 | #parameters 181 | learning_rate = 0.1 182 | training_epochs = 1 183 | batch_size = 20 184 | display_step = 1 185 | n_chains = 20 186 | n_samples = 10 187 | #define input 188 | x = tf.placeholder(tf.float32, [None, 784]) 189 | #network parameters 190 | n_visible = 784 191 | n_hidden = 500 192 | rbm = RBM(x, n_visible = n_visible, n_hidden = n_hidden) 193 | 194 | cost = rbm.get_reconstruction_cost() 195 | 196 | #create the persistent variable 197 | persistent_chain = tf.Variable(tf.zeros([batch_size, n_hidden]), dtype = tf.float32) 198 | cost_updates = rbm.get_cost_updates(lr = learning_rate, persistent = persistent_chain, k = 15) 199 | #initializing the variables 200 | init = tf.global_variables_initializer() 201 | 202 | #------------------ 203 | # Training RBM 204 | #------------------ 205 | 206 | with tf.Session() as sess: 207 | start_time = timeit.default_timer() 208 | sess.run(init) 209 | 210 | total_batch = int(mnist.train.num_examples/batch_size) 211 | for epoch in range(training_epochs): 212 | c = 0.0 213 | #loop over all batches 214 | for i in range(total_batch): 215 | batch_xs, batch_ys = mnist.train.next_batch(batch_size) 216 | 217 | #run optimization op (backprop) and cost op (get loss value) 218 | _ = sess.run(cost_updates, feed_dict = {x: batch_xs}) 219 | c += sess.run(cost, feed_dict = {x: batch_xs}) / total_batch 220 | #display logs per epoch step 221 | if epoch % display_step ==0: 222 | print("epoch", '%04d' % (epoch +1), "cost", "{:.4f}".format(c)) 223 | 224 | #construct image from the weight matrix 225 | 226 | #image = plt.imshow(tile_raster_images(X = rbm.W, img_shape = (28, 28), tile_shape = (10,10), tile_spacing = (1,1))) 227 | plt.imsave("new_filters_at_{0}.png".format(epoch),tile_raster_images(X = sess.run(tf.transpose(rbm.W)), img_shape = (28, 28), tile_shape = (10,10), tile_spacing = (1,1)), cmap = 'gray') 228 | plt.show() 229 | 230 | end_time = timeit.default_timer() 231 | training_time = end_time - start_time 232 | print("Finished!") 233 | print(" The training ran for {0} minutes.".format(training_time/60,)) 234 | 235 | 236 | ################################# 237 | # Sampling from the RBM # 238 | ################################# 239 | # Reconstruct the image by sampling 240 | print("...Sampling from the RBM") 241 | number_test_examples = mnist.test.num_examples 242 | #randomly select the n_chains examples 243 | 244 | test_indexs = np.random.randint(number_test_examples - n_chains) 245 | test_samples = mnist.test.images[test_indexs:test_indexs + n_chains] 246 | #create the persistent variable saving the visiable state 247 | persistent_v_chain = tf.Variable(tf.to_float(test_samples), dtype = tf.float32) 248 | # the step of gibbs 249 | step_every = 1000 250 | 251 | # implement the gibbs sampling 252 | cond = lambda j, h_mean, h_sample, v_mean, v_sample: j < step_every 253 | body = lambda j, h_mean, h_sample, v_mean, v_sample: (j+1, ) + rbm.gibbs_vhv(v_sample) 254 | j, h_mean, h_sample, v_mean, v_sample = tf.while_loop(cond, body, loop_vars=[tf.constant(0), tf.zeros([n_chains, n_hidden]), 255 | tf.zeros([n_chains, n_hidden]), tf.zeros(tf.shape(persistent_v_chain)), persistent_v_chain]) 256 | # Update the persistent_v_chain 257 | new_persistent_v_chain = tf.assign(persistent_v_chain, v_sample) 258 | # Store the image by sampling 259 | image_data = np.zeros((29*(n_samples+1)+1, 29*(n_chains)-1), 260 | dtype="uint8") 261 | # Add the original images 262 | image_data[0:28,:] = tile_raster_images(X=test_samples, 263 | img_shape=(28, 28), 264 | tile_shape=(1, n_chains), 265 | tile_spacing=(1, 1)) 266 | # Initialize the variable 267 | sess.run(tf.variables_initializer(var_list=[persistent_v_chain])) 268 | # Do successive sampling 269 | for idx in range(1, n_samples+1): 270 | sample = sess.run(v_mean) 271 | sess.run(new_persistent_v_chain) 272 | print("...plotting sample", idx) 273 | image_data[idx*29:idx*29+28,:] = tile_raster_images(X=sample, 274 | img_shape=(28, 28), 275 | tile_shape=(1, n_chains), 276 | tile_spacing=(1, 1)) 277 | #image = plt.imshow(image_data) 278 | plt.imsave("new_original_and_{0}samples.png".format(n_samples), image_data, cmap = 'gray') 279 | plt.show() 280 | 281 | 282 | if __name__ == '__main__': 283 | test_rbm() 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | -------------------------------------------------------------------------------- /rbm.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/rbm.pyc -------------------------------------------------------------------------------- /recurrent_network.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A Recurrent Neural Network (LSTM) implementation example using TensorFlow library. 3 | 4 | ''' 5 | 6 | from __future__ import print_function 7 | 8 | import tensorflow as tf 9 | from tensorflow.contrib import rnn 10 | 11 | #import Mnist data 12 | from tensorflow.examples.tutorials.mnist import input_data 13 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 14 | 15 | # to classify images using a recurrent neural network, we consider every image 16 | # row as a sequence of pixels. Because mnist image shape is 28*28px, 17 | # we will then handle 28 sequences of 28 steps for every sample. 18 | 19 | # parameters 20 | learning_rate = 0.001 21 | training_iters = 100000 22 | batch_size = 128 23 | display_step = 10 24 | 25 | #network parameters 26 | n_input = 28 #mnist data input (img shape 28*28) 27 | n_steps = 28 #timesteps 28 | n_hidden = 128 29 | n_classes = 10 30 | 31 | #tf graph input 32 | x = tf.placeholder("float", [None, n_steps, n_input]) 33 | y = tf.placeholder("float", [None, n_classes]) 34 | 35 | #Define weights 36 | weights = { 37 | 'out' : tf.Variable(tf.random_normal([n_hidden, n_classes])) 38 | } 39 | biases = { 40 | 'out' : tf.Variable(tf.random_normal([n_classes])) 41 | } 42 | 43 | def RNN(x, weights, biases): 44 | # prepare data shpe to match 'rnn' function requirements 45 | # current data input shape : (batch_size, n_steps, n_input) 46 | #required shape : 'n_steps' tensors list of shpe(batch_size, n_input) 47 | 48 | #unstack to get a list of n_steps tensors of shape (batch_size, n_input) 49 | 50 | x = tf.unstack(x, n_steps, 1) 51 | 52 | #define a lstm cell with tensorflow 53 | lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias = 1.0) 54 | #get lstm cell utput 55 | output, states = rnn.static_rnn(lstm_cell, x, dtype = tf.float32) 56 | 57 | #linear activation, using rnn inner loop last output 58 | return tf.matmul(output[-1], weights['out']) + biases['out'] 59 | 60 | pred = RNN(x, weights, biases) 61 | 62 | #Define loss and optimizer 63 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)) 64 | optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost) 65 | 66 | #evaluate model 67 | correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y,1)) 68 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 69 | 70 | #initializing the variables 71 | init = tf.global_variables_initializer() 72 | 73 | #launch the graph 74 | with tf.Session() as sess: 75 | sess.run(init) 76 | step = 1 77 | # keep training until reach max iterations 78 | while step * batch_size < training_iters: 79 | batch_x, batch_y = mnist.train.next_batch(batch_size) 80 | #reshape data to get 28 squences of 28 elements 81 | batch_x = batch_x.reshape((batch_size, n_steps, n_input)) 82 | #run optimization op (backprop) 83 | sess.run(optimizer, feed_dict = {x: batch_x, y: batch_y}) 84 | if step % display_step == 0 : 85 | #calculate batch accuracy 86 | acc = sess.run(accuracy, feed_dict = {x : batch_x, y: batch_y}) 87 | #loss 88 | loss = sess.run(cost, feed_dict = {x :batch_x,y: batch_y}) 89 | print("iter", str(step*batch_size), "minibatch loss", "{:.6f}".format(loss),"training accuracy =","{:.5f}".format(acc)) 90 | 91 | step += 1 92 | print("optimization finished") 93 | 94 | #calculate accuracy for 128 mnist test images 95 | test_len = 128 96 | test_data = mnist.test.images[:test_len].reshape((-1, n_steps,n_input)) 97 | test_label = mnist.test.labels[:test_len] 98 | print("testing accuracy:", sess.run(accuracy, feed_dict = {x:test_data, y:test_label})) 99 | -------------------------------------------------------------------------------- /tensorboard_basic.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Graph and Loss visualization using Tensorboard. 3 | This example is using the MNIST database of handwritten digits 4 | (http://yann.lecun.com/exdb/mnist/) 5 | ''' 6 | from __future__ import print_function 7 | 8 | import tensorflow as tf 9 | 10 | # import mnist data 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) 13 | 14 | 15 | # Parameters 16 | learning_rate = 0.01 17 | training_epochs = 25 18 | batch_size = 100 19 | display_step = 1 20 | logs_path = '/tmp/tensorflow_logs/example' 21 | 22 | 23 | # tf Graph Input 24 | # mnist data image of shape 28*28=784 25 | x = tf.placeholder(tf.float32, [None, 784], name='InputData') 26 | # 0-9 digits recognition => 10 classes 27 | y = tf.placeholder(tf.float32, [None, 10], name='LabelData') 28 | 29 | # Set model weights 30 | W = tf.Variable(tf.zeros([784, 10]), name='Weights') 31 | b = tf.Variable(tf.zeros([10]), name='Bias') 32 | 33 | 34 | 35 | # Construct model and encapsulating all ops into scopes, making 36 | # Tensorboard's Graph visualization more convenient 37 | with tf.name_scope('Model'): 38 | #model 39 | pred = tf.nn.softmax(tf.matmul(x, W) + b) 40 | -------------------------------------------------------------------------------- /test_filters_at_epoch_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/test_filters_at_epoch_0.png -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | """ This file contains different utility functions that are not connected 2 | in anyway to the networks presented in the tutorials, but rather help in 3 | processing the outputs into a more understandable way. 4 | For example ``tile_raster_images`` helps in generating a easy to grasp 5 | image from a set of samples or weights. 6 | """ 7 | 8 | import numpy 9 | 10 | 11 | def scale_to_unit_interval(ndar, eps=1e-8): 12 | """ Scales all values in the ndarray ndar to be between 0 and 1 """ 13 | ndar = ndar.copy() 14 | ndar -= ndar.min() 15 | ndar *= 1.0 / (ndar.max() + eps) 16 | return ndar 17 | 18 | 19 | def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), 20 | scale_rows_to_unit_interval=True, 21 | output_pixel_vals=True): 22 | """ 23 | Transform an array with one flattened image per row, into an array in 24 | which images are reshaped and layed out like tiles on a floor. 25 | This function is useful for visualizing datasets whose rows are images, 26 | and also columns of matrices for transforming those rows 27 | (such as the first layer of a neural net). 28 | :type X: a 2-D ndarray or a tuple of 4 channels, elements of which can 29 | be 2-D ndarrays or None; 30 | :param X: a 2-D array in which every row is a flattened image. 31 | :type img_shape: tuple; (height, width) 32 | :param img_shape: the original shape of each image 33 | :type tile_shape: tuple; (rows, cols) 34 | :param tile_shape: the number of images to tile (rows, cols) 35 | :param output_pixel_vals: if output should be pixel values (i.e. int8 36 | values) or floats 37 | :param scale_rows_to_unit_interval: if the values need to be scaled before 38 | being plotted to [0,1] or not 39 | :returns: array suitable for viewing as an image. 40 | (See:`Image.fromarray`.) 41 | :rtype: a 2-d array with same dtype as X. 42 | """ 43 | 44 | assert len(img_shape) == 2 45 | assert len(tile_shape) == 2 46 | assert len(tile_spacing) == 2 47 | 48 | # The expression below can be re-written in a more C style as 49 | # follows : 50 | # 51 | # out_shape = [0,0] 52 | # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - 53 | # tile_spacing[0] 54 | # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - 55 | # tile_spacing[1] 56 | out_shape = [ 57 | (ishp + tsp) * tshp - tsp 58 | for ishp, tshp, tsp in zip(img_shape, tile_shape, tile_spacing) 59 | ] 60 | 61 | if isinstance(X, tuple): 62 | assert len(X) == 4 63 | # Create an output numpy ndarray to store the image 64 | if output_pixel_vals: 65 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 66 | dtype='uint8') 67 | else: 68 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 69 | dtype=X.dtype) 70 | 71 | #colors default to 0, alpha defaults to 1 (opaque) 72 | if output_pixel_vals: 73 | channel_defaults = [0, 0, 0, 255] 74 | else: 75 | channel_defaults = [0., 0., 0., 1.] 76 | 77 | for i in range(4): 78 | if X[i] is None: 79 | # if channel is None, fill it with zeros of the correct 80 | # dtype 81 | dt = out_array.dtype 82 | if output_pixel_vals: 83 | dt = 'uint8' 84 | out_array[:, :, i] = numpy.zeros( 85 | out_shape, 86 | dtype=dt 87 | ) + channel_defaults[i] 88 | else: 89 | # use a recurrent call to compute the channel and store it 90 | # in the output 91 | out_array[:, :, i] = tile_raster_images( 92 | X[i], img_shape, tile_shape, tile_spacing, 93 | scale_rows_to_unit_interval, output_pixel_vals) 94 | return out_array 95 | 96 | else: 97 | # if we are dealing with only one channel 98 | H, W = img_shape 99 | Hs, Ws = tile_spacing 100 | 101 | # generate a matrix to store the output 102 | dt = X.dtype 103 | if output_pixel_vals: 104 | dt = 'uint8' 105 | out_array = numpy.zeros(out_shape, dtype=dt) 106 | 107 | for tile_row in range(tile_shape[0]): 108 | for tile_col in range(tile_shape[1]): 109 | if tile_row * tile_shape[1] + tile_col < X.shape[0]: 110 | this_x = X[tile_row * tile_shape[1] + tile_col] 111 | if scale_rows_to_unit_interval: 112 | # if we should scale values to be between 0 and 1 113 | # do this by calling the `scale_to_unit_interval` 114 | # function 115 | this_img = scale_to_unit_interval( 116 | this_x.reshape(img_shape)) 117 | else: 118 | this_img = this_x.reshape(img_shape) 119 | # add the slice to the corresponding position in the 120 | # output array 121 | c = 1 122 | if output_pixel_vals: 123 | c = 255 124 | out_array[ 125 | tile_row * (H + Hs): tile_row * (H + Hs) + H, 126 | tile_col * (W + Ws): tile_col * (W + Ws) + W 127 | ] = this_img * c 128 | return out_array -------------------------------------------------------------------------------- /utils.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khoink94/tensorflow-Deep-learning/35668dc4c946488f7568297bff6624c5519dbf40/utils.pyc --------------------------------------------------------------------------------