├── LICENSE ├── README.md ├── SparseAutoEncoder.py └── trained4000.png /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Dr. Zhiwei Lin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sparse Autoencoder with Tensorflow 2 | This is an example of using Tensorflow to build Sparse Autoencoder 3 | for representation learning. 4 | It is the implementation of the sparse autoencoder for 5 | 6 | https://web.stanford.edu/class/cs294a/sparseAutoencoder_2011new.pdf 7 | 8 | For any enquiry, please contact Dr. Zhiwei Lin at Ulster University 9 | 10 | http://scm.ulster.ac.uk/zhiwei.lin/ 11 | 12 | -------------------------------------------------------------------------------- /SparseAutoEncoder.py: -------------------------------------------------------------------------------- 1 | # This is an example of using Tensorflow to build Sparse Autoencoder 2 | # for representation learning. 3 | # It is the implementation of the sparse autoencoder for 4 | # https://web.stanford.edu/class/cs294a/sparseAutoencoder_2011new.pdf 5 | # 6 | # For any enquiry, please contact Dr. Zhiwei Lin at Ulster University 7 | # http://scm.ulster.ac.uk/zhiwei.lin/ 8 | # 9 | # 10 | # ============================================================================== 11 | import tensorflow as tf 12 | import matplotlib.pyplot 13 | import math 14 | 15 | 16 | 17 | 18 | 19 | class FeedforwardSparseAutoEncoder(): 20 | ''' 21 | This is the implementation of the sparse autoencoder for https://web.stanford.edu/class/cs294a/sparseAutoencoder_2011new.pdf 22 | 23 | ''' 24 | def __init__(self, n_input, n_hidden, rho=0.01, alpha=0.0001, beta=3, activation=tf.nn.sigmoid, optimizer=tf.train.AdamOptimizer()): 25 | self.n_input=n_input 26 | self.n_hidden=n_hidden 27 | self.rho=rho # sparse parameters 28 | self.alpha =alpha 29 | self.beta=beta 30 | self.optimizer=optimizer 31 | self.activation = activation 32 | 33 | self.W1=self.init_weights((self.n_input,self.n_hidden)) 34 | self.b1=self.init_weights((1,self.n_hidden)) 35 | 36 | self.W2=self.init_weights((self.n_hidden,self.n_input)) 37 | self.b2= self.init_weights((1,self.n_input)) 38 | init = tf.global_variables_initializer() 39 | self.sess = tf.Session() 40 | self.sess.run(init) 41 | 42 | def init_weights(self,shape): 43 | r= math.sqrt(6) / math.sqrt(self.n_input + self.n_hidden + 1) 44 | weights = tf.random_normal(shape, stddev=r) 45 | return tf.Variable(weights) 46 | 47 | def encode(self,X): 48 | l=tf.matmul(X, self.W1)+self.b1 49 | return self.activation(l) 50 | 51 | def decode(self,H): 52 | l=tf.matmul(H,self.W2)+self.b2 53 | return self.activation(l) 54 | 55 | 56 | def kl_divergence(self, rho, rho_hat): 57 | return rho * tf.log(rho) - rho * tf.log(rho_hat) + (1 - rho) * tf.log(1 - rho) - (1 - rho) * tf.log(1 - rho_hat) 58 | 59 | def regularization(self,weights): 60 | return tf.nn.l2_loss(weights) 61 | 62 | def loss(self,X): 63 | H = self.encode(X) 64 | rho_hat=tf.reduce_mean(H,axis=0) #Average hidden layer over all data points in X, Page 14 in https://web.stanford.edu/class/cs294a/sparseAutoencoder_2011new.pdf 65 | kl=self.kl_divergence(self.rho, rho_hat) 66 | X_=self.decode(H) 67 | diff=X-X_ 68 | cost= 0.5*tf.reduce_mean(tf.reduce_sum(diff**2,axis=1)) \ 69 | +0.5*self.alpha*(tf.nn.l2_loss(self.W1) + tf.nn.l2_loss(self.W2)) \ 70 | +self.beta*tf.reduce_sum(kl) 71 | return cost 72 | 73 | def training(self,training_data, n_iter=100): 74 | 75 | X=tf.placeholder("float",shape=[None,training_data.shape[1]]) 76 | var_list=[self.W1,self.W2] 77 | loss_=self.loss(X) 78 | train_step=tf.contrib.opt.ScipyOptimizerInterface(loss_, var_list=var_list, method='L-BFGS-B', options={'maxiter': n_iter}) 79 | train_step.minimize(self.sess, feed_dict={X: training_data}) 80 | 81 | 82 | def visualizeW1(images, vis_patch_side, hid_patch_side, iter, file_name="trained_"): 83 | """ Visual all images in one pane""" 84 | 85 | figure, axes = matplotlib.pyplot.subplots(nrows=hid_patch_side, ncols=hid_patch_side) 86 | index = 0 87 | 88 | for axis in axes.flat: 89 | """ Add row of weights as an image to the plot """ 90 | 91 | image = axis.imshow(images[index, :].reshape(vis_patch_side, vis_patch_side), 92 | cmap=matplotlib.pyplot.cm.gray, interpolation='nearest') 93 | axis.set_frame_on(False) 94 | axis.set_axis_off() 95 | index += 1 96 | 97 | """ Show the obtained plot """ 98 | file=file_name+str(iter)+".png" 99 | matplotlib.pyplot.savefig(file) 100 | print("Written into "+ file) 101 | matplotlib.pyplot.close() 102 | 103 | 104 | def main(): 105 | from tensorflow.examples.tutorials.mnist import input_data 106 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 107 | 108 | n_inputs=784 109 | n_hidden=100 110 | start=0 111 | lens=1000 112 | learning_rate=0.1 113 | 114 | sae= FeedforwardSparseAutoEncoder(n_inputs,n_hidden) 115 | n_iters=4000 116 | sae.training(mnist.train.images[start:start+lens],n_iter=n_iters) 117 | 118 | # After training the model, an image of the representations (W1) will be saved 119 | # Please check trained4000.png for example 120 | images=sae.W1.eval(sae.sess) 121 | images=images.transpose() 122 | visualizeW1(images,28,10,n_iters) 123 | 124 | 125 | 126 | if __name__=='__main__': 127 | main() 128 | -------------------------------------------------------------------------------- /trained4000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhiweiuk/sparse-autoencoder-tensorflow/024b3277d418deb4d47de8fc1c432cf35baae9c0/trained4000.png --------------------------------------------------------------------------------