├── LICENSE ├── LSTMAutoencoder.py ├── README.md └── test.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 iwyoo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LSTMAutoencoder.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn_cell import LSTMCell 3 | 4 | import numpy as np 5 | 6 | 7 | class LSTMAutoencoder(object): 8 | 9 | """Basic version of LSTM-autoencoder. 10 | (cf. http://arxiv.org/abs/1502.04681) 11 | 12 | Usage: 13 | ae = LSTMAutoencoder(hidden_num, inputs) 14 | sess.run(ae.train) 15 | """ 16 | 17 | def __init__( 18 | self, 19 | hidden_num, 20 | inputs, 21 | cell=None, 22 | optimizer=None, 23 | reverse=True, 24 | decode_without_input=False, 25 | ): 26 | """ 27 | Args: 28 | hidden_num : number of hidden elements of each LSTM unit. 29 | inputs : a list of input tensors with size 30 | (batch_num x elem_num) 31 | cell : an rnn cell object (the default option 32 | is `tf.python.ops.rnn_cell.LSTMCell`) 33 | optimizer : optimizer for rnn (the default option is 34 | `tf.train.AdamOptimizer`) 35 | reverse : Option to decode in reverse order. 36 | decode_without_input : Option to decode without input. 37 | """ 38 | 39 | self.batch_num = inputs[0].get_shape().as_list()[0] 40 | self.elem_num = inputs[0].get_shape().as_list()[1] 41 | 42 | if cell is None: 43 | self._enc_cell = LSTMCell(hidden_num) 44 | self._dec_cell = LSTMCell(hidden_num) 45 | else: 46 | self._enc_cell = cell 47 | self._dec_cell = cell 48 | 49 | with tf.variable_scope('encoder'): 50 | (self.z_codes, self.enc_state) = tf.contrib.rnn.static_rnn(self._enc_cell, inputs, dtype=tf.float32) 51 | 52 | with tf.variable_scope('decoder') as vs: 53 | dec_weight_ = tf.Variable(tf.truncated_normal([hidden_num, 54 | self.elem_num], dtype=tf.float32), name='dec_weight' 55 | ) 56 | dec_bias_ = tf.Variable(tf.constant(0.1, 57 | shape=[self.elem_num], 58 | dtype=tf.float32), name='dec_bias') 59 | 60 | if decode_without_input: 61 | dec_inputs = [tf.zeros(tf.shape(inputs[0]), 62 | dtype=tf.float32) for _ in 63 | range(len(inputs))] 64 | (dec_outputs, dec_state) = tf.contrib.rnn.static_rnn(self._dec_cell, dec_inputs, initial_state=self.enc_state, 65 | dtype=tf.float32) 66 | if reverse: 67 | dec_outputs = dec_outputs[::-1] 68 | dec_output_ = tf.transpose(tf.stack(dec_outputs), [1, 0, 69 | 2]) 70 | dec_weight_ = tf.tile(tf.expand_dims(dec_weight_, 0), 71 | [self.batch_num, 1, 1]) 72 | self.output_ = tf.matmul(dec_output_, dec_weight_) + dec_bias_ 73 | else: 74 | 75 | dec_state = self.enc_state 76 | dec_input_ = tf.zeros(tf.shape(inputs[0]), 77 | dtype=tf.float32) 78 | dec_outputs = [] 79 | for step in range(len(inputs)): 80 | if step > 0: 81 | vs.reuse_variables() 82 | (dec_input_, dec_state) = \ 83 | self._dec_cell(dec_input_, dec_state) 84 | dec_input_ = tf.matmul(dec_input_, dec_weight_) \ 85 | + dec_bias_ 86 | dec_outputs.append(dec_input_) 87 | if reverse: 88 | dec_outputs = dec_outputs[::-1] 89 | self.output_ = tf.transpose(tf.stack(dec_outputs), [1, 90 | 0, 2]) 91 | 92 | self.input_ = tf.transpose(tf.stack(inputs), [1, 0, 2]) 93 | self.loss = tf.reduce_mean(tf.square(self.input_ 94 | - self.output_)) 95 | 96 | if optimizer is None: 97 | self.train = tf.train.AdamOptimizer().minimize(self.loss) 98 | else: 99 | self.train = optimizer.minimize(self.loss) 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LSTM-autoencoder 2 | TensorFlow LSTM-autoencoder implementation 3 | 4 | ## Usage 5 | ```python 6 | 7 | # hidden_num : the number of hidden units in each RNN-cell 8 | # inputs : a list of tensor with size (batch_num x step_num x elem_num) 9 | ae = LSTMAutoencoder(hidden_num, inputs) 10 | ... 11 | sess.run(init) 12 | ... 13 | sess.run(ae.train, feed_dict={input_variable:input_array,...}) 14 | ``` 15 | 16 | ## Reference 17 | Unsupervised Learning of Video Representations using LSTMs (Nitish Srivastava, Elman Mansimov, Ruslan Salakhutdinov) 18 | 19 | http://arxiv.org/abs/1502.04681 20 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | # Basic libraries 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | tf.reset_default_graph() 6 | tf.set_random_seed(2016) 7 | np.random.seed(2016) 8 | 9 | # LSTM-autoencoder 10 | from LSTMAutoencoder import * 11 | 12 | # Constants 13 | batch_num = 128 14 | hidden_num = 12 15 | step_num = 8 16 | elem_num = 1 17 | iteration = 10000 18 | 19 | # placeholder list 20 | p_input = tf.placeholder(tf.float32, shape=(batch_num, step_num, elem_num)) 21 | p_inputs = [tf.squeeze(t, [1]) for t in tf.split(p_input, step_num, 1)] 22 | 23 | cell = tf.nn.rnn_cell.LSTMCell(hidden_num, use_peepholes=True) 24 | ae = LSTMAutoencoder(hidden_num, p_inputs, cell=cell, decode_without_input=True) 25 | 26 | with tf.Session() as sess: 27 | sess.run(tf.global_variables_initializer()) 28 | 29 | for i in range(iteration): 30 | """Random sequences. 31 | Every sequence has size batch_num * step_num * elem_num 32 | Each step number increases 1 by 1. 33 | An initial number of each sequence is in the range from 0 to 19. 34 | (ex. [8. 9. 10. 11. 12. 13. 14. 15]) 35 | """ 36 | r = np.random.randint(20, size=batch_num).reshape([batch_num, 1, 1]) 37 | r = np.tile(r, (1, step_num, elem_num)) 38 | d = np.linspace(0, step_num, step_num, endpoint=False).reshape([1, step_num, elem_num]) 39 | d = np.tile(d, (batch_num, 1, 1)) 40 | random_sequences = r + d 41 | 42 | (loss_val, _) = sess.run([ae.loss, ae.train], {p_input: random_sequences}) 43 | print('iter %d:' % (i + 1), loss_val) 44 | 45 | (input_, output_) = sess.run([ae.input_, ae.output_], {p_input: r + d}) 46 | print('train result :') 47 | print('input :', input_[0, :, :].flatten()) 48 | print('output :', output_[0, :, :].flatten()) 49 | --------------------------------------------------------------------------------