├── .gitignore ├── README.md ├── Results.ipynb ├── main.py ├── ptb_reader.py ├── results ├── cifg.csv ├── fgr.csv ├── nfg.csv ├── niaf.csv ├── nig.csv ├── noaf.csv ├── nog.csv ├── np.csv ├── sessions.csv └── vanilla.csv └── variants ├── __init__.py ├── cifg.py ├── fgr.py ├── nfg.py ├── niaf.py ├── nig.py ├── noaf.py ├── nog.py ├── np.py └── vanilla.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | .env/ 4 | .ipynb_checkpoints/ 5 | ptb/ 6 | logs/ 7 | models/ 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # An LSTM Odyssey 2 | 3 | Code for training variants of "LSTM: A Search Space Odyssey" on Fomoro. 4 | 5 | Check out the [blog post](https://medium.com/jim-fleming/implementing-lstm-a-search-space-odyssey-7d50c3bacf93). 6 | 7 | ## Training 8 | 9 | 1. [Install TensorFlow](https://www.tensorflow.org/versions/r0.7/get_started/os_setup.html#pip-installation). 10 | 2. Clone the repo: `git clone https://github.com/fomorians/lstm-odyssey.git && cd lstm-odyssey` 11 | 3. Run training: `python main.py` 12 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import os 6 | import time 7 | import numpy as np 8 | import pandas as pd 9 | import tensorflow as tf 10 | 11 | from tensorflow.python.ops import rnn_cell 12 | from tensorflow.python.ops.seq2seq import sequence_loss_by_example 13 | 14 | # parses the dataset 15 | import ptb_reader 16 | 17 | # import variants 18 | from variants.vanilla import VanillaLSTMCell 19 | from variants.nig import NIGLSTMCell 20 | from variants.nfg import NFGLSTMCell 21 | from variants.nog import NOGLSTMCell 22 | from variants.niaf import NIAFLSTMCell 23 | from variants.noaf import NOAFLSTMCell 24 | from variants.np import NPLSTMCell 25 | from variants.cifg import CIFGLSTMCell 26 | from variants.fgr import FGRLSTMCell 27 | 28 | # define artifact directories where results from the session can be saved 29 | model_path = os.environ.get('MODEL_PATH', 'models/') 30 | checkpoint_path = os.environ.get('CHECKPOINT_PATH', 'checkpoints/') 31 | summary_path = os.environ.get('SUMMARY_PATH', 'logs/') 32 | 33 | # load dataset 34 | train_data, valid_data, test_data, _ = ptb_reader.ptb_raw_data("ptb") 35 | 36 | def write_csv(arr, path): 37 | df = pd.DataFrame(arr) 38 | df.to_csv(path) 39 | 40 | class PTBModel(object): 41 | def __init__(self, CellType, is_training, config): 42 | self.batch_size = batch_size = config.batch_size 43 | self.num_steps = num_steps = config.num_steps 44 | size = config.hidden_size 45 | vocab_size = config.vocab_size 46 | 47 | self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps], name="input_data") 48 | self.targets = tf.placeholder(tf.int32, [batch_size, num_steps], name="targets") 49 | 50 | lstm_cell = CellType(size) 51 | if is_training and config.keep_prob < 1: 52 | lstm_cell = rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=config.keep_prob) 53 | cell = rnn_cell.MultiRNNCell([lstm_cell] * config.num_layers) 54 | self.initial_state = cell.zero_state(batch_size, tf.float32) 55 | 56 | # initializer used for reusable variable initializer (see `get_variable`) 57 | initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) 58 | 59 | with tf.device("/cpu:0"): 60 | embedding = tf.get_variable("embedding", [vocab_size, size], initializer=initializer) 61 | inputs = tf.nn.embedding_lookup(embedding, self.input_data) 62 | 63 | if is_training and config.keep_prob < 1: 64 | inputs = tf.nn.dropout(inputs, config.keep_prob) 65 | 66 | outputs = [] 67 | states = [] 68 | state = self.initial_state 69 | 70 | with tf.variable_scope("RNN", initializer=initializer): 71 | for time_step in range(num_steps): 72 | if time_step > 0: 73 | tf.get_variable_scope().reuse_variables() 74 | 75 | inputs_slice = inputs[:,time_step,:] 76 | (cell_output, state) = cell(inputs_slice, state) 77 | 78 | outputs.append(cell_output) 79 | states.append(state) 80 | 81 | self.final_state = states[-1] 82 | 83 | output = tf.reshape(tf.concat(1, outputs), [-1, size]) 84 | w = tf.get_variable("softmax_w", 85 | [size, vocab_size], 86 | initializer=initializer) 87 | b = tf.get_variable("softmax_b", [vocab_size], initializer=initializer) 88 | 89 | logits = tf.nn.xw_plus_b(output, w, b) # compute logits for loss 90 | targets = tf.reshape(self.targets, [-1]) # reshape our target outputs 91 | weights = tf.ones([batch_size * num_steps]) # used to scale the loss average 92 | 93 | # computes loss and performs softmax on our fully-connected output layer 94 | loss = sequence_loss_by_example([logits], [targets], [weights], vocab_size) 95 | self.cost = cost = tf.div(tf.reduce_sum(loss), batch_size, name="cost") 96 | 97 | if is_training: 98 | # setup learning rate variable to decay 99 | self.lr = tf.Variable(1.0, trainable=False) 100 | 101 | # define training operation and clip the gradients 102 | tvars = tf.trainable_variables() 103 | grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), config.max_grad_norm) 104 | optimizer = tf.train.GradientDescentOptimizer(self.lr) 105 | self.train_op = optimizer.apply_gradients(zip(grads, tvars), name="train") 106 | else: 107 | # if this model isn't for training (i.e. testing/validation) then we don't do anything here 108 | self.train_op = tf.no_op() 109 | 110 | def run_epoch(sess, model, data, verbose=False): 111 | epoch_size = ((len(data) // model.batch_size) - 1) // model.num_steps 112 | start_time = time.time() 113 | 114 | # accumulated counts 115 | costs = 0.0 116 | iters = 0 117 | 118 | # initial RNN state 119 | state = model.initial_state.eval() 120 | 121 | for step, (x, y) in enumerate(ptb_reader.ptb_iterator(data, model.batch_size, model.num_steps)): 122 | cost, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed_dict={ 123 | model.input_data: x, 124 | model.targets: y, 125 | model.initial_state: state 126 | }) 127 | costs += cost 128 | iters += model.num_steps 129 | 130 | perplexity = np.exp(costs / iters) 131 | 132 | if verbose and step % 10 == 0: 133 | progress = (step / epoch_size) * 100 134 | wps = iters * model.batch_size / (time.time() - start_time) 135 | print("%.1f%% Perplexity: %.3f (Cost: %.3f) Speed: %.0f wps" % (progress, perplexity, cost, wps)) 136 | 137 | return (costs / iters), perplexity 138 | 139 | class Config(object): 140 | batch_size = 20 141 | num_steps = 35 # number of unrolled time steps 142 | hidden_size = 450 # number of blocks in an LSTM cell 143 | vocab_size = 10000 144 | max_grad_norm = 5 # maximum gradient for clipping 145 | init_scale = 0.05 # scale between -0.1 and 0.1 for all random initialization 146 | keep_prob = 0.5 # dropout probability 147 | num_layers = 2 # number of LSTM layers 148 | learning_rate = 1.0 149 | lr_decay = 0.8 150 | lr_decay_epoch_offset = 6 # don't decay until after the Nth epoch 151 | 152 | # default settings for training 153 | train_config = Config() 154 | 155 | # our evaluation runs (validation and testing), use a batch size and time step of one 156 | eval_config = Config() 157 | eval_config.batch_size = 1 158 | eval_config.num_steps = 1 159 | 160 | # number of epochs to perform over the training data 161 | num_epochs = 39 162 | 163 | cell_types = { 164 | 'vanilla': VanillaLSTMCell, 165 | 'nig': NIGLSTMCell, 166 | 'nfg': NFGLSTMCell, 167 | 'nog': NOGLSTMCell, 168 | 'niaf': NIAFLSTMCell, 169 | 'noaf': NOAFLSTMCell, 170 | 'np': NPLSTMCell, 171 | 'cifg': CIFGLSTMCell, 172 | 'fgr': FGRLSTMCell, 173 | } 174 | 175 | model_name = "vanilla" 176 | CellType = cell_types[model_name] 177 | 178 | with tf.Graph().as_default(), tf.Session() as sess: 179 | # define our training model 180 | with tf.variable_scope("model", reuse=None): 181 | train_model = PTBModel(CellType, is_training=True, config=train_config) 182 | 183 | # we create a separate model for validation and testing to alter the batch size and time steps 184 | # reuse=True reuses variables from the previously defined `train_model` 185 | with tf.variable_scope("model", reuse=True): 186 | valid_model = PTBModel(CellType, is_training=False, config=train_config) 187 | test_model = PTBModel(CellType, is_training=False, config=eval_config) 188 | 189 | # create a saver instance to restore from the checkpoint 190 | saver = tf.train.Saver(max_to_keep=1) 191 | 192 | # initialize our variables 193 | sess.run(tf.initialize_all_variables()) 194 | 195 | # save the graph definition as a protobuf file 196 | tf.train.write_graph(sess.graph_def, model_path, '%s.pb'.format(model_name), as_text=False) 197 | 198 | train_costs = [] 199 | train_perps = [] 200 | valid_costs = [] 201 | valid_perps = [] 202 | 203 | for i in range(num_epochs): 204 | print("Epoch: %d Learning Rate: %.3f" % (i + 1, sess.run(train_model.lr))) 205 | 206 | # run training pass 207 | train_cost, train_perp = run_epoch(sess, train_model, train_data, verbose=True) 208 | print("Epoch: %i Training Perplexity: %.3f (Cost: %.3f)" % (i + 1, train_perp, train_cost)) 209 | train_costs.append(train_cost) 210 | train_perps.append(train_perp) 211 | 212 | # run validation pass 213 | valid_cost, valid_perp = run_epoch(sess, valid_model, valid_data) 214 | print("Epoch: %i Validation Perplexity: %.3f (Cost: %.3f)" % (i + 1, valid_perp, valid_cost)) 215 | valid_costs.append(valid_cost) 216 | valid_perps.append(valid_perp) 217 | 218 | saver.save(sess, checkpoint_path + 'checkpoint') 219 | 220 | # run test pass 221 | test_cost, test_perp = run_epoch(sess, test_model, test_data) 222 | print("Test Perplexity: %.3f (Cost: %.3f)" % (test_perp, test_cost)) 223 | 224 | write_csv(train_costs, os.path.join(summary_path, "train_costs.csv")) 225 | write_csv(train_perps, os.path.join(summary_path, "train_perps.csv")) 226 | write_csv(valid_costs, os.path.join(summary_path, "valid_costs.csv")) 227 | write_csv(valid_perps, os.path.join(summary_path, "valid_perps.csv")) 228 | -------------------------------------------------------------------------------- /ptb_reader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | # pylint: disable=unused-import,g-bad-import-order 17 | 18 | """Utilities for parsing PTB text files.""" 19 | from __future__ import absolute_import 20 | from __future__ import division 21 | from __future__ import print_function 22 | 23 | import collections 24 | import os 25 | import sys 26 | import time 27 | 28 | import tensorflow.python.platform 29 | 30 | import numpy as np 31 | from six.moves import xrange # pylint: disable=redefined-builtin 32 | import tensorflow as tf 33 | 34 | from tensorflow.python.platform import gfile 35 | 36 | 37 | def _read_words(filename): 38 | with gfile.GFile(filename, "r") as f: 39 | return f.read().replace("\n", "").split() 40 | 41 | 42 | def _build_vocab(filename): 43 | data = _read_words(filename) 44 | 45 | counter = collections.Counter(data) 46 | count_pairs = sorted(counter.items(), key=lambda x: -x[1]) 47 | 48 | words, _ = list(zip(*count_pairs)) 49 | word_to_id = dict(zip(words, range(len(words)))) 50 | 51 | return word_to_id 52 | 53 | 54 | def _file_to_word_ids(filename, word_to_id): 55 | data = _read_words(filename) 56 | return [word_to_id[word] for word in data] 57 | 58 | 59 | def ptb_raw_data(data_path=None): 60 | """Load PTB raw data from data directory "data_path". 61 | 62 | Reads PTB text files, converts strings to integer ids, 63 | and performs mini-batching of the inputs. 64 | 65 | The PTB dataset comes from Tomas Mikolov's webpage: 66 | 67 | http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz 68 | 69 | Args: 70 | data_path: string path to the directory where simple-examples.tgz has 71 | been extracted. 72 | 73 | Returns: 74 | tuple (train_data, valid_data, test_data, vocabulary) 75 | where each of the data objects can be passed to PTBIterator. 76 | """ 77 | 78 | train_path = os.path.join(data_path, "ptb.train.txt") 79 | valid_path = os.path.join(data_path, "ptb.valid.txt") 80 | test_path = os.path.join(data_path, "ptb.test.txt") 81 | 82 | word_to_id = _build_vocab(train_path) 83 | train_data = _file_to_word_ids(train_path, word_to_id) 84 | valid_data = _file_to_word_ids(valid_path, word_to_id) 85 | test_data = _file_to_word_ids(test_path, word_to_id) 86 | vocabulary = len(word_to_id) 87 | return train_data, valid_data, test_data, vocabulary 88 | 89 | 90 | def ptb_iterator(raw_data, batch_size, num_steps): 91 | """Iterate on the raw PTB data. 92 | 93 | This generates batch_size pointers into the raw PTB data, and allows 94 | minibatch iteration along these pointers. 95 | 96 | Args: 97 | raw_data: one of the raw data outputs from ptb_raw_data. 98 | batch_size: int, the batch size. 99 | num_steps: int, the number of unrolls. 100 | 101 | Yields: 102 | Pairs of the batched data, each a matrix of shape [batch_size, num_steps]. 103 | The second element of the tuple is the same data time-shifted to the 104 | right by one. 105 | 106 | Raises: 107 | ValueError: if batch_size or num_steps are too high. 108 | """ 109 | raw_data = np.array(raw_data, dtype=np.int32) 110 | 111 | data_len = len(raw_data) 112 | batch_len = data_len // batch_size 113 | data = np.zeros([batch_size, batch_len], dtype=np.int32) 114 | for i in range(batch_size): 115 | data[i] = raw_data[batch_len * i:batch_len * (i + 1)] 116 | 117 | epoch_size = (batch_len - 1) // num_steps 118 | 119 | if epoch_size == 0: 120 | raise ValueError("epoch_size == 0, decrease batch_size or num_steps") 121 | 122 | for i in range(epoch_size): 123 | x = data[:, i*num_steps:(i+1)*num_steps] 124 | y = data[:, i*num_steps+1:(i+1)*num_steps+1] 125 | yield (x, y) 126 | -------------------------------------------------------------------------------- /results/cifg.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "cifg@0.0.1-14","14","cifg@0.0.1","4.56571387881","cifg@0.0.1" 3 | "cifg@0.0.1-0","0","cifg@0.0.1","5.35973400298","cifg@0.0.1" 4 | "cifg@0.0.1-11","11","cifg@0.0.1","4.59295570841","cifg@0.0.1" 5 | "cifg@0.0.1-27","27","cifg@0.0.1","4.54497026301","cifg@0.0.1" 6 | "cifg@0.0.1-3","3","cifg@0.0.1","4.81261358352","cifg@0.0.1" 7 | "cifg@0.0.1-17","17","cifg@0.0.1","4.55408622067","cifg@0.0.1" 8 | "cifg@0.0.1-2","2","cifg@0.0.1","4.90735231153","cifg@0.0.1" 9 | "cifg@0.0.1-35","35","cifg@0.0.1","4.55179084622","cifg@0.0.1" 10 | "cifg@0.0.1-7","7","cifg@0.0.1","4.64509129946","cifg@0.0.1" 11 | "cifg@0.0.1-39","39","cifg@0.0.1","4.54742722284","cifg@0.0.1" 12 | "cifg@0.0.1-4","4","cifg@0.0.1","4.75222915338","cifg@0.0.1" 13 | "cifg@0.0.1-8","8","cifg@0.0.1","4.62649529074","cifg@0.0.1" 14 | "cifg@0.0.1-24","24","cifg@0.0.1","4.54727769086","cifg@0.0.1" 15 | "cifg@0.0.1-36","36","cifg@0.0.1","4.54584719288","cifg@0.0.1" 16 | "cifg@0.0.1-16","16","cifg@0.0.1","4.56260580673","cifg@0.0.1" 17 | "cifg@0.0.1-34","34","cifg@0.0.1","4.54168040996","cifg@0.0.1" 18 | "cifg@0.0.1-12","12","cifg@0.0.1","4.59481569459","cifg@0.0.1" 19 | "cifg@0.0.1-37","37","cifg@0.0.1","4.54339334293","cifg@0.0.1" 20 | "cifg@0.0.1-38","38","cifg@0.0.1","4.54453285269","cifg@0.0.1" 21 | "cifg@0.0.1-23","23","cifg@0.0.1","4.54897202265","cifg@0.0.1" 22 | "cifg@0.0.1-28","28","cifg@0.0.1","4.54159441682","cifg@0.0.1" 23 | "cifg@0.0.1-5","5","cifg@0.0.1","4.70623467478","cifg@0.0.1" 24 | "cifg@0.0.1-19","19","cifg@0.0.1","4.55321266641","cifg@0.0.1" 25 | "cifg@0.0.1-26","26","cifg@0.0.1","4.54528385085","cifg@0.0.1" 26 | "cifg@0.0.1-33","33","cifg@0.0.1","4.54197307561","cifg@0.0.1" 27 | "cifg@0.0.1-30","30","cifg@0.0.1","4.54992367284","cifg@0.0.1" 28 | "cifg@0.0.1-25","25","cifg@0.0.1","4.54612050582","cifg@0.0.1" 29 | "cifg@0.0.1-9","9","cifg@0.0.1","4.6119695173","cifg@0.0.1" 30 | "cifg@0.0.1-13","13","cifg@0.0.1","4.5792891771","cifg@0.0.1" 31 | "cifg@0.0.1-1","1","cifg@0.0.1","5.0519930968","cifg@0.0.1" 32 | "cifg@0.0.1-32","32","cifg@0.0.1","4.54441199478","cifg@0.0.1" 33 | "cifg@0.0.1-29","29","cifg@0.0.1","4.54200395857","cifg@0.0.1" 34 | "cifg@0.0.1-15","15","cifg@0.0.1","4.57298601008","cifg@0.0.1" 35 | "cifg@0.0.1-21","21","cifg@0.0.1","4.54161292251","cifg@0.0.1" 36 | "cifg@0.0.1-20","20","cifg@0.0.1","4.55281201421","cifg@0.0.1" 37 | "cifg@0.0.1-10","10","cifg@0.0.1","4.6006144787","cifg@0.0.1" 38 | "cifg@0.0.1-6","6","cifg@0.0.1","4.66987341226","cifg@0.0.1" 39 | "cifg@0.0.1-22","22","cifg@0.0.1","4.55058253282","cifg@0.0.1" 40 | "cifg@0.0.1-31","31","cifg@0.0.1","4.54762794962","cifg@0.0.1" 41 | "cifg@0.0.1-18","18","cifg@0.0.1","4.55826695864","cifg@0.0.1" 42 | -------------------------------------------------------------------------------- /results/fgr.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "fgr@0.0.1-9","9","fgr@0.0.1","5.57419463489","fgr@0.0.1" 3 | "fgr@0.0.1-26","26","fgr@0.0.1","5.10573539059","fgr@0.0.1" 4 | "fgr@0.0.1-15","15","fgr@0.0.1","5.24929324195","fgr@0.0.1" 5 | "fgr@0.0.1-32","32","fgr@0.0.1","5.06292984943","fgr@0.0.1" 6 | "fgr@0.0.1-25","25","fgr@0.0.1","5.11771704849","fgr@0.0.1" 7 | "fgr@0.0.1-29","29","fgr@0.0.1","5.08112738992","fgr@0.0.1" 8 | "fgr@0.0.1-19","19","fgr@0.0.1","5.17304012376","fgr@0.0.1" 9 | "fgr@0.0.1-4","4","fgr@0.0.1","7.43091932232","fgr@0.0.1" 10 | "fgr@0.0.1-7","7","fgr@0.0.1","5.92115276726","fgr@0.0.1" 11 | "fgr@0.0.1-34","34","fgr@0.0.1","5.06310319758","fgr@0.0.1" 12 | "fgr@0.0.1-16","16","fgr@0.0.1","5.23218988328","fgr@0.0.1" 13 | "fgr@0.0.1-35","35","fgr@0.0.1","5.04446169483","fgr@0.0.1" 14 | "fgr@0.0.1-14","14","fgr@0.0.1","5.26833354094","fgr@0.0.1" 15 | "fgr@0.0.1-27","27","fgr@0.0.1","5.09842462034","fgr@0.0.1" 16 | "fgr@0.0.1-18","18","fgr@0.0.1","5.20137509716","fgr@0.0.1" 17 | "fgr@0.0.1-13","13","fgr@0.0.1","5.33838695293","fgr@0.0.1" 18 | "fgr@0.0.1-38","38","fgr@0.0.1","5.04695339488","fgr@0.0.1" 19 | "fgr@0.0.1-20","20","fgr@0.0.1","5.15817199707","fgr@0.0.1" 20 | "fgr@0.0.1-37","37","fgr@0.0.1","5.04786122017","fgr@0.0.1" 21 | "fgr@0.0.1-8","8","fgr@0.0.1","5.68872257985","fgr@0.0.1" 22 | "fgr@0.0.1-0","0","fgr@0.0.1","6.8848703605","fgr@0.0.1" 23 | "fgr@0.0.1-33","33","fgr@0.0.1","5.05569784774","fgr@0.0.1" 24 | "fgr@0.0.1-23","23","fgr@0.0.1","5.13172213807","fgr@0.0.1" 25 | "fgr@0.0.1-11","11","fgr@0.0.1","5.38992589003","fgr@0.0.1" 26 | "fgr@0.0.1-5","5","fgr@0.0.1","6.69804911296","fgr@0.0.1" 27 | "fgr@0.0.1-30","30","fgr@0.0.1","5.07568364773","fgr@0.0.1" 28 | "fgr@0.0.1-31","31","fgr@0.0.1","5.10411486567","fgr@0.0.1" 29 | "fgr@0.0.1-28","28","fgr@0.0.1","5.08672402726","fgr@0.0.1" 30 | "fgr@0.0.1-36","36","fgr@0.0.1","5.04566630461","fgr@0.0.1" 31 | "fgr@0.0.1-2","2","fgr@0.0.1","6.77391976908","fgr@0.0.1" 32 | "fgr@0.0.1-17","17","fgr@0.0.1","5.20267718464","fgr@0.0.1" 33 | "fgr@0.0.1-22","22","fgr@0.0.1","5.14186635128","fgr@0.0.1" 34 | "fgr@0.0.1-12","12","fgr@0.0.1","5.3474358342","fgr@0.0.1" 35 | "fgr@0.0.1-21","21","fgr@0.0.1","5.14458252368","fgr@0.0.1" 36 | "fgr@0.0.1-39","39","fgr@0.0.1","5.15934032207","fgr@0.0.1" 37 | "fgr@0.0.1-10","10","fgr@0.0.1","5.45000143246","fgr@0.0.1" 38 | "fgr@0.0.1-6","6","fgr@0.0.1","6.3777493473","fgr@0.0.1" 39 | "fgr@0.0.1-3","3","fgr@0.0.1","6.645218182","fgr@0.0.1" 40 | "fgr@0.0.1-1","1","fgr@0.0.1","6.82028494283","fgr@0.0.1" 41 | "fgr@0.0.1-24","24","fgr@0.0.1","5.11634295613","fgr@0.0.1" 42 | -------------------------------------------------------------------------------- /results/nfg.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "nfg@0.0.1-33","33","nfg@0.0.1","6.73697249017","nfg@0.0.1" 3 | "nfg@0.0.1-2","2","nfg@0.0.1","10.7772650271","nfg@0.0.1" 4 | "nfg@0.0.1-15","15","nfg@0.0.1","7.05052340345","nfg@0.0.1" 5 | "nfg@0.0.1-37","37","nfg@0.0.1","6.57318869247","nfg@0.0.1" 6 | "nfg@0.0.1-7","7","nfg@0.0.1","7.58502580085","nfg@0.0.1" 7 | "nfg@0.0.1-30","30","nfg@0.0.1","6.9465462115","nfg@0.0.1" 8 | "nfg@0.0.1-36","36","nfg@0.0.1","6.55495053246","nfg@0.0.1" 9 | "nfg@0.0.1-12","12","nfg@0.0.1","7.27562576398","nfg@0.0.1" 10 | "nfg@0.0.1-31","31","nfg@0.0.1","6.65674984056","nfg@0.0.1" 11 | "nfg@0.0.1-26","26","nfg@0.0.1","6.5590266678","nfg@0.0.1" 12 | "nfg@0.0.1-14","14","nfg@0.0.1","6.92289278407","nfg@0.0.1" 13 | "nfg@0.0.1-20","20","nfg@0.0.1","6.96981757547","nfg@0.0.1" 14 | "nfg@0.0.1-10","10","nfg@0.0.1","7.34191184115","nfg@0.0.1" 15 | "nfg@0.0.1-22","22","nfg@0.0.1","6.6930575001","nfg@0.0.1" 16 | "nfg@0.0.1-27","27","nfg@0.0.1","6.83511791826","nfg@0.0.1" 17 | "nfg@0.0.1-1","1","nfg@0.0.1","9.60636258703","nfg@0.0.1" 18 | "nfg@0.0.1-34","34","nfg@0.0.1","6.68265999515","nfg@0.0.1" 19 | "nfg@0.0.1-25","25","nfg@0.0.1","6.81897217627","nfg@0.0.1" 20 | "nfg@0.0.1-38","38","nfg@0.0.1","6.56490313679","nfg@0.0.1" 21 | "nfg@0.0.1-32","32","nfg@0.0.1","7.29989512333","nfg@0.0.1" 22 | "nfg@0.0.1-35","35","nfg@0.0.1","6.89666105309","nfg@0.0.1" 23 | "nfg@0.0.1-19","19","nfg@0.0.1","7.19394393402","nfg@0.0.1" 24 | "nfg@0.0.1-21","21","nfg@0.0.1","7.78741690084","nfg@0.0.1" 25 | "nfg@0.0.1-24","24","nfg@0.0.1","6.83581714033","nfg@0.0.1" 26 | "nfg@0.0.1-5","5","nfg@0.0.1","8.4004106545","nfg@0.0.1" 27 | "nfg@0.0.1-6","6","nfg@0.0.1","8.76982587957","nfg@0.0.1" 28 | "nfg@0.0.1-9","9","nfg@0.0.1","7.78570335751","nfg@0.0.1" 29 | "nfg@0.0.1-8","8","nfg@0.0.1","7.92139685806","nfg@0.0.1" 30 | "nfg@0.0.1-28","28","nfg@0.0.1","6.80814289534","nfg@0.0.1" 31 | "nfg@0.0.1-18","18","nfg@0.0.1","6.91271603124","nfg@0.0.1" 32 | "nfg@0.0.1-0","0","nfg@0.0.1","8.51652220019","nfg@0.0.1" 33 | "nfg@0.0.1-13","13","nfg@0.0.1","8.22332325215","nfg@0.0.1" 34 | "nfg@0.0.1-3","3","nfg@0.0.1","8.3267098397","nfg@0.0.1" 35 | "nfg@0.0.1-39","39","nfg@0.0.1","6.92131826388","nfg@0.0.1" 36 | "nfg@0.0.1-17","17","nfg@0.0.1","6.65563921662","nfg@0.0.1" 37 | "nfg@0.0.1-29","29","nfg@0.0.1","6.84448328135","nfg@0.0.1" 38 | "nfg@0.0.1-16","16","nfg@0.0.1","6.94809958114","nfg@0.0.1" 39 | "nfg@0.0.1-4","4","nfg@0.0.1","8.486267804","nfg@0.0.1" 40 | "nfg@0.0.1-23","23","nfg@0.0.1","6.88019028437","nfg@0.0.1" 41 | "nfg@0.0.1-11","11","nfg@0.0.1","7.82899129232","nfg@0.0.1" 42 | -------------------------------------------------------------------------------- /results/niaf.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "niaf@0.0.1-18","18","niaf@0.0.1","6.09870429837","niaf@0.0.1" 3 | "niaf@0.0.1-34","34","niaf@0.0.1","6.49889642625","niaf@0.0.1" 4 | "niaf@0.0.1-0","0","niaf@0.0.1","5.38259327999","niaf@0.0.1" 5 | "niaf@0.0.1-39","39","niaf@0.0.1","6.46301138742","niaf@0.0.1" 6 | "niaf@0.0.1-29","29","niaf@0.0.1","6.54324906745","niaf@0.0.1" 7 | "niaf@0.0.1-26","26","niaf@0.0.1","6.60043739786","niaf@0.0.1" 8 | "niaf@0.0.1-21","21","niaf@0.0.1","7.80527049785","niaf@0.0.1" 9 | "niaf@0.0.1-3","3","niaf@0.0.1","4.93202389256","niaf@0.0.1" 10 | "niaf@0.0.1-33","33","niaf@0.0.1","6.25475331832","niaf@0.0.1" 11 | "niaf@0.0.1-37","37","niaf@0.0.1","6.38993363361","niaf@0.0.1" 12 | "niaf@0.0.1-5","5","niaf@0.0.1","4.88481721839","niaf@0.0.1" 13 | "niaf@0.0.1-20","20","niaf@0.0.1","6.51050514273","niaf@0.0.1" 14 | "niaf@0.0.1-7","7","niaf@0.0.1","4.926224905","niaf@0.0.1" 15 | "niaf@0.0.1-19","19","niaf@0.0.1","6.6580266296","niaf@0.0.1" 16 | "niaf@0.0.1-6","6","niaf@0.0.1","4.93154007477","niaf@0.0.1" 17 | "niaf@0.0.1-17","17","niaf@0.0.1","6.44447528865","niaf@0.0.1" 18 | "niaf@0.0.1-22","22","niaf@0.0.1","6.67975375013","niaf@0.0.1" 19 | "niaf@0.0.1-23","23","niaf@0.0.1","6.74290647338","niaf@0.0.1" 20 | "niaf@0.0.1-35","35","niaf@0.0.1","6.33469659714","niaf@0.0.1" 21 | "niaf@0.0.1-10","10","niaf@0.0.1","5.1298927276","niaf@0.0.1" 22 | "niaf@0.0.1-25","25","niaf@0.0.1","6.64688618147","niaf@0.0.1" 23 | "niaf@0.0.1-30","30","niaf@0.0.1","6.40274289501","niaf@0.0.1" 24 | "niaf@0.0.1-8","8","niaf@0.0.1","5.01341147909","niaf@0.0.1" 25 | "niaf@0.0.1-12","12","niaf@0.0.1","5.37862656366","niaf@0.0.1" 26 | "niaf@0.0.1-31","31","niaf@0.0.1","6.32741337575","niaf@0.0.1" 27 | "niaf@0.0.1-1","1","niaf@0.0.1","5.12564350985","niaf@0.0.1" 28 | "niaf@0.0.1-11","11","niaf@0.0.1","5.24035681485","niaf@0.0.1" 29 | "niaf@0.0.1-24","24","niaf@0.0.1","6.81309428312","niaf@0.0.1" 30 | "niaf@0.0.1-27","27","niaf@0.0.1","6.7691514494","niaf@0.0.1" 31 | "niaf@0.0.1-9","9","niaf@0.0.1","5.06914311623","niaf@0.0.1" 32 | "niaf@0.0.1-38","38","niaf@0.0.1","6.54618267449","niaf@0.0.1" 33 | "niaf@0.0.1-28","28","niaf@0.0.1","6.27318023474","niaf@0.0.1" 34 | "niaf@0.0.1-14","14","niaf@0.0.1","6.14624147584","niaf@0.0.1" 35 | "niaf@0.0.1-16","16","niaf@0.0.1","7.13847035518","niaf@0.0.1" 36 | "niaf@0.0.1-32","32","niaf@0.0.1","6.68451376986","niaf@0.0.1" 37 | "niaf@0.0.1-4","4","niaf@0.0.1","4.90387949003","niaf@0.0.1" 38 | "niaf@0.0.1-2","2","niaf@0.0.1","5.04349403848","niaf@0.0.1" 39 | "niaf@0.0.1-15","15","niaf@0.0.1","6.75808747791","niaf@0.0.1" 40 | "niaf@0.0.1-36","36","niaf@0.0.1","6.34581575355","niaf@0.0.1" 41 | "niaf@0.0.1-13","13","niaf@0.0.1","5.77299887313","niaf@0.0.1" 42 | -------------------------------------------------------------------------------- /results/nig.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "nig@0.0.1-4","4","nig@0.0.1","5.0634400286","nig@0.0.1" 3 | "nig@0.0.1-28","28","nig@0.0.1","5.11901716375","nig@0.0.1" 4 | "nig@0.0.1-24","24","nig@0.0.1","4.96274283273","nig@0.0.1" 5 | "nig@0.0.1-27","27","nig@0.0.1","4.95690269418","nig@0.0.1" 6 | "nig@0.0.1-10","10","nig@0.0.1","4.99553157469","nig@0.0.1" 7 | "nig@0.0.1-20","20","nig@0.0.1","5.00766572083","nig@0.0.1" 8 | "nig@0.0.1-33","33","nig@0.0.1","4.97766051","nig@0.0.1" 9 | "nig@0.0.1-5","5","nig@0.0.1","5.03391128229","nig@0.0.1" 10 | "nig@0.0.1-9","9","nig@0.0.1","4.97509119981","nig@0.0.1" 11 | "nig@0.0.1-11","11","nig@0.0.1","4.95187664836","nig@0.0.1" 12 | "nig@0.0.1-14","14","nig@0.0.1","4.95656396983","nig@0.0.1" 13 | "nig@0.0.1-39","39","nig@0.0.1","5.04389540899","nig@0.0.1" 14 | "nig@0.0.1-2","2","nig@0.0.1","5.17046821854","nig@0.0.1" 15 | "nig@0.0.1-7","7","nig@0.0.1","4.98839857037","nig@0.0.1" 16 | "nig@0.0.1-37","37","nig@0.0.1","4.99414971799","nig@0.0.1" 17 | "nig@0.0.1-31","31","nig@0.0.1","4.95984494579","nig@0.0.1" 18 | "nig@0.0.1-19","19","nig@0.0.1","4.97275236584","nig@0.0.1" 19 | "nig@0.0.1-17","17","nig@0.0.1","4.92884244543","nig@0.0.1" 20 | "nig@0.0.1-25","25","nig@0.0.1","4.99081471969","nig@0.0.1" 21 | "nig@0.0.1-26","26","nig@0.0.1","4.98007538465","nig@0.0.1" 22 | "nig@0.0.1-16","16","nig@0.0.1","4.99646701942","nig@0.0.1" 23 | "nig@0.0.1-8","8","nig@0.0.1","4.96714377475","nig@0.0.1" 24 | "nig@0.0.1-0","0","nig@0.0.1","5.84728838655","nig@0.0.1" 25 | "nig@0.0.1-3","3","nig@0.0.1","5.07785130092","nig@0.0.1" 26 | "nig@0.0.1-35","35","nig@0.0.1","5.0274908987","nig@0.0.1" 27 | "nig@0.0.1-30","30","nig@0.0.1","4.98458433398","nig@0.0.1" 28 | "nig@0.0.1-12","12","nig@0.0.1","4.98306028197","nig@0.0.1" 29 | "nig@0.0.1-22","22","nig@0.0.1","5.02418405546","nig@0.0.1" 30 | "nig@0.0.1-38","38","nig@0.0.1","5.03905231087","nig@0.0.1" 31 | "nig@0.0.1-34","34","nig@0.0.1","4.96260324491","nig@0.0.1" 32 | "nig@0.0.1-29","29","nig@0.0.1","4.99007783436","nig@0.0.1" 33 | "nig@0.0.1-15","15","nig@0.0.1","4.96798115218","nig@0.0.1" 34 | "nig@0.0.1-18","18","nig@0.0.1","4.96754955889","nig@0.0.1" 35 | "nig@0.0.1-21","21","nig@0.0.1","4.95532925353","nig@0.0.1" 36 | "nig@0.0.1-36","36","nig@0.0.1","5.00145846205","nig@0.0.1" 37 | "nig@0.0.1-32","32","nig@0.0.1","4.94433695475","nig@0.0.1" 38 | "nig@0.0.1-6","6","nig@0.0.1","4.99505435035","nig@0.0.1" 39 | "nig@0.0.1-1","1","nig@0.0.1","5.31560859109","nig@0.0.1" 40 | "nig@0.0.1-13","13","nig@0.0.1","4.93833229117","nig@0.0.1" 41 | "nig@0.0.1-23","23","nig@0.0.1","5.03039055857","nig@0.0.1" 42 | -------------------------------------------------------------------------------- /results/noaf.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "noaf@0.0.1-37","37","noaf@0.0.1","4290.20021817","noaf@0.0.1" 3 | "noaf@0.0.1-10","10","noaf@0.0.1","279.870795766","noaf@0.0.1" 4 | "noaf@0.0.1-27","27","noaf@0.0.1","4118.30538312","noaf@0.0.1" 5 | "noaf@0.0.1-8","8","noaf@0.0.1","1898.05338575","noaf@0.0.1" 6 | "noaf@0.0.1-29","29","noaf@0.0.1","9.43916148491","noaf@0.0.1" 7 | "noaf@0.0.1-33","33","noaf@0.0.1","9.61957276221","noaf@0.0.1" 8 | "noaf@0.0.1-39","39","noaf@0.0.1","6.65304548406","noaf@0.0.1" 9 | "noaf@0.0.1-2","2","noaf@0.0.1","550.44563768","noaf@0.0.1" 10 | "noaf@0.0.1-11","11","noaf@0.0.1","7.83392040019","noaf@0.0.1" 11 | "noaf@0.0.1-32","32","noaf@0.0.1","7.7068826813","noaf@0.0.1" 12 | "noaf@0.0.1-13","13","noaf@0.0.1","4211.0994596","noaf@0.0.1" 13 | "noaf@0.0.1-3","3","noaf@0.0.1","554.386627978","noaf@0.0.1" 14 | "noaf@0.0.1-36","36","noaf@0.0.1","6.9200514024","noaf@0.0.1" 15 | "noaf@0.0.1-24","24","noaf@0.0.1","7.64599560381","noaf@0.0.1" 16 | "noaf@0.0.1-34","34","noaf@0.0.1","9.98013323933","noaf@0.0.1" 17 | "noaf@0.0.1-22","22","noaf@0.0.1","3322.59726184","noaf@0.0.1" 18 | "noaf@0.0.1-26","26","noaf@0.0.1","3692.11154064","noaf@0.0.1" 19 | "noaf@0.0.1-9","9","noaf@0.0.1","1898.75770974","noaf@0.0.1" 20 | "noaf@0.0.1-20","20","noaf@0.0.1","4369.45756271","noaf@0.0.1" 21 | "noaf@0.0.1-30","30","noaf@0.0.1","8.33805747934","noaf@0.0.1" 22 | "noaf@0.0.1-0","0","noaf@0.0.1","529.017566043","noaf@0.0.1" 23 | "noaf@0.0.1-17","17","noaf@0.0.1","7.15123054349","noaf@0.0.1" 24 | "noaf@0.0.1-6","6","noaf@0.0.1","3264.55570143","noaf@0.0.1" 25 | "noaf@0.0.1-18","18","noaf@0.0.1","3883.66015867","noaf@0.0.1" 26 | "noaf@0.0.1-38","38","noaf@0.0.1","6.7566742317","noaf@0.0.1" 27 | "noaf@0.0.1-19","19","noaf@0.0.1","4766.62075182","noaf@0.0.1" 28 | "noaf@0.0.1-5","5","noaf@0.0.1","938.562468644","noaf@0.0.1" 29 | "noaf@0.0.1-31","31","noaf@0.0.1","11.09778929","noaf@0.0.1" 30 | "noaf@0.0.1-14","14","noaf@0.0.1","652.125144325","noaf@0.0.1" 31 | "noaf@0.0.1-1","1","noaf@0.0.1","425.874478776","noaf@0.0.1" 32 | "noaf@0.0.1-16","16","noaf@0.0.1","4546.26859365","noaf@0.0.1" 33 | "noaf@0.0.1-4","4","noaf@0.0.1","932.856996722","noaf@0.0.1" 34 | "noaf@0.0.1-7","7","noaf@0.0.1","2044.94882338","noaf@0.0.1" 35 | "noaf@0.0.1-35","35","noaf@0.0.1","7.59317197631","noaf@0.0.1" 36 | "noaf@0.0.1-28","28","noaf@0.0.1","13.8777147939","noaf@0.0.1" 37 | "noaf@0.0.1-12","12","noaf@0.0.1","3771.35240264","noaf@0.0.1" 38 | "noaf@0.0.1-25","25","noaf@0.0.1","9.08382430511","noaf@0.0.1" 39 | "noaf@0.0.1-21","21","noaf@0.0.1","7.49697055946","noaf@0.0.1" 40 | "noaf@0.0.1-15","15","noaf@0.0.1","8.64484022906","noaf@0.0.1" 41 | "noaf@0.0.1-23","23","noaf@0.0.1","7.01312223473","noaf@0.0.1" 42 | -------------------------------------------------------------------------------- /results/nog.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "nog@0.0.1-37","37","nog@0.0.1","7.412956298","nog@0.0.1" 3 | "nog@0.0.1-34","34","nog@0.0.1","6.99996446674","nog@0.0.1" 4 | "nog@0.0.1-30","30","nog@0.0.1","7.133992567","nog@0.0.1" 5 | "nog@0.0.1-25","25","nog@0.0.1","7.16936312513","nog@0.0.1" 6 | "nog@0.0.1-17","17","nog@0.0.1","7.19486801044","nog@0.0.1" 7 | "nog@0.0.1-11","11","nog@0.0.1","6.80988537432","nog@0.0.1" 8 | "nog@0.0.1-0","0","nog@0.0.1","7.3894141663","nog@0.0.1" 9 | "nog@0.0.1-36","36","nog@0.0.1","7.53731693605","nog@0.0.1" 10 | "nog@0.0.1-23","23","nog@0.0.1","6.96933742393","nog@0.0.1" 11 | "nog@0.0.1-5","5","nog@0.0.1","6.78902139054","nog@0.0.1" 12 | "nog@0.0.1-6","6","nog@0.0.1","6.86653206961","nog@0.0.1" 13 | "nog@0.0.1-13","13","nog@0.0.1","6.80426534017","nog@0.0.1" 14 | "nog@0.0.1-20","20","nog@0.0.1","7.29606951032","nog@0.0.1" 15 | "nog@0.0.1-39","39","nog@0.0.1","7.86915457589","nog@0.0.1" 16 | "nog@0.0.1-1","1","nog@0.0.1","7.18026752083","nog@0.0.1" 17 | "nog@0.0.1-27","27","nog@0.0.1","6.99274124665","nog@0.0.1" 18 | "nog@0.0.1-12","12","nog@0.0.1","6.88194942137","nog@0.0.1" 19 | "nog@0.0.1-29","29","nog@0.0.1","6.84864648936","nog@0.0.1" 20 | "nog@0.0.1-22","22","nog@0.0.1","6.90418495645","nog@0.0.1" 21 | "nog@0.0.1-21","21","nog@0.0.1","6.90020912638","nog@0.0.1" 22 | "nog@0.0.1-14","14","nog@0.0.1","6.99287436972","nog@0.0.1" 23 | "nog@0.0.1-8","8","nog@0.0.1","7.18486089382","nog@0.0.1" 24 | "nog@0.0.1-38","38","nog@0.0.1","6.99698644106","nog@0.0.1" 25 | "nog@0.0.1-15","15","nog@0.0.1","6.9204078311","nog@0.0.1" 26 | "nog@0.0.1-4","4","nog@0.0.1","6.84255480293","nog@0.0.1" 27 | "nog@0.0.1-24","24","nog@0.0.1","6.88128912893","nog@0.0.1" 28 | "nog@0.0.1-7","7","nog@0.0.1","6.91971702109","nog@0.0.1" 29 | "nog@0.0.1-33","33","nog@0.0.1","6.8841088452","nog@0.0.1" 30 | "nog@0.0.1-26","26","nog@0.0.1","7.01705987424","nog@0.0.1" 31 | "nog@0.0.1-28","28","nog@0.0.1","7.25507385669","nog@0.0.1" 32 | "nog@0.0.1-10","10","nog@0.0.1","6.86224379351","nog@0.0.1" 33 | "nog@0.0.1-18","18","nog@0.0.1","7.5729258427","nog@0.0.1" 34 | "nog@0.0.1-9","9","nog@0.0.1","7.38669598015","nog@0.0.1" 35 | "nog@0.0.1-19","19","nog@0.0.1","6.96028349792","nog@0.0.1" 36 | "nog@0.0.1-31","31","nog@0.0.1","7.4201335715","nog@0.0.1" 37 | "nog@0.0.1-16","16","nog@0.0.1","7.48384874824","nog@0.0.1" 38 | "nog@0.0.1-32","32","nog@0.0.1","6.9520984468","nog@0.0.1" 39 | "nog@0.0.1-35","35","nog@0.0.1","7.25752189377","nog@0.0.1" 40 | "nog@0.0.1-3","3","nog@0.0.1","6.83531997369","nog@0.0.1" 41 | "nog@0.0.1-2","2","nog@0.0.1","7.26428808926","nog@0.0.1" 42 | -------------------------------------------------------------------------------- /results/np.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "nph@0.0.1-33","33","nph@0.0.1","4.53145505736","nph@0.0.1" 3 | "nph@0.0.1-6","6","nph@0.0.1","4.65676885696","nph@0.0.1" 4 | "nph@0.0.1-7","7","nph@0.0.1","4.63731491815","nph@0.0.1" 5 | "nph@0.0.1-8","8","nph@0.0.1","4.60844982744","nph@0.0.1" 6 | "nph@0.0.1-1","1","nph@0.0.1","5.05092327092","nph@0.0.1" 7 | "nph@0.0.1-13","13","nph@0.0.1","4.56014743935","nph@0.0.1" 8 | "nph@0.0.1-34","34","nph@0.0.1","4.53155367274","nph@0.0.1" 9 | "nph@0.0.1-4","4","nph@0.0.1","4.74217485285","nph@0.0.1" 10 | "nph@0.0.1-21","21","nph@0.0.1","4.53244125678","nph@0.0.1" 11 | "nph@0.0.1-25","25","nph@0.0.1","4.53018298921","nph@0.0.1" 12 | "nph@0.0.1-18","18","nph@0.0.1","4.54054730681","nph@0.0.1" 13 | "nph@0.0.1-11","11","nph@0.0.1","4.57620711346","nph@0.0.1" 14 | "nph@0.0.1-38","38","nph@0.0.1","4.53817342953","nph@0.0.1" 15 | "nph@0.0.1-2","2","nph@0.0.1","4.89957033741","nph@0.0.1" 16 | "nph@0.0.1-20","20","nph@0.0.1","4.52846144903","nph@0.0.1" 17 | "nph@0.0.1-3","3","nph@0.0.1","4.80691271724","nph@0.0.1" 18 | "nph@0.0.1-9","9","nph@0.0.1","4.59063655931","nph@0.0.1" 19 | "nph@0.0.1-17","17","nph@0.0.1","4.53789104851","nph@0.0.1" 20 | "nph@0.0.1-16","16","nph@0.0.1","4.54482488308","nph@0.0.1" 21 | "nph@0.0.1-14","14","nph@0.0.1","4.55338537099","nph@0.0.1" 22 | "nph@0.0.1-37","37","nph@0.0.1","4.52955599207","nph@0.0.1" 23 | "nph@0.0.1-22","22","nph@0.0.1","4.53486252558","nph@0.0.1" 24 | "nph@0.0.1-12","12","nph@0.0.1","4.56727293676","nph@0.0.1" 25 | "nph@0.0.1-35","35","nph@0.0.1","4.53245268737","nph@0.0.1" 26 | "nph@0.0.1-31","31","nph@0.0.1","4.53206993882","nph@0.0.1" 27 | "nph@0.0.1-29","29","nph@0.0.1","4.52804873927","nph@0.0.1" 28 | "nph@0.0.1-26","26","nph@0.0.1","4.52728411824","nph@0.0.1" 29 | "nph@0.0.1-19","19","nph@0.0.1","4.53052622685","nph@0.0.1" 30 | "nph@0.0.1-36","36","nph@0.0.1","4.52756872476","nph@0.0.1" 31 | "nph@0.0.1-24","24","nph@0.0.1","4.529532861","nph@0.0.1" 32 | "nph@0.0.1-15","15","nph@0.0.1","4.55088257666","nph@0.0.1" 33 | "nph@0.0.1-10","10","nph@0.0.1","4.58736787757","nph@0.0.1" 34 | "nph@0.0.1-27","27","nph@0.0.1","4.52858718976","nph@0.0.1" 35 | "nph@0.0.1-23","23","nph@0.0.1","4.52951108764","nph@0.0.1" 36 | "nph@0.0.1-32","32","nph@0.0.1","4.53280504292","nph@0.0.1" 37 | "nph@0.0.1-0","0","nph@0.0.1","5.33801592976","nph@0.0.1" 38 | "nph@0.0.1-28","28","nph@0.0.1","4.53878545852","nph@0.0.1" 39 | "nph@0.0.1-5","5","nph@0.0.1","4.70111912319","nph@0.0.1" 40 | "nph@0.0.1-30","30","nph@0.0.1","4.52984692217","nph@0.0.1" 41 | -------------------------------------------------------------------------------- /results/sessions.csv: -------------------------------------------------------------------------------- 1 | "SessionId (S)","CompletedEpochs (N)","EndTime (S)","GlobalStep (N)","ModelId (S)","ShouldStop (BOOL)","StartTime (S)","State (S)","TotalEpochs (N)" 2 | "cifg@0.0.1","39","2016-01-24T02:26:55.088454","39","cifg@0.0.1","false","2016-01-23T05:45:42.758995","completed","40" 3 | "vanilla@0.0.1","39","2016-01-24T05:07:10.998134","39","vanilla@0.0.1","false","2016-01-22T22:06:11.263869","completed","40" 4 | "nig@0.0.1","39","2016-01-24T02:49:44.412907","39","nig@0.0.1","false","2016-01-23T05:54:24.204514","completed","40" 5 | "nfg@0.0.1","39","2016-01-24T13:15:16.833713","39","nfg@0.0.1","false","2016-01-23T05:45:42.743418","completed","40" 6 | "nog@0.0.1","39","2016-01-24T02:28:13.992627","39","nog@0.0.1","false","2016-01-23T05:45:43.719520","completed","40" 7 | "highway@0.0.1","2000","2016-01-25T05:26:37.636236","1999","highway@0.0.1","false","2016-01-25T05:24:56.915390","completed","2000" 8 | "nph@0.0.1","0","true","0","nph@0.0.1","false","true","queued","40" 9 | "niaf@0.0.1","39","2016-01-24T17:09:26.359602","39","niaf@0.0.1","false","2016-01-23T14:39:13.485475","completed","40" 10 | "fgr@0.0.1","39","2016-01-24T21:21:04.786900","39","fgr@0.0.1","false","2016-01-23T05:45:44.896618","completed","40" 11 | "noaf@0.0.1","39","2016-01-24T05:20:28.676130","39","noaf@0.0.1","false","2016-01-23T05:46:19.740328","completed","40" 12 | -------------------------------------------------------------------------------- /results/vanilla.csv: -------------------------------------------------------------------------------- 1 | "CheckpointId (S)","GlobalStep (N)","ModelId (S)","Performance (N)","SessionId (S)" 2 | "vanilla@0.0.1-15","15","vanilla@0.0.1","4.5538766085","vanilla@0.0.1" 3 | "vanilla@0.0.1-10","10","vanilla@0.0.1","4.60219004521","vanilla@0.0.1" 4 | "vanilla@0.0.1-21","21","vanilla@0.0.1","4.5553238351","vanilla@0.0.1" 5 | "vanilla@0.0.1-35","35","vanilla@0.0.1","4.56556541806","vanilla@0.0.1" 6 | "vanilla@0.0.1-6","6","vanilla@0.0.1","4.61753465717","vanilla@0.0.1" 7 | "vanilla@0.0.1-23","23","vanilla@0.0.1","4.55214406954","vanilla@0.0.1" 8 | "vanilla@0.0.1-12","12","vanilla@0.0.1","4.58706867659","vanilla@0.0.1" 9 | "vanilla@0.0.1-1","1","vanilla@0.0.1","5.06126384294","vanilla@0.0.1" 10 | "vanilla@0.0.1-11","11","vanilla@0.0.1","4.59029340056","vanilla@0.0.1" 11 | "vanilla@0.0.1-38","38","vanilla@0.0.1","4.56242011038","vanilla@0.0.1" 12 | "vanilla@0.0.1-22","22","vanilla@0.0.1","4.54922481615","vanilla@0.0.1" 13 | "vanilla@0.0.1-16","16","vanilla@0.0.1","4.54919675658","vanilla@0.0.1" 14 | "vanilla@0.0.1-30","30","vanilla@0.0.1","4.53836640442","vanilla@0.0.1" 15 | "vanilla@0.0.1-34","34","vanilla@0.0.1","4.55237950747","vanilla@0.0.1" 16 | "vanilla@0.0.1-18","18","vanilla@0.0.1","4.55532773388","vanilla@0.0.1" 17 | "vanilla@0.0.1-32","32","vanilla@0.0.1","4.5517229934","vanilla@0.0.1" 18 | "vanilla@0.0.1-39","39","vanilla@0.0.1","4.56770337994","vanilla@0.0.1" 19 | "vanilla@0.0.1-36","36","vanilla@0.0.1","4.54791005245","vanilla@0.0.1" 20 | "vanilla@0.0.1-2","2","vanilla@0.0.1","4.89098669351","vanilla@0.0.1" 21 | "vanilla@0.0.1-31","31","vanilla@0.0.1","4.56443549539","vanilla@0.0.1" 22 | "vanilla@0.0.1-14","14","vanilla@0.0.1","4.56607342986","vanilla@0.0.1" 23 | "vanilla@0.0.1-4","4","vanilla@0.0.1","4.71028458161","vanilla@0.0.1" 24 | "vanilla@0.0.1-26","26","vanilla@0.0.1","4.56121046001","vanilla@0.0.1" 25 | "vanilla@0.0.1-0","0","vanilla@0.0.1","5.39483115683","vanilla@0.0.1" 26 | "vanilla@0.0.1-37","37","vanilla@0.0.1","4.55772523633","vanilla@0.0.1" 27 | "vanilla@0.0.1-19","19","vanilla@0.0.1","4.5450539393","vanilla@0.0.1" 28 | "vanilla@0.0.1-13","13","vanilla@0.0.1","4.56868618297","vanilla@0.0.1" 29 | "vanilla@0.0.1-5","5","vanilla@0.0.1","4.65656550609","vanilla@0.0.1" 30 | "vanilla@0.0.1-33","33","vanilla@0.0.1","4.55278505494","vanilla@0.0.1" 31 | "vanilla@0.0.1-27","27","vanilla@0.0.1","4.5479845444","vanilla@0.0.1" 32 | "vanilla@0.0.1-9","9","vanilla@0.0.1","4.5446551223","vanilla@0.0.1" 33 | "vanilla@0.0.1-3","3","vanilla@0.0.1","4.78097672858","vanilla@0.0.1" 34 | "vanilla@0.0.1-29","29","vanilla@0.0.1","4.55700397933","vanilla@0.0.1" 35 | "vanilla@0.0.1-7","7","vanilla@0.0.1","4.58915808022","vanilla@0.0.1" 36 | "vanilla@0.0.1-20","20","vanilla@0.0.1","4.55254584695","vanilla@0.0.1" 37 | "vanilla@0.0.1-25","25","vanilla@0.0.1","4.55099895867","vanilla@0.0.1" 38 | "vanilla@0.0.1-28","28","vanilla@0.0.1","4.55108744719","vanilla@0.0.1" 39 | "vanilla@0.0.1-17","17","vanilla@0.0.1","4.54722175442","vanilla@0.0.1" 40 | "vanilla@0.0.1-24","24","vanilla@0.0.1","4.54537134287","vanilla@0.0.1" 41 | "vanilla@0.0.1-8","8","vanilla@0.0.1","4.55814789026","vanilla@0.0.1" 42 | -------------------------------------------------------------------------------- /variants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fomorians/lstm-odyssey/ddd0a411f0ce1b9a60ad23febdd2582462d19100/variants/__init__.py -------------------------------------------------------------------------------- /variants/cifg.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class CIFGLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 32 | 33 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 34 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 35 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 36 | 37 | b_z = get_variable("b_z", [1, self._num_blocks]) 38 | b_i = get_variable("b_i", [1, self._num_blocks]) 39 | b_o = get_variable("b_o", [1, self._num_blocks]) 40 | 41 | p_i = get_variable("p_i", [self._num_blocks]) 42 | p_o = get_variable("p_o", [self._num_blocks]) 43 | 44 | g = h = tf.tanh 45 | 46 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 47 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i) 48 | f = 1 - i 49 | c = tf.mul(i, z) + tf.mul(f, c_prev) 50 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o) 51 | y = tf.mul(h(c), o) 52 | 53 | return y, tf.concat(1, [c, y]) 54 | -------------------------------------------------------------------------------- /variants/fgr.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class FGRLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 5 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev, i_prev, f_prev, o_prev = tf.split(1, 5, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 32 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 33 | 34 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 35 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 36 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 37 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 38 | 39 | R_ii = get_variable("R_ii", [self._num_blocks, self._num_blocks]) 40 | R_fi = get_variable("R_fi", [self._num_blocks, self._num_blocks]) 41 | R_oi = get_variable("R_oi", [self._num_blocks, self._num_blocks]) 42 | 43 | R_if = get_variable("R_if", [self._num_blocks, self._num_blocks]) 44 | R_ff = get_variable("R_ff", [self._num_blocks, self._num_blocks]) 45 | R_of = get_variable("R_of", [self._num_blocks, self._num_blocks]) 46 | 47 | R_io = get_variable("R_io", [self._num_blocks, self._num_blocks]) 48 | R_fo = get_variable("R_fo", [self._num_blocks, self._num_blocks]) 49 | R_oo = get_variable("R_oo", [self._num_blocks, self._num_blocks]) 50 | 51 | b_z = get_variable("b_z", [1, self._num_blocks]) 52 | b_i = get_variable("b_i", [1, self._num_blocks]) 53 | b_f = get_variable("b_f", [1, self._num_blocks]) 54 | b_o = get_variable("b_o", [1, self._num_blocks]) 55 | 56 | p_i = get_variable("p_i", [self._num_blocks]) 57 | p_f = get_variable("p_f", [self._num_blocks]) 58 | p_o = get_variable("p_o", [self._num_blocks]) 59 | 60 | g = h = tf.tanh 61 | 62 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 63 | 64 | i_bar = tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i + tf.matmul(i_prev, R_ii) + tf.matmul(f_prev, R_fi) + tf.matmul(o_prev, R_oi) 65 | i = tf.sigmoid(i_bar) 66 | 67 | f_bar = tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + tf.mul(c_prev, p_f) + b_f + tf.matmul(i_prev, R_if) + tf.matmul(f_prev, R_ff) + tf.matmul(o_prev, R_of) 68 | f = tf.sigmoid(f_bar) 69 | 70 | c = tf.mul(i, z) + tf.mul(f, c_prev) 71 | 72 | o_bar = tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o + tf.matmul(i_prev, R_io) + tf.matmul(f_prev, R_fo) + tf.matmul(o_prev, R_oo) 73 | o = tf.sigmoid(o_bar) 74 | 75 | y = tf.mul(h(c), o) 76 | 77 | return y, tf.concat(1, [c, y, i, f, o]) 78 | -------------------------------------------------------------------------------- /variants/nfg.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class NFGLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 32 | 33 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 34 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 35 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 36 | 37 | b_z = get_variable("b_z", [1, self._num_blocks]) 38 | b_i = get_variable("b_i", [1, self._num_blocks]) 39 | b_o = get_variable("b_o", [1, self._num_blocks]) 40 | 41 | p_i = get_variable("p_i", [self._num_blocks]) 42 | p_o = get_variable("p_o", [self._num_blocks]) 43 | 44 | g = h = tf.tanh 45 | 46 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 47 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i) 48 | f = 1.0 49 | c = tf.mul(i, z) + tf.mul(f, c_prev) 50 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o) 51 | y = tf.mul(h(c), o) 52 | 53 | return y, tf.concat(1, [c, y]) 54 | -------------------------------------------------------------------------------- /variants/niaf.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class NIAFLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 32 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 33 | 34 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 35 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 36 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 37 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 38 | 39 | b_z = get_variable("b_z", [1, self._num_blocks]) 40 | b_i = get_variable("b_i", [1, self._num_blocks]) 41 | b_f = get_variable("b_f", [1, self._num_blocks]) 42 | b_o = get_variable("b_o", [1, self._num_blocks]) 43 | 44 | p_i = get_variable("p_i", [self._num_blocks]) 45 | p_f = get_variable("p_f", [self._num_blocks]) 46 | p_o = get_variable("p_o", [self._num_blocks]) 47 | 48 | g = h = tf.tanh 49 | 50 | z = tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z 51 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i) 52 | f = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + tf.mul(c_prev, p_f) + b_f) 53 | c = tf.mul(i, z) + tf.mul(f, c_prev) 54 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o) 55 | y = tf.mul(h(c), o) 56 | 57 | return y, tf.concat(1, [c, y]) 58 | -------------------------------------------------------------------------------- /variants/nig.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class NIGLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 31 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 32 | 33 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 34 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 35 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 36 | 37 | b_z = get_variable("b_z", [1, self._num_blocks]) 38 | b_f = get_variable("b_f", [1, self._num_blocks]) 39 | b_o = get_variable("b_o", [1, self._num_blocks]) 40 | 41 | p_f = get_variable("p_f", [self._num_blocks]) 42 | p_o = get_variable("p_o", [self._num_blocks]) 43 | 44 | g = h = tf.tanh 45 | 46 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 47 | i = 1.0 48 | f = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + tf.mul(c_prev, p_f) + b_f) 49 | c = tf.mul(i, z) + tf.mul(f, c_prev) 50 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o) 51 | y = tf.mul(h(c), o) 52 | 53 | return y, tf.concat(1, [c, y]) 54 | -------------------------------------------------------------------------------- /variants/noaf.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class NOAFLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 32 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 33 | 34 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 35 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 36 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 37 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 38 | 39 | b_z = get_variable("b_z", [1, self._num_blocks]) 40 | b_i = get_variable("b_i", [1, self._num_blocks]) 41 | b_f = get_variable("b_f", [1, self._num_blocks]) 42 | b_o = get_variable("b_o", [1, self._num_blocks]) 43 | 44 | p_i = get_variable("p_i", [self._num_blocks]) 45 | p_f = get_variable("p_f", [self._num_blocks]) 46 | p_o = get_variable("p_o", [self._num_blocks]) 47 | 48 | g = h = tf.tanh 49 | 50 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 51 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i) 52 | f = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + tf.mul(c_prev, p_f) + b_f) 53 | c = tf.mul(i, z) + tf.mul(f, c_prev) 54 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o) 55 | y = tf.mul(c, o) 56 | 57 | return y, tf.concat(1, [c, y]) 58 | -------------------------------------------------------------------------------- /variants/nog.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class NOGLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 32 | 33 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 34 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 35 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 36 | 37 | b_z = get_variable("b_z", [1, self._num_blocks]) 38 | b_i = get_variable("b_i", [1, self._num_blocks]) 39 | b_f = get_variable("b_f", [1, self._num_blocks]) 40 | 41 | p_i = get_variable("p_i", [self._num_blocks]) 42 | p_f = get_variable("p_f", [self._num_blocks]) 43 | 44 | g = h = tf.tanh 45 | 46 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 47 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i) 48 | f = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + tf.mul(c_prev, p_f) + b_f) 49 | c = tf.mul(i, z) + tf.mul(f, c_prev) 50 | o = 1.0 51 | y = tf.mul(h(c), o) 52 | 53 | return y, tf.concat(1, [c, y]) 54 | -------------------------------------------------------------------------------- /variants/np.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class NPLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 32 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 33 | 34 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 35 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 36 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 37 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 38 | 39 | b_z = get_variable("b_z", [1, self._num_blocks]) 40 | b_i = get_variable("b_i", [1, self._num_blocks]) 41 | b_f = get_variable("b_f", [1, self._num_blocks]) 42 | b_o = get_variable("b_o", [1, self._num_blocks]) 43 | 44 | g = h = tf.tanh 45 | 46 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 47 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + b_i) 48 | f = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + b_f) 49 | c = tf.mul(i, z) + tf.mul(f, c_prev) 50 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + b_o) 51 | y = tf.mul(h(c), o) 52 | 53 | return y, tf.concat(1, [c, y]) 54 | -------------------------------------------------------------------------------- /variants/vanilla.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.rnn import rnn_cell 3 | 4 | class VanillaLSTMCell(rnn_cell.RNNCell): 5 | def __init__(self, num_blocks): 6 | self._num_blocks = num_blocks 7 | 8 | @property 9 | def input_size(self): 10 | return self._num_blocks 11 | 12 | @property 13 | def output_size(self): 14 | return self._num_blocks 15 | 16 | @property 17 | def state_size(self): 18 | return 2 * self._num_blocks 19 | 20 | def __call__(self, inputs, state, scope=None): 21 | with tf.variable_scope(scope or type(self).__name__): 22 | initializer = tf.random_uniform_initializer(-0.1, 0.1) 23 | 24 | def get_variable(name, shape): 25 | return tf.get_variable(name, shape, initializer=initializer, dtype=inputs.dtype) 26 | 27 | c_prev, y_prev = tf.split(1, 2, state) 28 | 29 | W_z = get_variable("W_z", [self.input_size, self._num_blocks]) 30 | W_i = get_variable("W_i", [self.input_size, self._num_blocks]) 31 | W_f = get_variable("W_f", [self.input_size, self._num_blocks]) 32 | W_o = get_variable("W_o", [self.input_size, self._num_blocks]) 33 | 34 | R_z = get_variable("R_z", [self._num_blocks, self._num_blocks]) 35 | R_i = get_variable("R_i", [self._num_blocks, self._num_blocks]) 36 | R_f = get_variable("R_f", [self._num_blocks, self._num_blocks]) 37 | R_o = get_variable("R_o", [self._num_blocks, self._num_blocks]) 38 | 39 | b_z = get_variable("b_z", [1, self._num_blocks]) 40 | b_i = get_variable("b_i", [1, self._num_blocks]) 41 | b_f = get_variable("b_f", [1, self._num_blocks]) 42 | b_o = get_variable("b_o", [1, self._num_blocks]) 43 | 44 | p_i = get_variable("p_i", [self._num_blocks]) 45 | p_f = get_variable("p_f", [self._num_blocks]) 46 | p_o = get_variable("p_o", [self._num_blocks]) 47 | 48 | g = h = tf.tanh 49 | 50 | z = g(tf.matmul(inputs, W_z) + tf.matmul(y_prev, R_z) + b_z) 51 | i = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(y_prev, R_i) + tf.mul(c_prev, p_i) + b_i) 52 | f = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(y_prev, R_f) + tf.mul(c_prev, p_f) + b_f) 53 | c = tf.mul(i, z) + tf.mul(f, c_prev) 54 | o = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(y_prev, R_o) + tf.mul(c, p_o) + b_o) 55 | y = tf.mul(h(c), o) 56 | 57 | return y, tf.concat(1, [c, y]) 58 | --------------------------------------------------------------------------------