├── .gitignore ├── For-loops with theano.ipynb ├── LSTM_ReberGrammar.ipynb ├── README.md ├── RNN_LSTM_SMT.ipynb ├── RNN_sinus.ipynb ├── Sequence_Memory_Task.ipynb ├── binary_code.py ├── data ├── memory_task │ ├── long_input_data_test.npy │ ├── long_input_data_train.npy │ ├── long_input_data_validation.npy │ ├── long_output_data_test.npy │ ├── long_output_data_train.npy │ ├── long_output_data_validation.npy │ ├── long_sequence_test.json │ ├── long_sequence_train.json │ ├── long_sequence_validation.json │ ├── short_input_data_test.npy │ ├── short_input_data_train.npy │ ├── short_input_data_validation.npy │ ├── short_output_data_test.npy │ ├── short_output_data_train.npy │ ├── short_output_data_validation.npy │ ├── short_sequence_test.json │ ├── short_sequence_train.json │ └── short_sequence_validation.json └── running_max │ ├── long_input_data_test.npy │ ├── long_input_data_train.npy │ ├── long_input_data_validation.npy │ ├── long_output_data_test.npy │ ├── long_output_data_train.npy │ ├── long_output_data_validation.npy │ ├── long_sequence_test.json │ ├── long_sequence_train.json │ ├── long_sequence_validation.json │ ├── short_input_data_test.npy │ ├── short_input_data_train.npy │ ├── short_input_data_validation.npy │ ├── short_output_data_test.npy │ ├── short_output_data_train.npy │ ├── short_output_data_validation.npy │ ├── short_sequence_test.json │ ├── short_sequence_train.json │ └── short_sequence_validation.json ├── draft └── simple_lstm_tutorial_grammar.ipynb ├── img ├── rnn_simple.svg └── rnn_unfolded.svg ├── init.py ├── lstm.py ├── minimal_RNN.ipynb ├── rnn.py └── scan_tutorial_solutions ├── sol_pure_python_accumulate_squares.py ├── sol_pure_python_fibonacci.py ├── sol_pure_python_general_acc_squares.py ├── sol_pure_python_general_accumulate.py ├── sol_reduce_verbose_add.py ├── sol_theano_map_add_two_seq.py ├── sol_theano_map_binomial.py ├── sol_theano_map_square.py ├── sol_theano_map_univar_polynomial.py ├── sol_theano_reduce_awkward_multiple_powers.py ├── sol_theano_reduce_monomials.py ├── sol_theano_reduce_monomials_from_map.py ├── sol_theano_reduce_multi_output.py ├── sol_theano_reduce_sum_square_ints.py ├── sol_theano_reduce_vectorized_multiple_powers.py ├── sol_theano_scan_accumulate_ints_and_squares.py ├── sol_theano_scan_accumulate_squares.py ├── sol_theano_scan_fibonacci.py ├── sol_theano_scan_modulated_oscillation.py └── sol_theano_scan_oscillation.py /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | *.pyc 3 | -------------------------------------------------------------------------------- /LSTM_ReberGrammar.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import theano\n", 13 | "import theano.tensor as T\n", 14 | "from theano import shared \n", 15 | "from collections import OrderedDict\n", 16 | "\n", 17 | "dtype=T.config.floatX\n", 18 | "theano.config.optimizer='fast_compile'" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": { 25 | "collapsed": true 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "#Embedded Reber Grammar http://christianherta.de/lehre/dataScience/machineLearning/neuralNetworks/reberGrammar.php\n", 30 | "\n", 31 | "chars='BTSXPVE'\n", 32 | "\n", 33 | "graph = [[(1,5),('T','P')] , [(1,2),('S','X')], \\\n", 34 | " [(3,5),('S','X')], [(6,),('E')], \\\n", 35 | " [(3,2),('V','P')], [(4,5),('V','T')] ]\n", 36 | "\n", 37 | "\n", 38 | "def in_grammar(word):\n", 39 | " if word[0] != 'B':\n", 40 | " return False\n", 41 | " node = 0 \n", 42 | " for c in word[1:]:\n", 43 | " transitions = graph[node]\n", 44 | " try:\n", 45 | " node = transitions[0][transitions[1].index(c)]\n", 46 | " except ValueError: # using exceptions for flow control in python is common\n", 47 | " return False\n", 48 | " return True \n", 49 | " \n", 50 | "def sequenceToWord(sequence):\n", 51 | " \"\"\"\n", 52 | " converts a sequence (one-hot) in a reber string\n", 53 | " \"\"\"\n", 54 | " reberString = ''\n", 55 | " for s in sequence:\n", 56 | " index = np.where(s==1.)[0][0]\n", 57 | " reberString += chars[index]\n", 58 | " return reberString\n", 59 | " \n", 60 | "def generateSequences(minLength):\n", 61 | " while True:\n", 62 | " inchars = ['B']\n", 63 | " node = 0\n", 64 | " outchars = [] \n", 65 | " while node != 6:\n", 66 | " transitions = graph[node]\n", 67 | " i = np.random.randint(0, len(transitions[0]))\n", 68 | " inchars.append(transitions[1][i])\n", 69 | " outchars.append(transitions[1])\n", 70 | " node = transitions[0][i]\n", 71 | " if len(inchars) > minLength: \n", 72 | " return inchars, outchars\n", 73 | "\n", 74 | "\n", 75 | "def get_one_example(minLength):\n", 76 | " inchars, outchars = generateSequences(minLength)\n", 77 | " inseq = []\n", 78 | " outseq= []\n", 79 | " for i,o in zip(inchars, outchars): \n", 80 | " inpt = np.zeros(7)\n", 81 | " inpt[chars.find(i)] = 1. \n", 82 | " outpt = np.zeros(7)\n", 83 | " for oo in o:\n", 84 | " outpt[chars.find(oo)] = 1.\n", 85 | " inseq.append(inpt)\n", 86 | " outseq.append(outpt)\n", 87 | " return inseq, outseq\n", 88 | "\n", 89 | "\n", 90 | "def get_char_one_hot(char):\n", 91 | " char_oh = np.zeros(7)\n", 92 | " for c in char:\n", 93 | " char_oh[chars.find(c)] = 1.\n", 94 | " return [char_oh] \n", 95 | " \n", 96 | "def get_n_examples(n, minLength=10):\n", 97 | " examples = []\n", 98 | " for i in xrange(n):\n", 99 | " examples.append(get_one_example(minLength))\n", 100 | " return examples\n", 101 | "\n", 102 | "emb_chars = \"TP\"\n", 103 | "\n", 104 | "\n", 105 | "def get_one_embedded_example(minLength=10):\n", 106 | " i, o = get_one_example(minLength)\n", 107 | " emb_char = emb_chars[np.random.randint(0, len(emb_chars))]\n", 108 | " new_in = get_char_one_hot(('B',))\n", 109 | " new_in += get_char_one_hot((emb_char,))\n", 110 | " new_out= get_char_one_hot(emb_chars)\n", 111 | " new_out+= get_char_one_hot('B',)\n", 112 | " new_in += i\n", 113 | " new_out += o\n", 114 | " new_in += get_char_one_hot(('E',))\n", 115 | " new_in += get_char_one_hot((emb_char,))\n", 116 | " new_out += get_char_one_hot((emb_char, ))\n", 117 | " new_out += get_char_one_hot(('E',))\n", 118 | " return new_in, new_out\n", 119 | " \n", 120 | "def get_n_embedded_examples(n, minLength=10):\n", 121 | " examples = []\n", 122 | " for i in xrange(n):\n", 123 | " examples.append(get_one_embedded_example(minLength))\n", 124 | " return examples" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": { 131 | "collapsed": false 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "train_data = get_n_embedded_examples(1000)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": { 142 | "collapsed": false 143 | }, 144 | "outputs": [], 145 | "source": [ 146 | "def sample_weights(sizeX, sizeY):\n", 147 | " W = np.random.uniform(low=-1., high=1., size=(sizeX, sizeY))\n", 148 | " _, svs, _ = np.linalg.svd(W)\n", 149 | " values = np.asarray(W / svs[0], dtype=dtype)\n", 150 | " return shared(values, borrow=True) \n", 151 | "\n", 152 | "class LSTM:\n", 153 | " def __init__(self, n_in, n_lstm, n_out): \n", 154 | " self.n_in = n_in\n", 155 | " self.n_lstm = n_lstm\n", 156 | " self.n_out = n_out\n", 157 | " self.W_xi = sample_weights(n_in, n_lstm)\n", 158 | " self.W_hi = sample_weights(n_lstm, n_lstm)\n", 159 | " self.W_ci = sample_weights(n_lstm, n_lstm)\n", 160 | " self.b_i = shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_lstm)))\n", 161 | " self.W_xf = sample_weights(n_in, n_lstm)\n", 162 | " self.W_hf = sample_weights(n_lstm, n_lstm)\n", 163 | " self.W_cf = sample_weights(n_lstm, n_lstm)\n", 164 | " self.b_f = shared(np.cast[dtype](np.random.uniform(0, 1.,size = n_lstm)))\n", 165 | " self.W_xc = sample_weights(n_in, n_lstm)\n", 166 | " self.W_hc = sample_weights(n_lstm, n_lstm)\n", 167 | " self.b_c = shared(np.zeros(n_lstm, dtype=dtype))\n", 168 | " self.W_xo = sample_weights(n_in, n_lstm)\n", 169 | " self.W_ho = sample_weights(n_lstm, n_lstm)\n", 170 | " self.W_co = sample_weights(n_lstm, n_lstm)\n", 171 | " self.b_o = shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_lstm)))\n", 172 | " self.W_hy = sample_weights(n_lstm, n_out)\n", 173 | " self.b_y = shared(np.zeros(n_out, dtype=dtype))\n", 174 | " self.params = [self.W_xi, self.W_hi, self.W_ci, self.b_i, \n", 175 | " self.W_xf, self.W_hf, self.W_cf, self.b_f, \n", 176 | " self.W_xc, self.W_hc, self.b_c, \n", 177 | " self.W_ho, self.W_co, self.W_co, self.b_o, \n", 178 | " self.W_hy, self.b_y]\n", 179 | " \n", 180 | "\n", 181 | " def step_lstm(x_t, h_tm1, c_tm1):\n", 182 | " i_t = T.nnet.sigmoid(T.dot(x_t, self.W_xi) + T.dot(h_tm1, self.W_hi) + T.dot(c_tm1, self.W_ci) + self.b_i)\n", 183 | " f_t = T.nnet.sigmoid(T.dot(x_t, self.W_xf) + T.dot(h_tm1, self.W_hf) + T.dot(c_tm1, self.W_cf) + self.b_f)\n", 184 | " c_t = f_t * c_tm1 + i_t * T.tanh(T.dot(x_t, self.W_xc) + T.dot(h_tm1, self.W_hc) + self.b_c) \n", 185 | " o_t = T.nnet.sigmoid(T.dot(x_t, self.W_xo)+ T.dot(h_tm1, self.W_ho) + T.dot(c_t, self.W_co) + self.b_o)\n", 186 | " h_t = o_t * T.tanh(c_t)\n", 187 | " y_t = T.nnet.sigmoid(T.dot(h_t, self.W_hy) + self.b_y) \n", 188 | " return [h_t, c_t, y_t]\n", 189 | " \n", 190 | " X = T.matrix() # X is a sequence of vector \n", 191 | " Y = T.matrix() # Y is a sequence of vector\n", 192 | " h0 = shared(np.zeros(self.n_lstm, dtype=dtype)) # initial hidden state \n", 193 | " c0 = shared(np.zeros(self.n_lstm, dtype=dtype)) # initial cell state\n", 194 | " \n", 195 | " [h_vals, c_vals, y_vals], _ = theano.scan(fn=step_lstm, \n", 196 | " sequences=X,\n", 197 | " outputs_info=[h0, c0, None])\n", 198 | " \n", 199 | " self.output = y_vals\n", 200 | " \n", 201 | " cost = -T.mean(Y * T.log(y_vals)+ (1.- Y) * T.log(1. - y_vals))\n", 202 | " lr = shared(np.cast[dtype](0.1))\n", 203 | " gparams = T.grad(cost, self.params)\n", 204 | " updates = OrderedDict()\n", 205 | " for param, gparam in zip(self.params, gparams):\n", 206 | " updates[param] = param - gparam * lr\n", 207 | " self.train = theano.function(inputs = [X, Y], outputs = cost, updates=updates) \n", 208 | " \n", 209 | " self.pred = theano.function(inputs = [X], outputs = self.output) " 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": { 216 | "collapsed": false 217 | }, 218 | "outputs": [], 219 | "source": [ 220 | "model = LSTM(7, 50, 7)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": { 227 | "collapsed": false 228 | }, 229 | "outputs": [], 230 | "source": [ 231 | "nb_epochs = 100\n", 232 | "#stupid and naive sgd\n", 233 | "for x in range(nb_epochs):\n", 234 | " error = 0.\n", 235 | " for j in range(len(train_data)): \n", 236 | " index = np.random.randint(0, len(train_data))\n", 237 | " i, o = train_data[index]\n", 238 | " train_cost = model.train(i, o)\n", 239 | " error += train_cost\n", 240 | " if x%10==0:\n", 241 | " print \"epoch \"+str(x)+ \" error: \"+str(error)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": { 248 | "collapsed": false 249 | }, 250 | "outputs": [], 251 | "source": [ 252 | "test_data = get_n_embedded_examples(10)\n", 253 | "\n", 254 | "def print_out(test_data):\n", 255 | " for i,o in test_data:\n", 256 | " p = model.pred(i)\n", 257 | " print o[-2] # target\n", 258 | " print np.asarray([0. if x!=np.argmax(p[-2]) else 1. for x in range(7)]) # prediction\n", 259 | " print \n", 260 | "print_out(test_data)" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": { 267 | "collapsed": true 268 | }, 269 | "outputs": [], 270 | "source": [] 271 | } 272 | ], 273 | "metadata": { 274 | "kernelspec": { 275 | "display_name": "Python 2", 276 | "language": "python", 277 | "name": "python2" 278 | }, 279 | "language_info": { 280 | "codemirror_mode": { 281 | "name": "ipython", 282 | "version": 2 283 | }, 284 | "file_extension": ".py", 285 | "mimetype": "text/x-python", 286 | "name": "python", 287 | "nbconvert_exporter": "python", 288 | "pygments_lexer": "ipython2", 289 | "version": "2.7.6" 290 | } 291 | }, 292 | "nbformat": 4, 293 | "nbformat_minor": 0 294 | } 295 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dl-rnn 2 | Recurrent Deep Learning Tutorials and examples: RNN, GRU, LSTM 3 | 4 | All the ipython notebooks can run on the dl-machine AWS instance: https://github.com/deeplearningparis/dl-machine -------------------------------------------------------------------------------- /RNN_LSTM_SMT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "%load_ext autoreload\n", 12 | "%autoreload 2" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": { 19 | "collapsed": false 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "import rnn\n", 24 | "import lstm\n", 25 | "import numpy as np\n", 26 | "import theano\n", 27 | "import theano.tensor as T\n", 28 | "batch_size = 64\n", 29 | "theano.config.optimizer='fast_compile'" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": { 36 | "collapsed": false 37 | }, 38 | "outputs": [], 39 | "source": [ 40 | "#model = rnn.Rnn(4, 20, 4, lr = 0.01, single_output=False, output_activation='nll', cost_function=T.nnet.softmax)\n", 41 | "#model = rnn.RnnMiniBatch(4,20,4, 0.1, single_output=e_output=False, output_activation='nll', cost_function=T.nnet.softmax)\n", 42 | "#model = lstm.Lstm(4,20,4, 0.1, single_output=False, output_activation='nll', cost_function=T.nnet.softmax)\n", 43 | "model = lstm.LstmMiniBatch(4, 20, 4, 0.01, batch_size, single_output=False, output_activation='nll', cost_function=T.nnet.softmax)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": { 50 | "collapsed": false 51 | }, 52 | "outputs": [], 53 | "source": [ 54 | "X_short_train = np.load('data/memory_task/short_output_data_test.npy')\n", 55 | "Y_short_train = np.load('data/memory_task/short_output_data_train.npy')\n", 56 | "X_short_train = np.load('data/running_max/short_input_data_train.npy')\n", 57 | "Y_short_train = np.load('data/running_max/short_output_data_train.npy')\n", 58 | "#Y_short_train = build_preds(X_short_train)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": { 65 | "collapsed": true 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "print X_short_train[0],\"\\n\\n\",Y_short_train[0]" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": { 76 | "collapsed": true 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "def build_preds(X):\n", 81 | " Y = np.zeros(shape=(X.shape[0], X.shape[2]), dtype='float32')\n", 82 | " for idx, maxim in enumerate(np.argmax(np.sum(X, axis=1), axis=1)):\n", 83 | " Y[idx, maxim] = 1.\n", 84 | " return Y\n", 85 | "\n", 86 | "def output(X):\n", 87 | " Y = np.zeros(shape=X.shape, dtype='float32')\n", 88 | " for idx, maxim in enumerate(np.argmax(X, axis=1)):\n", 89 | " Y[idx, maxim] = 1.\n", 90 | " return Y" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": { 97 | "collapsed": true 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "epochs = 10\n", 102 | "#minibatch train\n", 103 | "for epoch in range(epochs):\n", 104 | " loss = 0\n", 105 | " num_batch = len(X_short_train)/batch_size\n", 106 | " for i in range(num_batch):\n", 107 | " start_idx = i * batch_size\n", 108 | " end_idx = min((i+1) * (batch_size), len(X_short_train) -1)\n", 109 | " X_train = X_short_train[start_idx:end_idx]\n", 110 | " Y_train = Y_short_train[start_idx:end_idx] \n", 111 | " #print X_train.shape, Y_train.shape, i, start_idx\n", 112 | " #print model.debug(X_train, Y_train) \n", 113 | " loss += model.train(X_train, Y_train)\n", 114 | " print \"epoch \"+str(epoch) + \" sum of batch loss \"+str(loss)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "outputs": [], 124 | "source": [ 125 | "nb_epochs = 100\n", 126 | "#naive sgd\n", 127 | "for x in range(nb_epochs):\n", 128 | " loss = 0.\n", 129 | " for j in range(len(X_short_train)): \n", 130 | " index = np.random.randint(0, len(X_short_train))\n", 131 | " X_train = X_short_train[index]\n", 132 | " Y_train = Y_short_train[index] \n", 133 | " loss += model.train(X_train, Y_train) \n", 134 | " print \"epoch \"+str(x)+ \" error: \"+str(loss)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": { 141 | "collapsed": true 142 | }, 143 | "outputs": [], 144 | "source": [ 145 | "output(model.predictions(X_short_train[0:64]))" 146 | ] 147 | } 148 | ], 149 | "metadata": { 150 | "kernelspec": { 151 | "display_name": "Python 2", 152 | "language": "python", 153 | "name": "python2" 154 | }, 155 | "language_info": { 156 | "codemirror_mode": { 157 | "name": "ipython", 158 | "version": 2 159 | }, 160 | "file_extension": ".py", 161 | "mimetype": "text/x-python", 162 | "name": "python", 163 | "nbconvert_exporter": "python", 164 | "pygments_lexer": "ipython2", 165 | "version": "2.7.6" 166 | } 167 | }, 168 | "nbformat": 4, 169 | "nbformat_minor": 0 170 | } 171 | -------------------------------------------------------------------------------- /RNN_sinus.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 26, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "loaded rnn.py\n", 15 | "The autoreload extension is already loaded. To reload it, use:\n", 16 | " %reload_ext autoreload\n", 17 | "(49, 5) (49, 5)\n" 18 | ] 19 | } 20 | ], 21 | "source": [ 22 | "%load_ext autoreload\n", 23 | "%autoreload 2\n", 24 | "\n", 25 | "import numpy as np\n", 26 | "import theano\n", 27 | "import theano.tensor as T\n", 28 | "from scipy import linalg\n", 29 | " \n", 30 | "# number of input units\n", 31 | "n_in = 5\n", 32 | "# number of hidden units\n", 33 | "n_hid = 10\n", 34 | "# number of output units\n", 35 | "n_out = 5\n", 36 | " \n", 37 | "# Generate sinewaves offset in phase\n", 38 | "n_timesteps = 50\n", 39 | "d1 = 3 * np.arange(n_timesteps) / (2 * np.pi)\n", 40 | "d2 = 3 * np.arange(n_in) / (2 * np.pi)\n", 41 | "all_sines = np.sin(np.array([d1] * n_in).T + d2)\n", 42 | " \n", 43 | "# Setup dataset and initial hidden vector of zeros\n", 44 | "X = all_sines[:-1]\n", 45 | "y = all_sines[1:]\n", 46 | "print X.shape, y.shape" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 27, 52 | "metadata": { 53 | "collapsed": false 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "import rnn\n", 58 | "model = rnn.Rnn(n_in,n_hid,n_out, 0.001, single_output=False)\n", 59 | "\n", 60 | "# RNN uses a mean squared error and a linear output activation. Feel free to change it in rnn.py" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 28, 66 | "metadata": { 67 | "collapsed": true 68 | }, 69 | "outputs": [ 70 | { 71 | "name": "stdout", 72 | "output_type": "stream", 73 | "text": [ 74 | "Epoch 0: err 123.894156\n", 75 | "Epoch 40: err 0.326533\n", 76 | "Epoch 80: err 0.166228\n", 77 | "Epoch 120: err 0.124947\n", 78 | "Epoch 160: err 0.096020\n", 79 | "Epoch 200: err 0.075262\n", 80 | "Epoch 240: err 0.060122\n", 81 | "Epoch 280: err 0.048925\n", 82 | "Epoch 320: err 0.040530\n", 83 | "Epoch 360: err 0.034145\n", 84 | "Epoch 400: err 0.029218\n", 85 | "Epoch 440: err 0.025358\n", 86 | "Epoch 480: err 0.022291\n", 87 | "Epoch 520: err 0.019819\n", 88 | "Epoch 560: err 0.017802\n", 89 | "Epoch 600: err 0.016136\n", 90 | "Epoch 640: err 0.014745\n", 91 | "Epoch 680: err 0.013573\n", 92 | "Epoch 720: err 0.012576\n", 93 | "Epoch 760: err 0.011721\n", 94 | "Epoch 800: err 0.010982\n", 95 | "Epoch 840: err 0.010340\n", 96 | "Epoch 880: err 0.009777\n", 97 | "Epoch 920: err 0.009281\n", 98 | "Epoch 960: err 0.008841\n", 99 | "Epoch 1000: err 0.008448\n", 100 | "Epoch 1040: err 0.008096\n", 101 | "Epoch 1080: err 0.007779\n", 102 | "Epoch 1120: err 0.007491\n", 103 | "Epoch 1160: err 0.007228\n", 104 | "Epoch 1200: err 0.006988\n", 105 | "Epoch 1240: err 0.006767\n", 106 | "Epoch 1280: err 0.006562\n", 107 | "Epoch 1320: err 0.006372\n", 108 | "Epoch 1360: err 0.006195\n", 109 | "Epoch 1400: err 0.006030\n", 110 | "Epoch 1440: err 0.005874\n", 111 | "Epoch 1480: err 0.005728\n", 112 | "Epoch 1520: err 0.005590\n", 113 | "Epoch 1560: err 0.005459\n", 114 | "Epoch 1600: err 0.005335\n", 115 | "Epoch 1640: err 0.005216\n", 116 | "Epoch 1680: err 0.005104\n", 117 | "Epoch 1720: err 0.004996\n", 118 | "Epoch 1760: err 0.004893\n", 119 | "Epoch 1800: err 0.004794\n", 120 | "Epoch 1840: err 0.004699\n", 121 | "Epoch 1880: err 0.004607\n", 122 | "Epoch 1920: err 0.004519\n", 123 | "Epoch 1960: err 0.004434\n", 124 | "Epoch 1999: err 0.004354\n" 125 | ] 126 | } 127 | ], 128 | "source": [ 129 | "epochs = 2000\n", 130 | "# Print 50 status updates along with last\n", 131 | "status_points = list(range(epochs))\n", 132 | "status_points = status_points[::epochs // 50] + [status_points[-1]]\n", 133 | "for i in range(epochs): \n", 134 | " pred = model.predictions(X)\n", 135 | " err = model.train(X, y)\n", 136 | " if i in status_points:\n", 137 | " print(\"Epoch %i: err %f\" % (i, err))\n", 138 | " " 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 29, 144 | "metadata": { 145 | "collapsed": true 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "import matplotlib.pyplot as plt\n", 150 | "plt.plot(pred, color=\"darkred\")\n", 151 | "plt.plot(y, color=\"steelblue\")\n", 152 | "plt.show()\n" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": { 159 | "collapsed": true 160 | }, 161 | "outputs": [], 162 | "source": [] 163 | } 164 | ], 165 | "metadata": { 166 | "kernelspec": { 167 | "display_name": "Python 2", 168 | "language": "python", 169 | "name": "python2" 170 | }, 171 | "language_info": { 172 | "codemirror_mode": { 173 | "name": "ipython", 174 | "version": 2 175 | }, 176 | "file_extension": ".py", 177 | "mimetype": "text/x-python", 178 | "name": "python", 179 | "nbconvert_exporter": "python", 180 | "pygments_lexer": "ipython2", 181 | "version": "2.7.6" 182 | } 183 | }, 184 | "nbformat": 4, 185 | "nbformat_minor": 0 186 | } 187 | -------------------------------------------------------------------------------- /binary_code.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | ALPHABET = np.asarray(list("abcd"), dtype=object) 5 | 6 | 7 | def symbols_to_binarray(s, alphabet=ALPHABET, dtype=np.float32): 8 | """One-hot encode a sequence of symbols 9 | 10 | This numerical representation of a string of symbols is useful 11 | to feed the data and expected labels to the input and output 12 | layers of recurrent networks. 13 | """ 14 | alphabet = np.asarray(list(alphabet), dtype=object) 15 | n_samples = len(s) 16 | n_features = len(alphabet) 17 | 18 | mapping = dict(zip(alphabet, range(n_features))) 19 | 20 | code = np.zeros((n_samples, n_features), dtype=dtype) 21 | for i, e in enumerate(s): 22 | code[i, mapping[e]] = 1.0 23 | return code 24 | 25 | 26 | def binarray_to_symbols(code, alphabet=ALPHABET): 27 | """Convert encoded data by to a string of symbols""" 28 | n_samples, n_features = code.shape 29 | if n_features != len(alphabet): 30 | raise ValueError( 31 | "code should have %d columns (instead of %d)." 32 | % (len(alphabet), n_features) 33 | ) 34 | 35 | # Make sure that the alphabet is a numpy array of symbols 36 | # to make it possible to leverage numpy fancy indexing 37 | if not isinstance(alphabet, np.ndarray): 38 | alphabet = np.asarray(list(alphabet), dtype='object') 39 | 40 | return "".join(alphabet[code.argmax(axis=1)]) 41 | 42 | 43 | def plot_binary_tape(encoded_sequence, alphabet=ALPHABET): 44 | plt.matshow(encoded_sequence, cmap=plt.cm.gray) 45 | plt.xticks(np.arange(len(alphabet)), alphabet) 46 | 47 | 48 | def plot_parallel_tapes(input_data, output_data, 49 | input_symbols, output_symbols): 50 | fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(5, 10)) 51 | for ax, data in zip(axes, [input_data, output_data]): 52 | ax.matshow(data, cmap=plt.cm.gray) 53 | ax.set_xticks(np.arange(len(ALPHABET))) 54 | ax.set_xticklabels(ALPHABET, fontsize=18) 55 | fig.tight_layout() 56 | plt.title("input: %r, output: %r" % (input_symbols, output_symbols)) -------------------------------------------------------------------------------- /data/memory_task/long_input_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/long_input_data_test.npy -------------------------------------------------------------------------------- /data/memory_task/long_input_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/long_input_data_train.npy -------------------------------------------------------------------------------- /data/memory_task/long_input_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/long_input_data_validation.npy -------------------------------------------------------------------------------- /data/memory_task/long_output_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/long_output_data_test.npy -------------------------------------------------------------------------------- /data/memory_task/long_output_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/long_output_data_train.npy -------------------------------------------------------------------------------- /data/memory_task/long_output_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/long_output_data_validation.npy -------------------------------------------------------------------------------- /data/memory_task/short_input_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/short_input_data_test.npy -------------------------------------------------------------------------------- /data/memory_task/short_input_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/short_input_data_train.npy -------------------------------------------------------------------------------- /data/memory_task/short_input_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/short_input_data_validation.npy -------------------------------------------------------------------------------- /data/memory_task/short_output_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/short_output_data_test.npy -------------------------------------------------------------------------------- /data/memory_task/short_output_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/short_output_data_train.npy -------------------------------------------------------------------------------- /data/memory_task/short_output_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/memory_task/short_output_data_validation.npy -------------------------------------------------------------------------------- /data/memory_task/short_sequence_test.json: -------------------------------------------------------------------------------- 1 | [{"input":"dbacdcdadcbdd","output":"bd"},{"input":"dddcaaabdd","output":"da"},{"input":"acdddccbcad","output":"cd"},{"input":"baaccddb","output":"ac"},{"input":"aacacac","output":"ac"},{"input":"ddcacb","output":"dc"},{"input":"badbcacbbbbcac","output":"ac"},{"input":"adaccaac","output":"dc"},{"input":"cabcbcccdacba","output":"ab"},{"input":"bbcbbbbcacdcd","output":"bb"},{"input":"bcacddaab","output":"cd"},{"input":"dbdbaaccbdbd","output":"ba"},{"input":"dddadcdbacdd","output":"dd"},{"input":"cacba","output":"aa"},{"input":"ddccbdbbc","output":"db"},{"input":"ddcadb","output":"dd"},{"input":"acdbdcccdadab","output":"cd"},{"input":"bcbddba","output":"cd"},{"input":"badcaddb","output":"aa"},{"input":"dddabadcbb","output":"db"},{"input":"dcacaacddbbabb","output":"ca"},{"input":"acdadddcaa","output":"cd"},{"input":"daaacbbdddd","output":"ac"},{"input":"cbbcbba","output":"bb"},{"input":"bababaaccdd","output":"ab"},{"input":"bbadbccbca","output":"bb"},{"input":"dbcdabcbdb","output":"ba"},{"input":"caccabcdb","output":"aa"},{"input":"bbbabdadac","output":"bb"},{"input":"cadddda","output":"ad"},{"input":"cbbdbadabbca","output":"bb"},{"input":"cccccbbc","output":"cc"},{"input":"cadbacdbccadbc","output":"aa"},{"input":"dcdaaadacbcc","output":"ca"},{"input":"bdbdbbbdaaac","output":"db"},{"input":"bdbacdcdcbacb","output":"dc"},{"input":"dbbcccd","output":"bc"},{"input":"dbcadd","output":"bd"},{"input":"ddcaccd","output":"dc"},{"input":"cdbaabcd","output":"da"},{"input":"cbdccc","output":"bc"},{"input":"adcdcb","output":"dc"},{"input":"bacacddb","output":"ac"},{"input":"dadbbaddcb","output":"ab"},{"input":"cbcccdbca","output":"bc"},{"input":"dbcbcb","output":"bc"},{"input":"accdaccaadddd","output":"ca"},{"input":"dbccc","output":"bc"},{"input":"cdbcaca","output":"da"},{"input":"ddaadada","output":"dd"},{"input":"dbdabddacc","output":"bb"},{"input":"ddaabccdbbcd","output":"db"},{"input":"adaadccccbd","output":"dd"},{"input":"cbbacdbabaccc","output":"bc"},{"input":"dbdbacaabbcccc","output":"ba"},{"input":"cdbdadddbab","output":"da"},{"input":"aaadaadbbbccc","output":"aa"},{"input":"cadddcacacadba","output":"ad"},{"input":"dbbbddcbdaab","output":"bd"},{"input":"cdadc","output":"dc"},{"input":"adacbbbabbccdc","output":"db"},{"input":"baabacb","output":"aa"},{"input":"bbdbcccacbc","output":"bc"},{"input":"ccbabdccdcbadc","output":"cb"},{"input":"daadbbdbdc","output":"ab"},{"input":"cbdcb","output":"bb"},{"input":"cbabcbbabd","output":"bc"},{"input":"bccbbbddaa","output":"cb"},{"input":"dbbda","output":"ba"},{"input":"bcabbdbcbab","output":"cb"},{"input":"bbaca","output":"ba"},{"input":"bacbca","output":"ac"},{"input":"ccdccdd","output":"cc"},{"input":"daccccdab","output":"ac"},{"input":"bccada","output":"cd"},{"input":"dbbacababbcabc","output":"bc"},{"input":"cbdccbbdd","output":"bc"},{"input":"aaacd","output":"ad"},{"input":"acabbbbca","output":"cb"},{"input":"adaacbadd","output":"dc"},{"input":"bdbaccdbc","output":"dc"},{"input":"cbcddddddbdbd","output":"bd"},{"input":"cdaabacdacada","output":"db"},{"input":"cbbbcba","output":"bc"},{"input":"daabb","output":"ab"},{"input":"acababcb","output":"ca"},{"input":"acdaccbaddac","output":"cc"},{"input":"ccdabc","output":"cb"},{"input":"dcadd","output":"cd"},{"input":"adbacdb","output":"dc"},{"input":"ddcbdaccdadb","output":"dd"},{"input":"cdadadbcd","output":"da"},{"input":"dbbbacbaca","output":"ba"},{"input":"dccaccaacab","output":"cc"},{"input":"abbdaaddab","output":"ba"},{"input":"cbcdcbda","output":"bc"},{"input":"ccdcb","output":"cb"},{"input":"dabdacccdaa","output":"aa"},{"input":"aadcd","output":"ad"},{"input":"acacdbbb","output":"cd"},{"input":"dbcbdbbbbacaa","output":"bd"},{"input":"dcbcbddbdaa","output":"cb"},{"input":"acbaaadabaabc","output":"ca"},{"input":"ddbcdbdbad","output":"dd"},{"input":"acdbda","output":"cd"},{"input":"adcaaccccb","output":"da"},{"input":"ccbbaacdaa","output":"ca"},{"input":"dddbc","output":"dc"},{"input":"dacacadca","output":"ac"},{"input":"acdbacddabbb","output":"ca"},{"input":"cbbdacbdcbd","output":"ba"},{"input":"bbcbdd","output":"bd"},{"input":"cbbdcdb","output":"bc"},{"input":"daacdcacbabcbc","output":"ad"},{"input":"bccddcdadbd","output":"cd"},{"input":"bcabdacbcdbc","output":"cd"},{"input":"cbcbcda","output":"bc"},{"input":"cdbba","output":"da"},{"input":"cabcadbad","output":"aa"},{"input":"ddaddcd","output":"dd"},{"input":"bbbdbaad","output":"bb"},{"input":"acdaadcd","output":"ca"},{"input":"dabcabbcdd","output":"aa"},{"input":"dcdbacccbac","output":"ca"},{"input":"bdddbdcadcbdd","output":"db"},{"input":"cbbabbcbbd","output":"bb"},{"input":"aabbad","output":"aa"},{"input":"badacccaac","output":"ac"},{"input":"cacbdbb","output":"ad"},{"input":"cacbaccdbaac","output":"aa"},{"input":"cacbbbddbbbaa","output":"ab"},{"input":"dcdba","output":"ca"},{"input":"dbdbbcbb","output":"bb"},{"input":"dbcdc","output":"bc"},{"input":"dccdcadd","output":"cc"},{"input":"aabdbdacb","output":"ab"},{"input":"dbbaaddbddaa","output":"ba"},{"input":"cbcdadb","output":"ba"},{"input":"aaacb","output":"ab"},{"input":"bacaa","output":"aa"},{"input":"bddcbbadcbaa","output":"db"},{"input":"adaabd","output":"db"},{"input":"dbbabbabaad","output":"bb"},{"input":"cbbabbbddacc","output":"bb"},{"input":"dccdcba","output":"cc"},{"input":"aaddadadcb","output":"aa"},{"input":"dccddcbadbdcd","output":"cd"},{"input":"dddcbbbabcdba","output":"db"},{"input":"cccdaadccd","output":"ca"},{"input":"abcbaabd","output":"ba"},{"input":"dcdbbbaabbad","output":"cb"},{"input":"bbccbadc","output":"bb"},{"input":"bbbabdbdaaddbd","output":"bb"},{"input":"ddbacddacabcbd","output":"dc"},{"input":"baaacbbcbb","output":"ac"},{"input":"dbdabdca","output":"bb"},{"input":"babcbaddbb","output":"ab"},{"input":"bdcdbc","output":"db"},{"input":"abcaac","output":"ba"},{"input":"dcacacbad","output":"ca"},{"input":"bdddadbaba","output":"da"},{"input":"abbbccdcd","output":"bc"},{"input":"dccdccd","output":"cc"},{"input":"acbbca","output":"cc"},{"input":"ddbbaccc","output":"da"},{"input":"cdaaab","output":"da"},{"input":"abbccabba","output":"bc"},{"input":"bbbcbccdaca","output":"bb"},{"input":"aadddbbcbb","output":"ad"},{"input":"aadba","output":"aa"},{"input":"bdcdb","output":"db"},{"input":"ddaababdd","output":"db"},{"input":"abbbdaca","output":"bd"},{"input":"dbddaddbadbbcc","output":"ba"},{"input":"cddadaccbbbd","output":"dd"},{"input":"addaacbcaacabd","output":"da"},{"input":"bdbcaacdabbb","output":"da"},{"input":"dcddcdabacbcd","output":"cc"},{"input":"acabbcba","output":"cb"},{"input":"dbcdcbccbcdc","output":"bc"},{"input":"abccabcd","output":"ba"},{"input":"ddcbaccd","output":"da"},{"input":"dbcabadcaa","output":"bb"},{"input":"ccdddccdabab","output":"cd"},{"input":"cbbabbb","output":"bb"},{"input":"bccdbcdbbac","output":"cb"},{"input":"cdbdbbdbdcaddd","output":"db"},{"input":"dddbbcbbdcbb","output":"db"},{"input":"caacdcc","output":"ad"},{"input":"bbbccdc","output":"bc"},{"input":"dcdacda","output":"cc"},{"input":"cccbbbbbcaabab","output":"cb"},{"input":"abdada","output":"bd"},{"input":"bccaacdbcabbb","output":"ca"},{"input":"dbdcacddaab","output":"ba"},{"input":"bcbaaccd","output":"ca"},{"input":"bcacbd","output":"cb"},{"input":"acacdabdcacc","output":"cd"},{"input":"ddacdcbdddc","output":"dd"},{"input":"bcccdad","output":"cd"},{"input":"bccddadaccdca","output":"cd"},{"input":"daaddaacdcd","output":"ad"},{"input":"cdcbb","output":"db"},{"input":"bcbaaddacccba","output":"ca"},{"input":"acabcccdd","output":"cc"},{"input":"abacbbdbbcbdb","output":"bb"},{"input":"cdcaa","output":"da"},{"input":"acbcdbaddcadba","output":"cd"},{"input":"dcbcd","output":"cd"},{"input":"bbbbcbd","output":"bc"},{"input":"daaca","output":"aa"},{"input":"dcaadbcbcc","output":"cd"},{"input":"bccabbdccbac","output":"cb"},{"input":"bcbacccdbbabba","output":"cc"},{"input":"dcdbddabaadcbd","output":"cd"},{"input":"cdaabacbabdcc","output":"db"},{"input":"dabcddbadcdd","output":"ad"},{"input":"babdbbdddcca","output":"ab"},{"input":"bbbcbc","output":"bb"},{"input":"dcaddcbcac","output":"cd"},{"input":"daadccb","output":"ac"},{"input":"dddccbccacc","output":"dc"},{"input":"dbdbaaba","output":"ba"},{"input":"aadcbaacbcddc","output":"ab"},{"input":"cbddabaadcb","output":"ba"},{"input":"adabdaba","output":"dd"},{"input":"bcccbcacccb","output":"cb"},{"input":"abaadbacccdabc","output":"bd"},{"input":"cdcbbbd","output":"db"},{"input":"dadacdacd","output":"ac"},{"input":"ccbabaddb","output":"cb"},{"input":"adcadbba","output":"dd"},{"input":"dcbbbcbacb","output":"cb"},{"input":"addbdcacaaddca","output":"dd"},{"input":"adbaabbcb","output":"da"},{"input":"cdbbabcbd","output":"da"},{"input":"cbdabbbddc","output":"bb"},{"input":"acbcdda","output":"cd"},{"input":"cdcbdccdbd","output":"dd"},{"input":"cabcacbcadb","output":"aa"},{"input":"dabcbcddaddbcc","output":"ab"},{"input":"bdcbcadbbcaa","output":"dc"},{"input":"bdcab","output":"db"},{"input":"acabbddadc","output":"cb"},{"input":"dacdbaa","output":"ab"},{"input":"bbcbadbabd","output":"ba"},{"input":"abbdb","output":"bb"},{"input":"abdbcaabccadcb","output":"bc"},{"input":"baaabcbc","output":"ab"},{"input":"baacdddca","output":"ad"},{"input":"ddaada","output":"dd"},{"input":"acdaabc","output":"ca"},{"input":"daacdacdbbb","output":"ad"},{"input":"cdabddbbc","output":"dd"},{"input":"abdcbccdd","output":"bb"},{"input":"cbada","output":"ba"},{"input":"cbabacbacbbd","output":"ba"},{"input":"adddcddddc","output":"dc"},{"input":"abdbadc","output":"ba"},{"input":"caadabc","output":"aa"},{"input":"acbbab","output":"ca"},{"input":"cdcddadbcca","output":"dd"},{"input":"ccaccddaccddbb","output":"cc"},{"input":"abccdbab","output":"bd"},{"input":"cccbaadba","output":"ca"},{"input":"dacccbbabcc","output":"ac"},{"input":"dcbcdddcdcda","output":"cd"},{"input":"babcd","output":"ad"},{"input":"aaccdab","output":"ad"},{"input":"abccccadbd","output":"bc"},{"input":"bdaaa","output":"da"},{"input":"ccdbcacdadab","output":"cc"},{"input":"abcbdcdabb","output":"bd"},{"input":"daddadc","output":"aa"},{"input":"dcbcdacbc","output":"cd"},{"input":"bddbc","output":"dc"},{"input":"dbbddcacdbdcbb","output":"bd"},{"input":"cabaddaadaaccc","output":"ad"},{"input":"dadcbac","output":"ab"},{"input":"abddcdcbbab","output":"bc"},{"input":"baaadabbcd","output":"ad"},{"input":"dacdcabdac","output":"ac"},{"input":"dcbbabbbb","output":"ca"},{"input":"dadadbdaa","output":"ad"},{"input":"adbacbdbadab","output":"dc"},{"input":"ccacdbcbdcac","output":"cd"},{"input":"addcdacddb","output":"dd"},{"input":"bccbda","output":"cd"},{"input":"dccaadacdcc","output":"ca"},{"input":"dbcdcaabb","output":"bc"},{"input":"cabadc","output":"ad"},{"input":"accaacccaacaba","output":"ca"},{"input":"cacaabdcdcb","output":"aa"},{"input":"adbbcddc","output":"dc"},{"input":"cbacbbcadbcac","output":"bb"},{"input":"bbaacbbadaaaab","output":"bc"},{"input":"dbbaddcbc","output":"bd"},{"input":"babdbc","output":"ab"},{"input":"abbbca","output":"bc"},{"input":"cacbbdabdbcbcb","output":"ab"},{"input":"bdddac","output":"da"},{"input":"cbbdbd","output":"bb"},{"input":"ccabcacbac","output":"cc"},{"input":"ddadaad","output":"da"},{"input":"cbbabdbdcdddc","output":"bb"},{"input":"bdaddad","output":"dd"},{"input":"ccaabdbaaaad","output":"cb"},{"input":"ddcdbcbdb","output":"db"},{"input":"bdbbacadbacd","output":"da"},{"input":"ddadbabcac","output":"db"},{"input":"badaabdd","output":"aa"},{"input":"bccbccbadacdcd","output":"cc"},{"input":"aacabdda","output":"ab"},{"input":"dcacddcbcbbbd","output":"cd"},{"input":"adcccbdbabdbb","output":"dc"},{"input":"cbdbbadccabdc","output":"bb"},{"input":"bbccacaacc","output":"ba"},{"input":"acadab","output":"ca"},{"input":"dbcbacc","output":"ba"},{"input":"ababaad","output":"ba"},{"input":"acdacaac","output":"cc"},{"input":"bdaccba","output":"dc"},{"input":"ccddda","output":"cd"},{"input":"dbbdbdcba","output":"bb"},{"input":"acddaabbbc","output":"ca"},{"input":"cbdbdabd","output":"bd"},{"input":"bcdaccdacc","output":"cc"},{"input":"bacbdac","output":"ad"},{"input":"cddacdacab","output":"dc"},{"input":"cacbacba","output":"aa"},{"input":"abdbacbaa","output":"ba"},{"input":"bccdbbddddca","output":"cb"},{"input":"bdacdbbcacad","output":"dd"},{"input":"cbcbdcacacbc","output":"bd"},{"input":"dbabcaabbd","output":"bc"},{"input":"daabcab","output":"ac"},{"input":"bccadabdcac","output":"cd"},{"input":"bbcccbaca","output":"bc"},{"input":"dacdcdaaacb","output":"ac"},{"input":"badcbd","output":"ab"},{"input":"ccbaabbbbd","output":"ca"},{"input":"accdbdbccc","output":"cb"},{"input":"dacab","output":"ab"},{"input":"cddabaabbb","output":"db"},{"input":"dacddddd","output":"ad"},{"input":"cdbddcadddab","output":"dd"},{"input":"dbcbcddcdaca","output":"bc"},{"input":"cbacbcccbaabcc","output":"bb"},{"input":"ddcadbbdbabb","output":"dd"},{"input":"cdabcccaa","output":"dc"},{"input":"cbdcccdcbbccc","output":"bc"},{"input":"cadbbddda","output":"ab"},{"input":"bdcaddbaa","output":"dd"},{"input":"daacdccbdbcbaa","output":"ad"},{"input":"bbdbaadda","output":"ba"},{"input":"adababdbbadcb","output":"da"},{"input":"caabbaaadcd","output":"ab"},{"input":"dabbbcbdabb","output":"ab"},{"input":"badbaccddbb","output":"aa"},{"input":"cabbcad","output":"ac"},{"input":"bbbddcdaaacd","output":"bd"},{"input":"cbbdcda","output":"bc"},{"input":"cbbbca","output":"bc"},{"input":"adadabbaad","output":"da"},{"input":"cbbbdaabdbda","output":"bd"},{"input":"bcaaaddb","output":"ca"},{"input":"bacbdaada","output":"ad"},{"input":"bcbcdcadbad","output":"cd"},{"input":"bcdbdba","output":"cd"},{"input":"bbbdc","output":"bc"},{"input":"caacacdabadd","output":"aa"},{"input":"aacbcc","output":"ac"},{"input":"aaacbccb","output":"ab"},{"input":"bcadacbbddca","output":"ca"},{"input":"ccacdb","output":"cd"},{"input":"aacadbb","output":"ad"},{"input":"bcdddcaabbcbb","output":"cd"},{"input":"ddcbabdddaab","output":"da"},{"input":"aadcd","output":"ad"},{"input":"bcdbbdbb","output":"cb"},{"input":"cbdba","output":"ba"},{"input":"ddacddddadccd","output":"dd"},{"input":"cdccb","output":"db"},{"input":"dadbadaccdcdbc","output":"aa"},{"input":"aabbac","output":"aa"},{"input":"dabba","output":"aa"},{"input":"bbdadc","output":"bd"},{"input":"bacdcd","output":"ac"},{"input":"bdddbdbcacc","output":"db"},{"input":"cddabd","output":"db"},{"input":"bbddad","output":"ba"},{"input":"dcaccbad","output":"cc"},{"input":"dbabacb","output":"ba"},{"input":"ddaaccad","output":"dc"},{"input":"ababbbabacdacc","output":"bb"},{"input":"cdabaccddba","output":"da"},{"input":"abcbababb","output":"ba"},{"input":"bcbbbbc","output":"cb"},{"input":"cabacac","output":"ac"},{"input":"dbbdadad","output":"ba"},{"input":"dbdccab","output":"bc"},{"input":"ccbbdadbaaba","output":"cd"},{"input":"ccacbaac","output":"cb"},{"input":"aadccadacbcddb","output":"ac"},{"input":"bcbbbb","output":"cb"},{"input":"aabaca","output":"ac"},{"input":"daadbb","output":"ab"},{"input":"bbcdabccdcb","output":"ba"},{"input":"cbacd","output":"bd"},{"input":"dabbabdbcb","output":"aa"},{"input":"acbbbbbcdbcb","output":"cb"},{"input":"aaccdddadc","output":"ad"},{"input":"ddaccbacdbdbbc","output":"dc"},{"input":"daabab","output":"aa"},{"input":"abcdaddd","output":"ba"},{"input":"acbcdca","output":"cd"},{"input":"cbbcaca","output":"ba"},{"input":"aabddcdadbd","output":"ad"},{"input":"cadcba","output":"ab"},{"input":"dccabcb","output":"cb"},{"input":"dadcc","output":"ac"},{"input":"acbaab","output":"ca"},{"input":"ddadd","output":"dd"},{"input":"adddcdabdddb","output":"dc"},{"input":"aadddc","output":"ad"},{"input":"ddcaa","output":"da"},{"input":"bacdbc","output":"ab"},{"input":"aacacaabaadccb","output":"ac"},{"input":"adbcccabaadb","output":"dc"},{"input":"adadabddca","output":"da"},{"input":"baadb","output":"ab"},{"input":"bccddbd","output":"cd"},{"input":"dccbbd","output":"cb"},{"input":"dcbbabdacab","output":"ca"},{"input":"dcccdad","output":"cd"},{"input":"dbdabcaabc","output":"bb"},{"input":"dabdb","output":"ab"},{"input":"cbadd","output":"bd"},{"input":"bbaca","output":"ba"},{"input":"ddddbbcadbc","output":"db"},{"input":"ccdddadab","output":"cd"},{"input":"ddcacb","output":"dc"},{"input":"bcdaddacaaabba","output":"cd"},{"input":"bddcbbbcbacd","output":"db"},{"input":"bdacabdbaadbad","output":"da"},{"input":"babda","output":"aa"},{"input":"baabcccbbbdda","output":"ac"},{"input":"ddcbddaaacdbdb","output":"dd"},{"input":"badddcddcab","output":"ad"},{"input":"ddcaabacdbaca","output":"da"},{"input":"cddaaaa","output":"da"},{"input":"cdbbbcdaacbc","output":"db"},{"input":"baddccddccd","output":"ac"},{"input":"aacddddac","output":"ad"},{"input":"cbbadda","output":"bd"},{"input":"bdaccdddabcaca","output":"dc"},{"input":"dabdbabacbd","output":"ab"},{"input":"cddacdcddcdbc","output":"dc"},{"input":"acbabadcabbcc","output":"cb"},{"input":"cdbca","output":"da"},{"input":"addaaacbdbaab","output":"da"},{"input":"baccb","output":"ab"},{"input":"abdaacdbab","output":"ba"},{"input":"ababc","output":"bc"},{"input":"adcdbcba","output":"db"},{"input":"cdbccba","output":"dc"},{"input":"ddacdaccdba","output":"dd"},{"input":"dcbdcd","output":"cc"},{"input":"bacccdaad","output":"ac"},{"input":"cbadaad","output":"ba"},{"input":"caadd","output":"ad"},{"input":"adccacbba","output":"da"},{"input":"cabbaaadbcad","output":"aa"},{"input":"aaabdbbbc","output":"ad"},{"input":"dadbbabbbdcccc","output":"ab"},{"input":"cdccad","output":"da"},{"input":"bcdbddbdbbddcb","output":"cd"},{"input":"accdbdbccbb","output":"cb"},{"input":"daaabaccbba","output":"ab"},{"input":"cadbbbcdabccba","output":"ab"},{"input":"bcbbbbcacb","output":"cb"},{"input":"adbacadcaadcd","output":"dc"},{"input":"addbcbddddddbb","output":"dc"},{"input":"acdaacccdccba","output":"ca"},{"input":"accbddccac","output":"cd"},{"input":"cbbbd","output":"bd"},{"input":"bbcbadcccadab","output":"ba"},{"input":"bdaccdabca","output":"dc"},{"input":"abaabc","output":"bb"},{"input":"adcabcd","output":"db"},{"input":"abbdaaaadc","output":"ba"},{"input":"abddddca","output":"bd"},{"input":"ddbbcbbdbbdc","output":"dc"},{"input":"cbaaab","output":"ba"},{"input":"ddabd","output":"dd"},{"input":"abbbaacdbda","output":"ba"},{"input":"bbcbd","output":"bd"},{"input":"bcdadccdc","output":"cd"},{"input":"baaaddddad","output":"ad"},{"input":"cdbaabc","output":"da"},{"input":"bbdddda","output":"bd"},{"input":"bccddbcd","output":"cd"},{"input":"dadcbbabbdaac","output":"ab"},{"input":"dcaccbb","output":"cc"},{"input":"abbddbbdabdcd","output":"bd"},{"input":"acbbabc","output":"ca"},{"input":"ddbddacabadd","output":"dd"},{"input":"bdaaddbcdabcaa","output":"dd"},{"input":"ccddadad","output":"ca"},{"input":"badcdcabd","output":"ad"},{"input":"bcdbaaccbbac","output":"ca"},{"input":"cbcbdbbcbcc","output":"bd"},{"input":"dbcda","output":"ba"},{"input":"dbabbdbd","output":"bb"},{"input":"bbcdabacc","output":"ba"},{"input":"dacba","output":"aa"},{"input":"cddaddaaad","output":"dd"},{"input":"babcbccacc","output":"ab"},{"input":"ddcbacacbdcbd","output":"da"},{"input":"dacbda","output":"ad"},{"input":"caddac","output":"aa"},{"input":"cbadbccbcb","output":"bb"},{"input":"bbaacca","output":"bc"},{"input":"abbbdc","output":"bd"},{"input":"cdccabdc","output":"da"},{"input":"dadbdcda","output":"ad"},{"input":"daacadcdcba","output":"aa"},{"input":"dcbabcda","output":"cb"},{"input":"bbbaccbca","output":"bc"},{"input":"cbabaaabcab","output":"ba"},{"input":"dccdcabc","output":"cc"},{"input":"dbbcbbbdccb","output":"bb"},{"input":"bbccaadcaaab","output":"ba"},{"input":"ddcbccaaaaad","output":"dc"},{"input":"dcdaaccaacb","output":"ca"},{"input":"dcbacddadad","output":"cc"},{"input":"bdaadc","output":"dd"},{"input":"bcadadbdcbaac","output":"ca"},{"input":"acdbacacacbbcb","output":"ca"},{"input":"acbdbcbbbba","output":"cb"},{"input":"dacbcbbbbaba","output":"ac"},{"input":"caddcadb","output":"ac"},{"input":"cdcdcb","output":"dc"},{"input":"cddbbdabbad","output":"db"},{"input":"dcaaababdbddb","output":"ca"},{"input":"abcbd","output":"bd"},{"input":"dbddc","output":"bc"},{"input":"adaadbad","output":"dd"},{"input":"dbbccbabbacaa","output":"bc"},{"input":"daaaabcdc","output":"aa"},{"input":"ddcca","output":"da"},{"input":"cccdccc","output":"cc"},{"input":"baacc","output":"ac"},{"input":"ddcdc","output":"dc"},{"input":"baccbadd","output":"ab"},{"input":"ddbccadda","output":"dc"},{"input":"acdddbdbcb","output":"cd"},{"input":"ccdddab","output":"cd"},{"input":"cacabccbdcdcbb","output":"ab"},{"input":"cbccc","output":"bc"},{"input":"daabcccd","output":"ac"},{"input":"bbdbd","output":"bd"},{"input":"abadcadaa","output":"bc"},{"input":"ababdba","output":"bd"},{"input":"bbcbbaabcdd","output":"bb"},{"input":"bcbdcddcacccaa","output":"cc"},{"input":"bbcaaabcda","output":"ba"},{"input":"cccdc","output":"cc"},{"input":"acddbc","output":"cb"},{"input":"abbabcaadaba","output":"bb"},{"input":"aaabadbbc","output":"aa"},{"input":"dcbcbadb","output":"cb"},{"input":"acbacacacdca","output":"cc"},{"input":"aaccdddc","output":"ad"},{"input":"aacbbcbb","output":"ab"},{"input":"dbada","output":"ba"},{"input":"bccbddacbadbcd","output":"cd"},{"input":"baadcbaaccbdd","output":"ac"},{"input":"abddada","output":"ba"},{"input":"baabdbda","output":"ad"},{"input":"dddbabadaaacb","output":"da"},{"input":"acbdbbcbcd","output":"cb"},{"input":"bdbccbddada","output":"dc"},{"input":"addadacb","output":"dd"},{"input":"cdccacc","output":"da"},{"input":"bcbcd","output":"cd"},{"input":"bbccddcc","output":"bd"},{"input":"babbbbdbbc","output":"ab"},{"input":"abdbdccada","output":"bd"},{"input":"abbaa","output":"ba"},{"input":"ddcbdbaadbca","output":"dd"},{"input":"acbba","output":"ca"},{"input":"baddacacdadadd","output":"aa"},{"input":"acbdaaabd","output":"ca"},{"input":"bcdacbbddbb","output":"cc"},{"input":"dcddadddab","output":"ca"},{"input":"baadcabbddcbc","output":"ac"},{"input":"dbdac","output":"bc"},{"input":"cadbaddbdcddc","output":"aa"},{"input":"dcbaadddbbc","output":"ca"},{"input":"abbcacbdaddb","output":"ba"},{"input":"acaacdaacbabdb","output":"cc"},{"input":"bcabaacddada","output":"ca"},{"input":"bccddbdddb","output":"cd"},{"input":"ddadbcdbbdb","output":"db"},{"input":"aacabddccdd","output":"ab"},{"input":"ddcbbcadcdc","output":"db"},{"input":"adacaccbddbda","output":"da"},{"input":"cdbcac","output":"da"},{"input":"cadda","output":"aa"},{"input":"bcbddb","output":"cd"},{"input":"ccbbbadac","output":"cb"},{"input":"dbcacadac","output":"bc"},{"input":"dbdaad","output":"ba"},{"input":"adcbcabdacacad","output":"dc"},{"input":"bccabda","output":"cb"},{"input":"ddcbba","output":"db"},{"input":"ababcac","output":"bc"},{"input":"ddbaabbdadad","output":"da"},{"input":"badbaddb","output":"aa"},{"input":"abbad","output":"bd"},{"input":"cbaccdb","output":"bc"},{"input":"ccacaaacbac","output":"ca"},{"input":"adaab","output":"db"},{"input":"aaabacdbaadaab","output":"aa"},{"input":"baaaac","output":"aa"},{"input":"abdccdccadbdd","output":"bc"},{"input":"cccdbbddab","output":"cb"},{"input":"bdddddcccb","output":"dd"},{"input":"dabcaadacdcdba","output":"aa"},{"input":"cbbbdbbd","output":"bd"},{"input":"bbacab","output":"ba"},{"input":"cdbdcacbb","output":"dc"},{"input":"cccaabcbddb","output":"ca"},{"input":"cadddaccdbdccb","output":"ad"},{"input":"cdbdacac","output":"da"},{"input":"dcdccbccadbb","output":"cc"},{"input":"aaabbbbb","output":"ab"},{"input":"aabcbacbb","output":"ab"},{"input":"dabbdbcccdacc","output":"ad"},{"input":"aacdcbccdcbdbd","output":"ac"},{"input":"dbbbbdcc","output":"bb"},{"input":"adcbbdcc","output":"db"},{"input":"cdacccacaa","output":"dc"},{"input":"cdcbddabcddcab","output":"dd"},{"input":"ccacaccacddddc","output":"ca"},{"input":"adbdada","output":"da"},{"input":"ddaabbdddadcc","output":"db"},{"input":"aadaadaccd","output":"aa"},{"input":"bacaa","output":"aa"},{"input":"adcbdbb","output":"dd"},{"input":"abdcabc","output":"ba"},{"input":"abaaaabccdb","output":"ba"},{"input":"abddbbdccaa","output":"bb"},{"input":"acabacdbbb","output":"ca"},{"input":"ccacbbd","output":"cb"},{"input":"acccccbaccdbac","output":"cc"},{"input":"cdddaaaa","output":"da"},{"input":"ddada","output":"da"},{"input":"bdcccbbbca","output":"dc"},{"input":"bdccda","output":"dd"},{"input":"adcbd","output":"dd"},{"input":"abdcbdbb","output":"bb"},{"input":"bdccbcbdcad","output":"db"},{"input":"bcdbdabdbcbaa","output":"cd"},{"input":"cdcbbcaab","output":"db"},{"input":"cdaddaacc","output":"dd"},{"input":"aadccc","output":"ac"},{"input":"cccbbbbd","output":"cb"},{"input":"dcacbddddc","output":"cb"},{"input":"accbbdb","output":"cb"},{"input":"dabcbbd","output":"ab"},{"input":"dadbdadcc","output":"ad"},{"input":"aadccadb","output":"ac"},{"input":"cddbadd","output":"da"},{"input":"addacabdabbcd","output":"dc"},{"input":"cadaccc","output":"ac"},{"input":"bbccddbcdc","output":"bd"},{"input":"dcbcbdcabbbd","output":"cb"},{"input":"aacbd","output":"ad"},{"input":"bddabac","output":"db"},{"input":"baaaab","output":"aa"},{"input":"cabadddccacd","output":"ad"},{"input":"dddacadcbd","output":"dc"},{"input":"cbcacaadaca","output":"bc"},{"input":"bbbdcac","output":"bc"},{"input":"bdaaadaccbc","output":"da"},{"input":"cbdabdcad","output":"bb"},{"input":"ccbabccbc","output":"cb"},{"input":"caddb","output":"ab"},{"input":"acbdcbbc","output":"cc"},{"input":"cacabdddcbc","output":"ab"},{"input":"abddcbccbabb","output":"bc"},{"input":"adddbcdccc","output":"db"},{"input":"adacdbaccddabc","output":"dd"},{"input":"dbcaabbcacac","output":"ba"},{"input":"abbdad","output":"ba"},{"input":"cacccbabcdabc","output":"ac"},{"input":"dcdbcbdabdac","output":"cc"},{"input":"adbca","output":"da"},{"input":"cabcbabaddccd","output":"ab"},{"input":"dcacda","output":"cd"},{"input":"abacdcbbacad","output":"bd"},{"input":"bbadacdcdcd","output":"ba"},{"input":"babbddaddddda","output":"ad"},{"input":"bbcbdd","output":"bd"},{"input":"badaacbbaccb","output":"aa"},{"input":"baabcd","output":"ac"},{"input":"aaccbd","output":"ab"},{"input":"bacdbbad","output":"ab"},{"input":"ddcadcbcdddcd","output":"dd"},{"input":"bacbaba","output":"aa"},{"input":"acdaa","output":"ca"},{"input":"acdbcadccbadb","output":"cc"},{"input":"cabcbababbcb","output":"ab"},{"input":"bdabcdbdabbb","output":"dc"},{"input":"dbabbc","output":"bb"},{"input":"adacccd","output":"dc"},{"input":"ddadacdab","output":"da"},{"input":"cccadcbaada","output":"cd"},{"input":"dbbbbdbdca","output":"bb"},{"input":"dcbbcbdcab","output":"cc"},{"input":"dcbbaaac","output":"ca"},{"input":"abdbdcccbccdc","output":"bd"},{"input":"aacdcdcd","output":"ac"},{"input":"cdbbcbd","output":"dc"},{"input":"baababcb","output":"aa"},{"input":"bbbaaacdcaba","output":"ba"},{"input":"bdbcdbbbbca","output":"dd"},{"input":"bccccd","output":"cc"},{"input":"dabadbcc","output":"ad"},{"input":"dbdaadcabb","output":"ba"},{"input":"dcadddcdaddada","output":"cd"},{"input":"cddcbcdcbaddaa","output":"db"},{"input":"aabcadbccddab","output":"aa"},{"input":"dacacccd","output":"ac"},{"input":"dbbcacdbba","output":"ba"},{"input":"aadadccc","output":"ad"},{"input":"bbcdcaddabbbbd","output":"bc"},{"input":"bdabacbbb","output":"da"},{"input":"daddcadb","output":"ac"},{"input":"cdacbdaddddd","output":"db"},{"input":"ddcaabada","output":"da"},{"input":"ccaaad","output":"ca"},{"input":"adbbb","output":"db"},{"input":"adbcdcdca","output":"dd"},{"input":"accbbd","output":"cb"},{"input":"dacbd","output":"ad"},{"input":"ababc","output":"bc"},{"input":"cabcdcc","output":"ad"},{"input":"cbcbc","output":"bc"},{"input":"bcddccbbaaa","output":"cc"},{"input":"accdca","output":"cc"},{"input":"acaaa","output":"ca"},{"input":"dbcdd","output":"bd"},{"input":"cdcbbdaccac","output":"db"},{"input":"acdba","output":"ca"},{"input":"acccab","output":"ca"},{"input":"cbbaaacaaaba","output":"ba"},{"input":"ccdabccdcbdcca","output":"cb"},{"input":"dcdddbcacba","output":"cd"},{"input":"aacdaad","output":"aa"},{"input":"bbdddbccdbcdac","output":"bd"},{"input":"aacbbcdbabcdb","output":"ab"},{"input":"cccbdba","output":"cd"},{"input":"daadcccac","output":"ac"},{"input":"cbbacbcacc","output":"bc"},{"input":"aacaaca","output":"aa"},{"input":"bbdddc","output":"bd"},{"input":"abbbddd","output":"bd"},{"input":"babccaabdacd","output":"ac"},{"input":"abcaa","output":"ba"},{"input":"bcabcababc","output":"cc"},{"input":"cdccdcadbdda","output":"dd"},{"input":"aadcbcca","output":"ab"},{"input":"acbcababacbdcb","output":"ca"},{"input":"ababdbcbbad","output":"bd"},{"input":"abbbcccabdddd","output":"bc"},{"input":"abbddaa","output":"bd"},{"input":"cddbbbc","output":"db"},{"input":"badcdda","output":"ad"},{"input":"dbcbad","output":"ba"},{"input":"ddccabdabdcba","output":"da"},{"input":"aacbbdd","output":"ab"},{"input":"dbadcccdbaca","output":"bc"},{"input":"baddcdc","output":"ac"},{"input":"ccccad","output":"ca"},{"input":"dccdcdaddca","output":"cc"},{"input":"cdabaabacd","output":"da"},{"input":"cbaabadbdb","output":"bb"},{"input":"ccbccdad","output":"cc"},{"input":"acddbbd","output":"cb"},{"input":"abcabbbabbb","output":"bb"},{"input":"cadbcbbdcddaa","output":"ac"},{"input":"bcbcdcadb","output":"cd"},{"input":"cabdcbdbacadb","output":"ac"},{"input":"aabacabcdaacb","output":"ac"},{"input":"dbdbabbcdddadc","output":"ba"},{"input":"dcbabacdc","output":"cb"},{"input":"abbda","output":"ba"},{"input":"adbddbbdbc","output":"dd"},{"input":"ddabddbcbb","output":"dd"},{"input":"cbabbacb","output":"bb"},{"input":"abdabbac","output":"bb"},{"input":"bdbcada","output":"da"},{"input":"ddacdacacbb","output":"dd"},{"input":"aabadaa","output":"ad"},{"input":"dadcccdacc","output":"ac"},{"input":"dbdcdbabbdbadc","output":"bd"},{"input":"acaabccdbb","output":"cb"},{"input":"ccdaba","output":"cb"},{"input":"ccbcbbdcbab","output":"cb"},{"input":"dddbbcdbcdddad","output":"db"},{"input":"cbabadcbaa","output":"ba"},{"input":"bcddd","output":"cd"},{"input":"cdcbb","output":"db"},{"input":"bccaddadacbaa","output":"cd"},{"input":"bacacaa","output":"ac"},{"input":"abdbcdaabbcba","output":"bc"},{"input":"bbcbbbcb","output":"bb"},{"input":"aacaccbcbbdaa","output":"ac"},{"input":"baadccba","output":"ac"},{"input":"dacadbcabcbda","output":"ad"},{"input":"dbbaadbbbbcdcb","output":"ba"},{"input":"aabccbdc","output":"ac"},{"input":"bbbddd","output":"bd"},{"input":"abcacdcdabcb","output":"bc"},{"input":"cbbadbcdcccbaa","output":"bd"},{"input":"cbbbdbdacdb","output":"bd"},{"input":"dbcddbdaa","output":"bd"},{"input":"cdcbccbacdcb","output":"dc"},{"input":"cdddb","output":"db"},{"input":"adbcbdadaccdc","output":"db"},{"input":"dadbddbc","output":"ad"},{"input":"dbbddbdaaa","output":"bd"},{"input":"bababddbd","output":"ab"},{"input":"abcaddd","output":"bd"},{"input":"dccbbccbd","output":"cb"},{"input":"bbcaadcccada","output":"ba"},{"input":"abbcbcaccacdbd","output":"bb"},{"input":"bccdacdaacacbd","output":"ca"},{"input":"acadadbadb","output":"ca"},{"input":"bddcaddcba","output":"da"},{"input":"cbbddbcdd","output":"bd"},{"input":"addbad","output":"da"},{"input":"bbcabbcaa","output":"bb"},{"input":"dccbdd","output":"cd"},{"input":"bdaaa","output":"da"},{"input":"daccacadc","output":"aa"},{"input":"dcacadbbaadbbb","output":"ca"},{"input":"adacdcabddd","output":"dd"},{"input":"baaaabdcaabad","output":"aa"},{"input":"bccbcdb","output":"cc"},{"input":"bddda","output":"da"},{"input":"ddbcc","output":"dc"},{"input":"bdcdcd","output":"dc"},{"input":"dcbccdbdca","output":"cc"},{"input":"cdcbcabbbccdd","output":"dc"},{"input":"abbabdbbbaacb","output":"bb"},{"input":"badbdd","output":"ad"},{"input":"dccad","output":"cd"},{"input":"dbacdbadbcbda","output":"bd"},{"input":"abcdaca","output":"ba"},{"input":"dbbacaddbabb","output":"bc"},{"input":"badbabbcdb","output":"aa"},{"input":"bbaaccbb","output":"bc"},{"input":"cbdbbdbabaad","output":"bb"},{"input":"cbcadabcdc","output":"bd"},{"input":"abaccdaa","output":"bc"},{"input":"addbacdd","output":"da"},{"input":"dddacd","output":"dc"},{"input":"acaccdccd","output":"cc"},{"input":"cccdbcbaaabbac","output":"cb"},{"input":"ccbadcbabdac","output":"cd"},{"input":"dbcdbbabbadba","output":"bb"},{"input":"ddababddbabcdd","output":"da"},{"input":"abbdaacaa","output":"ba"},{"input":"dcdbca","output":"cc"},{"input":"bcbcbbbab","output":"cb"},{"input":"dbddabdbaab","output":"ba"},{"input":"addddacbdb","output":"dd"},{"input":"babbcbbbccc","output":"ac"},{"input":"caaddbd","output":"ad"},{"input":"cabaadab","output":"aa"},{"input":"cdadcdbaddaac","output":"dc"},{"input":"abaca","output":"ba"},{"input":"dcdccdddcbd","output":"cc"},{"input":"aadaadaacbcab","output":"aa"},{"input":"acddabaddacdb","output":"ca"},{"input":"ccadbddcdccdc","output":"cb"},{"input":"abdcccdba","output":"bc"},{"input":"bbccacadaadaa","output":"ba"},{"input":"cccdcacccbcccb","output":"cc"},{"input":"adbcdaabadc","output":"dd"},{"input":"cdbbcacbdcdbad","output":"dc"},{"input":"caacdbbabc","output":"ad"},{"input":"bdccdbdbaccdad","output":"dd"},{"input":"ccacbb","output":"cb"},{"input":"aadabd","output":"ab"},{"input":"accaadcbc","output":"ca"},{"input":"cdbdadcddcdada","output":"da"},{"input":"dbcab","output":"bb"},{"input":"ddbbddccacdd","output":"dd"},{"input":"dadbddbad","output":"ad"},{"input":"daccbdcdbad","output":"ab"},{"input":"dadddbaa","output":"ad"},{"input":"acbbbccbca","output":"cb"},{"input":"caaacdcaaadd","output":"ac"},{"input":"bacacbbbabdb","output":"ac"},{"input":"dcbdabddbdd","output":"ca"},{"input":"babadaacdadbdc","output":"ad"},{"input":"ddabbcacbaabda","output":"db"},{"input":"dbcabbdddbaada","output":"bb"},{"input":"aacbdc","output":"ad"},{"input":"cbbcaad","output":"ba"},{"input":"caaabcddbbd","output":"ab"},{"input":"caccbcbabdda","output":"ab"},{"input":"daadddadacbb","output":"ad"},{"input":"ccbaa","output":"ca"},{"input":"daababbad","output":"aa"},{"input":"addaccdabcd","output":"dc"},{"input":"ddcaaabacacc","output":"da"},{"input":"dabacabaa","output":"ac"},{"input":"ddaabadaada","output":"db"},{"input":"baadcaaddddaca","output":"ac"},{"input":"daabdccadadbc","output":"ad"},{"input":"ccada","output":"ca"},{"input":"dccabdbcadc","output":"cb"},{"input":"dcbdabadb","output":"ca"},{"input":"baddbcdcacaa","output":"ab"},{"input":"bbbcacbaaddab","output":"ba"},{"input":"abbabacd","output":"bb"},{"input":"cbdcabdcab","output":"ba"},{"input":"dcdbaccacaca","output":"ca"},{"input":"aaccdcbbdbc","output":"ad"},{"input":"bcbdaaba","output":"ca"},{"input":"cdacccab","output":"dc"},{"input":"bcbbabcbadda","output":"ca"},{"input":"ccbabbbaaad","output":"cb"},{"input":"cddacd","output":"dc"},{"input":"ddcdc","output":"dc"},{"input":"adbdcbbdacdcdb","output":"dc"},{"input":"dbbaa","output":"ba"},{"input":"bbdbbdddbddc","output":"bb"},{"input":"dccdaa","output":"ca"},{"input":"cabbbaacbcabbd","output":"ab"},{"input":"daabacadaaaa","output":"aa"},{"input":"adbbddb","output":"dd"},{"input":"dcddbbcdbabbbd","output":"cb"},{"input":"bbaadccccd","output":"bd"},{"input":"abcada","output":"bd"},{"input":"bbcadba","output":"bd"},{"input":"dbddddbb","output":"bd"},{"input":"aadcbddbac","output":"ab"},{"input":"abadab","output":"ba"},{"input":"ddccdcbada","output":"dd"},{"input":"dbabbbcd","output":"bb"},{"input":"dcbaccabcdd","output":"cc"},{"input":"dcdcddbccacdb","output":"cd"},{"input":"ccdbbdbccbad","output":"cb"},{"input":"dcbddcdbaccd","output":"cd"},{"input":"dbdbcbcbc","output":"bc"},{"input":"cdddadabbcaca","output":"da"},{"input":"adabdcadb","output":"dd"},{"input":"bbacaaddcaadb","output":"ba"},{"input":"adaaaaddc","output":"da"},{"input":"ddccccbbbd","output":"dc"},{"input":"cdbbcacdd","output":"dc"},{"input":"ccacdbdbcaa","output":"cd"},{"input":"aabacbcacdb","output":"ac"},{"input":"bbbbba","output":"bb"},{"input":"ddbbadaca","output":"da"},{"input":"caabaccada","output":"aa"},{"input":"bdacaca","output":"da"},{"input":"bbdcbab","output":"bb"},{"input":"dbcdccbaab","output":"bc"},{"input":"bbabac","output":"ba"},{"input":"daddddccbada","output":"ad"},{"input":"accbadccdac","output":"ca"},{"input":"bbddcccddbbd","output":"bc"},{"input":"bacca","output":"aa"},{"input":"ababdbcaccd","output":"bd"},{"input":"adaadcb","output":"dd"},{"input":"bccdbbbd","output":"cb"},{"input":"aaabadc","output":"aa"},{"input":"caacabaccacdc","output":"aa"},{"input":"daadd","output":"ad"},{"input":"cdadac","output":"da"},{"input":"cdacdcbcdacc","output":"dd"},{"input":"dbbbbaadbbad","output":"bb"},{"input":"badbabbaaaaca","output":"aa"},{"input":"aadba","output":"aa"},{"input":"ddcbca","output":"dc"},{"input":"bcaaba","output":"cb"},{"input":"bcadcddc","output":"cc"},{"input":"abbddca","output":"bd"},{"input":"abdcdbbcd","output":"bd"},{"input":"acaababaaac","output":"cb"},{"input":"cbbdbcddddacd","output":"bb"},{"input":"aadcdbaaddabaa","output":"ad"}] -------------------------------------------------------------------------------- /data/memory_task/short_sequence_validation.json: -------------------------------------------------------------------------------- 1 | [{"input":"daadbdbdaa","output":"ab"},{"input":"adbacb","output":"dc"},{"input":"acbcada","output":"ca"},{"input":"addbbdc","output":"db"},{"input":"cbbbd","output":"bd"},{"input":"bbaabaabddc","output":"bb"},{"input":"acddcbbbdaabda","output":"cc"},{"input":"abdbcdaab","output":"bc"},{"input":"ddcbacb","output":"da"},{"input":"dcdbadbb","output":"ca"},{"input":"caaccddd","output":"ac"},{"input":"bddacccaaddb","output":"dc"},{"input":"bdaabb","output":"db"},{"input":"abccadbddba","output":"ba"},{"input":"caddbabaa","output":"ab"},{"input":"abcdabd","output":"ba"},{"input":"acaaaabcabacdb","output":"ca"},{"input":"aadccca","output":"ac"},{"input":"ccccddacdbbcac","output":"cd"},{"input":"cbbbc","output":"bc"},{"input":"bacadb","output":"ad"},{"input":"bbdabdbbcbaad","output":"bb"},{"input":"baddadbcdbb","output":"aa"},{"input":"bababdbbd","output":"ab"},{"input":"cbdaacbdaccca","output":"ba"},{"input":"bbbcadac","output":"ba"},{"input":"badcd","output":"ad"},{"input":"bdaaabcdbaada","output":"da"},{"input":"caadcd","output":"ac"},{"input":"ddcdcddb","output":"dc"},{"input":"caaaddad","output":"ad"},{"input":"ccddbddbda","output":"cb"},{"input":"accbaaa","output":"ca"},{"input":"cbddbaccb","output":"bb"},{"input":"bcaaccabcac","output":"cc"},{"input":"adbaaddcdadc","output":"da"},{"input":"cadaaaccdd","output":"aa"},{"input":"babcbccbabcdca","output":"ab"},{"input":"dcadddbdaccda","output":"cd"},{"input":"cbdcabbabd","output":"ba"},{"input":"cdcaca","output":"dc"},{"input":"accadcadcdda","output":"cd"},{"input":"badcccabcdb","output":"ac"},{"input":"bbabcbaaccdbaa","output":"bc"},{"input":"cdcab","output":"db"},{"input":"bbcbbabc","output":"bb"},{"input":"daacd","output":"ad"},{"input":"baaacbcddcabda","output":"ac"},{"input":"ccacdcccab","output":"cd"},{"input":"abbba","output":"ba"},{"input":"cacadddddac","output":"ad"},{"input":"ccacbc","output":"cb"},{"input":"cdabacccb","output":"da"},{"input":"cdbadbd","output":"dd"},{"input":"caaacdcaadabc","output":"ac"},{"input":"cabdbb","output":"ab"},{"input":"bccabacdcb","output":"cb"},{"input":"caabda","output":"ad"},{"input":"baaacdcadcacaa","output":"ac"},{"input":"bdacc","output":"dc"},{"input":"cdbadaa","output":"dd"},{"input":"aacdbdbb","output":"ab"},{"input":"dadcccccc","output":"ac"},{"input":"cacbcbccdba","output":"ac"},{"input":"bbdcbbcbddbcd","output":"bb"},{"input":"dddcbddaababbc","output":"db"},{"input":"caacacddbcab","output":"aa"},{"input":"dacbddcdacabd","output":"ad"},{"input":"adadaaccdacc","output":"da"},{"input":"ddbcaabccc","output":"da"},{"input":"acdbcdb","output":"cc"},{"input":"cddaaddb","output":"da"},{"input":"dcaccdbb","output":"cc"},{"input":"ccbddccb","output":"cd"},{"input":"caaabdacaabc","output":"ab"},{"input":"dacacabaabcc","output":"ac"},{"input":"adaccbddba","output":"dc"},{"input":"babaddcbc","output":"ad"},{"input":"cabbcdcabcab","output":"ac"},{"input":"badadbb","output":"ad"},{"input":"bbbacbdcabbc","output":"bc"},{"input":"bccadcdddc","output":"cd"},{"input":"acabbabc","output":"cb"},{"input":"daaabaababbdc","output":"ab"},{"input":"cacbabba","output":"aa"},{"input":"dcbbaabccbd","output":"ca"},{"input":"dbddadbcdbda","output":"ba"},{"input":"acccbab","output":"cb"},{"input":"baaaaccc","output":"aa"},{"input":"cabaacbdcbcbd","output":"aa"},{"input":"bddbdbdbcdc","output":"dd"},{"input":"dbcadaaccaaa","output":"bd"},{"input":"acadbbad","output":"cb"},{"input":"bbcdcdbaada","output":"bc"},{"input":"dbccbd","output":"bb"},{"input":"aacbcccbcd","output":"ac"},{"input":"abdbbbab","output":"bb"},{"input":"dadaaacbcaca","output":"aa"},{"input":"cbababdca","output":"ba"},{"input":"bbbccaacbb","output":"bc"},{"input":"acddcddbcaacd","output":"cc"},{"input":"cbacbdac","output":"bb"},{"input":"cadbbbb","output":"ab"},{"input":"dccbbcddbbc","output":"cb"},{"input":"abddddddbbacdb","output":"bd"},{"input":"cbacbddca","output":"bb"},{"input":"dbacbdabcdabc","output":"bb"},{"input":"bdccbdb","output":"db"},{"input":"cbcdbdbabccbdb","output":"bb"},{"input":"babcccbaddadb","output":"ac"},{"input":"acddb","output":"cb"},{"input":"dccbbddccd","output":"cb"},{"input":"cddddcbbda","output":"dd"},{"input":"bacacabbdaddc","output":"ac"},{"input":"dcacdcaabdbc","output":"cd"},{"input":"dbabcbadd","output":"bc"},{"input":"bbdccb","output":"bc"},{"input":"bcdbcc","output":"cc"},{"input":"dbdaa","output":"ba"},{"input":"dcdabcad","output":"cb"},{"input":"bccdbbbca","output":"cb"},{"input":"acadcddbbdccda","output":"cc"},{"input":"daaccccaddccd","output":"ac"},{"input":"bdccdcabc","output":"dd"},{"input":"bbccba","output":"bb"},{"input":"ddabdccbdbc","output":"dd"},{"input":"dabbccada","output":"ac"},{"input":"dacbcacc","output":"ac"},{"input":"acabcca","output":"cc"},{"input":"adaabbadbddcb","output":"db"},{"input":"dadbaab","output":"aa"},{"input":"dbbacabcbdcda","output":"bc"},{"input":"ddabaacd","output":"da"},{"input":"dddaaaccd","output":"da"},{"input":"daccc","output":"ac"},{"input":"bcdaccdbc","output":"cc"},{"input":"acbcacacbabada","output":"ca"},{"input":"caccbbb","output":"ab"},{"input":"cabccdbbccdcd","output":"ac"},{"input":"bbdcddabb","output":"bd"},{"input":"dcbaaaacaccab","output":"ca"},{"input":"bdcbdccbcdcc","output":"dd"},{"input":"adbaddbbccdbc","output":"dd"},{"input":"adbccbccb","output":"dc"},{"input":"daaddddcbac","output":"ad"},{"input":"badcbdbdbaddbd","output":"ab"},{"input":"bdbdbdda","output":"db"},{"input":"abcbdacbcbca","output":"bd"},{"input":"dbdbaccb","output":"ba"},{"input":"dcddbdc","output":"cb"},{"input":"dccadacddabdc","output":"cd"},{"input":"cdddaca","output":"da"},{"input":"daccc","output":"ac"},{"input":"cacbddadaaddcc","output":"ad"},{"input":"badbaaddbcac","output":"aa"},{"input":"abcbdcadcdabdb","output":"bd"},{"input":"addcbdbbdcac","output":"db"},{"input":"cadacb","output":"ac"},{"input":"bdddcccb","output":"dc"},{"input":"bdcbaccda","output":"da"},{"input":"cddbd","output":"dd"},{"input":"cadaccaddbbad","output":"ac"},{"input":"cbccadbdbbcb","output":"ba"},{"input":"abddcc","output":"bc"},{"input":"cdccd","output":"dd"},{"input":"ddcdddaac","output":"dd"},{"input":"cbdacdbcdbca","output":"bc"},{"input":"badadcccbbaa","output":"ad"},{"input":"cabdcdcacbaa","output":"ac"},{"input":"abbdc","output":"bc"},{"input":"acbacdd","output":"cc"},{"input":"cabcad","output":"aa"},{"input":"cacaac","output":"aa"},{"input":"ddbbda","output":"dd"},{"input":"adccdaabacaa","output":"dd"},{"input":"cdcadadac","output":"dd"},{"input":"bbabddaa","output":"bd"},{"input":"daddcaaabaa","output":"ac"},{"input":"ddccbcacbdcdd","output":"db"},{"input":"addddbdc","output":"dd"},{"input":"dadbb","output":"ab"},{"input":"adcabbacaacca","output":"db"},{"input":"aaccabdbdbdcd","output":"aa"},{"input":"bacbdd","output":"ad"},{"input":"bdbbdc","output":"dd"},{"input":"bddbcdaddda","output":"dc"},{"input":"acbbaddaacaacc","output":"ca"},{"input":"ccdddaccbdccc","output":"cd"},{"input":"bbdbbabb","output":"bb"},{"input":"acbdbaabc","output":"cb"},{"input":"caccaaaabab","output":"aa"},{"input":"acdcadd","output":"ca"},{"input":"bdbdddc","output":"dd"},{"input":"cdbbccc","output":"dc"},{"input":"bbbbaaa","output":"ba"},{"input":"cddabcacbaca","output":"db"},{"input":"ddabc","output":"dc"},{"input":"caaacabcaaac","output":"ac"},{"input":"bbaaaa","output":"ba"},{"input":"dcdbbc","output":"cb"},{"input":"dcbacbcbbab","output":"cc"},{"input":"ccdada","output":"cd"},{"input":"babcd","output":"ad"},{"input":"caabadcb","output":"aa"},{"input":"adbbbdadc","output":"db"},{"input":"bbccbcdcabddd","output":"bb"},{"input":"bcdbac","output":"ca"},{"input":"daaabbdaacbbb","output":"ab"},{"input":"dadddcccbccbb","output":"ad"},{"input":"dcdddddabb","output":"cd"},{"input":"dbaccadb","output":"bc"},{"input":"dbbccbaccd","output":"bc"},{"input":"cadddcabaada","output":"ad"},{"input":"cababcb","output":"ab"},{"input":"cbbacbcabdc","output":"bc"},{"input":"adcdacbdca","output":"da"},{"input":"bcbbccccb","output":"cc"},{"input":"daddacdcad","output":"aa"},{"input":"dbcacd","output":"bc"},{"input":"addadbdca","output":"dd"},{"input":"acdddacad","output":"cd"},{"input":"adaaacdaad","output":"da"},{"input":"baabcaabcd","output":"ac"},{"input":"addccbadbcdba","output":"dc"},{"input":"bcabdbc","output":"cd"},{"input":"dbdcbbbabcbad","output":"bb"},{"input":"acdbac","output":"ca"},{"input":"dcbcdcbb","output":"cd"},{"input":"bacadddbb","output":"ad"},{"input":"ddbbabddc","output":"da"},{"input":"daacddacbcba","output":"ad"},{"input":"ccacd","output":"cd"},{"input":"ddacbbcccddc","output":"db"},{"input":"babdddcbcccbc","output":"ad"},{"input":"dacaacccdb","output":"aa"},{"input":"bccabdd","output":"cb"},{"input":"badbbacddbcdc","output":"ab"},{"input":"bdbcadbacba","output":"da"},{"input":"adcbacab","output":"da"},{"input":"cadac","output":"ac"},{"input":"dbccdc","output":"bd"},{"input":"dabbbdc","output":"ab"},{"input":"adbccbcbaac","output":"dc"},{"input":"dabcacacab","output":"aa"},{"input":"ddbbaaabcba","output":"da"},{"input":"abdbbabbb","output":"bb"},{"input":"cabdd","output":"ad"},{"input":"acbabcbdcdc","output":"cb"},{"input":"baddccdca","output":"ac"},{"input":"addabccdca","output":"db"},{"input":"addbcd","output":"dc"},{"input":"bcbaddcbcaa","output":"cd"},{"input":"dcbacdcbdcc","output":"cc"},{"input":"cabccccdbaad","output":"ac"},{"input":"aabcbdcada","output":"ab"},{"input":"dbacccbbcdbc","output":"bc"},{"input":"addbdbd","output":"dd"},{"input":"aabadb","output":"ad"},{"input":"abcbb","output":"bb"},{"input":"adacaaba","output":"da"},{"input":"aaccdabcd","output":"ad"},{"input":"babbcdd","output":"ac"},{"input":"ccabb","output":"cb"},{"input":"dccbcadbbbb","output":"cc"},{"input":"dbcabaaa","output":"bb"},{"input":"cdcbbcddbd","output":"db"},{"input":"dcccbddcdbb","output":"cb"},{"input":"bcaad","output":"cd"},{"input":"cbabbbbbccccbc","output":"bb"},{"input":"ccdba","output":"ca"},{"input":"cadaccacadabc","output":"ac"},{"input":"acbdbdaccaab","output":"cb"},{"input":"acaacadb","output":"cc"},{"input":"ddaaabbbadbaa","output":"da"},{"input":"bcacdcaddbabd","output":"cd"},{"input":"daaaaaacbbcdc","output":"aa"},{"input":"babddcbbaadabd","output":"ad"},{"input":"ccdacbaadbdb","output":"cc"},{"input":"acaaabcdabcd","output":"ca"},{"input":"cacacccc","output":"ac"},{"input":"baaaacdbcbdb","output":"aa"},{"input":"cdcdaadcdc","output":"da"},{"input":"dbcbbacadabbc","output":"bb"},{"input":"aaddbbdb","output":"ab"},{"input":"acacabb","output":"ca"},{"input":"cccacbcbca","output":"cc"},{"input":"badbbac","output":"ab"},{"input":"bddcdadbadabda","output":"dd"},{"input":"bcbddaabcb","output":"cd"},{"input":"aadbbdbcc","output":"ab"},{"input":"abcaddccbadcd","output":"bd"},{"input":"dbddacaddab","output":"ba"},{"input":"ababdc","output":"bd"},{"input":"ddbcadadbdc","output":"da"},{"input":"abccbbcabcdacb","output":"bb"},{"input":"babdcaada","output":"ac"},{"input":"ccabc","output":"cc"},{"input":"daadadcbddd","output":"aa"},{"input":"dbdabb","output":"bb"},{"input":"dabbdadbcac","output":"ad"},{"input":"accdbbd","output":"cb"},{"input":"dbddccdbccdda","output":"bc"},{"input":"ababbbdbdbab","output":"bb"},{"input":"adcddadaacacba","output":"dd"},{"input":"adcacdbdcbb","output":"dc"},{"input":"acddcbabbc","output":"cc"},{"input":"aacddcbcbdac","output":"ad"},{"input":"bcaadabdaaacad","output":"cd"},{"input":"adbdbccdaaaaa","output":"db"},{"input":"abadbdc","output":"bb"},{"input":"bcadabbcccccd","output":"ca"},{"input":"abacbcca","output":"bb"},{"input":"bbbadb","output":"bd"},{"input":"dcdbcddbcbbbb","output":"cc"},{"input":"ccbca","output":"ca"},{"input":"ddccaadcbdba","output":"da"},{"input":"daacbdcaaadbc","output":"ab"},{"input":"bbddadbd","output":"ba"},{"input":"bdddaadddadcaa","output":"da"},{"input":"cababcbbcdbdbd","output":"ab"},{"input":"bacba","output":"aa"},{"input":"adadcaababbdac","output":"dc"},{"input":"cbbacccd","output":"bc"},{"input":"bddbacda","output":"da"},{"input":"dbbdcbb","output":"bc"},{"input":"cbbbdaa","output":"bd"},{"input":"dcdbada","output":"ca"},{"input":"cbdbbbdbbbb","output":"bb"},{"input":"abbaacdbbd","output":"ba"},{"input":"baddbccaaaaa","output":"ab"},{"input":"acdcadda","output":"ca"},{"input":"acdccd","output":"cc"},{"input":"ddababbdcba","output":"da"},{"input":"ccdaadbddcbdc","output":"ca"},{"input":"addddbbaacacb","output":"dd"},{"input":"bdddcadbc","output":"dc"},{"input":"daccdbccddcb","output":"ad"},{"input":"caccc","output":"ac"},{"input":"ccbaadbb","output":"ca"},{"input":"accdbcbbcddb","output":"cb"},{"input":"bacdbcd","output":"ab"},{"input":"bbcadddbcbdd","output":"bd"},{"input":"bbadcbb","output":"bc"},{"input":"adcbbacdad","output":"db"},{"input":"bdcbdabccdac","output":"dd"},{"input":"bbdcdcbcbb","output":"bd"},{"input":"adadbadbb","output":"db"},{"input":"bcccc","output":"cc"},{"input":"dbdab","output":"bb"},{"input":"acbcabc","output":"ca"},{"input":"cabbdacabdaa","output":"ad"},{"input":"dabddacdaddda","output":"ad"},{"input":"adbdddcbbaadad","output":"dd"},{"input":"caadc","output":"ac"},{"input":"dcaddb","output":"cd"},{"input":"ddbddccdcbdca","output":"dd"},{"input":"dcdab","output":"cb"},{"input":"abcdacacdb","output":"ba"},{"input":"dcdaccc","output":"cc"},{"input":"bddcc","output":"dc"},{"input":"bbaaddbdad","output":"bd"},{"input":"dabad","output":"ad"},{"input":"babcdacdbbc","output":"ad"},{"input":"aaaacacbbc","output":"ac"},{"input":"cddcacaa","output":"da"},{"input":"ccddcdabc","output":"cc"},{"input":"adddbbccdbdd","output":"db"},{"input":"bdbbd","output":"dd"},{"input":"acdaccbdddccbb","output":"cc"},{"input":"cbbbbda","output":"bb"},{"input":"dbcdbadbbdb","output":"bb"},{"input":"abdacddcdaaab","output":"bc"},{"input":"abcaccbbcbdabc","output":"bc"},{"input":"bbdbbdbbabccaa","output":"bb"},{"input":"adcbcccdcdbd","output":"dc"},{"input":"cabba","output":"aa"},{"input":"cbdcaddaaabd","output":"ba"},{"input":"dbcdcaabcddbac","output":"bc"},{"input":"ccadccd","output":"cc"},{"input":"bdaaccc","output":"dc"},{"input":"acaadbcdd","output":"cd"},{"input":"bbcddcba","output":"bd"},{"input":"bdcbbacdcddbd","output":"db"},{"input":"ddabbada","output":"db"},{"input":"abcdabddadddac","output":"ba"},{"input":"bcdddaa","output":"cd"},{"input":"acccccccbadb","output":"cc"},{"input":"abdaaacdcbcbcc","output":"ba"},{"input":"bacdcdaacc","output":"ac"},{"input":"aaabaccabb","output":"aa"},{"input":"ddbdcc","output":"dc"},{"input":"dcaabcdb","output":"cb"},{"input":"bbdaa","output":"ba"},{"input":"dccdaacbba","output":"ca"},{"input":"ccccbccb","output":"cb"},{"input":"dcaadbadbaacda","output":"cd"},{"input":"dcadcbbbdcaa","output":"cc"},{"input":"dabbbcdcbbbd","output":"ab"},{"input":"dbbababbdacba","output":"bb"},{"input":"bbabc","output":"bc"},{"input":"cdbadcdcbdbba","output":"dd"},{"input":"cacacaccccbadc","output":"ac"},{"input":"acbbdcbddabd","output":"cd"},{"input":"abbbcdacbda","output":"bc"},{"input":"cbcbddcbaddcd","output":"bd"},{"input":"cdaccbb","output":"dc"},{"input":"dacdcabacc","output":"ac"},{"input":"aabdbcb","output":"ab"},{"input":"bcabcbaa","output":"cc"},{"input":"aaddabbacdc","output":"aa"},{"input":"dddda","output":"da"},{"input":"adcaabbdabbcad","output":"da"},{"input":"ddbca","output":"da"},{"input":"baddaaacbcdbc","output":"aa"},{"input":"dcddcabdccbaab","output":"cc"},{"input":"addbcbaad","output":"dc"},{"input":"ddbabab","output":"db"},{"input":"bcccccacdc","output":"cc"},{"input":"bcaccdccbbca","output":"cc"},{"input":"cdbbcbbda","output":"dc"},{"input":"dbbbaabdadcdc","output":"ba"},{"input":"cabcdbaaacdd","output":"ad"},{"input":"bdddacb","output":"da"},{"input":"bababdacccdb","output":"ab"},{"input":"bcdadcbdbadd","output":"cd"},{"input":"cddbdcbaacb","output":"dd"},{"input":"adbcaabbdbcba","output":"da"},{"input":"cbbacada","output":"bc"},{"input":"cbacaccbab","output":"ba"},{"input":"cdbdbccccdc","output":"db"},{"input":"bbaccaabcc","output":"bc"},{"input":"accaccdc","output":"cc"},{"input":"addccca","output":"dc"},{"input":"abdcadbdbdba","output":"ba"},{"input":"cdcccdbda","output":"dc"},{"input":"cbccbdadb","output":"bb"},{"input":"bbccccdaaacdd","output":"bc"},{"input":"dccacdccd","output":"cc"},{"input":"ddacbaaaccdcdd","output":"db"},{"input":"dcbdda","output":"cd"},{"input":"ccacbacdcbd","output":"cb"},{"input":"ddcdbcd","output":"db"},{"input":"daaaaacbcdd","output":"aa"},{"input":"ddbbd","output":"dd"},{"input":"cccccabc","output":"cc"},{"input":"bbadadc","output":"ba"},{"input":"aadadccacbaab","output":"ad"},{"input":"accbbbbccba","output":"cb"},{"input":"ddcddccbbddaa","output":"dd"},{"input":"bbbaaccabcb","output":"ba"},{"input":"ccacba","output":"cb"},{"input":"bbdbacababbc","output":"ba"},{"input":"bcbbc","output":"cc"},{"input":"bcbadcca","output":"cd"},{"input":"addcdaacc","output":"dd"},{"input":"ccbddaabcbabab","output":"cd"},{"input":"ccbbbbbabdccd","output":"cb"},{"input":"dbbdcacbbac","output":"bc"},{"input":"cdbbcccba","output":"dc"},{"input":"cdaddc","output":"dd"},{"input":"bbcacbaab","output":"bc"},{"input":"ababd","output":"bd"},{"input":"dbcdadbbb","output":"ba"},{"input":"ddcbcbbbdb","output":"dc"},{"input":"adbababcb","output":"db"},{"input":"dabcdcdaadb","output":"ad"},{"input":"bacbcaccbbc","output":"ac"},{"input":"bdacbbadcabbab","output":"db"},{"input":"acbbbbbbbda","output":"cb"},{"input":"cabbdccddd","output":"ad"},{"input":"abbabdaacacdc","output":"bb"},{"input":"baaacacdcac","output":"ac"},{"input":"cbbbaddbbbc","output":"ba"},{"input":"cbdaadbc","output":"ba"},{"input":"acddbbcabdddaa","output":"cb"},{"input":"cdbaaabdabc","output":"da"},{"input":"dadca","output":"aa"},{"input":"acddaadbaccc","output":"ca"},{"input":"bcabcbcdbd","output":"cc"},{"input":"adabddaddcbcd","output":"dd"},{"input":"aaadccadbbb","output":"ac"},{"input":"caacbdd","output":"ab"},{"input":"dacbcddd","output":"ac"},{"input":"bdcab","output":"db"},{"input":"dbadbcdaba","output":"bb"},{"input":"abbbbdcbaabd","output":"bb"},{"input":"cadbdadcba","output":"ad"},{"input":"bcbdbaadba","output":"cb"},{"input":"dbcdbbabcdbbd","output":"bb"},{"input":"dcadb","output":"cb"},{"input":"abdcdbabbbc","output":"bd"},{"input":"bdbadddcb","output":"dd"},{"input":"ddbdadddcdbdbc","output":"da"},{"input":"bbbbdbcbabbcc","output":"bd"},{"input":"abcaccaaadacab","output":"bc"},{"input":"abacb","output":"bb"},{"input":"badbbcccc","output":"ab"},{"input":"bdcddb","output":"dd"},{"input":"dbdbdcacacccb","output":"bd"},{"input":"bdcadcd","output":"dd"},{"input":"ddccdbb","output":"dd"},{"input":"dacdadbbbcd","output":"aa"},{"input":"bbadccdc","output":"bc"},{"input":"ddcbcbabbd","output":"dc"},{"input":"accbbacb","output":"cb"},{"input":"accdabaaacdd","output":"ca"},{"input":"badcdabddccb","output":"ad"},{"input":"bdcbbb","output":"db"},{"input":"acddbddabdcdc","output":"cb"},{"input":"bbcdbdbdadd","output":"bb"},{"input":"adbaaada","output":"da"},{"input":"acbcdcaddccd","output":"cd"},{"input":"adbdd","output":"dd"},{"input":"cbabbdabccb","output":"bb"},{"input":"abccbaccdcbd","output":"bb"},{"input":"babacdc","output":"ac"},{"input":"ccccddba","output":"cd"},{"input":"badddbc","output":"ad"},{"input":"cbdbddd","output":"bd"},{"input":"bcbabadbcddcb","output":"cb"},{"input":"cbbaabbcb","output":"ba"},{"input":"bcbcacbcabdac","output":"ca"},{"input":"dbcdbcacdcca","output":"bb"},{"input":"dabbccddd","output":"ac"},{"input":"addacabbcda","output":"dc"},{"input":"ddadadcadaabb","output":"da"},{"input":"bdadbbbbdd","output":"db"},{"input":"accccdddccaaca","output":"cc"},{"input":"ccaca","output":"ca"},{"input":"bdddabcabcbd","output":"da"},{"input":"cdabdaabcdacab","output":"dd"},{"input":"dbaabbcdcd","output":"bb"},{"input":"abdbbcadbdaaab","output":"bb"},{"input":"acadcda","output":"cc"},{"input":"dcadca","output":"cc"},{"input":"cdcabcddcaadac","output":"db"},{"input":"daccbbdbbabdab","output":"ab"},{"input":"dabaadcddbc","output":"aa"},{"input":"dabacaddccdc","output":"ac"},{"input":"dbddaccdccd","output":"ba"},{"input":"dbccacbdacbcbd","output":"ba"},{"input":"acacbcdab","output":"cb"},{"input":"dbcdaaddd","output":"ba"},{"input":"cdaccaaad","output":"dc"},{"input":"badadabaacbdc","output":"ad"},{"input":"ccdcacdbca","output":"ca"},{"input":"baccbaba","output":"ab"},{"input":"ccbbcccdccb","output":"cc"},{"input":"dbbdbaccdcadbb","output":"bb"},{"input":"abdabbad","output":"bb"},{"input":"bacabdac","output":"ab"},{"input":"cbbdcdcad","output":"bc"},{"input":"bbbcaa","output":"ba"},{"input":"badbcca","output":"ac"},{"input":"abaacbd","output":"bc"},{"input":"bdddc","output":"dc"},{"input":"aacbacbdaabc","output":"aa"},{"input":"aadcaac","output":"aa"},{"input":"bbddcacd","output":"bc"},{"input":"bdacdccdbcdbdc","output":"dd"},{"input":"baccdcadcb","output":"ad"},{"input":"bbbab","output":"bb"},{"input":"adadac","output":"da"},{"input":"ccabcdbdaad","output":"cc"},{"input":"dbaccdadcad","output":"bc"},{"input":"cbdadbdcad","output":"bd"},{"input":"bcbabccadac","output":"cb"},{"input":"dcaaccccb","output":"cc"},{"input":"dbdbcddacbb","output":"bc"},{"input":"cccadab","output":"cd"},{"input":"bacacaabbd","output":"ac"},{"input":"bcadcdcaca","output":"cc"},{"input":"cbdbdbcabbcbcb","output":"bd"},{"input":"addccbbcbcabb","output":"dc"},{"input":"bdddcacdbbbc","output":"dc"},{"input":"dbcdad","output":"ba"},{"input":"bdcbacb","output":"da"},{"input":"ddcbaaad","output":"da"},{"input":"babdcaabb","output":"ac"},{"input":"bcbaabcba","output":"ca"},{"input":"cadacdbcbbcccb","output":"ac"},{"input":"baabadc","output":"aa"},{"input":"dbdac","output":"bc"},{"input":"accaac","output":"ca"},{"input":"daccabdcbc","output":"aa"},{"input":"dccacdcc","output":"cc"},{"input":"acdccadacdbbc","output":"cc"},{"input":"bbdcdad","output":"bd"},{"input":"acccacab","output":"ca"},{"input":"abbaaab","output":"ba"},{"input":"bbdccac","output":"bc"},{"input":"bbabc","output":"bc"},{"input":"bcccaaab","output":"ca"},{"input":"dddcbd","output":"db"},{"input":"bcbcdcbcdb","output":"cd"},{"input":"dacaccaa","output":"ac"},{"input":"ddcbbbcacb","output":"db"},{"input":"abcbcbbbdcb","output":"bc"},{"input":"baadcdb","output":"ac"},{"input":"bddcbcd","output":"db"},{"input":"caaddacc","output":"ad"},{"input":"ccadaadd","output":"ca"},{"input":"ddbcc","output":"dc"},{"input":"acadd","output":"cd"},{"input":"aabbbdcdcdccbd","output":"ab"},{"input":"caabbacaabbbc","output":"ab"},{"input":"cbccaa","output":"ba"},{"input":"aacdaab","output":"aa"},{"input":"abbcdcccccacc","output":"bd"},{"input":"ccbcbd","output":"cb"},{"input":"aadcd","output":"ad"},{"input":"dbbdcbababbdba","output":"bc"},{"input":"bbacaddcba","output":"ba"},{"input":"ddbdaabba","output":"da"},{"input":"ccabbddddcdd","output":"cb"},{"input":"dadda","output":"aa"},{"input":"cbcbcbdacc","output":"bc"},{"input":"aaccacdabaa","output":"aa"},{"input":"dcadda","output":"cd"},{"input":"ababdabadca","output":"bd"},{"input":"cdcddccdcbac","output":"dd"},{"input":"bddabcaccddbad","output":"db"},{"input":"daaadddc","output":"ad"},{"input":"dadccbadd","output":"ac"},{"input":"bdbbcadbacdac","output":"dc"},{"input":"badbcaadbcdab","output":"ac"},{"input":"aadcbacbbc","output":"ab"},{"input":"cdbaacabb","output":"da"},{"input":"dcdacbcbcaacdc","output":"cc"},{"input":"cdbddbcb","output":"dd"},{"input":"adcdddb","output":"dd"},{"input":"ddcdabdbdaaaa","output":"da"},{"input":"bacbabcb","output":"aa"},{"input":"cdbccaaabacc","output":"dc"},{"input":"dbdddbcb","output":"bd"},{"input":"ccbcc","output":"cc"},{"input":"ddacadcba","output":"da"},{"input":"acbdbcdbdaccad","output":"cb"},{"input":"dadbaadad","output":"aa"},{"input":"dabdccdcd","output":"ac"},{"input":"dcdacaddac","output":"cc"},{"input":"bacbcadbdbccd","output":"ac"},{"input":"bbbcbda","output":"bb"},{"input":"bbdcbddaaaad","output":"bb"},{"input":"dbbacaaccbd","output":"bc"},{"input":"dccabdd","output":"cb"},{"input":"cbcccaddd","output":"bc"},{"input":"adcaacacacb","output":"da"},{"input":"abddcacac","output":"bc"},{"input":"dbdddcd","output":"bd"},{"input":"bcccbdadcaaa","output":"cb"},{"input":"cdcdbbcbbcb","output":"db"},{"input":"adabab","output":"da"},{"input":"bcbddaccdaa","output":"cd"},{"input":"bddacdb","output":"dc"},{"input":"dbddbadbbccbc","output":"bb"},{"input":"bcbbaab","output":"ca"},{"input":"abbcbbcdad","output":"bb"},{"input":"bbabadb","output":"ba"},{"input":"dbaddbaadcdbad","output":"bd"},{"input":"acadadabd","output":"ca"},{"input":"cbcbbaadaaabba","output":"bb"},{"input":"adbbbabac","output":"db"},{"input":"abbdbbab","output":"bb"},{"input":"accdc","output":"cc"},{"input":"bcddcddcddd","output":"cc"},{"input":"acbbcbaad","output":"cc"},{"input":"cdbaa","output":"da"},{"input":"cbabbdadd","output":"bb"},{"input":"babdaccdcaac","output":"aa"},{"input":"dcbdacccdddcad","output":"ca"},{"input":"cabbcbbddbcd","output":"ac"},{"input":"cbbdbacbbc","output":"bb"},{"input":"bbabccbdddb","output":"bc"},{"input":"ccdccddbcc","output":"cc"},{"input":"cabacdac","output":"ac"},{"input":"adcda","output":"da"},{"input":"aaccdccdbaacad","output":"ad"},{"input":"dbcadbdbdb","output":"bd"},{"input":"bbabdacb","output":"bd"},{"input":"bdcccdbdcaa","output":"dc"},{"input":"bdcacaccaaacb","output":"dc"},{"input":"accdcdaccbdb","output":"cc"},{"input":"acacabadd","output":"ca"},{"input":"acddbccbcd","output":"cb"},{"input":"cabacda","output":"ac"},{"input":"aadbbccdc","output":"ab"},{"input":"bcdbdaabbc","output":"cd"},{"input":"bbbcccbc","output":"bc"},{"input":"babbc","output":"ac"},{"input":"bbbccadb","output":"bc"},{"input":"cabca","output":"aa"},{"input":"abdaabbdaa","output":"ba"},{"input":"acdadda","output":"cd"},{"input":"cbddccb","output":"bc"},{"input":"cdacdadc","output":"dd"},{"input":"bcacbdbcb","output":"cb"},{"input":"aabadc","output":"ad"},{"input":"cccdaacbaba","output":"ca"},{"input":"ddcbbdcccbb","output":"db"},{"input":"cadadddbc","output":"ad"},{"input":"bccadada","output":"cd"},{"input":"bdbabd","output":"db"},{"input":"dcdcbbbcbcc","output":"cb"},{"input":"cacbbacabdc","output":"ab"},{"input":"caada","output":"aa"},{"input":"dadaccb","output":"ac"},{"input":"cacabcaadcba","output":"ab"},{"input":"aaccdcc","output":"ad"},{"input":"cdaab","output":"db"},{"input":"cbcbbaaabab","output":"bb"},{"input":"ddcbdccccadcad","output":"dd"},{"input":"bccabaabbbdb","output":"cb"},{"input":"cddbdaddd","output":"dd"},{"input":"bacbbcdddbbd","output":"ab"},{"input":"cbbcccdcbcdddc","output":"bc"},{"input":"dacddddbbc","output":"ad"},{"input":"dcccddcd","output":"cd"},{"input":"bcabdbabbdbba","output":"cd"},{"input":"dcdbdcabd","output":"cd"},{"input":"dacabadcdc","output":"ab"},{"input":"baccdaccc","output":"ad"},{"input":"aaddabbdacbbd","output":"aa"},{"input":"dddcddcc","output":"dd"},{"input":"bdcacaa","output":"dc"},{"input":"dabcdaaaadcd","output":"ad"},{"input":"ccdccdbdbccc","output":"cc"},{"input":"bdbccac","output":"dc"},{"input":"abbcddacab","output":"bd"},{"input":"abcbada","output":"ba"},{"input":"dadadbdda","output":"ad"},{"input":"aaacbcdac","output":"ab"},{"input":"bbbca","output":"ba"},{"input":"bcabba","output":"cb"},{"input":"caabdaddba","output":"ad"},{"input":"bbaacacc","output":"bc"},{"input":"dbbbbbdadddddb","output":"bb"},{"input":"ddabc","output":"dc"},{"input":"bcadaaddda","output":"ca"},{"input":"bdacdbdacc","output":"dd"},{"input":"cbcdbcccc","output":"bb"},{"input":"acacacaccabdaa","output":"ca"},{"input":"aabdbcd","output":"ab"},{"input":"daabbb","output":"ab"},{"input":"bbbbcbbdacd","output":"bc"},{"input":"cbdcac","output":"ba"},{"input":"cdccdacdddabca","output":"dd"},{"input":"cabdd","output":"ad"},{"input":"abcacbaaab","output":"bc"},{"input":"cadcabcdd","output":"aa"},{"input":"dcbccbcd","output":"cc"},{"input":"dbcbacab","output":"ba"},{"input":"cbcbabcacac","output":"ba"},{"input":"daddcacdb","output":"ac"},{"input":"babbc","output":"ac"},{"input":"cacdddad","output":"ad"},{"input":"daddbacab","output":"ab"},{"input":"addbbbadbdb","output":"db"},{"input":"dddac","output":"dc"},{"input":"bbcabccccbdac","output":"bb"},{"input":"caabaabddb","output":"aa"},{"input":"bdadadcdc","output":"da"},{"input":"dbaabbbaadd","output":"bb"},{"input":"bbaadcbcbaaac","output":"bd"},{"input":"acdaa","output":"ca"},{"input":"dadabaabcddcc","output":"ab"},{"input":"dbabd","output":"bd"},{"input":"cdbaadcaaab","output":"da"},{"input":"accdb","output":"cb"},{"input":"acadcdbaac","output":"cc"},{"input":"ccaabbb","output":"cb"},{"input":"bbacbcaaaacbd","output":"bb"},{"input":"bddacabacbdadb","output":"dc"},{"input":"aadbbccb","output":"ab"},{"input":"bcaddadc","output":"cd"},{"input":"abdcccbadabb","output":"bc"},{"input":"bccddcdcaa","output":"cd"},{"input":"acacacbcab","output":"ca"},{"input":"abbaa","output":"ba"},{"input":"bacccbabacbb","output":"ac"},{"input":"aabad","output":"ad"},{"input":"acdacdabb","output":"cc"},{"input":"dbaddabbd","output":"bd"},{"input":"bcbbdbcc","output":"cd"},{"input":"bdabbdbabbdad","output":"db"},{"input":"ddacaadb","output":"da"},{"input":"dddcabad","output":"da"},{"input":"cdddbcbbb","output":"db"},{"input":"cbcadbbbc","output":"bd"},{"input":"bdacabbacab","output":"da"},{"input":"addcacc","output":"da"},{"input":"aadcdbaccc","output":"ad"},{"input":"dccdcddada","output":"cc"},{"input":"dbdacbacbb","output":"bc"},{"input":"dbddddaac","output":"bd"},{"input":"dddcddddcdcd","output":"dd"},{"input":"bbadabab","output":"ba"},{"input":"dcdabbd","output":"cb"},{"input":"abdaabbb","output":"ba"},{"input":"dddababaada","output":"db"},{"input":"abddcdd","output":"bc"},{"input":"aaadaccadc","output":"aa"},{"input":"accabadbab","output":"cb"},{"input":"dabababcd","output":"ab"},{"input":"aaabacbdbcbdc","output":"aa"},{"input":"cabcdcaacdbdba","output":"ad"},{"input":"dbdcdd","output":"bd"},{"input":"bdccabdadbd","output":"da"},{"input":"acdcbcab","output":"cb"},{"input":"cbbdcdbdbcdd","output":"bc"},{"input":"bdbddbbcbbdd","output":"dd"},{"input":"cbaabddcbb","output":"bb"},{"input":"bbbcbcdddabdca","output":"bb"},{"input":"bccbaddbbd","output":"ca"},{"input":"bbbcbacacbacac","output":"bb"},{"input":"bacabbcba","output":"ab"},{"input":"bcbacb","output":"cc"},{"input":"bacdbbcbcb","output":"ab"},{"input":"abcbdb","output":"bd"},{"input":"cddadcddcadab","output":"dd"},{"input":"ccabdcabda","output":"cd"},{"input":"dcdca","output":"ca"},{"input":"ccddcbd","output":"cc"},{"input":"bacbbdddbddca","output":"ab"},{"input":"bbbcbdbddccc","output":"bb"},{"input":"dcdbcacccabdac","output":"cc"},{"input":"dcddcabdccaa","output":"cc"},{"input":"dbdadabdbcbbac","output":"bd"},{"input":"ddbbabc","output":"da"},{"input":"baacacbaa","output":"aa"},{"input":"acabb","output":"cb"},{"input":"dabbadccaccc","output":"aa"},{"input":"ccadaddacaa","output":"ca"},{"input":"adbaa","output":"da"},{"input":"cbdbdcaaaccbcc","output":"bd"},{"input":"dcddbaa","output":"cb"},{"input":"ccbddddb","output":"cd"},{"input":"adacacbbdbd","output":"da"},{"input":"cacaadaabcc","output":"aa"},{"input":"adcdbdacadbcc","output":"db"},{"input":"dbcbca","output":"bc"},{"input":"dcbccbdddcbcad","output":"cc"},{"input":"cdbcbcaabdac","output":"db"},{"input":"bdcadbbbabaaa","output":"dd"},{"input":"abcbdcaacbcccd","output":"bd"},{"input":"dadacbbcb","output":"ac"},{"input":"adabadaa","output":"da"},{"input":"bcddbadbaacda","output":"cb"},{"input":"cbacdcaaacadd","output":"bd"},{"input":"dcdccddc","output":"cc"},{"input":"babbcacdadbc","output":"ac"},{"input":"abbaaab","output":"ba"},{"input":"cdadacbdccdab","output":"da"},{"input":"baacdbdccd","output":"ad"},{"input":"ababacad","output":"ba"},{"input":"babbccdddb","output":"ac"},{"input":"cbbaacbcdccaad","output":"ba"},{"input":"dabcdbabdcdbbb","output":"ad"},{"input":"babadcbaa","output":"ad"},{"input":"ddbddddadccdda","output":"dd"},{"input":"abaaabcca","output":"ba"},{"input":"cdddcccaddccc","output":"dc"},{"input":"bacdb","output":"ab"},{"input":"dadaabdb","output":"aa"},{"input":"baaddcbdcbadcb","output":"ad"},{"input":"adcaacbbcbb","output":"da"},{"input":"cadacdbbbbbb","output":"ac"},{"input":"acdcccabdab","output":"cc"},{"input":"acadbbab","output":"cb"},{"input":"baabcb","output":"ac"},{"input":"cabaaad","output":"aa"},{"input":"bcadddb","output":"cd"},{"input":"cbcbcadcadbbb","output":"bc"},{"input":"dacddbab","output":"ad"},{"input":"bcdddd","output":"cd"},{"input":"acdbadaddba","output":"ca"},{"input":"dbbdbcadbcdcda","output":"bb"},{"input":"daacdb","output":"ad"},{"input":"cdcaabacadb","output":"da"},{"input":"caacababdaa","output":"aa"},{"input":"dbacbdaada","output":"bb"},{"input":"bdbbbadb","output":"db"},{"input":"dbdacbd","output":"bc"},{"input":"aaacaabacdbcb","output":"aa"},{"input":"cddbdbdcba","output":"dd"},{"input":"acdacabdab","output":"cc"},{"input":"ddddaadc","output":"da"},{"input":"cabcddbd","output":"ad"},{"input":"bbcadccdbc","output":"bd"},{"input":"adacababacdada","output":"da"},{"input":"ccdbad","output":"ca"},{"input":"dbadcabdbbd","output":"bc"},{"input":"bddabadd","output":"db"},{"input":"dcbbdcbdbcabba","output":"cd"},{"input":"dcbcbab","output":"cb"},{"input":"aaccaacbd","output":"aa"},{"input":"adbcdcbbd","output":"dd"},{"input":"acdabdcbcdca","output":"cb"},{"input":"cddcbcaadcb","output":"db"},{"input":"cccadbd","output":"cd"},{"input":"bcaabcc","output":"cb"},{"input":"bbcbcdbbb","output":"bc"},{"input":"bbddbddacc","output":"bb"},{"input":"cddcdbbcda","output":"dd"},{"input":"ddcbb","output":"db"},{"input":"dabdba","output":"ab"},{"input":"cbbcbcadbbabad","output":"bb"},{"input":"bdccbcdbdaddc","output":"db"},{"input":"ccdbbbda","output":"cb"},{"input":"bbabbcdaabbbb","output":"bb"},{"input":"ccbdadbcbccb","output":"ca"},{"input":"dcddabbd","output":"ca"},{"input":"aadbd","output":"ad"},{"input":"bbcbd","output":"bd"},{"input":"aaccbabdca","output":"ab"},{"input":"dcadadabcdb","output":"ca"},{"input":"bbdcbdcacb","output":"bb"},{"input":"cacabbaacc","output":"ab"},{"input":"cdaabddcbbdd","output":"db"},{"input":"bcabadccabcbd","output":"ca"},{"input":"bbdcaaba","output":"ba"},{"input":"adacbaadddb","output":"db"},{"input":"aaaacccdbc","output":"ac"},{"input":"cbcab","output":"bb"},{"input":"ccbdabbad","output":"ca"},{"input":"baccbddcccddbd","output":"ab"},{"input":"dcbdaac","output":"ca"},{"input":"bbbbacdbdbac","output":"ba"},{"input":"cddcb","output":"db"},{"input":"ddabc","output":"dc"},{"input":"bdcdbddb","output":"db"},{"input":"acdadccdd","output":"cd"},{"input":"adbbbaadddbd","output":"db"},{"input":"dbadbaaba","output":"bb"},{"input":"ccbaddaaaddcba","output":"cd"},{"input":"dbdcc","output":"bc"},{"input":"cdcca","output":"da"},{"input":"dbbaab","output":"ba"},{"input":"babcb","output":"ab"},{"input":"ddbbcbcabb","output":"dc"},{"input":"bbcbadbaadabaa","output":"ba"},{"input":"ccabdda","output":"cd"},{"input":"cdbbbbcdabca","output":"db"},{"input":"ddcbbc","output":"db"},{"input":"acbddddddddbcb","output":"cd"},{"input":"cbccca","output":"bc"},{"input":"cccdcaccd","output":"cc"},{"input":"bbdbddbbccdb","output":"bd"},{"input":"bddbb","output":"db"},{"input":"babdbccdba","output":"ab"},{"input":"dddaddcaacd","output":"dd"},{"input":"dabdcbcd","output":"ac"},{"input":"cacbdb","output":"ad"},{"input":"baabbdadddcbc","output":"ab"},{"input":"aaccdc","output":"ad"},{"input":"dcabdb","output":"cd"},{"input":"dccbbacacdcdba","output":"cb"},{"input":"caacccd","output":"ac"},{"input":"acbcbadbdb","output":"cb"},{"input":"dcaadcbaac","output":"cd"},{"input":"bcaabc","output":"cb"},{"input":"dbbbdbcadb","output":"bd"},{"input":"cbabacbddbab","output":"ba"},{"input":"daaddabacd","output":"ad"},{"input":"cabbc","output":"ac"},{"input":"adcdbcaad","output":"db"},{"input":"cbbdbcccdd","output":"bb"},{"input":"bbbdda","output":"bd"},{"input":"ccaccabcdcbbcb","output":"cc"},{"input":"cbaabbadcdc","output":"bb"},{"input":"dddadc","output":"dd"},{"input":"dbadbabcb","output":"bb"},{"input":"cdddaccdcdaa","output":"da"},{"input":"cbdddccacbccb","output":"bd"},{"input":"baaacddbbbcad","output":"ac"},{"input":"dbbcdbaacad","output":"bd"},{"input":"abddcbdbb","output":"bc"},{"input":"bddbdddaacb","output":"dd"},{"input":"aabdccd","output":"ac"},{"input":"cbabbacdcd","output":"bb"},{"input":"dadaacbadbcdb","output":"aa"},{"input":"bdaccbcc","output":"dc"},{"input":"bcccdaac","output":"cd"},{"input":"bbcadacbabad","output":"bd"},{"input":"daabcaacacbb","output":"ac"},{"input":"acdaabdac","output":"ca"},{"input":"cabdc","output":"ac"},{"input":"cbdcaabca","output":"ba"},{"input":"acabbbbcbc","output":"cb"},{"input":"daccbdbba","output":"ab"},{"input":"cbdcaa","output":"ba"},{"input":"dacacdccaaccdb","output":"ac"},{"input":"dacdbba","output":"ab"},{"input":"bcabca","output":"cc"},{"input":"bbcda","output":"ba"},{"input":"cacddbcd","output":"ad"},{"input":"cbcacd","output":"bc"},{"input":"accbcacc","output":"cc"},{"input":"bdccc","output":"dc"},{"input":"aaada","output":"aa"}] -------------------------------------------------------------------------------- /data/running_max/long_input_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/long_input_data_test.npy -------------------------------------------------------------------------------- /data/running_max/long_input_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/long_input_data_train.npy -------------------------------------------------------------------------------- /data/running_max/long_input_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/long_input_data_validation.npy -------------------------------------------------------------------------------- /data/running_max/long_output_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/long_output_data_test.npy -------------------------------------------------------------------------------- /data/running_max/long_output_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/long_output_data_train.npy -------------------------------------------------------------------------------- /data/running_max/long_output_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/long_output_data_validation.npy -------------------------------------------------------------------------------- /data/running_max/short_input_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/short_input_data_test.npy -------------------------------------------------------------------------------- /data/running_max/short_input_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/short_input_data_train.npy -------------------------------------------------------------------------------- /data/running_max/short_input_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/short_input_data_validation.npy -------------------------------------------------------------------------------- /data/running_max/short_output_data_test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/short_output_data_test.npy -------------------------------------------------------------------------------- /data/running_max/short_output_data_train.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/short_output_data_train.npy -------------------------------------------------------------------------------- /data/running_max/short_output_data_validation.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplearningparis/dl-rnn/73cddbf97c298588f2d872575784eca54e2473ac/data/running_max/short_output_data_validation.npy -------------------------------------------------------------------------------- /data/running_max/short_sequence_test.json: -------------------------------------------------------------------------------- 1 | [{"input":"dbacdcdadcbdd","output":"aaaaacdadabbd"},{"input":"dddcaaabdd","output":"aaddaaaaad"},{"input":"acdddccbcad","output":"aaadddcccaa"},{"input":"baaccddb","output":"aaaaccdd"},{"input":"aacacac","output":"aaaacac"},{"input":"ddcacb","output":"aadaca"},{"input":"badbcacbbbbcac","output":"aaaabacabbbbac"},{"input":"adaccaac","output":"aaaaccaa"},{"input":"cabcbcccdacba","output":"aaaabccccaaaa"},{"input":"bbcbbbbcacdcd","output":"aabbbbbbacacd"},{"input":"bcacddaab","output":"aaacaddaa"},{"input":"dbdbaaccbdbd","output":"aadbaaaccbbd"},{"input":"dddadcdbacdd","output":"aadddadbaaad"},{"input":"cacba","output":"aacaa"},{"input":"ddccbdbbc","output":"aadccbbbb"},{"input":"ddcadb","output":"aadaaa"},{"input":"acdbdcccdadab","output":"aaabdbcccadaa"},{"input":"bcbddba","output":"aabbdda"},{"input":"badcaddb","output":"aaaaaadd"},{"input":"dddabadcbb","output":"aaddaaaabb"},{"input":"dcacaacddbbabb","output":"aaacaaaaddbbbb"},{"input":"acdadddcaa","output":"aaaaddddaa"},{"input":"daaacbbdddd","output":"aaaaaabbddd"},{"input":"cbbcbba","output":"aabbbbb"},{"input":"bababaaccdd","output":"aababaaaccd"},{"input":"bbadbccbca","output":"aabaabccca"},{"input":"dbcdabcbdb","output":"aabbaaabbb"},{"input":"caccabcdb","output":"aacccaabb"},{"input":"bbbabdadac","output":"aabbbaadaa"},{"input":"cadddda","output":"aaadddd"},{"input":"cbbdbadabbca","output":"aabbbaaaabba"},{"input":"cccccbbc","output":"aaccccbb"},{"input":"cadbacdbccadbc","output":"aaaaaaabbccaab"},{"input":"dcdaaadacbcc","output":"aadaaaaaaacc"},{"input":"bdbdbbbdaaac","output":"aabdbbbbaaaa"},{"input":"bdbacdcdcbacb","output":"aabaaacdcbaaa"},{"input":"dbbcccd","output":"aabbccc"},{"input":"dbcadd","output":"aabaad"},{"input":"ddcaccd","output":"aadaccc"},{"input":"cdbaabcd","output":"aabaaaab"},{"input":"cbdccc","output":"aabbcc"},{"input":"adcdcb","output":"aaadcb"},{"input":"bacacddb","output":"aaaacadd"},{"input":"dadbbaddcb","output":"aadabbaddb"},{"input":"cbcccdbca","output":"aaccccbba"},{"input":"dbcbcb","output":"aabbcb"},{"input":"accdaccaadddd","output":"aaccaaccaaddd"},{"input":"dbccc","output":"aabcc"},{"input":"cdbcaca","output":"aabbaca"},{"input":"ddaadada","output":"aadaaada"},{"input":"dbdabddacc","output":"aadaaaddac"},{"input":"ddaabccdbbcd","output":"aadaaaccbbbb"},{"input":"adaadccccbd","output":"aaaaaaccccb"},{"input":"cbbacdbabaccc","output":"aabbaababaacc"},{"input":"dbdbacaabbcccc","output":"aadbaaaaabbccc"},{"input":"cdbdadddbab","output":"aabdaddddab"},{"input":"aaadaadbbbccc","output":"aaaaaaaabbbcc"},{"input":"cadddcacacadba","output":"aaadddacacaaaa"},{"input":"dbbbddcbdaab","output":"aabbbddbbaaa"},{"input":"cdadc","output":"aaada"},{"input":"adacbbbabbccdc","output":"aaaaabbbbbbccc"},{"input":"baabacb","output":"aaaaaaa"},{"input":"bbdbcccacbc","output":"aabbbccccac"},{"input":"ccbabdccdcbadc","output":"aacababcccbaaa"},{"input":"daadbbdbdc","output":"aaaaabbbdb"},{"input":"cbdcb","output":"aabbb"},{"input":"cbabcbbabd","output":"aaababbbba"},{"input":"bccbbbddaa","output":"aaccbbbdda"},{"input":"dbbda","output":"aabba"},{"input":"bcabbdbcbab","output":"aaaabbbbbab"},{"input":"bbaca","output":"aabaa"},{"input":"bacbca","output":"aaaaca"},{"input":"ccdccdd","output":"aaccccd"},{"input":"daccccdab","output":"aaaccccaa"},{"input":"bccada","output":"aaccaa"},{"input":"dbbacababbcabc","output":"aabbaaaabbbaaa"},{"input":"cbdccbbdd","output":"aabbccbbd"},{"input":"aaacd","output":"aaaaa"},{"input":"acabbbbca","output":"aaaabbbba"},{"input":"adaacbadd","output":"aaaaaaaad"},{"input":"bdbaccdbc","output":"aabaaccbb"},{"input":"cbcddddddbdbd","output":"aacbdddddddbd"},{"input":"cdaabacdacada","output":"aaaaaaaaaaaaa"},{"input":"cbbbcba","output":"aabbbba"},{"input":"daabb","output":"aaaab"},{"input":"acababcb","output":"aaaaabab"},{"input":"acdaccbaddac","output":"aaaaaccaadda"},{"input":"ccdabc","output":"aacaaa"},{"input":"dcadd","output":"aaaad"},{"input":"adbacdb","output":"aaaaaab"},{"input":"ddcbdaccdadb","output":"aadbbaaccada"},{"input":"cdadadbcd","output":"aaadadabb"},{"input":"dbbbacbaca","output":"aabbbaaaaa"},{"input":"dccaccaacab","output":"aacccccaaaa"},{"input":"abbdaaddab","output":"aabbaaadda"},{"input":"cbcdcbda","output":"aacbcbba"},{"input":"ccdcb","output":"aaccb"},{"input":"dabdacccdaa","output":"aaaaaacccaa"},{"input":"aadcd","output":"aaaad"},{"input":"acacdbbb","output":"aaacabbb"},{"input":"dbcbdbbbbacaa","output":"aabbbbbbbbaaa"},{"input":"dcbcbddbdaa","output":"aabcbbdddaa"},{"input":"acbaaadabaabc","output":"aaaaaaaaaaaaa"},{"input":"ddbcdbdbad","output":"aadbbbdbaa"},{"input":"acdbda","output":"aaabda"},{"input":"adcaaccccb","output":"aaaaaacccc"},{"input":"ccbbaacdaa","output":"aacbbaaaaa"},{"input":"dddbc","output":"aaddb"},{"input":"dacacadca","output":"aaaacaaaa"},{"input":"acdbacddabbb","output":"aaabaaaddabb"},{"input":"cbbdacbdcbd","output":"aabbaaabbbb"},{"input":"bbcbdd","output":"aabbbd"},{"input":"cbbdcdb","output":"aabbbdb"},{"input":"daacdcacbabcbc","output":"aaaaacacaababc"},{"input":"bccddcdadbd","output":"aaccdddadad"},{"input":"bcabdacbcdbc","output":"aaaaaaaacbbb"},{"input":"cbcbcda","output":"aacbcba"},{"input":"cdbba","output":"aabbb"},{"input":"cabcadbad","output":"aaaaaaaaa"},{"input":"ddaddcd","output":"aaddddd"},{"input":"bbbdbaad","output":"aabbbaaa"},{"input":"acdaadcd","output":"aaaaaaad"},{"input":"dabcabbcdd","output":"aaaaaabbbd"},{"input":"dcdbacccbac","output":"aadbaacccaa"},{"input":"bdddbdcadcbdd","output":"aaddddbaaabbd"},{"input":"cbbabbcbbd","output":"aabbbbbbbb"},{"input":"aabbad","output":"aaabba"},{"input":"badacccaac","output":"aaaaacccaa"},{"input":"cacbdbb","output":"aacabbb"},{"input":"cacbaccdbaac","output":"aacaaaccbaaa"},{"input":"cacbbbddbbbaa","output":"aacabbbddbbba"},{"input":"dcdba","output":"aadba"},{"input":"dbdbbcbb","output":"aadbbbbb"},{"input":"dbcdc","output":"aabbc"},{"input":"dccdcadd","output":"aacccaad"},{"input":"aabdbdacb","output":"aaaabdaaa"},{"input":"dbbaaddbddaa","output":"aabbaaddddda"},{"input":"cbcdadb","output":"aacbada"},{"input":"aaacb","output":"aaaaa"},{"input":"bacaa","output":"aaaaa"},{"input":"bddcbbadcbaa","output":"aaddbbbaabaa"},{"input":"adaabd","output":"aaaaaa"},{"input":"dbbabbabaad","output":"aabbbbbbaaa"},{"input":"cbbabbbddacc","output":"aabbbbbbddac"},{"input":"dccdcba","output":"aacccba"},{"input":"aaddadadcb","output":"aaadddadab"},{"input":"dccddcbadbdcd","output":"aaccddbaaadbd"},{"input":"dddcbbbabcdba","output":"aaddbbbbbabba"},{"input":"cccdaadccd","output":"aaccaaaacc"},{"input":"abcbaabd","output":"aaabaaaa"},{"input":"dcdbbbaabbad","output":"aadbbbbaabba"},{"input":"bbccbadc","output":"aabccaaa"},{"input":"bbbabdbdaaddbd","output":"aabbbabdaaaddd"},{"input":"ddbacddacabcbd","output":"aadaaaddaaaabb"},{"input":"baaacbbcbb","output":"aaaaaabbbb"},{"input":"dbdabdca","output":"aadaaaba"},{"input":"babcbaddbb","output":"aababaaddb"},{"input":"bdcdbc","output":"aabdbb"},{"input":"abcaac","output":"aaaaaa"},{"input":"dcacacbad","output":"aaacacaaa"},{"input":"bdddadbaba","output":"aaddddaaba"},{"input":"abbbccdcd","output":"aabbbcccd"},{"input":"dccdccd","output":"aaccccc"},{"input":"acbbca","output":"aaabba"},{"input":"ddbbaccc","output":"aadbbacc"},{"input":"cdaaab","output":"aaaaaa"},{"input":"abbccabba","output":"aabbccabb"},{"input":"bbbcbccdaca","output":"aabbbcccaaa"},{"input":"aadddbbcbb","output":"aaadddbbbb"},{"input":"aadba","output":"aaaaa"},{"input":"bdcdb","output":"aabdb"},{"input":"ddaababdd","output":"aadaaabad"},{"input":"abbbdaca","output":"aabbbaaa"},{"input":"dbddaddbadbbcc","output":"aaddddddaaabbc"},{"input":"cddadaccbbbd","output":"aadddaaccbbb"},{"input":"addaacbcaacabd","output":"aaddaaacaaaaaa"},{"input":"bdbcaacdabbb","output":"aabbaaaaaabb"},{"input":"dcddcdabacbcd","output":"aaddddaaaaacb"},{"input":"acabbcba","output":"aaaabbba"},{"input":"dbcdcbccbcdc","output":"aabbcbccccbc"},{"input":"abccabcd","output":"aaaccaab"},{"input":"ddcbaccd","output":"aadbaacc"},{"input":"dbcabadcaa","output":"aabaaaaaaa"},{"input":"ccdddccdabab","output":"aacdddccaaab"},{"input":"cbbabbb","output":"aabbbbb"},{"input":"bccdbcdbbac","output":"aaccbbbbbba"},{"input":"cdbdbbdbdcaddd","output":"aabdbbbbdbaadd"},{"input":"dddbbcbbdcbb","output":"aaddbbbbbbbb"},{"input":"caacdcc","output":"aaaaacc"},{"input":"bbbccdc","output":"aabbccc"},{"input":"dcdacda","output":"aadaaaa"},{"input":"cccbbbbbcaabab","output":"aaccbbbbbaaaab"},{"input":"abdada","output":"aaaada"},{"input":"bccaacdbcabbb","output":"aaccaaabbaabb"},{"input":"dbdcacddaab","output":"aadbacaddaa"},{"input":"bcbaaccd","output":"aabaaacc"},{"input":"bcacbd","output":"aaacab"},{"input":"acacdabdcacc","output":"aaacaaaabacc"},{"input":"ddacdcbdddc","output":"aadaacbbddd"},{"input":"bcccdad","output":"aacccad"},{"input":"bccddadaccdca","output":"aaccdddaaccca"},{"input":"daaddaacdcd","output":"aaaaddaaacd"},{"input":"cdcbb","output":"aacbb"},{"input":"bcbaaddacccba","output":"aabaaaddaccca"},{"input":"acabcccdd","output":"aaaaacccd"},{"input":"abacbbdbbcbdb","output":"aaaaabbbbbbbb"},{"input":"cdcaa","output":"aacaa"},{"input":"acbcdbaddcadba","output":"aaacbbaaddaaaa"},{"input":"dcbcd","output":"aabcb"},{"input":"bbbbcbd","output":"aabbbbb"},{"input":"daaca","output":"aaaaa"},{"input":"dcaadbcbcc","output":"aaaaaabbcc"},{"input":"bccabbdccbac","output":"aaccabbbccaa"},{"input":"bcbacccdbbabba","output":"aabaacccbbbbbb"},{"input":"dcdbddabaadcbd","output":"aadbdddaaaaabb"},{"input":"cdaabacbabdcc","output":"aaaaaaaaababc"},{"input":"dabcddbadcdd","output":"aaaabddaaadd"},{"input":"babdbbdddcca","output":"aababbbdddcc"},{"input":"bbbcbc","output":"aabbbc"},{"input":"dcaddcbcac","output":"aaaaddbcac"},{"input":"daadccb","output":"aaaaacc"},{"input":"dddccbccacc","output":"aaddccccccc"},{"input":"dbdbaaba","output":"aadbaaaa"},{"input":"aadcbaacbcddc","output":"aaaabaaaacbdd"},{"input":"cbddabaadcb","output":"aabddaaaaab"},{"input":"adabdaba","output":"aaaaaaaa"},{"input":"bcccbcacccb","output":"aaccccacccc"},{"input":"abaadbacccdabc","output":"aaaaaaaacccaaa"},{"input":"cdcbbbd","output":"aacbbbb"},{"input":"dadacdacd","output":"aadaaaaaa"},{"input":"ccbabaddb","output":"aacabaadd"},{"input":"adcadbba","output":"aaaaaabb"},{"input":"dcbbbcbacb","output":"aabbbbbaaa"},{"input":"addbdcacaaddca","output":"aadddbacaaadda"},{"input":"adbaabbcb","output":"aaaaaabbb"},{"input":"cdbbabcbd","output":"aabbbbabb"},{"input":"cbdabbbddc","output":"aabaabbbdd"},{"input":"acbcdda","output":"aaacbdd"},{"input":"cdcbdccdbd","output":"aacbbbccbd"},{"input":"cabcacbcadb","output":"aaaaacacaaa"},{"input":"dabcbcddaddbcc","output":"aaaabcbdddddbc"},{"input":"bdcbcadbbcaa","output":"aabbcaaabbaa"},{"input":"bdcab","output":"aabaa"},{"input":"acabbddadc","output":"aaaabbddda"},{"input":"dacdbaa","output":"aaaabaa"},{"input":"bbcbadbabd","output":"aabbaaaaba"},{"input":"abbdb","output":"aabbb"},{"input":"abdbcaabccadcb","output":"aaabbaaaaccaab"},{"input":"baaabcbc","output":"aaaaaabc"},{"input":"baacdddca","output":"aaaaaddda"},{"input":"ddaada","output":"aadaaa"},{"input":"acdaabc","output":"aaaaaaa"},{"input":"daacdacdbbb","output":"aaaaaaaabbb"},{"input":"cdabddbbc","output":"aaaaaddbb"},{"input":"abdcbccdd","output":"aaabbcccd"},{"input":"cbada","output":"aaaaa"},{"input":"cbabacbacbbd","output":"aaabaaaaaabb"},{"input":"adddcddddc","output":"aadddddddd"},{"input":"abdbadc","output":"aaabaaa"},{"input":"caadabc","output":"aaaaaaa"},{"input":"acbbab","output":"aaabbb"},{"input":"cdcddadbcca","output":"aacddddabcc"},{"input":"ccaccddaccddbb","output":"aaccccddaccddb"},{"input":"abccdbab","output":"aaaccbab"},{"input":"cccbaadba","output":"aaccaaaaa"},{"input":"dacccbbabcc","output":"aaacccbbbac"},{"input":"dcbcdddcdcda","output":"aabcbddddcda"},{"input":"babcd","output":"aabab"},{"input":"aaccdab","output":"aaaccaa"},{"input":"abccccadbd","output":"aaaccccaad"},{"input":"bdaaa","output":"aaaaa"},{"input":"ccdbcacdadab","output":"aacbbacaadaa"},{"input":"abcbdcdabb","output":"aaabbbdaab"},{"input":"daddadc","output":"aadddda"},{"input":"dcbcdacbc","output":"aabcbaaac"},{"input":"bddbc","output":"aaddb"},{"input":"dbbddcacdbdcbb","output":"aabbddacabdbbb"},{"input":"cabaddaadaaccc","output":"aaaaaddaaaaacc"},{"input":"dadcbac","output":"aadabaa"},{"input":"abddcdcbbab","output":"aaadddcbbbb"},{"input":"baaadabbcd","output":"aaaaaaabbb"},{"input":"dacdcabdac","output":"aaaacaaaaa"},{"input":"dcbbabbbb","output":"aabbbbbbb"},{"input":"dadadbdaa","output":"aadadadaa"},{"input":"adbacbdbadab","output":"aaaaaabbaaaa"},{"input":"ccacdbcbdcac","output":"aaccabbbbbac"},{"input":"addcdacddb","output":"aadddaaadd"},{"input":"bccbda","output":"aaccba"},{"input":"dccaadacdcc","output":"aaccaaaaacc"},{"input":"dbcdcaabb","output":"aabbcaaab"},{"input":"cabadc","output":"aaaaaa"},{"input":"accaacccaacaba","output":"aaccaacccaaaaa"},{"input":"cacaabdcdcb","output":"aacaaaabdcb"},{"input":"adbbcddc","output":"aaabbbdd"},{"input":"cbacbbcadbcac","output":"aaaaabbaaabac"},{"input":"bbaacbbadaaaab","output":"aabaaabbaaaaaa"},{"input":"dbbaddcbc","output":"aabbaddbc"},{"input":"babdbc","output":"aababb"},{"input":"abbbca","output":"aabbba"},{"input":"cacbbdabdbcbcb","output":"aacabbaaabbbcb"},{"input":"bdddac","output":"aaddda"},{"input":"cbbdbd","output":"aabbbd"},{"input":"ccabcacbac","output":"aacaaacaaa"},{"input":"ddadaad","output":"aaddaaa"},{"input":"cbbabdbdcdddc","output":"aabbbabdbdddd"},{"input":"bdaddad","output":"aaadddd"},{"input":"ccaabdbaaaad","output":"aacaaabaaaaa"},{"input":"ddcdbcbdb","output":"aaddbbbbb"},{"input":"bdbbacadbacd","output":"aabbbaaaaaaa"},{"input":"ddadbabcac","output":"aaddaabaac"},{"input":"badaabdd","output":"aaaaaaad"},{"input":"bccbccbadacdcd","output":"aacccccaaaaacd"},{"input":"aacabdda","output":"aaaaaadd"},{"input":"dcacddcbcbbbd","output":"aaacaddbcbbbb"},{"input":"adcccbdbabdbb","output":"aaacccbbababb"},{"input":"cbdbbadccabdc","output":"aabbbbaaccaab"},{"input":"bbccacaacc","output":"aabcccaaac"},{"input":"acadab","output":"aaaaaa"},{"input":"dbcbacc","output":"aabbaac"},{"input":"ababaad","output":"aaabaaa"},{"input":"acdacaac","output":"aaaaaaaa"},{"input":"bdaccba","output":"aaaacca"},{"input":"ccddda","output":"aacddd"},{"input":"dbbdbdcba","output":"aabbbdbba"},{"input":"acddaabbbc","output":"aaaddaabbb"},{"input":"cbdbdabd","output":"aabbdaaa"},{"input":"bcdaccdacc","output":"aabaaccaac"},{"input":"bacbdac","output":"aaaabaa"},{"input":"cddacdacab","output":"aaddaaaaaa"},{"input":"cacbacba","output":"aacaaaaa"},{"input":"abdbacbaa","output":"aaabaaaaa"},{"input":"bccdbbddddca","output":"aaccbbbdddda"},{"input":"bdacdbbcacad","output":"aaaaabbbacaa"},{"input":"cbcbdcacacbc","output":"aacbbbacacac"},{"input":"dbabcaabbd","output":"aaabaaaabb"},{"input":"daabcab","output":"aaaaaaa"},{"input":"bccadabdcac","output":"aaccaaaabac"},{"input":"bbcccbaca","output":"aabcccaaa"},{"input":"dacdcdaaacb","output":"aaaacdaaaaa"},{"input":"badcbd","output":"aaaabb"},{"input":"ccbaabbbbd","output":"aacaaabbbb"},{"input":"accdbdbccc","output":"aaccbdbbcc"},{"input":"dacab","output":"aaaaa"},{"input":"cddabaabbb","output":"aaddaaaabb"},{"input":"dacddddd","output":"aaaadddd"},{"input":"cdbddcadddab","output":"aabdddaaddda"},{"input":"dbcbcddcdaca","output":"aabbcbdddaaa"},{"input":"cbacbcccbaabcc","output":"aaaaaccccaaaac"},{"input":"ddcadbbdbabb","output":"aadaaabbbabb"},{"input":"cdabcccaa","output":"aaaaaccca"},{"input":"cbdcccdcbbccc","output":"aabbccccbbbcc"},{"input":"cadbbddda","output":"aaaabbddd"},{"input":"bdcaddbaa","output":"aabaaddaa"},{"input":"daacdccbdbcbaa","output":"aaaaacccbbbbaa"},{"input":"bbdbaadda","output":"aabbaaadd"},{"input":"adababdbbadcb","output":"aaaaababbbaab"},{"input":"caabbaaadcd","output":"aaaabbaaaad"},{"input":"dabbbcbdabb","output":"aaabbbbbaab"},{"input":"badbaccddbb","output":"aaaaaaccddb"},{"input":"cabbcad","output":"aaabbaa"},{"input":"bbbddcdaaacd","output":"aabbdddaaaaa"},{"input":"cbbdcda","output":"aabbbda"},{"input":"cbbbca","output":"aabbba"},{"input":"adadabbaad","output":"aaadaabbaa"},{"input":"cbbbdaabdbda","output":"aabbbaaaabda"},{"input":"bcaaaddb","output":"aaaaaadd"},{"input":"bacbdaada","output":"aaaabaaaa"},{"input":"bcbcdcadbad","output":"aabcbcaaaaa"},{"input":"bcdbdba","output":"aabbdba"},{"input":"bbbdc","output":"aabbb"},{"input":"caacacdabadd","output":"aaaaacaaaaad"},{"input":"aacbcc","output":"aaaacc"},{"input":"aaacbccb","output":"aaaaaccc"},{"input":"bcadacbbddca","output":"aaaaaaabbdda"},{"input":"ccacdb","output":"aaccab"},{"input":"aacadbb","output":"aaaaaab"},{"input":"bcdddcaabbcbb","output":"aabdddaaabbbb"},{"input":"ddcbabdddaab","output":"aadbabadddaa"},{"input":"aadcd","output":"aaaad"},{"input":"bcdbbdbb","output":"aabbbbbb"},{"input":"cbdba","output":"aabba"},{"input":"ddacddddadccd","output":"aadaadddddacc"},{"input":"cdccb","output":"aaccc"},{"input":"dadbadaccdcdbc","output":"aadaaaaacccdbb"},{"input":"aabbac","output":"aaabba"},{"input":"dabba","output":"aaabb"},{"input":"bbdadc","output":"aabada"},{"input":"bacdcd","output":"aaaacd"},{"input":"bdddbdbcacc","output":"aaddddbbacc"},{"input":"cddabd","output":"aaddaa"},{"input":"bbddad","output":"aabddd"},{"input":"dcaccbad","output":"aaacccaa"},{"input":"dbabacb","output":"aaabaaa"},{"input":"ddaaccad","output":"aadaacca"},{"input":"ababbbabacdacc","output":"aaabbbbbaaaaac"},{"input":"cdabaccddba","output":"aaaaaaccdda"},{"input":"abcbababb","output":"aaabababb"},{"input":"bcbbbbc","output":"aabbbbb"},{"input":"cabacac","output":"aaaaaac"},{"input":"dbbdadad","output":"aabbadad"},{"input":"dbdccab","output":"aadbcca"},{"input":"ccbbdadbaaba","output":"aacbbadaaaaa"},{"input":"ccacbaac","output":"aaccaaaa"},{"input":"aadccadacbcddb","output":"aaaaccaaaacbdd"},{"input":"bcbbbb","output":"aabbbb"},{"input":"aabaca","output":"aaaaaa"},{"input":"daadbb","output":"aaaaab"},{"input":"bbcdabccdcb","output":"aabbaaacccb"},{"input":"cbacd","output":"aaaaa"},{"input":"dabbabdbcb","output":"aaabbbabbb"},{"input":"acbbbbbcdbcb","output":"aaabbbbbbbbb"},{"input":"aaccdddadc","output":"aaaccdddda"},{"input":"ddaccbacdbdbbc","output":"aadaccaaabdbbb"},{"input":"daabab","output":"aaaaab"},{"input":"abcdaddd","output":"aaabaddd"},{"input":"acbcdca","output":"aaacbca"},{"input":"cbbcaca","output":"aabbaca"},{"input":"aabddcdadbd","output":"aaaadddadad"},{"input":"cadcba","output":"aaaaba"},{"input":"dccabcb","output":"aaccaab"},{"input":"dadcc","output":"aadac"},{"input":"acbaab","output":"aaaaaa"},{"input":"ddadd","output":"aaddd"},{"input":"adddcdabdddb","output":"aaddddaaaddd"},{"input":"aadddc","output":"aaaddd"},{"input":"ddcaa","output":"aadaa"},{"input":"bacdbc","output":"aaaabb"},{"input":"aacacaabaadccb","output":"aaaacaaaaaaacc"},{"input":"adbcccabaadb","output":"aaabcccaaaaa"},{"input":"adadabddca","output":"aaadaaadda"},{"input":"baadb","output":"aaaaa"},{"input":"bccddbd","output":"aaccddd"},{"input":"dccbbd","output":"aaccbb"},{"input":"dcbbabdacab","output":"aabbbbaaaaa"},{"input":"dcccdad","output":"aacccad"},{"input":"dbdabcaabc","output":"aadaaaaaaa"},{"input":"dabdb","output":"aaaab"},{"input":"cbadd","output":"aaaad"},{"input":"bbaca","output":"aabaa"},{"input":"ddddbbcadbc","output":"aadddbbaaab"},{"input":"ccdddadab","output":"aacddddaa"},{"input":"ddcacb","output":"aadaca"},{"input":"bcdaddacaaabba","output":"aabadddaaaaabb"},{"input":"bddcbbbcbacd","output":"aaddbbbbbaaa"},{"input":"bdacabdbaadbad","output":"aaaaaaabaaaaaa"},{"input":"babda","output":"aabaa"},{"input":"baabcccbbbdda","output":"aaaaacccbbbdd"},{"input":"ddcbddaaacdbdb","output":"aadbbddaaaabdb"},{"input":"badddcddcab","output":"aaaddddddaa"},{"input":"ddcaabacdbaca","output":"aadaaaaaabaaa"},{"input":"cddaaaa","output":"aaddaaa"},{"input":"cdbbbcdaacbc","output":"aabbbbbaaaac"},{"input":"baddccddccd","output":"aaaddccddcc"},{"input":"aacddddac","output":"aaaadddda"},{"input":"cbbadda","output":"aabbadd"},{"input":"bdaccdddabcaca","output":"aaaaccdddaaaca"},{"input":"dabdbabacbd","output":"aaaababaaab"},{"input":"cddacdcddcdbc","output":"aaddaacddddbb"},{"input":"acbabadcabbcc","output":"aaaabaaaaabbc"},{"input":"cdbca","output":"aabba"},{"input":"addaaacbdbaab","output":"aaddaaaabbaaa"},{"input":"baccb","output":"aaacc"},{"input":"abdaacdbab","output":"aaaaaaabab"},{"input":"ababc","output":"aaaba"},{"input":"adcdbcba","output":"aaadbbba"},{"input":"cdbccba","output":"aabbcca"},{"input":"ddacdaccdba","output":"aadaaaaccba"},{"input":"dcbdcd","output":"aabbbd"},{"input":"bacccdaad","output":"aaacccaaa"},{"input":"cbadaad","output":"aaaaaaa"},{"input":"caadd","output":"aaaad"},{"input":"adccacbba","output":"aaacccabb"},{"input":"cabbaaadbcad","output":"aaabbaaaabaa"},{"input":"aaabdbbbc","output":"aaaaabbbb"},{"input":"dadbbabbbdcccc","output":"aadabbbbbbbccc"},{"input":"cdccad","output":"aaccca"},{"input":"bcdbddbdbbddcb","output":"aabbddddbbbddb"},{"input":"accdbdbccbb","output":"aaccbdbbccb"},{"input":"daaabaccbba","output":"aaaaaaaccbb"},{"input":"cadbbbcdabccba","output":"aaaabbbbaaacca"},{"input":"bcbbbbcacb","output":"aabbbbbaca"},{"input":"adbacadcaadcd","output":"aaaaaaaaaaaad"},{"input":"addbcbddddddbb","output":"aaddbbbddddddb"},{"input":"acdaacccdccba","output":"aaaaaacccccca"},{"input":"accbddccac","output":"aaccbddccc"},{"input":"cbbbd","output":"aabbb"},{"input":"bbcbadcccadab","output":"aabbaaacccaaa"},{"input":"bdaccdabca","output":"aaaaccaaaa"},{"input":"abaabc","output":"aaaaaa"},{"input":"adcabcd","output":"aaaaaab"},{"input":"abbdaaaadc","output":"aabbaaaaaa"},{"input":"abddddca","output":"aaadddda"},{"input":"ddbbcbbdbbdc","output":"aadbbbbbbbbb"},{"input":"cbaaab","output":"aaaaaa"},{"input":"ddabd","output":"aadaa"},{"input":"abbbaacdbda","output":"aabbbaaabda"},{"input":"bbcbd","output":"aabbb"},{"input":"bcdadccdc","output":"aabadaccc"},{"input":"baaaddddad","output":"aaaaaddddd"},{"input":"cdbaabc","output":"aabaaaa"},{"input":"bbdddda","output":"aabdddd"},{"input":"bccddbcd","output":"aaccddbb"},{"input":"dadcbbabbdaac","output":"aadabbbbbbaaa"},{"input":"dcaccbb","output":"aaacccb"},{"input":"abbddbbdabdcd","output":"aabbddbbaaabd"},{"input":"acbbabc","output":"aaabbba"},{"input":"ddbddacabadd","output":"aaddddaaaaad"},{"input":"bdaaddbcdabcaa","output":"aaaaaddbbaaaaa"},{"input":"ccddadad","output":"aacdddad"},{"input":"badcdcabd","output":"aaaadcaaa"},{"input":"bcdbaaccbbac","output":"aabbaaaccbba"},{"input":"cbcbdbbcbcc","output":"aacbbbbbbcc"},{"input":"dbcda","output":"aabba"},{"input":"dbabbdbd","output":"aaabbbbd"},{"input":"bbcdabacc","output":"aabbaaaac"},{"input":"dacba","output":"aaaaa"},{"input":"cddaddaaad","output":"aadddddaaa"},{"input":"babcbccacc","output":"aababccccc"},{"input":"ddcbacacbdcbd","output":"aadbaaacabbbb"},{"input":"dacbda","output":"aaaaba"},{"input":"caddac","output":"aaadda"},{"input":"cbadbccbcb","output":"aaaaabcccb"},{"input":"bbaacca","output":"aabaacc"},{"input":"abbbdc","output":"aabbbb"},{"input":"cdccabdc","output":"aacccaab"},{"input":"dadbdcda","output":"aadadbda"},{"input":"daacadcdcba","output":"aaaaaaadcba"},{"input":"dcbabcda","output":"aabababa"},{"input":"bbbaccbca","output":"aabbaccca"},{"input":"cbabaaabcab","output":"aaabaaaaaaa"},{"input":"dccdcabc","output":"aacccaaa"},{"input":"dbbcbbbdccb","output":"aabbbbbbbcc"},{"input":"bbccaadcaaab","output":"aabccaaaaaaa"},{"input":"ddcbccaaaaad","output":"aadbcccaaaaa"},{"input":"dcdaaccaacb","output":"aadaaaccaaa"},{"input":"dcbacddadad","output":"aabaaadddad"},{"input":"bdaadc","output":"aaaaaa"},{"input":"bcadadbdcbaac","output":"aaaaadadbbaaa"},{"input":"acdbacacacbbcb","output":"aaabaaacacabbb"},{"input":"acbdbcbbbba","output":"aaabbbbbbbb"},{"input":"dacbcbbbbaba","output":"aaaacbbbbbba"},{"input":"caddcadb","output":"aaaddaaa"},{"input":"cdcdcb","output":"aacdcb"},{"input":"cddbbdabbad","output":"aaddbbaabba"},{"input":"dcaaababdbddb","output":"aaaaaaababddd"},{"input":"abcbd","output":"aaabb"},{"input":"dbddc","output":"aaddd"},{"input":"adaadbad","output":"aaaaaaaa"},{"input":"dbbccbabbacaa","output":"aabbccabbbaaa"},{"input":"daaaabcdc","output":"aaaaaaabc"},{"input":"ddcca","output":"aadcc"},{"input":"cccdccc","output":"aaccccc"},{"input":"baacc","output":"aaaac"},{"input":"ddcdc","output":"aaddc"},{"input":"baccbadd","output":"aaaccaad"},{"input":"ddbccadda","output":"aadbccadd"},{"input":"acdddbdbcb","output":"aaaddddbbb"},{"input":"ccdddab","output":"aacddda"},{"input":"cacabccbdcdcbb","output":"aacaaaccbbdcbb"},{"input":"cbccc","output":"aaccc"},{"input":"daabcccd","output":"aaaaaccc"},{"input":"bbdbd","output":"aabbd"},{"input":"abadcadaa","output":"aaaaaaaaa"},{"input":"ababdba","output":"aaababa"},{"input":"bbcbbaabcdd","output":"aabbbbaaabd"},{"input":"bcbdcddcacccaa","output":"aabbbdddacccca"},{"input":"bbcaaabcda","output":"aabaaaaaba"},{"input":"cccdc","output":"aaccc"},{"input":"acddbc","output":"aaaddb"},{"input":"abbabcaadaba","output":"aabbbaaaaaaa"},{"input":"aaabadbbc","output":"aaaaaaabb"},{"input":"dcbcbadb","output":"aabcbaaa"},{"input":"acbacacacdca","output":"aaaaaacacaca"},{"input":"aaccdddc","output":"aaaccddd"},{"input":"aacbbcbb","output":"aaaabbbb"},{"input":"dbada","output":"aaaaa"},{"input":"bccbddacbadbcd","output":"aaccbddaaaaabb"},{"input":"baadcbaaccbdd","output":"aaaaabaaaccbd"},{"input":"abddada","output":"aaaddda"},{"input":"baabdbda","output":"aaaaabda"},{"input":"dddbabadaaacb","output":"aaddabaaaaaaa"},{"input":"acbdbbcbcd","output":"aaabbbbbcb"},{"input":"bdbccbddada","output":"aabbccbddda"},{"input":"addadacb","output":"aadddaaa"},{"input":"cdccacc","output":"aaccccc"},{"input":"bcbcd","output":"aabcb"},{"input":"bbccddcc","output":"aabccddc"},{"input":"babbbbdbbc","output":"aabbbbbbbb"},{"input":"abdbdccada","output":"aaabdbccaa"},{"input":"abbaa","output":"aabba"},{"input":"ddcbdbaadbca","output":"aadbbbaaaaba"},{"input":"acbba","output":"aaabb"},{"input":"baddacacdadadd","output":"aaaddaacaadadd"},{"input":"acbdaaabd","output":"aaabaaaaa"},{"input":"bcdacbbddbb","output":"aabaaabbddb"},{"input":"dcddadddab","output":"aaddddddda"},{"input":"baadcabbddcbc","output":"aaaaaaabbddbc"},{"input":"dbdac","output":"aadaa"},{"input":"cadbaddbdcddc","output":"aaaaaadddbddd"},{"input":"dcbaadddbbc","output":"aabaaadddbb"},{"input":"abbcacbdaddb","output":"aabbacabaddd"},{"input":"acaacdaacbabdb","output":"aaaaaaaaaaabab"},{"input":"bcabaacddada","output":"aaaaaaaaddda"},{"input":"bccddbdddb","output":"aaccdddddd"},{"input":"ddadbcdbbdb","output":"aaddabbbbbb"},{"input":"aacabddccdd","output":"aaaaaaddccd"},{"input":"ddcbbcadcdc","output":"aadbbbaaadc"},{"input":"adacaccbddbda","output":"aaaaacccbddda"},{"input":"cdbcac","output":"aabbac"},{"input":"cadda","output":"aaadd"},{"input":"bcbddb","output":"aabbdd"},{"input":"ccbbbadac","output":"aacbbbaaa"},{"input":"dbcacadac","output":"aabacaaaa"},{"input":"dbdaad","output":"aadaaa"},{"input":"adcbcabdacacad","output":"aaabcaaaaaacaa"},{"input":"bccabda","output":"aaccaaa"},{"input":"ddcbba","output":"aadbbb"},{"input":"ababcac","output":"aaabaac"},{"input":"ddbaabbdadad","output":"aadaaabbadad"},{"input":"badbaddb","output":"aaaaaadd"},{"input":"abbad","output":"aabba"},{"input":"cbaccdb","output":"aaaaccb"},{"input":"ccacaaacbac","output":"aaccaaaaaaa"},{"input":"adaab","output":"aaaaa"},{"input":"aaabacdbaadaab","output":"aaaaaaabaaaaaa"},{"input":"baaaac","output":"aaaaaa"},{"input":"abdccdccadbdd","output":"aaabcccccaadd"},{"input":"cccdbbddab","output":"aaccbbbdda"},{"input":"bdddddcccb","output":"aadddddccc"},{"input":"dabcaadacdcdba","output":"aaaaaaaaaacdba"},{"input":"cbbbdbbd","output":"aabbbbbb"},{"input":"bbacab","output":"aabaaa"},{"input":"cdbdcacbb","output":"aabdbacab"},{"input":"cccaabcbddb","output":"aaccaaabbdd"},{"input":"cadddaccdbdccb","output":"aaadddaccbdbcc"},{"input":"cdbdacac","output":"aabdaaac"},{"input":"dcdccbccadbb","output":"aadccccccaab"},{"input":"aaabbbbb","output":"aaaabbbb"},{"input":"aabcbacbb","output":"aaaabaaab"},{"input":"dabbdbcccdacc","output":"aaabbbbcccaac"},{"input":"aacdcbccdcbdbd","output":"aaaacbccccbbbd"},{"input":"dbbbbdcc","output":"aabbbbbc"},{"input":"adcbbdcc","output":"aaabbbbc"},{"input":"cdacccacaa","output":"aaaaccccaa"},{"input":"cdcbddabcddcab","output":"aacbbddaabddaa"},{"input":"ccacaccacddddc","output":"aaccaccccadddd"},{"input":"adbdada","output":"aaadada"},{"input":"ddaabbdddadcc","output":"aadaabbddddac"},{"input":"aadaadaccd","output":"aaaaaaaacc"},{"input":"bacaa","output":"aaaaa"},{"input":"adcbdbb","output":"aaabbbb"},{"input":"abdcabc","output":"aaabaaa"},{"input":"abaaaabccdb","output":"aaaaaaaaccb"},{"input":"abddbbdccaa","output":"aaaddbbbcca"},{"input":"acabacdbbb","output":"aaaaaaabbb"},{"input":"ccacbbd","output":"aaccabb"},{"input":"acccccbaccdbac","output":"aacccccaaccbaa"},{"input":"cdddaaaa","output":"aadddaaa"},{"input":"ddada","output":"aadda"},{"input":"bdcccbbbca","output":"aabcccbbba"},{"input":"bdccda","output":"aabcca"},{"input":"adcbd","output":"aaabb"},{"input":"abdcbdbb","output":"aaabbbbb"},{"input":"bdccbcbdcad","output":"aabcccbbbaa"},{"input":"bcdbdabdbcbaa","output":"aabbdaaabbbaa"},{"input":"cdcbbcaab","output":"aacbbbaaa"},{"input":"cdaddaacc","output":"aaadddaac"},{"input":"aadccc","output":"aaaacc"},{"input":"cccbbbbd","output":"aaccbbbb"},{"input":"dcacbddddc","output":"aaacabdddd"},{"input":"accbbdb","output":"aaccbbb"},{"input":"dabcbbd","output":"aaaabbb"},{"input":"dadbdadcc","output":"aadadadac"},{"input":"aadccadb","output":"aaaaccaa"},{"input":"cddbadd","output":"aaddaad"},{"input":"addacabdabbcd","output":"aaddaaaaaabbb"},{"input":"cadaccc","output":"aaaaacc"},{"input":"bbccddbcdc","output":"aabccddbbc"},{"input":"dcbcbdcabbbd","output":"aabcbbbaabbb"},{"input":"aacbd","output":"aaaab"},{"input":"bddabac","output":"aaddaaa"},{"input":"baaaab","output":"aaaaaa"},{"input":"cabadddccacd","output":"aaaaadddccca"},{"input":"dddacadcbd","output":"aaddaaaabb"},{"input":"cbcacaadaca","output":"aacacaaaaaa"},{"input":"bbbdcac","output":"aabbbac"},{"input":"bdaaadaccbc","output":"aaaaaaaaccc"},{"input":"cbdabdcad","output":"aabaaabaa"},{"input":"ccbabccbc","output":"aacabaccc"},{"input":"caddb","output":"aaadd"},{"input":"acbdcbbc","output":"aaabbbbb"},{"input":"cacabdddcbc","output":"aacaaadddbc"},{"input":"abddcbccbabb","output":"aaaddbcccabb"},{"input":"adddbcdccc","output":"aadddbbccc"},{"input":"adacdbaccddabc","output":"aaaaabaaccddaa"},{"input":"dbcaabbcacac","output":"aabaaabbacac"},{"input":"abbdad","output":"aabbad"},{"input":"cacccbabcdabc","output":"aaccccababaaa"},{"input":"dcdbcbdabdac","output":"aadbbbbaaaaa"},{"input":"adbca","output":"aaaba"},{"input":"cabcbabaddccd","output":"aaaababaaddcc"},{"input":"dcacda","output":"aaacaa"},{"input":"abacdcbbacad","output":"aaaaacbbbaaa"},{"input":"bbadacdcdcd","output":"aabaaaacdcd"},{"input":"babbddaddddda","output":"aabbbdddddddd"},{"input":"bbcbdd","output":"aabbbd"},{"input":"badaacbbaccb","output":"aaaaaaabbacc"},{"input":"baabcd","output":"aaaaab"},{"input":"aaccbd","output":"aaaccb"},{"input":"bacdbbad","output":"aaaabbba"},{"input":"ddcadcbcdddcd","output":"aadaaabcbdddd"},{"input":"bacbaba","output":"aaaaaba"},{"input":"acdaa","output":"aaaaa"},{"input":"acdbcadccbadb","output":"aaabbaaaccaaa"},{"input":"cabcbababbcb","output":"aaaabababbbb"},{"input":"bdabcdbdabbb","output":"aaaaabbdaabb"},{"input":"dbabbc","output":"aaabbb"},{"input":"adacccd","output":"aaaaccc"},{"input":"ddadacdab","output":"aaddaaaaa"},{"input":"cccadcbaada","output":"aaccaabaaaa"},{"input":"dbbbbdbdca","output":"aabbbbbdba"},{"input":"dcbbcbdcab","output":"aabbbbbbaa"},{"input":"dcbbaaac","output":"aabbbaaa"},{"input":"abdbdcccbccdc","output":"aaabdbccccccc"},{"input":"aacdcdcd","output":"aaaacdcd"},{"input":"cdbbcbd","output":"aabbbbb"},{"input":"baababcb","output":"aaaaabab"},{"input":"bbbaaacdcaba","output":"aabbaaaacaaa"},{"input":"bdbcdbbbbca","output":"aabbbbbbbba"},{"input":"bccccd","output":"aacccc"},{"input":"dabadbcc","output":"aaaaaabc"},{"input":"dbdaadcabb","output":"aadaaaaaab"},{"input":"dcadddcdaddada","output":"aaaaddddadddda"},{"input":"cddcbcdcbaddaa","output":"aaddbcbcbaadda"},{"input":"aabcadbccddab","output":"aaaaaaabccdda"},{"input":"dacacccd","output":"aaaacccc"},{"input":"dbbcacdbba","output":"aabbacabbb"},{"input":"aadadccc","output":"aaaadacc"},{"input":"bbcdcaddabbbbd","output":"aabbcaaddabbbb"},{"input":"bdabacbbb","output":"aaaaaaabb"},{"input":"daddcadb","output":"aadddaaa"},{"input":"cdacbdaddddd","output":"aaaaabaddddd"},{"input":"ddcaabada","output":"aadaaaaaa"},{"input":"ccaaad","output":"aacaaa"},{"input":"adbbb","output":"aaabb"},{"input":"adbcdcdca","output":"aaabbcdca"},{"input":"accbbd","output":"aaccbb"},{"input":"dacbd","output":"aaaab"},{"input":"ababc","output":"aaaba"},{"input":"cabcdcc","output":"aaaabcc"},{"input":"cbcbc","output":"aacbc"},{"input":"bcddccbbaaa","output":"aabddccbbaa"},{"input":"accdca","output":"aaccca"},{"input":"acaaa","output":"aaaaa"},{"input":"dbcdd","output":"aabbd"},{"input":"cdcbbdaccac","output":"aacbbbaaccc"},{"input":"acdba","output":"aaaba"},{"input":"acccab","output":"aaccca"},{"input":"cbbaaacaaaba","output":"aabbaaaaaaaa"},{"input":"ccdabccdcbdcca","output":"aacaaacccbbbcc"},{"input":"dcdddbcacba","output":"aaddddbacaa"},{"input":"aacdaad","output":"aaaaaaa"},{"input":"bbdddbccdbcdac","output":"aabdddbccbbbaa"},{"input":"aacbbcdbabcdb","output":"aaaabbbbababb"},{"input":"cccbdba","output":"aaccbba"},{"input":"daadcccac","output":"aaaaacccc"},{"input":"cbbacbcacc","output":"aabbaacacc"},{"input":"aacaaca","output":"aaaaaaa"},{"input":"bbdddc","output":"aabddd"},{"input":"abbbddd","output":"aabbbdd"},{"input":"babccaabdacd","output":"aabaccaaaaaa"},{"input":"abcaa","output":"aaaaa"},{"input":"bcabcababc","output":"aaaaaaaaba"},{"input":"cdccdcadbdda","output":"aaccccaaaddd"},{"input":"aadcbcca","output":"aaaabccc"},{"input":"acbcababacbdcb","output":"aaacaaabaaabbb"},{"input":"ababdbcbbad","output":"aaababbbbba"},{"input":"abbbcccabdddd","output":"aabbbcccaaddd"},{"input":"abbddaa","output":"aabbdda"},{"input":"cddbbbc","output":"aaddbbb"},{"input":"badcdda","output":"aaaaddd"},{"input":"dbcbad","output":"aabbaa"},{"input":"ddccabdabdcba","output":"aadccaaaaabba"},{"input":"aacbbdd","output":"aaaabbd"},{"input":"dbadcccdbaca","output":"aaaaacccbaaa"},{"input":"baddcdc","output":"aaadddc"},{"input":"ccccad","output":"aaccca"},{"input":"dccdcdaddca","output":"aacccdaddda"},{"input":"cdabaabacd","output":"aaaaaaaaaa"},{"input":"cbaabadbdb","output":"aaaaaaaadb"},{"input":"ccbccdad","output":"aaccccad"},{"input":"acddbbd","output":"aaaddbb"},{"input":"abcabbbabbb","output":"aaaaabbbbbb"},{"input":"cadbcbbdcddaa","output":"aaaabbbbbddda"},{"input":"bcbcdcadb","output":"aabcbcaaa"},{"input":"cabdcbdbacadb","output":"aaaabbbbaaaaa"},{"input":"aabacabcdaacb","output":"aaaaaaaabaaaa"},{"input":"dbdbabbcdddadc","output":"aadbabbbbdddda"},{"input":"dcbabacdc","output":"aababaaac"},{"input":"abbda","output":"aabba"},{"input":"adbddbbdbc","output":"aaadddbbbb"},{"input":"ddabddbcbb","output":"aadaaddbbb"},{"input":"cbabbacb","output":"aaabbbaa"},{"input":"abdabbac","output":"aaaaabba"},{"input":"bdbcada","output":"aabbaaa"},{"input":"ddacdacacbb","output":"aadaaaaacab"},{"input":"aabadaa","output":"aaaaaaa"},{"input":"dadcccdacc","output":"aadacccaac"},{"input":"dbdcdbabbdbadc","output":"aadbdbabbbbaaa"},{"input":"acaabccdbb","output":"aaaaaaccbb"},{"input":"ccdaba","output":"aacaaa"},{"input":"ccbcbbdcbab","output":"aaccbbbbbab"},{"input":"dddbbcdbcdddad","output":"aaddbbbbbbdddd"},{"input":"cbabadcbaa","output":"aaabaaabaa"},{"input":"bcddd","output":"aabdd"},{"input":"cdcbb","output":"aacbb"},{"input":"bccaddadacbaa","output":"aaccadddaaaaa"},{"input":"bacacaa","output":"aaaacaa"},{"input":"abdbcdaabbcba","output":"aaabbbaaabbba"},{"input":"bbcbbbcb","output":"aabbbbbb"},{"input":"aacaccbcbbdaa","output":"aaaaccccbbbaa"},{"input":"baadccba","output":"aaaaacca"},{"input":"dacadbcabcbda","output":"aaaaaabaaabba"},{"input":"dbbaadbbbbcdcb","output":"aabbaaabbbbbcb"},{"input":"aabccbdc","output":"aaaaccbb"},{"input":"bbbddd","output":"aabbdd"},{"input":"abcacdcdabcb","output":"aaaacacdaaab"},{"input":"cbbadbcdcccbaa","output":"aabbaabbccccaa"},{"input":"cbbbdbdacdb","output":"aabbbbdaaab"},{"input":"dbcddbdaa","output":"aabbdddaa"},{"input":"cdcbccbacdcb","output":"aacbcccaaacb"},{"input":"cdddb","output":"aaddd"},{"input":"adbcbdadaccdc","output":"aaabbbadaaccc"},{"input":"dadbddbc","output":"aadadddb"},{"input":"dbbddbdaaa","output":"aabbdddaaa"},{"input":"bababddbd","output":"aababaddd"},{"input":"abcaddd","output":"aaaaadd"},{"input":"dccbbccbd","output":"aaccbbccb"},{"input":"bbcaadcccada","output":"aabaaaacccaa"},{"input":"abbcbcaccacdbd","output":"aabbbcaccccabd"},{"input":"bccdacdaacacbd","output":"aaccaaaaaaacab"},{"input":"acadadbadb","output":"aaaaadaaaa"},{"input":"bddcaddcba","output":"aaddaaddba"},{"input":"cbbddbcdd","output":"aabbddbbd"},{"input":"addbad","output":"aaddaa"},{"input":"bbcabbcaa","output":"aabaabbaa"},{"input":"dccbdd","output":"aaccbd"},{"input":"bdaaa","output":"aaaaa"},{"input":"daccacadc","output":"aaacccaaa"},{"input":"dcacadbbaadbbb","output":"aaacaaabbaaabb"},{"input":"adacdcabddd","output":"aaaaacaaadd"},{"input":"baaaabdcaabad","output":"aaaaaaabaaaaa"},{"input":"bccbcdb","output":"aacccbb"},{"input":"bddda","output":"aaddd"},{"input":"ddbcc","output":"aadbc"},{"input":"bdcdcd","output":"aabdcd"},{"input":"dcbccdbdca","output":"aabcccbdba"},{"input":"cdcbcabbbccdd","output":"aacbcaabbbccd"},{"input":"abbabdbbbaacb","output":"aabbbabbbbaaa"},{"input":"badbdd","output":"aaaadd"},{"input":"dccad","output":"aacca"},{"input":"dbacdbadbcbda","output":"aaaaabaaabbba"},{"input":"abcdaca","output":"aaabaaa"},{"input":"dbbacaddbabb","output":"aabbaaaddabb"},{"input":"badbabbcdb","output":"aaaaabbbbb"},{"input":"bbaaccbb","output":"aabaaccb"},{"input":"cbdbbdbabaad","output":"aabbbbbabaaa"},{"input":"cbcadabcdc","output":"aacaaaaabc"},{"input":"abaccdaa","output":"aaaaccaa"},{"input":"addbacdd","output":"aaddaaad"},{"input":"dddacd","output":"aaddaa"},{"input":"acaccdccd","output":"aaacccccc"},{"input":"cccdbcbaaabbac","output":"aaccbbbaaaabba"},{"input":"ccbadcbabdac","output":"aacaaababaaa"},{"input":"dbcdbbabbadba","output":"aabbbbbbbbaaa"},{"input":"ddababddbabcdd","output":"aadaabaddababd"},{"input":"abbdaacaa","output":"aabbaaaaa"},{"input":"dcdbca","output":"aadbba"},{"input":"bcbcbbbab","output":"aabcbbbbb"},{"input":"dbddabdbaab","output":"aadddaabaaa"},{"input":"addddacbdb","output":"aaddddaabb"},{"input":"babbcbbbccc","output":"aabbbbbbbcc"},{"input":"caaddbd","output":"aaaaddd"},{"input":"cabaadab","output":"aaaaaaaa"},{"input":"cdadcdbaddaac","output":"aaadadbaaddaa"},{"input":"abaca","output":"aaaaa"},{"input":"dcdccdddcbd","output":"aadcccdddbb"},{"input":"aadaadaacbcab","output":"aaaaaaaaaacaa"},{"input":"acddabaddacdb","output":"aaaddaaaddaab"},{"input":"ccadbddcdccdc","output":"aacaaddddcccc"},{"input":"abdcccdba","output":"aaabcccba"},{"input":"bbccacadaadaa","output":"aabcccaaaaaaa"},{"input":"cccdcacccbcccb","output":"aacccacccccccc"},{"input":"adbcdaabadc","output":"aaabbaaaaaa"},{"input":"cdbbcacbdcdbad","output":"aabbbacabbdbaa"},{"input":"caacdbbabc","output":"aaaaabbbba"},{"input":"bdccdbdbaccdad","output":"aabccbdbaaccad"},{"input":"ccacbb","output":"aaccab"},{"input":"aadabd","output":"aaaaaa"},{"input":"accaadcbc","output":"aaccaaabc"},{"input":"cdbdadcddcdada","output":"aabdadaddddada"},{"input":"dbcab","output":"aabaa"},{"input":"ddbbddccacdd","output":"aadbbddcccad"},{"input":"dadbddbad","output":"aadadddaa"},{"input":"daccbdcdbad","output":"aaaccbbdbaa"},{"input":"dadddbaa","output":"aaddddaa"},{"input":"acbbbccbca","output":"aaabbbccca"},{"input":"caaacdcaaadd","output":"aaaaaacaaaad"},{"input":"bacacbbbabdb","output":"aaaacabbbbab"},{"input":"dcbdabddbdd","output":"aabbaaadddd"},{"input":"babadaacdadbdc","output":"aabaaaaaaadadb"},{"input":"ddabbcacbaabda","output":"aadabbacaaaaaa"},{"input":"dbcabbdddbaada","output":"aabaabbdddaaaa"},{"input":"aacbdc","output":"aaaabb"},{"input":"cbbcaad","output":"aabbaaa"},{"input":"caaabcddbbd","output":"aaaaaabddbb"},{"input":"caccbcbabdda","output":"aaccccbabadd"},{"input":"daadddadacbb","output":"aaaaddddaaab"},{"input":"ccbaa","output":"aacaa"},{"input":"daababbad","output":"aaaaabbba"},{"input":"addaccdabcd","output":"aaddaccaaab"},{"input":"ddcaaabacacc","output":"aadaaaaaaacc"},{"input":"dabacabaa","output":"aaaaaaaaa"},{"input":"ddaabadaada","output":"aadaaaaaaaa"},{"input":"baadcaaddddaca","output":"aaaaaaaaddddaa"},{"input":"daabdccadadbc","output":"aaaaabccaadab"},{"input":"ccada","output":"aacaa"},{"input":"dccabdbcadc","output":"aaccaabbaaa"},{"input":"dcbdabadb","output":"aabbaaaaa"},{"input":"baddbcdcacaa","output":"aaaddbbcacaa"},{"input":"bbbcacbaaddab","output":"aabbacaaaadda"},{"input":"abbabacd","output":"aabbbaaa"},{"input":"cbdcabdcab","output":"aabbaaabaa"},{"input":"dcdbaccacaca","output":"aadbaacccaca"},{"input":"aaccdcbbdbc","output":"aaacccbbbbb"},{"input":"bcbdaaba","output":"aabbaaaa"},{"input":"cdacccab","output":"aaaaccca"},{"input":"bcbbabcbadda","output":"aabbbbabaadd"},{"input":"ccbabbbaaad","output":"aacabbbbaaa"},{"input":"cddacd","output":"aaddaa"},{"input":"ddcdc","output":"aaddc"},{"input":"adbdcbbdacdcdb","output":"aaadbbbbaaacdb"},{"input":"dbbaa","output":"aabba"},{"input":"bbdbbdddbddc","output":"aabbbbdddddd"},{"input":"dccdaa","output":"aaccaa"},{"input":"cabbbaacbcabbd","output":"aaabbbaaacaabb"},{"input":"daabacadaaaa","output":"aaaaaaaaaaaa"},{"input":"adbbddb","output":"aaabbdd"},{"input":"dcddbbcdbabbbd","output":"aadddbbbbabbbb"},{"input":"bbaadccccd","output":"aabaaacccc"},{"input":"abcada","output":"aaaaaa"},{"input":"bbcadba","output":"aabaaaa"},{"input":"dbddddbb","output":"aadddddb"},{"input":"aadcbddbac","output":"aaaabbddaa"},{"input":"abadab","output":"aaaaaa"},{"input":"ddccdcbada","output":"aadcccbaaa"},{"input":"dbabbbcd","output":"aaabbbbb"},{"input":"dcbaccabcdd","output":"aabaaccaabd"},{"input":"dcdcddbccacdb","output":"aadcdddbcccab"},{"input":"ccdbbdbccbad","output":"aacbbbbbccaa"},{"input":"dcbddcdbaccd","output":"aabbdddbaacc"},{"input":"dbdbcbcbc","output":"aadbbbcbc"},{"input":"cdddadabbcaca","output":"aaddddaabbaca"},{"input":"adabdcadb","output":"aaaaabaaa"},{"input":"bbacaaddcaadb","output":"aabaaaaddaaaa"},{"input":"adaaaaddc","output":"aaaaaaadd"},{"input":"ddccccbbbd","output":"aadccccbbb"},{"input":"cdbbcacdd","output":"aabbbacad"},{"input":"ccacdbdbcaa","output":"aaccabdbbaa"},{"input":"aabacbcacdb","output":"aaaaaacacab"},{"input":"bbbbba","output":"aabbbb"},{"input":"ddbbadaca","output":"aadbbaaaa"},{"input":"caabaccada","output":"aaaaaaccaa"},{"input":"bdacaca","output":"aaaaaca"},{"input":"bbdcbab","output":"aabbbab"},{"input":"dbcdccbaab","output":"aabbcccaaa"},{"input":"bbabac","output":"aabbaa"},{"input":"daddddccbada","output":"aadddddccaaa"},{"input":"accbadccdac","output":"aaccaaaccaa"},{"input":"bbddcccddbbd","output":"aabddcccddbb"},{"input":"bacca","output":"aaacc"},{"input":"ababdbcaccd","output":"aaababbaccc"},{"input":"adaadcb","output":"aaaaaab"},{"input":"bccdbbbd","output":"aaccbbbb"},{"input":"aaabadc","output":"aaaaaaa"},{"input":"caacabaccacdc","output":"aaaaaaaacccac"},{"input":"daadd","output":"aaaad"},{"input":"cdadac","output":"aaadaa"},{"input":"cdacdcbcdacc","output":"aaaaacbcbaac"},{"input":"dbbbbaadbbad","output":"aabbbbaaabba"},{"input":"badbabbaaaaca","output":"aaaaabbbaaaaa"},{"input":"aadba","output":"aaaaa"},{"input":"ddcbca","output":"aadbca"},{"input":"bcaaba","output":"aaaaaa"},{"input":"bcadcddc","output":"aaaaaddd"},{"input":"abbddca","output":"aabbdda"},{"input":"abdcdbbcd","output":"aaabdbbbb"},{"input":"acaababaaac","output":"aaaaaabaaaa"},{"input":"cbbdbcddddacd","output":"aabbbbbddddaa"},{"input":"aadcdbaaddabaa","output":"aaaadbaaaddaaa"}] -------------------------------------------------------------------------------- /draft/simple_lstm_tutorial_grammar.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "\n", 13 | "chars='BTSXPVE'\n", 14 | "\n", 15 | "graph = [[(1,5),('T','P')] , [(1,2),('S','X')], \\\n", 16 | " [(3,5),('S','X')], [(6,),('E')], \\\n", 17 | " [(3,2),('V','P')], [(4,5),('V','T')] ]\n", 18 | "\n", 19 | "\n", 20 | "def in_grammar(word):\n", 21 | " if word[0] != 'B':\n", 22 | " return False\n", 23 | " node = 0 \n", 24 | " for c in word[1:]:\n", 25 | " transitions = graph[node]\n", 26 | " try:\n", 27 | " node = transitions[0][transitions[1].index(c)]\n", 28 | " except ValueError: # using exceptions for flow control in python is common\n", 29 | " return False\n", 30 | " return True \n", 31 | " \n", 32 | "def sequenceToWord(sequence):\n", 33 | " \"\"\"\n", 34 | " converts a sequence (one-hot) in a reber string\n", 35 | " \"\"\"\n", 36 | " reberString = ''\n", 37 | " for s in sequence:\n", 38 | " index = np.where(s==1.)[0][0]\n", 39 | " reberString += chars[index]\n", 40 | " return reberString\n", 41 | " \n", 42 | "def generateSequences(minLength):\n", 43 | " while True:\n", 44 | " inchars = ['B']\n", 45 | " node = 0\n", 46 | " outchars = [] \n", 47 | " while node != 6:\n", 48 | " transitions = graph[node]\n", 49 | " i = np.random.randint(0, len(transitions[0]))\n", 50 | " inchars.append(transitions[1][i])\n", 51 | " outchars.append(transitions[1])\n", 52 | " node = transitions[0][i]\n", 53 | " if len(inchars) > minLength: \n", 54 | " return inchars, outchars\n", 55 | "\n", 56 | "\n", 57 | "def get_one_example(minLength):\n", 58 | " inchars, outchars = generateSequences(minLength)\n", 59 | " inseq = []\n", 60 | " outseq= []\n", 61 | " for i,o in zip(inchars, outchars): \n", 62 | " inpt = np.zeros(7)\n", 63 | " inpt[chars.find(i)] = 1. \n", 64 | " outpt = np.zeros(7)\n", 65 | " for oo in o:\n", 66 | " outpt[chars.find(oo)] = 1.\n", 67 | " inseq.append(inpt)\n", 68 | " outseq.append(outpt)\n", 69 | " return inseq, outseq\n", 70 | "\n", 71 | "\n", 72 | "def get_char_one_hot(char):\n", 73 | " char_oh = np.zeros(7)\n", 74 | " for c in char:\n", 75 | " char_oh[chars.find(c)] = 1.\n", 76 | " return [char_oh] \n", 77 | " \n", 78 | "def get_n_examples(n, minLength=10):\n", 79 | " examples = []\n", 80 | " for i in xrange(n):\n", 81 | " examples.append(get_one_example(minLength))\n", 82 | " return examples\n", 83 | "\n", 84 | "emb_chars = \"TP\"\n", 85 | "\n", 86 | "\n", 87 | "def get_one_embedded_example(minLength=10):\n", 88 | " i, o = get_one_example(minLength)\n", 89 | " emb_char = emb_chars[np.random.randint(0, len(emb_chars))]\n", 90 | " new_in = get_char_one_hot(('B',))\n", 91 | " new_in += get_char_one_hot((emb_char,))\n", 92 | " new_out= get_char_one_hot(emb_chars)\n", 93 | " new_out+= get_char_one_hot('B',)\n", 94 | " new_in += i\n", 95 | " new_out += o\n", 96 | " new_in += get_char_one_hot(('E',))\n", 97 | " new_in += get_char_one_hot((emb_char,))\n", 98 | " new_out += get_char_one_hot((emb_char, ))\n", 99 | " new_out += get_char_one_hot(('E',))\n", 100 | " return new_in, new_out\n", 101 | " \n", 102 | "def get_n_embedded_examples(n, minLength=10):\n", 103 | " examples = []\n", 104 | " for i in xrange(n):\n", 105 | " examples.append(get_one_embedded_example(minLength))\n", 106 | " return examples" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": { 113 | "collapsed": true 114 | }, 115 | "outputs": [], 116 | "source": [ 117 | "import numpy as np\n", 118 | "import theano\n", 119 | "import theano.tensor as T\n", 120 | "\n", 121 | "dtype=theano.config.floatX" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": { 128 | "collapsed": true 129 | }, 130 | "outputs": [], 131 | "source": [ 132 | "# squashing of the gates should result in values between 0 and 1\n", 133 | "# therefore we use the logistic function\n", 134 | "sigma = lambda x: 1 / (1 + T.exp(-x))\n", 135 | "\n", 136 | "\n", 137 | "# for the other activation function we use the tanh\n", 138 | "act = T.tanh\n", 139 | "\n", 140 | "# sequences: x_t\n", 141 | "# prior results: h_tm1, c_tm1\n", 142 | "# non-sequences: W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xy, W_hy, W_cy, b_y\n", 143 | "def one_lstm_step(x_t, h_tm1, c_tm1, W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xy, W_ho, W_cy, b_o, W_hy, b_y):\n", 144 | " i_t = sigma(theano.dot(x_t, W_xi) + theano.dot(h_tm1, W_hi) + theano.dot(c_tm1, W_ci) + b_i)\n", 145 | " f_t = sigma(theano.dot(x_t, W_xf) + theano.dot(h_tm1, W_hf) + theano.dot(c_tm1, W_cf) + b_f)\n", 146 | " c_t = f_t * c_tm1 + i_t * act(theano.dot(x_t, W_xc) + theano.dot(h_tm1, W_hc) + b_c) \n", 147 | " o_t = sigma(theano.dot(x_t, W_xo)+ theano.dot(h_tm1, W_ho) + theano.dot(c_t, W_co) + b_o)\n", 148 | " h_t = o_t * act(c_t)\n", 149 | " y_t = sigma(theano.dot(h_t, W_hy) + b_y) \n", 150 | " return [h_t, c_t, y_t]" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": { 157 | "collapsed": true 158 | }, 159 | "outputs": [], 160 | "source": [ 161 | "#TODO: Use a more appropriate initialization method\n", 162 | "def sample_weights(sizeX, sizeY):\n", 163 | " values = np.ndarray([sizeX, sizeY], dtype=dtype)\n", 164 | " for dx in xrange(sizeX):\n", 165 | " vals = np.random.uniform(low=-1., high=1., size=(sizeY,))\n", 166 | " #vals_norm = np.sqrt((vals**2).sum())\n", 167 | " #vals = vals / vals_norm\n", 168 | " values[dx,:] = vals\n", 169 | " _,svs,_ = np.linalg.svd(values)\n", 170 | " #svs[0] is the largest singular value \n", 171 | " values = values / svs[0]\n", 172 | " return values " 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": { 179 | "collapsed": true 180 | }, 181 | "outputs": [], 182 | "source": [ 183 | "n_in = 7 # for embedded reber grammar\n", 184 | "n_hidden = n_i = n_c = n_o = n_f = 100\n", 185 | "n_y = 7 # for embedded reber grammar\n", 186 | "\n", 187 | "# initialize weights\n", 188 | "# i_t and o_t should be \"open\" or \"closed\"\n", 189 | "# f_t should be \"open\" (don't forget at the beginning of training)\n", 190 | "# we try to archive this by appropriate initialization of the corresponding biases \n", 191 | "\n", 192 | "W_xi = theano.shared(sample_weights(n_in, n_i)) \n", 193 | "W_hi = theano.shared(sample_weights(n_hidden, n_i)) \n", 194 | "W_ci = theano.shared(sample_weights(n_c, n_i)) \n", 195 | "b_i = theano.shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_i)))\n", 196 | "W_xf = theano.shared(sample_weights(n_in, n_f)) \n", 197 | "W_hf = theano.shared(sample_weights(n_hidden, n_f))\n", 198 | "W_cf = theano.shared(sample_weights(n_c, n_f))\n", 199 | "b_f = theano.shared(np.cast[dtype](np.random.uniform(0, 1.,size = n_f)))\n", 200 | "W_xc = theano.shared(sample_weights(n_in, n_c)) \n", 201 | "W_hc = theano.shared(sample_weights(n_hidden, n_c))\n", 202 | "b_c = theano.shared(np.zeros(n_c, dtype=dtype))\n", 203 | "W_xo = theano.shared(sample_weights(n_in, n_o))\n", 204 | "W_ho = theano.shared(sample_weights(n_hidden, n_o))\n", 205 | "W_co = theano.shared(sample_weights(n_c, n_o))\n", 206 | "b_o = theano.shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_o)))\n", 207 | "W_hy = theano.shared(sample_weights(n_hidden, n_y))\n", 208 | "b_y = theano.shared(np.zeros(n_y, dtype=dtype))\n", 209 | "\n", 210 | "c0 = theano.shared(np.zeros(n_hidden, dtype=dtype))\n", 211 | "h0 = T.tanh(c0)\n", 212 | "\n", 213 | "params = [W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xo, W_ho, W_co, b_o, W_hy, b_y, c0]" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": { 220 | "collapsed": true 221 | }, 222 | "outputs": [], 223 | "source": [ 224 | "#first dimension is time\n", 225 | "\n", 226 | "#input \n", 227 | "v = T.matrix(dtype=dtype)\n", 228 | "\n", 229 | "# target\n", 230 | "target = T.matrix(dtype=dtype)\n", 231 | "\n", 232 | "# hidden and outputs of the entire sequence\n", 233 | "[h_vals, _, y_vals], _ = theano.scan(fn=one_lstm_step, \n", 234 | " sequences = dict(input=v, taps=[0]), \n", 235 | " outputs_info = [h0, c0, None ], # corresponds to return type of fn\n", 236 | "\n", 237 | " \n", 238 | " non_sequences = [W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xo, W_ho, W_co, b_o, W_hy, b_y] )" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": { 245 | "collapsed": false 246 | }, 247 | "outputs": [], 248 | "source": [ 249 | "print \"this step may take time: compute all gradients relative to the cost\"\n", 250 | "\n", 251 | "cost = -T.mean(target * T.log(y_vals)+ (1.- target) * T.log(1. - y_vals))\n", 252 | "\n", 253 | "# learning rate\n", 254 | "lr = np.cast[dtype](.1)\n", 255 | "learning_rate = theano.shared(lr)\n", 256 | "\n", 257 | "print \"numparams: \" + str(len(params))\n", 258 | "\n", 259 | "gparams = []\n", 260 | "for param in params:\n", 261 | " gparam = T.grad(cost, param)\n", 262 | " gparams.append(gparam)\n", 263 | "\n", 264 | " \n", 265 | "print \"appending updates\"\n", 266 | "updates=[]\n", 267 | "for param, gparam in zip(params, gparams):\n", 268 | " updates.append((param, param - gparam * learning_rate))" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": { 275 | "collapsed": false 276 | }, 277 | "outputs": [], 278 | "source": [ 279 | "train_data = get_n_embedded_examples(1000)" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": null, 285 | "metadata": { 286 | "collapsed": false 287 | }, 288 | "outputs": [], 289 | "source": [ 290 | "learn_rnn_fn = theano.function(inputs = [v, target],\n", 291 | " outputs = cost,\n", 292 | " updates = updates)" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "metadata": { 299 | "collapsed": false 300 | }, 301 | "outputs": [], 302 | "source": [ 303 | "nb_epochs=250\n", 304 | "train_errors = np.ndarray(nb_epochs)\n", 305 | "def train_rnn(train_data): \n", 306 | " for x in range(nb_epochs):\n", 307 | " error = 0.\n", 308 | " for j in range(len(train_data)): \n", 309 | " index = np.random.randint(0, len(train_data))\n", 310 | " i, o = train_data[index]\n", 311 | " train_cost = learn_rnn_fn(i, o)\n", 312 | " #train_cost = train_f2(i, o)\n", 313 | " error += train_cost\n", 314 | " train_errors[x] = error \n", 315 | " if x%10==0:\n", 316 | " print \"epoch \"+str(x)+ \" error: \"+str(error)\n", 317 | " \n", 318 | " \n", 319 | "train_rnn(train_data)" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "metadata": { 326 | "collapsed": false 327 | }, 328 | "outputs": [], 329 | "source": [ 330 | "%matplotlib inline\n", 331 | "import matplotlib.pyplot as plt\n", 332 | "plt.plot(np.arange(nb_epochs), train_errors, 'b-')\n", 333 | "plt.xlabel('epochs')\n", 334 | "plt.ylabel('error')\n", 335 | "plt.ylim(0., 50)" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": { 342 | "collapsed": false 343 | }, 344 | "outputs": [], 345 | "source": [ 346 | "predictions = theano.function(inputs = [v], outputs = y_vals)\n", 347 | "\n", 348 | "test_data = get_n_embedded_examples(10)\n", 349 | "\n", 350 | "def print_out(test_data):\n", 351 | " for i,o in test_data:\n", 352 | " p = predictions(i)\n", 353 | " print o[-2] # target\n", 354 | " print np.asarray([0. if x!=np.argmax(p[-2]) else 1. for x in range(len(p[-2]))]) # prediction\n", 355 | " print \n", 356 | "print_out(test_data)" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "metadata": { 363 | "collapsed": true 364 | }, 365 | "outputs": [], 366 | "source": [] 367 | } 368 | ], 369 | "metadata": { 370 | "kernelspec": { 371 | "display_name": "Python 2", 372 | "language": "python", 373 | "name": "python2" 374 | }, 375 | "language_info": { 376 | "codemirror_mode": { 377 | "name": "ipython", 378 | "version": 2 379 | }, 380 | "file_extension": ".py", 381 | "mimetype": "text/x-python", 382 | "name": "python", 383 | "nbconvert_exporter": "python", 384 | "pygments_lexer": "ipython2", 385 | "version": "2.7.6" 386 | } 387 | }, 388 | "nbformat": 4, 389 | "nbformat_minor": 0 390 | } 391 | -------------------------------------------------------------------------------- /img/rnn_simple.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 27 | 33 | 34 | 41 | 47 | 48 | 49 | 71 | 73 | 74 | 76 | image/svg+xml 77 | 79 | 80 | 81 | 82 | 83 | 88 | 96 | 104 | 112 | 117 | 122 | input 133 | hidden 144 | output 155 | 162 | W_in 173 | W_out 184 | W_rec 195 | 196 | 197 | -------------------------------------------------------------------------------- /img/rnn_unfolded.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 27 | 33 | 34 | 41 | 47 | 48 | 49 | 71 | 73 | 74 | 76 | image/svg+xml 77 | 79 | 80 | 81 | 82 | 83 | 88 | 96 | 104 | 112 | 117 | 122 | x_t-1 133 | h_t-1 144 | y_t-1 155 | 161 | W_in 172 | W_out 183 | W_rec 194 | 202 | 210 | 218 | 223 | 228 | x_t 239 | h_t 250 | y_t 261 | W_in 272 | W_out 283 | 289 | W_rec 300 | 308 | 316 | 324 | 329 | 334 | x_t+1 345 | h_t+1 356 | y_t+1 367 | W_in 378 | W_out 389 | 394 | time sequence 405 | 406 | 407 | -------------------------------------------------------------------------------- /init.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | import theano.tensor as T 4 | from theano import shared 5 | 6 | dtype = T.config.floatX 7 | 8 | print("loading init.py") 9 | 10 | 11 | def init_weight(shape, name, sample='uni', seed=None): 12 | rng = np.random.RandomState(seed) 13 | 14 | if sample == 'unishape': 15 | values = rng.uniform( 16 | low=-np.sqrt(6. / (shape[0] + shape[1])), 17 | high=np.sqrt(6. / (shape[0] + shape[1])), 18 | size=shape).astype(dtype) 19 | 20 | elif sample == 'svd': 21 | values = rng.uniform(low=-1., high=1., size=shape).astype(dtype) 22 | _, svs, _ = np.linalg.svd(values) 23 | # svs[0] is the largest singular value 24 | values = values / svs[0] 25 | 26 | elif sample == 'uni': 27 | values = rng.uniform(low=-0.1, high=0.1, size=shape).astype(dtype) 28 | 29 | elif sample == 'zero': 30 | values = np.zeros(shape=shape, dtype=dtype) 31 | 32 | else: 33 | raise ValueError("Unsupported initialization scheme: %s" 34 | % sample) 35 | 36 | return shared(values, name=name, borrow=True) 37 | 38 | -------------------------------------------------------------------------------- /lstm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | import theano.tensor as T 4 | from theano import shared 5 | from collections import OrderedDict 6 | from init import init_weight 7 | 8 | dtype=T.config.floatX 9 | 10 | print "loaded lstm.py" 11 | 12 | class Lstm: 13 | def __init__(self, n_in, n_lstm, n_out, lr=0.05, single_output=True, output_activation=T.nnet.softmax, cost_function='nll'): 14 | self.n_in = n_in 15 | self.n_lstm = n_lstm 16 | self.n_out = n_out 17 | self.W_xi = init_weight((self.n_in, self.n_lstm),'W_xi') 18 | self.W_hi = init_weight((self.n_lstm, self.n_lstm),'W_hi', 'svd') 19 | self.W_ci = init_weight((self.n_lstm, self.n_lstm),'W_ci', 'svd') 20 | self.b_i = shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_lstm))) 21 | self.W_xf = init_weight((self.n_in, self.n_lstm),'W_xf') 22 | self.W_hf = init_weight((self.n_lstm, self.n_lstm),'W_hf', 'svd') 23 | self.W_cf = init_weight((self.n_lstm, self.n_lstm),'W_cf', 'svd') 24 | self.b_f = shared(np.cast[dtype](np.random.uniform(0, 1.,size = n_lstm))) 25 | self.W_xc = init_weight((self.n_in, self.n_lstm),'W_xc') 26 | self.W_hc = init_weight((self.n_lstm, self.n_lstm),'W_hc', 'svd') 27 | self.b_c = shared(np.zeros(n_lstm, dtype=dtype)) 28 | self.W_xo = init_weight((self.n_in, self.n_lstm),'W_xo') 29 | self.W_ho = init_weight((self.n_lstm, self.n_lstm),'W_ho', 'svd') 30 | self.W_co = init_weight((self.n_lstm, self.n_lstm),'W_co', 'svd') 31 | self.b_o = shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_lstm))) 32 | self.W_hy = init_weight((self.n_lstm, self.n_out),'W_hy') 33 | self.b_y = shared(np.zeros(n_out, dtype=dtype)) 34 | self.params = [self.W_xi, self.W_hi, self.W_ci, self.b_i, 35 | self.W_xf, self.W_hf, self.W_cf, self.b_f, 36 | self.W_xc, self.W_hc, self.b_c, 37 | self.W_ho, self.W_co, self.W_co, self.b_o, 38 | self.W_hy, self.b_y] 39 | 40 | 41 | def step_lstm(x_t, h_tm1, c_tm1): 42 | i_t = T.nnet.sigmoid(T.dot(x_t, self.W_xi) + T.dot(h_tm1, self.W_hi) + T.dot(c_tm1, self.W_ci) + self.b_i) 43 | f_t = T.nnet.sigmoid(T.dot(x_t, self.W_xf) + T.dot(h_tm1, self.W_hf) + T.dot(c_tm1, self.W_cf) + self.b_f) 44 | c_t = f_t * c_tm1 + i_t * T.tanh(T.dot(x_t, self.W_xc) + T.dot(h_tm1, self.W_hc) + self.b_c) 45 | o_t = T.nnet.sigmoid(T.dot(x_t, self.W_xo)+ T.dot(h_tm1, self.W_ho) + T.dot(c_t, self.W_co) + self.b_o) 46 | h_t = o_t * T.tanh(c_t) 47 | y_t = T.nnet.softmax(T.dot(h_t, self.W_hy) + self.b_y) 48 | return [h_t, c_t, y_t] 49 | 50 | X = T.matrix() # batch of sequence of vector 51 | Y = T.matrix() # batch of sequence of vector (should be 0 when X is not null) 52 | if single_output: 53 | Y = T.vector() 54 | h0 = shared(np.zeros(shape=self.n_lstm, dtype=dtype)) # initial hidden state 55 | c0 = shared(np.zeros(shape=self.n_lstm, dtype=dtype)) # initial hidden state 56 | lr = shared(np.cast[dtype](lr)) 57 | 58 | [h_vals, c_vals, y_vals], _ = theano.scan(fn=step_lstm, 59 | sequences=X, 60 | outputs_info=[h0, c0, None]) 61 | 62 | if single_output: 63 | self.output = y_vals[-1] 64 | else: 65 | self.output = y_vals 66 | 67 | cxe = T.mean(T.nnet.binary_crossentropy(self.output, Y)) 68 | nll = -T.mean(Y * T.log(self.output)+ (1.- Y) * T.log(1. - self.output)) 69 | mse = T.mean((self.output - Y) ** 2) 70 | 71 | cost = 0 72 | if cost_function == 'mse': 73 | cost = mse 74 | elif cost_function == 'cxe': 75 | cost = cxe 76 | else: 77 | cost = nll 78 | 79 | gparams = T.grad(cost, self.params) 80 | updates = OrderedDict() 81 | for param, gparam in zip(self.params, gparams): 82 | updates[param] = param - gparam * lr 83 | 84 | self.loss = theano.function(inputs = [X, Y], outputs = cost) 85 | self.train = theano.function(inputs = [X, Y], outputs = cost, updates=updates) 86 | self.predictions = theano.function(inputs = [X], outputs = self.output) 87 | self.debug = theano.function(inputs = [X, Y], outputs = [X.shape, Y.shape, y_vals.shape, cost.shape]) 88 | 89 | 90 | class LstmMiniBatch: 91 | def __init__(self, n_in, n_lstm, n_out, lr=0.05, batch_size=64, single_output=True, output_activation=T.nnet.softmax, cost_function='nll'): 92 | self.n_in = n_in 93 | self.n_lstm = n_lstm 94 | self.n_out = n_out 95 | self.W_xi = init_weight((self.n_in, self.n_lstm),'W_xi') 96 | self.W_hi = init_weight((self.n_lstm, self.n_lstm),'W_hi', 'svd') 97 | self.W_ci = init_weight((self.n_lstm, self.n_lstm),'W_ci', 'svd') 98 | self.b_i = shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_lstm))) 99 | self.W_xf = init_weight((self.n_in, self.n_lstm),'W_xf') 100 | self.W_hf = init_weight((self.n_lstm, self.n_lstm),'W_hf', 'svd') 101 | self.W_cf = init_weight((self.n_lstm, self.n_lstm),'W_cf', 'svd') 102 | self.b_f = shared(np.cast[dtype](np.random.uniform(0, 1.,size = n_lstm))) 103 | self.W_xc = init_weight((self.n_in, self.n_lstm),'W_xc') 104 | self.W_hc = init_weight((self.n_lstm, self.n_lstm),'W_hc', 'svd') 105 | self.b_c = shared(np.zeros(n_lstm, dtype=dtype)) 106 | self.W_xo = init_weight((self.n_in, self.n_lstm),'W_xo') 107 | self.W_ho = init_weight((self.n_lstm, self.n_lstm),'W_ho', 'svd') 108 | self.W_co = init_weight((self.n_lstm, self.n_lstm),'W_co', 'svd') 109 | self.b_o = shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_lstm))) 110 | self.W_hy = init_weight((self.n_lstm, self.n_out),'W_hy') 111 | self.b_y = shared(np.zeros(n_out, dtype=dtype)) 112 | self.params = [self.W_xi, self.W_hi, self.W_ci, self.b_i, 113 | self.W_xf, self.W_hf, self.W_cf, self.b_f, 114 | self.W_xc, self.W_hc, self.b_c, 115 | self.W_ho, self.W_co, self.W_co, self.b_o, 116 | self.W_hy, self.b_y] 117 | 118 | 119 | def step_lstm(x_t, h_tm1, c_tm1): 120 | i_t = T.nnet.sigmoid(T.dot(x_t, self.W_xi) + T.dot(h_tm1, self.W_hi) + T.dot(c_tm1, self.W_ci) + self.b_i) 121 | f_t = T.nnet.sigmoid(T.dot(x_t, self.W_xf) + T.dot(h_tm1, self.W_hf) + T.dot(c_tm1, self.W_cf) + self.b_f) 122 | c_t = f_t * c_tm1 + i_t * T.tanh(T.dot(x_t, self.W_xc) + T.dot(h_tm1, self.W_hc) + self.b_c) 123 | o_t = T.nnet.sigmoid(T.dot(x_t, self.W_xo)+ T.dot(h_tm1, self.W_ho) + T.dot(c_t, self.W_co) + self.b_o) 124 | h_t = o_t * T.tanh(c_t) 125 | y_t = T.nnet.softmax(T.dot(h_t, self.W_hy) + self.b_y) 126 | return [h_t, c_t, y_t] 127 | 128 | X = T.tensor3() # batch of sequence of vector 129 | Y = T.tensor3() # batch of sequence of vector (should be 0 when X is not null) 130 | h0 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state 131 | c0 = shared(np.zeros(shape=(batch_size,self.n_lstm), dtype=dtype)) # initial hidden state 132 | lr = shared(np.cast[dtype](lr)) 133 | 134 | [h_vals, c_vals, y_vals], _ = theano.scan(fn=step_lstm, 135 | sequences=X.dimshuffle(1,0,2), 136 | outputs_info=[h0, c0, None]) 137 | 138 | if single_output: 139 | self.output = y_vals[-1] 140 | else: 141 | self.output = y_vals.dimshuffle(1,0,2) 142 | 143 | cxe = T.mean(T.nnet.binary_crossentropy(self.output, Y)) 144 | nll = -T.mean(Y * T.log(self.output)+ (1.- Y) * T.log(1. - self.output)) 145 | mse = T.mean((self.output - Y) ** 2) 146 | 147 | cost = 0 148 | if cost_function == 'mse': 149 | cost = mse 150 | elif cost_function == 'cxe': 151 | cost = cxe 152 | else: 153 | cost = nll 154 | 155 | gparams = T.grad(cost, self.params) 156 | updates = OrderedDict() 157 | for param, gparam in zip(self.params, gparams): 158 | updates[param] = param - gparam * lr 159 | 160 | self.loss = theano.function(inputs = [X, Y], outputs = [cxe, mse, cost]) 161 | self.train = theano.function(inputs = [X, Y], outputs = cost, updates=updates) 162 | self.predictions = theano.function(inputs = [X], outputs = y_vals.dimshuffle(1,0,2)) 163 | self.debug = theano.function(inputs = [X, Y], outputs = [X.shape, Y.shape, y_vals.shape, cxe.shape]) 164 | -------------------------------------------------------------------------------- /minimal_RNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Minimal RNN notebook\n", 8 | "\n", 9 | "### RNN representation\n", 10 | "\n", 11 | "Contrary to a Feed Forward Neural Network, an RNN is a recurrent neural network, in which the information flow is not linear. A general representation can be seen as follows:\n", 12 | "\n", 13 | "![Representation](img/rnn_simple.svg)\n", 14 | "\n", 15 | "An RNN is useful to deal with sequential information: a sequence of inputs is fed through the network and the hidden state is updated at each step of the sequence. The sequence is commonly represented as a time sequence, and the most straight forward learning algorithm is backpropagation through time (BPTT) http://en.wikipedia.org/wiki/Backpropagation_through_time.\n", 16 | "\n", 17 | "To understand properly BPTT, a better representation of the RNN is its unfolded version:\n", 18 | "\n", 19 | "![Representation](img/rnn_unfolded.svg)\n", 20 | "\n", 21 | "The input X is a sequence $x_0, x_1, ... x_t$, at each time-step t a new input $x_t$ is fed to the network.\n", 22 | "\n", 23 | "### Equations\n", 24 | "\n", 25 | "The most simple forward equations for a RNN are as follows:\n", 26 | "\n", 27 | "$$h_t = \\tanh(x_t . W_{in} + h_{t-1} . W_{rec})$$\n", 28 | "$$y_t = softmax(h_t . W_{out})$$\n", 29 | "\n", 30 | "Depending on the problem, all the outputs $y_0, ... y_t$ might be useful, or just $y_t$ the last one." 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 4, 36 | "metadata": { 37 | "collapsed": false 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "import numpy as np\n", 42 | "import theano\n", 43 | "import theano.tensor as T\n", 44 | "from theano import shared \n", 45 | "from collections import OrderedDict\n", 46 | "\n", 47 | "dtype=T.config.floatX\n", 48 | "theano.config.optimizer='fast_compile'" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "metadata": { 55 | "collapsed": true 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "def init_weight(shape, name, sample='uni'):\n", 60 | " if sample=='unishape':\n", 61 | " return shared(value=np.asarray(np.random.uniform(\n", 62 | " low=-np.sqrt(6. / (shape[0] + shape[1])),\n", 63 | " high=np.sqrt(6. / (shape[0] + shape[1])),\n", 64 | " size=shape), dtype=dtype), \n", 65 | " name=name, borrow=True)\n", 66 | " \n", 67 | " if sample=='svd':\n", 68 | " values = np.ndarray(shape, dtype=dtype)\n", 69 | " for dx in xrange(shape[0]):\n", 70 | " vals = np.random.uniform(low=-1., high=1., size=(shape[1],))\n", 71 | " values[dx,:] = vals\n", 72 | " _,svs,_ = np.linalg.svd(values)\n", 73 | " #svs[0] is the largest singular value \n", 74 | " values = values / svs[0]\n", 75 | " return shared(values, name=name, borrow=True)\n", 76 | " \n", 77 | " if sample=='uni':\n", 78 | " return shared(value=np.asarray(np.random.uniform(low=-0.1,high=0.1, size=shape), dtype=dtype), \n", 79 | " name=name, borrow=True)\n", 80 | " \n", 81 | " if sample=='zero':\n", 82 | " return shared(value=np.zeros(shape=shape, dtype=dtype), \n", 83 | " name=name, borrow=True)\n", 84 | " \n", 85 | " \n", 86 | " raise \"error bad sample technique\"" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 14, 92 | "metadata": { 93 | "collapsed": false 94 | }, 95 | "outputs": [], 96 | "source": [ 97 | "class Rnn:\n", 98 | " def __init__(self, n_in, n_hid, n_out, lr): \n", 99 | " self.n_in = n_in\n", 100 | " self.n_hid = n_hid\n", 101 | " self.n_out = n_out\n", 102 | " self.W_in = init_weight((self.n_in, self.n_hid),'W_in', 'svd')\n", 103 | " self.W_out = init_weight((self.n_hid, self.n_out),'W_out', 'svd')\n", 104 | " self.W_rec = init_weight((self.n_hid, self.n_hid),'W_rec', 'svd')\n", 105 | " self.b_out = init_weight((self.n_out), 'b_out','zero')\n", 106 | " self.params = [self.W_in,self.W_out,self.W_rec, self.b_out]\n", 107 | " \n", 108 | " def step(x_t, h_tm1):\n", 109 | " h_t = T.tanh(T.dot(x_t, self.W_in) + T.dot(h_tm1, self.W_rec))\n", 110 | " y_t = T.nnet.softmax(- (T.dot(h_t, self.W_out) + self.b_out)) \n", 111 | " return [h_t, y_t]\n", 112 | "\n", 113 | " X = T.matrix() # X is a sequence of vectors\n", 114 | " Y = T.matrix() # Y is a sequence of vectors\n", 115 | " h0 = shared(np.zeros(self.n_hid, dtype=dtype)) # initial hidden state \n", 116 | " lr = shared(np.cast[dtype](lr))\n", 117 | " \n", 118 | " [h_vals, y_vals], _ = theano.scan(fn=step, \n", 119 | " sequences=X,\n", 120 | " outputs_info=[h0, None])\n", 121 | " \n", 122 | " #h_vals is a sequence of hidden states\n", 123 | " #y_vals is a sequence of outputs\n", 124 | " \n", 125 | " # compute cost : cross entropy cost\n", 126 | " cost = -T.mean(Y * T.log(y_vals)+ (1.- Y) * T.log(1. - y_vals)) \n", 127 | " # for mean squared error, use \n", 128 | " # cost = -T.mean((Y - y_vals) ** 2)\n", 129 | " \n", 130 | " gparams = T.grad(cost, self.params)\n", 131 | " updates = OrderedDict()\n", 132 | " for param, gparam in zip(self.params, gparams):\n", 133 | " updates[param] = param - gparam * lr\n", 134 | " \n", 135 | " self.train = theano.function(inputs = [X, Y], outputs = cost, updates=updates)\n", 136 | " self.predictions = theano.function(inputs = [X], outputs = y_vals)\n", 137 | " self.debug = theano.function(inputs = [X, Y], outputs = [X.shape, Y.shape, h_vals.shape, y_vals.shape])\n", 138 | " " 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 15, 144 | "metadata": { 145 | "collapsed": false 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "model = Rnn(7, 50, 7, 0.1)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": { 156 | "collapsed": false 157 | }, 158 | "outputs": [], 159 | "source": [ 160 | "#sequences of 100 elements and vector size 7\n", 161 | "X = np.random.uniform(low=-0.1, high=0.1, size=(100,7)).astype(dtype=dtype) \n", 162 | "Y = np.random.uniform(low=-0.1, high=0.1, size=(100,7)).astype(dtype=dtype)\n", 163 | "\n", 164 | "print(model.debug(X,Y))\n", 165 | "model.predictions(X)" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "model.train(X,Y)" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "metadata": { 183 | "collapsed": false 184 | }, 185 | "outputs": [], 186 | "source": [ 187 | "nb_epochs = 100\n", 188 | "#stupid and naive sgd\n", 189 | "for x in range(nb_epochs):\n", 190 | " error = 0.\n", 191 | " for j in range(len(train_data)): \n", 192 | " index = np.random.randint(0, len(train_data))\n", 193 | " i, o = train_data[index]\n", 194 | " train_cost = model.train(i, o)\n", 195 | " error += train_cost\n", 196 | " if x%10==0:\n", 197 | " print \"epoch \"+str(x)+ \" error: \"+str(error)" 198 | ] 199 | } 200 | ], 201 | "metadata": { 202 | "kernelspec": { 203 | "display_name": "Python 2", 204 | "language": "python", 205 | "name": "python2" 206 | }, 207 | "language_info": { 208 | "codemirror_mode": { 209 | "name": "ipython", 210 | "version": 2 211 | }, 212 | "file_extension": ".py", 213 | "mimetype": "text/x-python", 214 | "name": "python", 215 | "nbconvert_exporter": "python", 216 | "pygments_lexer": "ipython2", 217 | "version": "2.7.6" 218 | } 219 | }, 220 | "nbformat": 4, 221 | "nbformat_minor": 0 222 | } 223 | -------------------------------------------------------------------------------- /rnn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | import theano.tensor as T 4 | from theano import shared 5 | from collections import OrderedDict 6 | from init import init_weight 7 | 8 | dtype=T.config.floatX 9 | 10 | print "loaded rnn.py" 11 | 12 | # Simple RNN class 13 | # optional parameters: 14 | # - activation: lambda x: x ; T.nnet.softmax ;T.nnet.sigmoid 15 | # - cost function: 'mse' 'bce' 'cce' 16 | 17 | class Rnn: 18 | def __init__(self, n_in, n_hid, n_out, lr=0.05, single_output=True, output_activation=T.nnet.softmax, cost_function='nll'): 19 | self.n_in = n_in 20 | self.n_hid = n_hid 21 | self.n_out = n_out 22 | self.W_in = init_weight((self.n_in, self.n_hid),'W_in') 23 | self.W_out = init_weight((self.n_hid, self.n_out),'W_out') 24 | self.W_rec = init_weight((self.n_hid, self.n_hid),'W_rec', 'svd') 25 | self.b_hid = shared(np.zeros(shape = n_hid, dtype=dtype)) 26 | self.b_out = shared(np.zeros(shape = n_out, dtype=dtype)) 27 | 28 | self.params = [self.W_in,self.W_out,self.W_rec,self.b_out,self.b_hid] 29 | 30 | self.activation = output_activation 31 | 32 | def step(x_t, h_tm1): 33 | h_t = T.tanh(T.dot(x_t, self.W_in) + T.dot(h_tm1, self.W_rec) + self.b_hid) 34 | y_t = self.activation(T.dot(h_t, self.W_out) + self.b_out) 35 | return [h_t, y_t] 36 | 37 | X = T.matrix() # sequence of vector 38 | Y = T.matrix() # sequence of vector 39 | if single_output: 40 | Y = T.vector() 41 | 42 | h0 = shared(np.zeros(shape=self.n_hid, dtype=dtype)) # initial hidden state 43 | lr = shared(np.cast[dtype](lr)) 44 | 45 | [h_vals, y_vals], _ = theano.scan(fn=step, 46 | sequences=X, 47 | outputs_info=[h0, None]) 48 | 49 | if single_output: 50 | self.output = y_vals[-1] 51 | else: 52 | self.output = y_vals 53 | 54 | cxe = T.mean(T.nnet.binary_crossentropy(self.output, Y)) 55 | nll = -T.mean(Y * T.log(self.output)+ (1.- Y) * T.log(1. - self.output)) 56 | mse = T.mean((self.output - Y) ** 2) 57 | 58 | cost = 0 59 | if cost_function == 'mse': 60 | cost = mse 61 | elif cost_function == 'cxe': 62 | cost = cxe 63 | else: 64 | cost = nll 65 | 66 | 67 | gparams = T.grad(cost, self.params) 68 | updates = OrderedDict() 69 | for param, gparam in zip(self.params, gparams): 70 | updates[param] = param - gparam * lr 71 | 72 | self.loss = theano.function(inputs = [X, Y], outputs = cost) 73 | self.train = theano.function(inputs = [X, Y], outputs = cost, updates=updates) 74 | self.predictions = theano.function(inputs = [X], outputs = self.output) 75 | self.debug = theano.function(inputs = [X, Y], outputs = [X.shape, Y.shape, y_vals.shape, self.output.shape]) 76 | 77 | 78 | # Same class with MiniBatch support 79 | class RnnMiniBatch: 80 | def __init__(self, n_in, n_hid, n_out, lr=0.05, batch_size=64, single_output=True, output_activation=T.nnet.softmax, cost_function='nll'): 81 | self.n_in = n_in 82 | self.n_hid = n_hid 83 | self.n_out = n_out 84 | self.W_in = init_weight((self.n_in, self.n_hid),'W_in') 85 | self.W_out = init_weight((self.n_hid, self.n_out),'W_out') 86 | self.W_rec = init_weight((self.n_hid, self.n_hid),'W_rec', 'svd') 87 | self.b_hid = shared(np.zeros(shape = n_hid, dtype=dtype)) 88 | self.b_out = shared(np.zeros(shape = n_out, dtype=dtype)) 89 | 90 | self.params = [self.W_in,self.W_out,self.W_rec,self.b_out,self.b_hid] 91 | 92 | self.activation = output_activation 93 | 94 | def step(x_t, h_tm1): 95 | h_t = T.tanh(T.dot(x_t, self.W_in) + T.dot(h_tm1, self.W_rec) + self.b_hid) 96 | y_t = T.nnet.softmax(T.dot(h_t, self.W_out) + self.b_out) 97 | return [h_t, y_t] 98 | 99 | X = T.tensor3() # batch of sequence of vector 100 | Y = T.tensor3() # batch of sequence of vector (should be 0 when X is not null) 101 | if single_output: 102 | Y = T.matrix() 103 | else: 104 | Y = T.tensor3() 105 | h0 = shared(np.zeros(shape=(batch_size,self.n_hid), dtype=dtype)) # initial hidden state 106 | lr = shared(np.cast[dtype](lr)) 107 | 108 | [h_vals, y_vals], _ = theano.scan(fn=step, 109 | sequences=X.dimshuffle(1,0,2), 110 | outputs_info=[h0, None]) 111 | 112 | if single_output: 113 | self.output = y_vals[-1] 114 | else: 115 | self.output = y_vals.dimshuffle(1,0,2) 116 | 117 | cxe = T.mean(T.nnet.binary_crossentropy(self.output, Y)) 118 | nll = -T.mean(Y * T.log(self.output)+ (1.- Y) * T.log(1. - self.output)) 119 | mse = T.mean((self.output - Y) ** 2) 120 | 121 | cost = 0 122 | if cost_function == 'mse': 123 | cost = mse 124 | elif cost_function == 'cxe': 125 | cost = cxe 126 | else: 127 | cost = nll 128 | 129 | gparams = T.grad(cost, self.params) 130 | updates = OrderedDict() 131 | for param, gparam in zip(self.params, gparams): 132 | updates[param] = param - gparam * lr 133 | 134 | self.loss = theano.function(inputs = [X, Y], outputs = cost) 135 | self.train = theano.function(inputs = [X, Y], outputs = cost, updates=updates) 136 | self.predictions = theano.function(inputs = [X], outputs = self.output) 137 | self.debug = theano.function(inputs = [X, Y], outputs = [X.shape, Y.shape, y_vals.shape, self.output.shape]) 138 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_pure_python_accumulate_squares.py: -------------------------------------------------------------------------------- 1 | def accumulate_squares(n): 2 | output = [0] 3 | for i in range(1, n): 4 | output.append(output[-1] + i ** 2) 5 | return output 6 | 7 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_pure_python_fibonacci.py: -------------------------------------------------------------------------------- 1 | def fibonacci(n): 2 | output = [0, 1] 3 | for _ in range(2, n): 4 | output.append(output[-1] + output[-2]) 5 | return output[:n] 6 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_pure_python_general_acc_squares.py: -------------------------------------------------------------------------------- 1 | def accumulate_squares(sequence): # to be removed 2 | output = [sequence[0] ** 2] 3 | for item in sequence[1:]: 4 | output.append(output[-1] + item ** 2) 5 | return output 6 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_pure_python_general_accumulate.py: -------------------------------------------------------------------------------- 1 | def accumulate(func, sequence, starting_point): # to be removed 2 | output = [starting_point] 3 | for element in sequence: 4 | output.append(func(element, output[-1])) 5 | return output[1:] 6 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_reduce_verbose_add.py: -------------------------------------------------------------------------------- 1 | def verbose_add(a, b): 2 | print "a={}, b={}".format(str(a), str(b)) 3 | return a + b 4 | reduce(verbose_add, map(lambda x: [x], np.arange(10)), [100]) 5 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_map_add_two_seq.py: -------------------------------------------------------------------------------- 1 | arr1 = T.fvector() 2 | arr2 = T.fvector() 3 | 4 | expression, updates = theano.map(add, sequences=[arr1, arr2]) 5 | f_add = theano.function([arr1, arr2], expression) -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_map_binomial.py: -------------------------------------------------------------------------------- 1 | def generate_double_monomial(coefficient, counter, xx, yy, n): 2 | return coefficient * xx ** counter * yy ** (n - counter - 1) 3 | 4 | double_monomial_expr, updates = theano.map(generate_double_monomial, 5 | sequences=[coefs, T.arange(coefs.shape[0])], 6 | non_sequences=[x, y, coefs.shape[0]]) 7 | 8 | f_binomial = theano.function([x, y], double_monomial_expr) -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_map_square.py: -------------------------------------------------------------------------------- 1 | arr = T.fvector() # to be removed 2 | expression, updates = theano.map(square, sequences=arr) 3 | 4 | f_square_arr = theano.function([arr], expression) -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_map_univar_polynomial.py: -------------------------------------------------------------------------------- 1 | def generate_monomial(coefficient, counter, xx): 2 | return coefficient * xx ** counter 3 | 4 | monomial_expr, updates = theano.map(generate_monomial, 5 | sequences=[coefs, T.arange(coefs.shape[0])], 6 | non_sequences=x) 7 | 8 | f_monomials = theano.function([x], monomial_expr) 9 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_reduce_awkward_multiple_powers.py: -------------------------------------------------------------------------------- 1 | def accumulate_all_powers(i, *args): 2 | # half of args is current values, the other half is non_sequences 3 | current_values = args[:len(args) / 2] 4 | powers = args[len(args) / 2:] 5 | 6 | output = [i ** power + cur_val 7 | for power, cur_val in zip(powers, current_values)] 8 | return output 9 | 10 | all_expressions, updates = theano.reduce(accumulate_all_powers, 11 | sequences=T.arange(n), 12 | outputs_info=[np.int32(0)] * len(powers), 13 | non_sequences=powers) 14 | 15 | f_all_powers = theano.function([n], all_expressions) 16 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_reduce_monomials.py: -------------------------------------------------------------------------------- 1 | def accumulate_polynomial(coef, cur_polynomial, cur_power, xx): 2 | return cur_polynomial + coef * cur_power, cur_power * xx 3 | 4 | (r_poly_expr, _), updates = theano.reduce(accumulate_polynomial, 5 | sequences=coefs, 6 | outputs_info=[np.float32(0.), np.float32(1.)], 7 | non_sequences=x) 8 | 9 | f_r_poly = theano.function([x], r_poly_expr) -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_reduce_monomials_from_map.py: -------------------------------------------------------------------------------- 1 | test, updates = theano.reduce(add, 2 | sequences=monomial_expr, 3 | outputs_info=np.float64(0)) 4 | 5 | f_poly = theano.function([x], test) 6 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_reduce_multi_output.py: -------------------------------------------------------------------------------- 1 | both_expressions, updates = theano.reduce(return_args_and_accumulate_squares_and_ints, 2 | sequences=T.arange(n), 3 | outputs_info=[ 4 | None, None, None, np.int32(0), np.cast['int32'](0)]) 5 | f_both = theano.function([n], both_expressions) 6 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_reduce_sum_square_ints.py: -------------------------------------------------------------------------------- 1 | def add_square_a_to_b(a, b): # to be removed 2 | return a ** 2 + b 3 | 4 | expr_square_sum, updates = theano.reduce(add_square_a_to_b, 5 | sequences=T.arange(n), 6 | outputs_info=np.cast['int32'](0)) 7 | f_square_sum = theano.function([n], expr_square_sum) 8 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_reduce_vectorized_multiple_powers.py: -------------------------------------------------------------------------------- 1 | def accumulate_powers(i, cur_result, powers): 2 | return cur_result + i ** powers 3 | 4 | acc_powers, updates = theano.reduce(accumulate_powers, 5 | sequences=T.arange(n), 6 | outputs_info=T.zeros_like(powvec, dtype='int32'), 7 | non_sequences=powvec) 8 | 9 | f_acc_powers = theano.function([n, powvec], acc_powers) 10 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_scan_accumulate_ints_and_squares.py: -------------------------------------------------------------------------------- 1 | def acc_ints_and_squares(i, cur_ints, cur_squares): 2 | return (i ** 2 + i) / 2 + 1, i + cur_ints, i ** 2 + cur_squares 3 | 4 | scan_ints_and_squares, updates = theano.scan(acc_ints_and_squares, 5 | sequences=T.arange(n), 6 | outputs_info=[None, np.int32(0), np.int32(0)]) 7 | 8 | f_scan_ints_and_squares = theano.function([n], scan_ints_and_squares) 9 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_scan_accumulate_squares.py: -------------------------------------------------------------------------------- 1 | scan_squares_expr, updates = theano.scan(add_square_a_to_b, 2 | sequences=T.arange(n), 3 | outputs_info=np.int32(0)) 4 | f_scan_squares = theano.function([n], scan_squares_expr) 5 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_scan_fibonacci.py: -------------------------------------------------------------------------------- 1 | def fib_acc(old, older): 2 | return old + older 3 | 4 | fib_expr, updates = theano.scan( 5 | fib_acc, 6 | sequences=None, 7 | outputs_info=[dict(initial=np.int32([0, 1]), taps=[-1, -2])], 8 | n_steps=n) 9 | 10 | f_fib_scan = theano.function([n], fib_expr) 11 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_scan_modulated_oscillation.py: -------------------------------------------------------------------------------- 1 | a_vector = T.fvector() 2 | 3 | def controlled_rotation(a, old): 4 | return T.exp(a) * rot.dot(old) 5 | 6 | controlled_oscillation, updates = theano.scan(controlled_rotation, 7 | sequences=a_vector, 8 | outputs_info=np.float32([1., 0.])) 9 | 10 | f_controlled_oscillation = theano.function([a_vector], controlled_oscillation) 11 | -------------------------------------------------------------------------------- /scan_tutorial_solutions/sol_theano_scan_oscillation.py: -------------------------------------------------------------------------------- 1 | def rotation_increment(old): 2 | return rot.dot(old) 3 | 4 | oscillation, updates = theano.scan( 5 | rotation_increment, 6 | sequences=None, 7 | outputs_info=T.constant(np.float32([1., 0.])), 8 | n_steps=n) 9 | 10 | f_oscillation = theano.function([n], oscillation) --------------------------------------------------------------------------------