├── .gitattributes └── RNN Case Study └── RNNs_in_Python.ipynb /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /RNN Case Study/RNNs_in_Python.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "RNN in Python.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [] 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | }, 14 | "language_info": { 15 | "name": "python" 16 | } 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "thmejNEWhkk3" 23 | }, 24 | "source": [ 25 | "# RNN in Python\n", 26 | "In this colab notebook, you will get to know how various types of RNNs implemented. Feel free to experiment and also keep an eye on number of paramters shown in the model summary, try calculating it on your own." 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "metadata": { 32 | "id": "558BehfxXD_3" 33 | }, 34 | "source": [ 35 | "# import libraries\n", 36 | "import tensorflow as tf\n", 37 | "from tensorflow.keras.models import Sequential\n", 38 | "from tensorflow.keras.layers import Dense\n", 39 | "from tensorflow.keras.layers import SimpleRNN, LSTM, GRU\n", 40 | "from tensorflow.keras.layers import TimeDistributed, RepeatVector, Bidirectional" 41 | ], 42 | "execution_count": 13, 43 | "outputs": [] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "metadata": { 48 | "id": "Bcd21SkpYhkl" 49 | }, 50 | "source": [ 51 | "# define parameters\n", 52 | "n_output = 4 # number of classes in case of classification, 1 in case of regression\n", 53 | "output_activation = \"sigmoid\" # “softmax” or “sigmoid” in case of classification, “linear” in case of regression" 54 | ], 55 | "execution_count": 14, 56 | "outputs": [] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": { 61 | "id": "fW9YA4sEY4UE" 62 | }, 63 | "source": [ 64 | "#### First, let's look at how to build a vanilla RNN in Keras. We will be using Simple RNN to build sample models for different types but you may replace it by GRU or LSTM without changing anything else." 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "metadata": { 70 | "id": "PZtocp_3YxzB" 71 | }, 72 | "source": [ 73 | "vanilla_model = Sequential()\n", 74 | "\n", 75 | "# add the first hidden layer\n", 76 | "n_cells = 5 # number of cells to add in the hidden layer\n", 77 | "time_steps = 10 # length of sequences\n", 78 | "features = 4 # number of features of each entity in the sequence\n", 79 | "\n", 80 | "vanilla_model.add(SimpleRNN(n_cells, input_shape=(time_steps, features)))\n", 81 | "\n", 82 | "# add output layer\n", 83 | "vanilla_model.add(Dense(n_output, activation=output_activation))" 84 | ], 85 | "execution_count": 15, 86 | "outputs": [] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "metadata": { 91 | "colab": { 92 | "base_uri": "https://localhost:8080/" 93 | }, 94 | "id": "s3N57TGtZBwZ", 95 | "outputId": "0e2ff7fc-ac36-409f-d85d-a2bb802abf41" 96 | }, 97 | "source": [ 98 | "vanilla_model.summary()" 99 | ], 100 | "execution_count": 16, 101 | "outputs": [ 102 | { 103 | "output_type": "stream", 104 | "text": [ 105 | "Model: \"sequential_4\"\n", 106 | "_________________________________________________________________\n", 107 | "Layer (type) Output Shape Param # \n", 108 | "=================================================================\n", 109 | "simple_rnn_4 (SimpleRNN) (None, 5) 50 \n", 110 | "_________________________________________________________________\n", 111 | "dense_3 (Dense) (None, 4) 24 \n", 112 | "=================================================================\n", 113 | "Total params: 74\n", 114 | "Trainable params: 74\n", 115 | "Non-trainable params: 0\n", 116 | "_________________________________________________________________\n" 117 | ], 118 | "name": "stdout" 119 | } 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": { 125 | "id": "g1X2eZlph6IJ" 126 | }, 127 | "source": [ 128 | "Let's also learn to build multi-layer RNN model with two hidden RNN layers." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "metadata": { 134 | "id": "zPewur-8iH0y" 135 | }, 136 | "source": [ 137 | "multilayer_model = Sequential()\n", 138 | "\n", 139 | "# add the first hidden layer\n", 140 | "n_cells = 5 # number of cells to add in the hidden layer\n", 141 | "time_steps = 10 # length of sequences\n", 142 | "features = 4 # number of features of each entity in the sequence\n", 143 | "\n", 144 | "multilayer_model.add(SimpleRNN(n_cells, input_shape=(time_steps, features), return_sequences=True))\n", 145 | "\n", 146 | "multilayer_model.add(SimpleRNN(n_cells, input_shape=(time_steps, features)))\n", 147 | "\n", 148 | "# add output layer\n", 149 | "multilayer_model.add(Dense(n_output, activation=output_activation))" 150 | ], 151 | "execution_count": 17, 152 | "outputs": [] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "metadata": { 157 | "colab": { 158 | "base_uri": "https://localhost:8080/" 159 | }, 160 | "id": "XTv5Fn27iXBf", 161 | "outputId": "e197c9db-46f6-49d7-ac32-67fa6cc14837" 162 | }, 163 | "source": [ 164 | "multilayer_model.summary()" 165 | ], 166 | "execution_count": 18, 167 | "outputs": [ 168 | { 169 | "output_type": "stream", 170 | "text": [ 171 | "Model: \"sequential_5\"\n", 172 | "_________________________________________________________________\n", 173 | "Layer (type) Output Shape Param # \n", 174 | "=================================================================\n", 175 | "simple_rnn_5 (SimpleRNN) (None, 10, 5) 50 \n", 176 | "_________________________________________________________________\n", 177 | "simple_rnn_6 (SimpleRNN) (None, 5) 55 \n", 178 | "_________________________________________________________________\n", 179 | "dense_4 (Dense) (None, 4) 24 \n", 180 | "=================================================================\n", 181 | "Total params: 129\n", 182 | "Trainable params: 129\n", 183 | "Non-trainable params: 0\n", 184 | "_________________________________________________________________\n" 185 | ], 186 | "name": "stdout" 187 | } 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": { 193 | "id": "BsUh3EA9ZfCn" 194 | }, 195 | "source": [ 196 | "#### Let's look at how to build a many-to-one architecture in Keras." 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": { 202 | "id": "Ix5DMSGhIKbU" 203 | }, 204 | "source": [ 205 | "
\n", 206 | "\n", 207 | "![m20.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUsAAAIjCAYAAACK+Dk0AAAAAXNSR0IArs4c6QAAAHhlWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACWAAAAAQAAAJYAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAUugAwAEAAAAAQAAAiMAAAAAPTBjAwAAAAlwSFlzAAAXEgAAFxIBZ5/SUgAAHmBJREFUeAHt3UGPXFeVB/BXHXYgBTaJRiBiNBuQEGGDWU3S/gbwBTxZJlZ/B7dX8wVajpeOWY9gP9LQyY5kMQqDBKuBIFgkQgIjIoGQXFOv3Dcu293t+/rVqzrvnl9LdlVXv3frnN+5/vfrrrbddd4IECBA4IUCixce4QACBAhsSeCtt48Ot7TUVpa5f+/ktHYhYVkr5TgCBK4k0AfkctHdXnbd4ZUW2MVJi+5O/zQP7p4cX/R0wvIiGY8TIDBK4LyQfOUbL49ac9snf/aHh08tuQrE08Wyu3PeFaewfIrKOwQIbEPg5q2j427Z3e7X6gPyuz/8ZvfK12MF5Wafv/rF77vP/viw+yI8V1eaz15lCstNMfcJEBgtsBmUfUj2v+by1odm/6t/O1h2NzavMA/m0oQ6CRCYicDZFeXcgrLX3ay5/z7rpvhLm++4T4AAgTEC66vK1Qs5m6EzZr19nNt/26D/kvzzv/7j2uvXry8+/ujD074OV5b7mIbnJNCqwMZV5ZxbLN86WL3Y82bpQ1gWCbcECIwSKD9DGe0V76s0VV6M2vxxJ2F5FUnnECDwnMCjg+6wf7AEzXMHzOyBEvrlk4CwnNkAlUuAwH4EhOV+3D0rAQLBBcoVcrliFpbBB6Y8AgRiCAjLGHNQBQECwQWEZfABKY8AgRgCwjLGHFRBgEBwAWEZfEDKI0AghoCwjDEHVRAgEFxAWAYfkPIIEIghICxjzEEVBAgEFxCWwQekPAIEYggIyxhzUAUBAsEFhGXwASmPAIEYAsIyxhxUQYBAcAFhGXxAyiNAIIaAsIwxB1UQIBBcQFgGH5DyCBCIISAsY8xBFQQIBBcQlsEHpDwCBGIICMsYc1AFAQLBBYRl8AEpjwCBGALCMsYcVEGAQHABYRl8QMojQCCGgLCMMQdVECAQXEBYBh+Q8ggQiCEgLGPMQRUECAQXEJbBB6Q8AgRiCAjLGHNQBQECwQWEZfABKY8AgRgCwjLGHFRBgEBwAWEZfEDKI0AghoCwjDEHVRAgEFxAWAYfkPIIEIghICxjzEEVBAgEFxCWwQekPAIEYggIyxhzUAUBAsEFhGXwASmPAIEYAsIyxhxUQYBAcAFhGXxAyiNAIIaAsIwxB1UQIBBcQFgGH5DyCBCIISAsY8xBFQQIBBcQlsEHpDwCBGIICMsYc1AFAQLBBYRl8AEpjwCBGALCMsYcVEGAQHABYRl8QMojQCCGgLCMMQdVECAQXEBYBh+Q8ggQiCEgLGPMQRUECAQXEJbBB6Q8AgRiCAjLGHNQBQECwQWEZfABKY8AgRgCwjLGHFRBgEBwAWEZfEDKI0AghoCwjDEHVRAgEFxAWAYfkPIIEIghICxjzEEVBAgEFxCWwQekPAIEYggIyxhzUAUBAsEFhGXwASmPAIEYAsIyxhxUQYBAcAFhGXxAyiNAIIaAsIwxB1UQIBBcQFgGH5DyCBCIISAsY8xBFQQIBBcQlsEHpDwCBGIICMsYc1AFAQLBBYRl8AEpjwCBGALCMsYcVEGAQHABYRl8QMojQCCGgLCMMQdVECAQXEBYBh+Q8ggQiCEgLGPMQRUECAQXEJbBB6Q8AgRiCAjLGHNQBQECwQWEZfABKY8AgRgCwjLGHFRBgEBwAWEZfEDKI0AghoCwjDEHVRAgEFxAWAYfkPIIEIghICxjzEEVBAgEFxCWwQekPAIEYggIyxhzUAUBAsEFhGXwASmPAIEYAsIyxhxUQYBAcAFhGXxAyiNAIIaAsIwxB1UQIBBcQFgGH5DyCBCIISAsY8xBFQQIBBcQlsEHpDwCBGIICMsYc1AFAQLBBYRl8AEpjwCBGALCMsYcVEGAQHABYRl8QMojQCCGgLCMMQdVECAQXEBYBh+Q8ggQiCEgLGPMQRUECAQXEJbBB6Q8AgRiCAjLGHNQBQECwQWEZfABKY8AgRgCwjLGHFRBgEBwAWEZfEDKI0AghoCwjDEHVRAgEFxAWAYfkPIIEIghICxjzEEVBAgEFxCWwQekPAIEYggIyxhzUAUBAsEFhGXwASmPAIEYAsIyxhxUQYBAcAFhGXxAyiNAIIaAsIwxB1UQIBBcQFgGH5DyCBCIISAsY8xBFQQIBBcQlsEHpDwCBGIICMsYc1AFAQLBBYRl8AEpjwCBGALCMsYcVEFg9gIHj7rTvonP/vhw9r30DfzqF79f9/Hg7slxf0dYrjn8RoDAWIH7905O+zU++0MbYfmsh7B8VsT7BAhcWWDRtXF1Wa4qSz89iLC88rZwIgECzwosF937/WP//Z//++yHZvN+/22EEpaln774l2bTgUIJEAgv8PFHH55+/wfXD1eFXutD51vfeTV8zZsFlqD8/K//6LpFd6d8v7I/ZnWV6Y0AAQLbFbj5ztGyrPjdH36z639Ff+uDslwR919+v/fuyY3NmoXlpob7BAhsTeDmraPjbtnd7hd85Rsvd698/eX12v39SG/lS+7ywtR5QdnXKywjTU0tBBoT2AzMObR2sOxulFf1n61XWD4r4n0CBLYu8NbbR4ePDrrDfuHFsntz608wYsFVPXf60y8KyRFLO5UAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQOBZgf6Fo2cfm+p9f91xKlnrEiAwqUD/Y0mrv464/jnOSZ/obHFhuQtlz0GAwPYFVj/wvvprQoe7uroUltsfoRUJEJhYYP3D7mfPsaurS2E58VAtT4DABAJnf42yX3lXV5fCcoI5WpIAgekENq8qy7Ps4upSWBZttwQIzENg46qyFLyLq0thWbTdEiAQXuC8q8pS9NRXl8KySLslQCC+wDlXlaXoqa8uhWWRdkuAQGiBy64qS+FTXl0Ky6LslgCB2AKXXFWWwqe8uhSWRdktAQJhBWquKkvxU11dfqk8gVsCBAhEFOj/hs7y8T8YfFrq668gy/3+v4Eo96e89S+lT6lrbQIEJhFYX2mWL8uf+V8YJ3nC1aK+DJ9K1roECDQlICybGqdmCBCYSkBYTiVrXQIEmhIQlk2NUzMECEwlICynkrUuAQKTCWz+d7oHj3bzariwnGycFiZAoCUBYdnSNPVCgMBkAsJyMloLEyDQkoCwbGmaeiFAYDIBYTkZrYUJEGhJQFi2NE29ECAwmYCwnIzWwgQItCQgLFuapl4IEJhMQFhORmthAgRaEhCWLU1TLwQITCYgLCejtTABAi0JCMuWpqkXAkkENv+l9Pv3Tk530baw3IWy5yBAYPYCwnL2I9QAAQK7EBCWu1D2HAQIzF5AWM5+hBogQGAXAsJyF8qegwCB2QsIy9mPUAMECOxCQFjuQtlzECAwewFhOfsRaoAAgV0ICMtdKHsOAgRmLyAsZz9CDRAgsAsBYbkLZc9BgMDsBYTl7EeoAQIEdiEgLHeh7DkIEJi9gLCc/Qg1QCCXwFtvHx2Wjhddd1ruT30rLKcWtj4BAk0ICMsmxqgJAgSmFhCWUwtbnwCBJgSEZRNj1AQBAlMLCMupha1PgEATAsKyiTFqggCBqQWE5dTC1idAoAkBYdnEGDVBgMDUAsJyamHrEyDQhICwbGKMmiBAYGoBYTm1sPUJEGhCQFg2MUZNECAwtYCwnFrY+gQIbFXg0UF3WBZcLrr3y/2pb4Xl1MLWJ0CgCQFh2cQYNUGAwNQCwnJqYesTINCEgLBsYoyaIEBgagFhObWw9QkQaEJAWDYxRk0QIDC1gLCcWtj6BAg0ISAsmxijJggQmFpg9Z+jvfht839Te/HR0x9x/97J6fTP4hmGCNgjQ7TyHruNfbL+ofRld7tX7P93x8Wyu3NV0SFZcmFY9k2tfjr+9rLrDq9ayOTnLR4jPbh7cjz5c3mC5wTskedIPHCOQCv75LmwPK+xr7762jkE+3voL59+8tSTr5pYf3YZ8lniqQW8M0jAHhnElfbg1vbJU2F589bRcXd2edsH5Le+90YXLSg3d97vfvlB9+dVcH4RnqsrTVeZm0Lbv2+PbN+0xRVb3CdfhOVmc31IXlv9mstbH5q/Xf3q3w6W3Q1XmNNMzh6ZxrW1VVvdJ09eDT+7opxbUPYbrQ/2vu7+rf8+6/qO37YvYI9s37TFFRvdJy/1s1p/Jli9kDPHoCx7rf92Qf/l+N8/f3jt9evXFx9/9OFp+Zjb8QL2yHjDDCu0vE8eX1mefSaY05fe5228cnW5+lGCN8/7uMdGCNgjI/ASndrwPjnoX7HqRxn5hZzarVZ6CP3jTrXNBDrOHgk0jMCltL5PDsq/Ovy1YD8edNU9UQKzDO6q6zjviYA98sTCvYsFWt8nT17gudjARwgQIJBeoLmwLFfI5bNc+gkDeE7AHnmOxAPnCDy7T5oLy3N69hABAgRGCwjL0YQWIEAgg4CwzDBlPRIgMFpAWI4mtAABAhkEhGWGKeuRAIHRAsJyNKEFCBDIICAsM0xZjwQIjBYQlqMJLUCAQAYBYZlhynokQGC0gLAcTWgBAgQyCAjLDFPWIwECowWE5WhCCxAgkEFAWGaYsh4JEBgtICxHE1qAAIEMAsIyw5T1SIDAaAFhOZrQAgQIZBAQlhmmrEcCBEYLCMvRhBYgQCCDgLDMMGU9EiAwWkBYjia0AAECGQSEZYYp65EAgdECwnI0oQUIEMggICwzTFmPBAiMFhCWowktQIBABgFhmWHKeiRAYLSAsBxNaAECBDIICMsMU9YjAQKjBYTlaEILECCQQUBYZpiyHgkQGC0gLEcTWoAAgQwCwjLDlPVIgMBoAWE5mtACBAhkEBCWGaasRwIERgsIy9GEFiBAIIOAsMwwZT0SIDBaQFiOJrQAAQIZBIRlhinrkQCB0QLCcjShBQgQyCAgLDNMWY8ECIwWEJajCS1AgEAGAWGZYcp6JEBgtICwHE1oAQIEMggIywxT1iMBAqMFhOVoQgsQIJBBQFhmmLIeCRAYLSAsRxNagACBDALCMsOU9UiAwGgBYTma0AIECGQQEJYZpqxHAgRGCwjL0YQWIEAgg4CwbGzKN28dHfe/GmtLO1sWsE+GgwrL4Wbxz1h2t2++c7QUmvFHtdcK7ZNB/MJyENfMDvaHYWYD21O59kkVvLCsYpr5Qf4wzHyAOyrfPrkU+kuXfrSRD7719tFhI628sI1Hlx3x+A/D7W7R3Xlw9+T4skMzfsw+OZu6fXLu9k8Rlo8W3c/P7b7FB5cVTfnDcC6SffIMi33yFIgvw5/iSPZO/4fBK+fJhn6Fdu2TNVqKK8tF151eYYvM8pTVheVhVeG+FH+OyT55jqTzLZsnJinC8r13T248abnte+srxdWVwIVdCskLaeyTDRr7ZAPj8d0UYflc1xkfsPkzTn14z/bJhWbC8kKaRj5g8zcyyInbsE9eCCwsX0g00wNs/pkObsdl2yfV4MKymmoeB579/OTxPKpV5b4E7JPh8n50aLiZMwgQSCggLBMOXcsECAwXEJbDzZxBgEBCAWGZcOhaJkBguICwHG7mDAIEEgoIy4RD1zIBAsMFhOVwM2cQIJBQQFgmHLqWCRAYLiAsh5s5gwCBhALCMuHQtUyAwHABYTnczBkECCQUEJYJh65lAgSGCwjL4WbOIEAgoYCwTDh0LRMgMFxAWA43cwYBAgkFhGXCoWuZAIHhAsJyuJkzCBBIKCAsEw5dywQIDBcQlsPNnEGAQEIBYZlw6FomQGC4gLAcbuYMAgQSCgjLhEPXMgECwwWE5XAzZxAgkFBAWCYcupYJEBguICyHmzmDAIGEAsIy4dC1TIDAcAFhOdzMGQQIJBQQlgmHrmUCBIYLCMvhZs4gQCChgLBMOHQtEyAwXEBYDjdzBgECCQWEZcKha5kAgeECwnK4mTMIEEgoICwTDl3LBAgMFxCWw82cQYBAQgFhmXDoWiZAYLiAsBxu5gwCBBIKCMuEQ9cyAQLDBYTlcDNnECCQUEBYJhy6lgkQGC4gLIebOYMAgYQCwjLh0LVMgMBwAWE53MwZBAgkFBCWCYeuZQIEhgsIy+FmziBAIKGAsEw4dC0TIDBcQFgON3MGAQIJBYRlwqFrmQCB4QLCcriZMwgQSCggLBMOXcsECAwXEJbDzZxBgEBCgYODR91p3/efP/2kifZ/+8sP1n08uHty3ERDAZqwRwIMYQYltL5PDu7fOznt5/CXRsJyBntqdiXaI7Mb2V4Kbn2frL8MX3SPry7nHpi/O7uqLP3sZcc0+qTF1B5pdMBbaqvlfbIOy+Wie7+3+p//+smWyHa/TP+HuHwJXvrZfRXtPmMxtUfanfE2Omt5n7zUA3380Yen3//B9cPV3Wt96PzLv77ePzybtxKUf//8Ydctuju+X7n90dkj2zdtccWW98nqqvnJ2813jpblvW99743u2upX9Lc+KMvVTv8lwHvvntyIXvOc67NH5jy93dXe4j55Kix7ypu3jo67ZXe7v//VV1/rvrb6Ve6v7wT5rXzJ3Ydl/yYodzcYe2R31nN+ptb2yXNh2Q9ns8k5DOtg2d0or8TNod4WarRHWpji9D20tE/ODctC+NbbR4ePDrrD/v3FsnuzPB7hdlXPnb4OIbnfadgj+/Wfy7PbJ1eYVP+Z5t/fOfr5FU51ShIBeyTJoEe2uet9sv7RoZE1Dzt99f3QZdcd9p9php3o6DQC9kiaUY9qdMf7ZKdhuf7+xZnO6uexbo+CcnKTAvZIk2PdelP72Cc7DcvyKnsv5+py6/unjQXPfhLDHmljnJN1sYd9srOw3PxMUABdXRYJt72APWIf1Ajsa5/sLCw3ryoLiKvLIuF2LbBxtVBE7JEi4fYLgT3tk52E5XmfCUrjri6LRO5beyT3/Gu73+c+2UlYnndVWXBcORSJ5LfnXC0UEXukSLjdZ5ZMHpaXfSYoo3d1WSRy3tojOec+tOt975PJw/KyzwQFy5VDkUh6e8lVZRGxR4pE4ts975NJw7LmM0EZvavLIpHr1h7JNe+rdhthn3zpqsW/6Lz+b+gsH/998tNybH91UO73/0pQue82p4A9knPuQ7uOsk8u/Yc0hjb1ouPXnx3KpbR/pPdFXCk/bo+kHPvgpvexTyb9MnywgBMIECAQVEBYBh2MsggQiCUgLGPNQzUECAQVEJZBB6MsAgRiCew0LDf/tfWDR14Nj7UVYlRjj8SYQ/Qq9rFPdhqW0QegPgIECFwkICwvkvE4AQIENgSE5QaGuwQIELhIQFheJONxAgQIbAgIyw0MdwkQIHCRgLC8SMbjBAgQ2BAQlhsY7hIgQOAiAWF5kYzHCRAgsCEgLDcw3CVAgMBFAsLyIhmPEyBAYENAWG5guEuAAIGLBHYalpv/Uvr9eyenFxXl8bwC9kje2Q/pfB/7ZKdhOQTDsQQIEIgkICwjTUMtBAiEFRCWYUejMAIEIgkIy0jTUAsBAmEFhGXY0SiMAIFIAsIy0jTUQoBAWAFhGXY0CiNAIJKAsIw0DbUQIBBWQFiGHY3CCBCIJCAsI01DLQQIhBUQlmFHozACBCIJCMtI01ALAQJhBYRl2NEojACBSAI7C8u33j46LI0vuu603HdLoAjYI0XC7WUC+9onOwvLy5r3MQIECEQXEJbRJ6Q+AgRCCAjLEGNQBAEC0QWEZfQJqY8AgRACwjLEGBRBgEB0AWEZfULqI0AghICwDDEGRRAgEF1AWEafkPoIEAghICxDjEERBAhEFxCW0SekPgIEQggIyxBjUAQBAtEFhGX0CamPAIEQAjsLy0cH3WHpeLno3i/33RIoAvZIkXB7mcC+9snOwvKy5n2MAAEC0QWEZfQJqY8AgRACwjLEGBRBgEB0AWEZfULqI0AghICwDDEGRRAgEF1AWEafkPoIEAghICxDjEERBAhEFxCW0SekPgIEQggIyxBjUAQBAtEFhGX0CamPAIG8Apv/729eBZ1fJmCPXKbjY0XAPikSbgkQIECAAAECBAgQIECAQFMCi5puon1f4P69k9Oauh2zOwF7ZHfWc36mOe+TC8Oyb2r1707eXnbdYdjhLLo7fW0P7p4ch62x4cLskYaHu8XWWtknz4XleY19+7Uvb5Fu/FK/+eTzpxZZNXG6WHZ3XHE+xTLZO/bIZLRNLdzaPnkqLG/eOjrult3tfmJ9QP74jVfXt1En+LMPPut+/cnfui/Cc3Wl6Spz2mnZI9P6trJ6i/vki7DcbK4PyR+98cps5taH5k8/+HRd78Gyu+EKc5rR2SPTuLa2aqv75Mnf4Dm7opxbUPYbrQ/2vu7+rf8+6/qO37YvYI9s37TFFRvdJy/1s1p/Jli9kDPHoCx7rf+2Qf/l+J8e/vPa69evLz7+6MPT8jG34wXskfGGGVZoeZ88vrI8+0wwpy+9z9t45epy9WLPm+d93GMjBOyREXiJTm14nxz0r1j1o4z2ivdVtlfpIfSPO12lsT2fY4/seQAzefrW98lB+T94v/PaV2YyksvLLIFZBnf50T5aI2CP1Cg5pvV98uQFHrMmQIAAgQsFmgvLcoVcPstd2LkPpBWwR9KOflDjz+6T5sJykIaDCRAgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG4BYZl7/ronQKBSQFhWQjmMAIHcAsIy9/x1T4BApYCwrIRyGAECuQWEZe75654AgUoBYVkJ5TACBHILCMvc89c9AQKVAsKyEsphBAjkFhCWueevewIEKgWEZSWUwwgQyC0gLHPPX/cECFQKCMtKKIcRIJBbQFjmnr/uCRCoFBCWlVAOI0Agt4CwzD1/3RMgUCkgLCuhHEaAQG6Bg4NH3WlP8OtP/taExE8/+HTdx4O7J8dNNBSgCXskwBBmUELr++Tg/r2T034Ov/nk8xmMQ4n7ELBH9qE+v+dsfZ+svwxfdI+vLucemD/74LP1Div9zG+7xa24mNojcWcUobKW98k6LJeL7v0e+j9+8n8RvK9UQ/+HuHwJXvq50kJOOlegmNoj5/J48Eyg5X3yUt/jxx99ePr9H1w/XN291ofOv73+tf7h2byVoPzTw3923aK74/uV2x+dPbJ90xZXbHmfrK6an7zdfOdoWd778Ruvdj9645XybtjbPijL1U7/JcB7757cCFtsA4XZIw0McQcttLhPngrL3vDmraPjbtnd7u9/+7Uvd9957Sv93fX99Z0gv5Uvufuw7N8E5e4GY4/sznrOz9TaPnkuLPvhbDY5h2EdLLsb5ZW4OdTbQo32SAtTnL6HlvbJ/wMl8RqxQHmBpAAAAABJRU5ErkJggg==)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "metadata": { 213 | "id": "kSoGSVHpZMZ8" 214 | }, 215 | "source": [ 216 | "# instantiate model\n", 217 | "m2o_model = Sequential()\n", 218 | "\n", 219 | "# time_steps: multiple input, that is, one input at each timestep\n", 220 | "# A normal implementation returns only the RNN output at last time step and it is defined by the return_sequences, which is False by default.\n", 221 | "# In order to get output at each time step, you need to set return_sequences=True\n", 222 | "# Hence this is a many-to-one RNN model.\n", 223 | "m2o_model.add(SimpleRNN(n_cells, input_shape=(time_steps, features)))\n", 224 | "\n", 225 | "# single output at output layer\n", 226 | "m2o_model.add(Dense(n_output, activation=output_activation))" 227 | ], 228 | "execution_count": 19, 229 | "outputs": [] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "metadata": { 234 | "colab": { 235 | "base_uri": "https://localhost:8080/" 236 | }, 237 | "id": "8ICY-hf1aAjD", 238 | "outputId": "879230ec-f168-4403-f688-54ccddebe6d9" 239 | }, 240 | "source": [ 241 | "m2o_model.summary()" 242 | ], 243 | "execution_count": 20, 244 | "outputs": [ 245 | { 246 | "output_type": "stream", 247 | "text": [ 248 | "Model: \"sequential_6\"\n", 249 | "_________________________________________________________________\n", 250 | "Layer (type) Output Shape Param # \n", 251 | "=================================================================\n", 252 | "simple_rnn_7 (SimpleRNN) (None, 5) 50 \n", 253 | "_________________________________________________________________\n", 254 | "dense_5 (Dense) (None, 4) 24 \n", 255 | "=================================================================\n", 256 | "Total params: 74\n", 257 | "Trainable params: 74\n", 258 | "Non-trainable params: 0\n", 259 | "_________________________________________________________________\n" 260 | ], 261 | "name": "stdout" 262 | } 263 | ] 264 | }, 265 | { 266 | "cell_type": "markdown", 267 | "metadata": { 268 | "id": "0-CD0CRgaHyc" 269 | }, 270 | "source": [ 271 | "#### Let's look at how to build a many-to-many (with equal length of input and output sequences) architecture in Keras. TimeDistributed() funcion allows the dense layer to access output at each time step one by one.\n", 272 | "\n", 273 | "#### If you remove the Timedistributed() function and use plain dense layer as we did previously in many-to-one, the dense layer will be applied only at the last time step and get inputs of all the time steps at once. For example Sentiment analysis where you just need to apply dense layer once at the last time step.\n", 274 | "\n", 275 | "#### Where as in many-to-many RNN, Timedistributed() makes sure that dense layers gets applied on RNN outputs at each time step. Hence dense layer is active here at each time step. For example POS tagger where you need to have a dense layer at each time step." 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "metadata": { 281 | "id": "KudDswp1SxS8" 282 | }, 283 | "source": [ 284 | "
\n", 285 | "\n", 286 | "![m2m.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUkAAAIjCAYAAACODekJAAAAAXNSR0IArs4c6QAAAHhlWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACWAAAAAQAAAJYAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAUmgAwAEAAAAAQAAAiMAAAAA6QzzxAAAAAlwSFlzAAAXEgAAFxIBZ5/SUgAAIXBJREFUeAHt3U9vZNd5J+DLshFkYGOiAIm8SBBlZuUAmcEYcDw7ix0g+2SyjtRLSWA+QFbd+gCzI9QCspH6CyRAlrMQpc3AQTYJAthAFrYCz8JCgImByWqAZuqQfY4ueYunb5N1/5z7PmXIvDzn8tZ5n/PWr4pFdnfXuREgQIDAnQInd870Jh6/d3ba+3TRw08+Pr9YdAHu/KDAmnokLVCfHNymxQfX1Cdje+RgSKZCLk+6J0n0sutOF5c9tICT7sM0/Pyj86eHpo1NK9BEjyQCfTJtI7zi6k30ySt6ZBCS73xw9nSfjFcBmet/87d/LR8u/vGrn//yxhr2BVx8+uz80Y1Bn0wqsPYeScXf7pMUlp5QJ22LwcXX3ie3e+SuLLkRku++f/ZZfuX4+//9d7oUjm/+1noCsr8L//ijf+7Sf+XmQVAopjzo90jqj6s+WWmPJIfbfbK77B6N/TZrSsetX7vfJ61nSQnJnPotNH6/wfoPAg+Avszxj3OPpCv/4Z/+l9U+gR6qvN8nz5+dl74/dK6xhwnkPtlKluwSRy4qHf/h/2ir+dOzVPov3fL7qFef+L+jCtzokcYCMkHkVzPpOL3KSR/dji9wo082kiVXIZnfg8xhc3y6aa+YHwDprYI1/fRs2qpnvvrL96mvrFf87XVNJb0ASDd9UlN64FyvTx54pUW+/FCW7HKo5JfGi6zsCHeaA96rySNg3rpE7pE0nJ1vndLMp+ltgnTTJ8ffstwnW8uS3Yvd9a/4rPUHNMffSld8XYHcI60H5OvW7fzXE8h9srUsuf52+/UsVnl23pj80/lVLtKiFhfQJ4tvweoXcLtHdieX3dtp1eklcuu3XEN+2d96PWtZ/5Z6JJnqk2k6a0t90u+RzbySnGbbXZUAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6wO7ypPs8IXz18182b5Fr+OTj84vmi1lRAVvqkcSqT6Zpri31Sb9HdrsX3cVV4/yftkPyq5frP+mu65mmDWJedSs90u9zfXL8Xt5Kn9zOkl1+1ZWSM08en2/6K+bkz89m099jnHvYSo+kHdMn0/XtVvrkdo9cvSeZn1X/8Uf/PJ3ghFdO685rz89mE95dzEufdB+mwrNziwj9Pnn+0fnTFmtY+5q3mCVXIfnps/NHqbiUoK09CNKr37Lm/QM5P5utvZlaW18KlVZ7JFnf7pPW/FtZ7xaz5BsZ/3vf/8GX+29VH+dvud/87V/LU6v9mMLxR//rn67Xtw9Irw6m3ap+j6Q++dZ//NWr/6a914dfXZ883PB1rtDvk/R1rWfJ/sXB17d3Pjh72l12T/JIKu7N37oOyzUVml855vcOOgGZt2zyj4/fOzvdP5k+uey603Rna+2RtDZ9khSWuW0pS26EZOa8XWAeX+PH3WX3yLfY8+9MSz2S3iY4ufRWzPxd0nUt9cldWXIwJDNmetXwYnf9imHfZG/n8aU/poZPaxCOS+9E1621R5KMPlm+P/IK1toneiTvkI8ECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQJDgfRTruGoEQI3BfTJTQ+fDQXm6pGrP5Y4vPtpRtLvTKVfRJ7m6q66FQF9spWdnK6OOXtk1pBMf5on/UmNuZ4BptsiV55UQJ9MyruJi8/YI7OF5NVv3r/cHa8mN9GmkxShTyZh3dRF5+6RWULyqqjenwnPf+53UzunmAcL6JMHE27+Akv0yCwheWjn+s8Gh+aNEUgC+kQfvEpg6h6ZPCRvJ38puPfKsow5CCugT8Ju/ejCl+qRyUOyJjD1M0Dtvs21I6BP2tmrpVY6ZY9MGpJ3Jn+W9GoyS4T+qE9Cb/+o4pfskUlDcsxfr+bXgUb1yKZP0ieb3t6jFLdkj0wWkin5x/wU268DHaWHmr2IPml262Zb+NI9MllI9v8ZiJqmXy6v6QSYG/mWiz4J0At3lbhwj3zzrnU9ZDx9C71v6ov+NfqvKtNfp9+fe/m3n98Y68873qaAPtnmvh6zqjX0SPWfbzhmse++f/ZZDsq7/i2JY96fa7UpoE/a3Lc5Vz13j0z37facau6LAAECEwkIyYlgXZYAgW0ICMlt7KMqCBCYSEBITgTrsgQIbENASG5jH1VBgMBEAkJyIliXJUBgGwJCchv7qAoCBCYSEJITwbosAQLbEJgtJPMvkie2Tz4+v9gGnyqOLaBPji26vevN3SOzheT2tkpFBAhEEBCSEXZZjQQI3FtASN6bzhcSIBBBQEhG2GU1EiBwbwEheW86X0iAQAQBIRlhl9VIgMC9BYTkvel8IQECEQSEZIRdViMBAvcWEJL3pvOFBAhEEBCSEXZZjQQI3FtASN6bzhcSIBBBQEhG2GU1EiBwb4FZQjL9s5B5hbf/Odk87iMBfaIHXiWwRI/MEpKvKtw8AQIE1iogJNe6M9ZFgMAqBITkKrbBIggQWKuAkFzrzlgXAQKrEBCSq9gGiyBAYK0CQnKtO2NdBAisQkBIrmIbLIIAgbUKCMm17ox1ESCwCgEhuYptsAgCBNYqICTXujPWRYDAKgSE5Cq2wSIIEFirgJBc685YFwECqxCYJSRf7LrTXO3lSfd5PvaRQF9An/Q1HB8SWKJHZgnJQ8UaI0CAQAsCQrKFXbJGAgQWExCSi9G7YwIEWhAQki3skjUSILCYgJBcjN4dEyDQgoCQbGGXrJEAgcUEhORi9O6YAIEWBIRkC7tkjQQILCYgJBejd8cECLQgICRb2CVrJEBgMQEhuRi9OyZAoAUBIdnCLlkjAQKLCQjJxejdMQECLQjMEpInl93bGWP3orvIxz4S6Avok76G40MCS/TILCF5qFhjBAgQaEFASLawS9ZIgMBiAkJyMXp3TIBACwLfHLPIx++dnY45765zLnsT6W8W3l+vN/J6h598fH7xel/h7DkEHtojaY36ZI6dWvY+HtonS/TIySGyVMj+n1l4kub2izo9dM7iYyfdh2kNzz86f7r4WgIuoIkeSfuiTxbtzib65BU9MgjJdz44e7pPxquAzLpvfOetfLj4x3/9xZc31rAv4OLTZ+ePbgz6ZFKBtfdIKv52n6Sw9IQ6aVsMLr72PrndI3dlyX7869u77599ll85/qf/+sMuheOaAvLrlXbdz/7hi+6n+//KzYOgUEx50O+R1Bu5T6a8z4dc+3af7C67R96yeYjouK/t90nukVazpIRkTv0WGr+/Tf0HgQdAX+b4x7lH0pW/90d/tton0EOV9/vk+bPz0veHzjX2MIHcJ1vJkqufbueiEk1rzf+7+1e86Zkq3fL7qFef+L+jCrTcIwki9Ul+JZNe5RwVx8WKQMt9cleWXP8K0Mv3IHPYlIobOcgPgPRWwUN/etZIyfMvs9cjOWzmX8TD7jG9AEg3ffIwx+pX9/qket5KJw9lyS6HSmr8dEKrtxzwXk0efwdzj6Qrt9wjaf0lKF/+9kYaczuOQO6TrWXJLv3eYiL69RX9BPs4W+YqxxLIPZKfiI51XdfZlkDuk61lyfW32xvYq/wtYP7p/AZKUsIEAvpkAtSNXfJ2j+zy36qRJ1quN9eQX/a3XMua1r6lHkmu+mSa7tpSn/R7ZDOvJKfZdlclQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAko3eA+gkQqAoIySqPSQIEogsIyegdoH4CBKoCQrLKY5IAgegCQjJ6B6ifAIGqgJCs8pgkQCC6gJCM3gHqJ0CgKiAkqzwmCRCILiAkN9YB775/9tk7H5w93VhZyjmygD4ZDyokx1u1c+Zl9+Sd988uhWU7W7bISvXJKHYhOYqp0ZM8CBrduJmXrU+q4EKyyrORSQ+CjWzkxGXok4PA3zw4urHBx++dnW6spDvLubxzZj9x/SB40p10Hz7/6Pxp7dSIc/rk5a7rkxvtv/mQTI3/4qT77EbV0T/xIBh0gD4ZkHhSfUni2+0DvRFl6OSyezvSq6co+3rsOqP3yeZfSaaGOem6i2M3zlqvt/92+/RVa0se+8b/8JOPzy9edW6keX1yc7f1ybXH5kPyZRBc3Nz+7X6Wfv/trqDU9Hfvuz752kaffG2RjjYfkjfLjfmZpo+5769btT45LCYkD7tsYlTTb2IbJy9Cn9SJhWTdp8lZTd/kts2+aH0yjlxIjnNq5iw/kGlmqxZdqD4Zz+9XgMZbNXGmn1g3sU2LL1KfjN8CITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMFxCS462cSYBAQAEhGXDTlUyAwHgBITneypkECAQUEJIBN13JBAiMF9hdnnSfp9P/9Rdfjv+qlZ6Za/jk4/OLlS6xyWVtqUfSBuiTadpwS33S75Hd7kV3kcj+b+MhmYs66a7rmaYNYl51Kz2Sdk+fTNfDW+mT2z2yy6+60kSenI5xuivntedns+nuKd6Vt9Ijaef0yXT9u5U+ud0jV+9J5ldfP/2HL6YTnPDKP9uvO689P5tNeHcxL33SfZgKz84tIvT75PlH509brGHta95illyF5KfPzh+l4lKCpkZq6ZbWXB64+wdyfjZrqYYW1ppCpdUeSb63+6QF8xbXuMUs+UbeiO99/wdf7r9VfZyaaf9g6N74zlt5arUfU6D/+H//zfX69gHp1cG0W9XvkdQn/+Hbb3S/uv9v7Td9Mu8O9ftkC1mSaii3dz44e9pddk/yQArKX38ZlmsKzfzKMT1Qr24CMm/Z5B8fv3d2un8yfXLZdafpztbaI2lt+iQpLHPbUpbcCMnMebvAPL7Gj7vL7pFvseffmZZ6ZN/kFyeX3oqZv0u6rqU+uStLDoZkxkyvGl7srl8x7Jvs7Ty+9MfU8GkNwnHpnei6tfZIktEny/dHXsFa+2R1PZKeVd59/+yzDOcjgUMC+uSQirG+wJw9cvXT7f6dT3q8f78zvZeVnlUmvR8Xb1tAn7S9f3OsfsYemS0kr96beImX3vifw9F9tCegT9rbs7lXPHePzBKSV0X1fmqeXk3ODev+1i+gT9a/R0uvcIkemSUkD8H2nw0OzRsjkAT0iT54lcDUPTJ5SN5O/lJw75VlGXMQVkCfhN360YUv1SOTh2RNYOpngNp9m2tHQJ+0s1dLrXTKHpk0JO9M/izp1WSWCP1Rn4Te/lHFL9kjk4bkmF9A9+tAo3pk0yfpk01v71GKW7JHJgvJlPxjfort14GO0kPNXkSfNLt1sy186R6ZLCT7f1FGTdMvl9d0AsyNfMtFnwTohbtKXLhHvnnXuh4ynr6F3jf1Rf8a/VeV6S8c6M+9/PPhN8b68463KaBPtrmvx6xqDT1S/Qsujlls+jPbOSjv+ts2jnl/rtWmgD5pc9/mXPXcPTLdt9tzqrkvAgQITCQgJCeCdVkCBLYhICS3sY+qIEBgIgEhORGsyxIgsA0BIbmNfVQFAQITCQjJiWBdlgCBbQgIyW3soyoIEJhIQEhOBOuyBAhsQ2C2kMy/SJ7Y/CuH22ieKarQJ1Oobuuac/fIbCG5rW1SDQECUQSEZJSdVicBAvcSEJL3YvNFBAhEERCSUXZanQQI3EtASN6LzRcRIBBFQEhG2Wl1EiBwLwEheS82X0SAQBQBIRllp9VJgMC9BITkvdh8EQECUQSEZJSdVicBAvcSEJL3YvNFBAhEERCSUXZanQQI3EtglpBM/yxkXt3tf042j/tIQJ/ogVcJLNEjs4Tkqwo3T4AAgbUKCMm17ox1ESCwCgEhuYptsAgCBNYqICTXujPWRYDAKgSE5Cq2wSIIEFirgJBc685YFwECqxAQkqvYBosgQGCtAkJyrTtjXQQIrEJASK5iGyyCAIG1CgjJte6MdREgsAoBIbmKbbAIAgTWKiAk17oz1kWAwCoEZgnJF7vuNFd7edJ9no99JNAX0Cd9DceHBJbokVlC8lCxxggQINCCgJBsYZeskQCBxQSE5GL07pgAgRYEhGQLu2SNBAgsJiAkF6N3xwQItCAgJFvYJWskQGAxASG5GL07JkCgBQEh2cIuWSMBAosJCMnF6N0xAQItCAjJFnbJGgkQWExASC5G744JEGhBQEi2sEvWSIDAYgJCcjF6d0yAQAsCs4TkyWX3dsbYvegu8rGPBPoC+qSv4fiQwBI9MktIHirWGAECBFoQEJIt7JI1EiCwmICQXIzeHRMg0IKAkGxhl6yRAIHFBITkYvTumAABAgQIECBAgAABAgQIECBAgEBIgZMxVT9+7+x0zHlznPPJx+cXc9yP+3g9gTX1SFq5Pnm9/Zvr7DX1ydgeORiSqZD9v4/9JMFddt3pXICvdT8n3Yfp/OcfnT99ra9z8lEEmuiRVKk+Ocp+3/ciTfTJK3pkEJLvfHD2dJ+MVwGZYb771rfy4eIff/Llv91Yw76Ai0+fnT+6MeiTSQXW3iOp+Nt9ksLSE+qkbTG4+Nr75HaP3JUlN0Ly3ffPPsuvHP/kh9/pUjiuKSD7u/DXX3zV/dUXv/h6yIPga4sJj/o9knoj98mEd/mgS9/uk91l92jst1kPuuPgX9zvk9wjrWZJCcmc+i00fr//+g8CD4C+zPGPc4+kK//Fn/3n1T6BHqq83yfPn52Xvj90rrGHCeQ+2UqWfCNx5KLS8f/88+92v/HGr6TDJm5pI072/7t66XzS/e7f/93fftrEwhtbZL9HWgvIRJ36JPXIv/zy/3f/7Q9+cKpPpmnAfp9sJUuu/8TNy/cg08viFm9//MM3rx4E6a2CNf30rEXLO9fc65EUOC3eUrinmz6ZcPd6fTLhvUx26UNZssuhkho/ndDqLQd8/ql8q3Wscd25R9LaWu6RtP4SlC9/eyONuR1HIPfJ1rJk92J3/Ss+v/fWt48j5SqbE8g9kp+INleggo4ikPtka1ly/e32UYiWvUj+FjD/dH7Z1bj3tQrok7XuzHrWdbtHdvmvQ88T61nq668k15Bf9r/+FXzFIYEt9UiqT58c2uWHj22pT/o9splXkg/fYlcgQIDAUEBIDk2MECBAoAgIyULhgAABAkMBITk0MUKAAIEiICQLhQMCBAgMBYTk0MQIAQIEioCQLBQOCBAgMBQQkkMTIwQIECgCQrJQOCBAgMBQQEgOTYwQIECgCAjJQuGAAAECQwEhOTQxQoAAgSIgJAuFAwIECAwFhOTQxAgBAgSKgJAsFA4IECAwFBCSQxMjBAgQKAJCslA4IECAwFBASA5NjBAgQKAICMlC4YAAAQJDASE5NDFCgACBIiAkC4UDAgQIDAWE5NDECAECBIqAkCwUDggQIDAUEJJDEyMECBAoAkKyUDggQIDAUEBIDk2MECBAoAgIyULhgAABAkMBITk0MUKAAIEiICQLhQMCBAgMBYTk0MQIAQIEioCQLBQOCBAgMBQQkkMTIwQIECgCQrJQOCBAgMBQQEgOTYwQIECgCAjJQuGAAAECQwEhOTQxQoAAgSIgJAuFAwIECAwFhOTQxAgBAgSKgJAsFA4IECAwFBCSQxMjBAgQKAJCslA4IECAwFBASA5NjBAgQKAICMlC4YAAAQJDASE5NDFCgACBIiAkC4UDAgQIDAWE5NDECAECBIqAkCwUDggQIDAUEJJDEyMECBAoAkKyUDggQIDAUEBIDk2MECBAoAgIyULhgAABAkMBITk0MUKAAIEiICQLhQMCBAgMBYTk0MQIAQIEioCQLBQOCBAgMBQQkkMTIwQIECgCQrJQOCBAgMBQQEgOTYwQIECgCAjJQuGAAAECQwEhOTQxQoAAgSIgJAuFAwIECAwFhOTQxAgBAgSKgJAsFA4IECAwFBCSQxMjBAgQKAJCslA4IECAwFBASA5NjBAgQKAICMlC4YAAAQJDASE5NDFCgACBIiAkC4UDAgQIDAWE5NDECAECBIqAkCwUDggQIDAUEJJDEyMECBAoAkKyUDggQIDAUEBIDk2MECBAoAgIyULhgAABAkMBITk0MUKAAIEiICQLhQMCBAgMBYTk0MQIAQIEioCQLBQOCBAgMBQQkkMTIwQIECgCQrJQOCBAgMBQQEgOTYwQIECgCAjJQuGAAAECQwEhOTQxQoAAgSIgJAuFAwIECAwFhOTQxAgBAgSKgJAsFA4IECAwFBCSQxMjBAgQKAJCslA4IECAwFBASA5NjBAgQKAICMlC4YAAAQJDASE5NDFCgACBIiAkC4UDAgQIDAWE5NDECAECBIqAkCwUDggQIDAUEJJDEyMECBAoAkKyUDggQIDAUEBIDk2MECBAoAgIyULhgAABAkMBITk0MUKAAIEiICQLhQMCBAgMBYTk0MQIAQIEioCQLBQOCBAgMBQQkkMTIwQIECgCQrJQOCBAgMBQQEgOTYwQIECgCAjJQuGAAAECQwEhOTQxQoAAgSIgJAuFAwIECAwFhOTQxAgBAgSKgJAsFA4IECAwFBCSQxMjBAgQKAJCslA4IECAwFBASA5NjBAgQKAICMlC4YAAAQJDASE5NDFCgACBIrC7POk+T5/95Mt/K4OtHuQaPvn4/KLVGta47i31SPLVJ9N02Zb6pN8ju92L7iKR/fjL/zeN3ExXzUWddNf1zHS3Ie5mKz2SNkufTNeyW+mT2z2yy6+60kSenI5xuivntedns+nuKd6Vt9Ijaef0yXT9u5U+ud0jV+9J5ldff/XFL6YTnPDKf/3FV11ee342m/DuYl76pPswFZ6dW0To98nzj86ftljD2te8xSy5CslPn50/SsWlBE2N1NItrbk8cPcP5Pxs1lINLaw1hUqrPZJ8b/dJC+YtrnGLWfKNvBHf+/4Pvtx/q/o4NdPJ/n/ffetbeWq1H1Og/+Xf/Px6ffuA9Opg2q3q90jqk99841e639j/t/abPpl3h/p9soUs2b84+Pr2zgdnT7vL7kkeSUH5e299++rTNYVmfuWYHqhXNwGZt2zyj4/fOzvdP5k+uey603Rna+2RtDZ9khSWuW0pS26EZOa8XWAeX+PH3WX3yLfY8+9MSz2S3iY4ufRWzPxd0nUt9cldWfLvIVYDCTRMz2QAAAAASUVORK5CYII=)" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "metadata": { 292 | "id": "jhYSMcHpaD0u" 293 | }, 294 | "source": [ 295 | "# instantiate model\n", 296 | "m2m_model = Sequential()\n", 297 | "\n", 298 | "# time_steps: multiple input, that is, one input at each timestep\n", 299 | "# return_sequences=True returns all the outputs processed by RNN\n", 300 | "m2m_model.add(SimpleRNN(n_cells, input_shape=(time_steps, features), return_sequences=True))\n", 301 | "\n", 302 | "# TimeDistributed(): This function is used when you want your neural network \n", 303 | "# to provide an separate output at each timestep which is exactly what we want in the many-to-many RNN model.\n", 304 | "m2m_model.add(TimeDistributed(Dense(n_output, activation=output_activation)))" 305 | ], 306 | "execution_count": 21, 307 | "outputs": [] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "metadata": { 312 | "colab": { 313 | "base_uri": "https://localhost:8080/" 314 | }, 315 | "id": "ghlWLq1Za3uL", 316 | "outputId": "ca836538-b5a2-419a-ea01-bc94ebb1ac45" 317 | }, 318 | "source": [ 319 | "m2m_model.summary()" 320 | ], 321 | "execution_count": 22, 322 | "outputs": [ 323 | { 324 | "output_type": "stream", 325 | "text": [ 326 | "Model: \"sequential_7\"\n", 327 | "_________________________________________________________________\n", 328 | "Layer (type) Output Shape Param # \n", 329 | "=================================================================\n", 330 | "simple_rnn_8 (SimpleRNN) (None, 10, 5) 50 \n", 331 | "_________________________________________________________________\n", 332 | "time_distributed (TimeDistri (None, 10, 4) 24 \n", 333 | "=================================================================\n", 334 | "Total params: 74\n", 335 | "Trainable params: 74\n", 336 | "Non-trainable params: 0\n", 337 | "_________________________________________________________________\n" 338 | ], 339 | "name": "stdout" 340 | } 341 | ] 342 | }, 343 | { 344 | "cell_type": "markdown", 345 | "metadata": { 346 | "id": "ecuLOmh2d598" 347 | }, 348 | "source": [ 349 | "#### Let's look at how to build a one-to-many RNN in Keras." 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": { 355 | "id": "S6Ja0TQYS8ck" 356 | }, 357 | "source": [ 358 | "
\n", 359 | "\n", 360 | "![o2m.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUsAAAIjCAYAAACK+Dk0AAAAAXNSR0IArs4c6QAAAHhlWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACWAAAAAQAAAJYAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAUugAwAEAAAAAQAAAiMAAAAAPTBjAwAAAAlwSFlzAAAXEgAAFxIBZ5/SUgAAIE9JREFUeAHt3c1vJMd5B+Ae2sglBmwDiYzAhiUnFxtwHBuI5ZvEDZB7gpy92qMk8JZTTss95R8gpD2u1+fEvgewKN1sHwJ/AHEusWXYBwkBbAURkCAAJ1NDFreX5HDZy+rqfqsfAtLMzjSr6n3emh+Lw/3oOh8ECBAg8EyB1TOvOLvg3usH+ze9duzrHj08Oh57DuMPF5jTHkmrt0+G97DGZ8xpnwzZI9eG5WtvHLyT8NZdt18Dcegcm8Ufr1fdu4/fOjoc+rmuLyOQNv6mB/fnuke2Va66B+nWPinT8+cZpYUsuTIs0wvgZNVtgzLDvPCFT+e7k99++JuPLq9h84LwYrjMMtYjV4XknPZIqvviPklfXFfr7sGQ08RYfksZt6UsuRSWd988ONwcJe+nZqbN/9VvfbF74fPzCcr+Jvvwtx9tXxA//+GvTx8WmH2e0e5H2iMJIe2PvFe2KPbJaHujP3CkfZL3x3VZ8lRY9ov7q7/789mGZL8h6X4qMBe5t+7uODlcFCr36/4eSV9I039RPuyTep3q75NWsmQv86Xjcj5Rzvk0mdfbv+2/aC++fdC/zv0CAmffdfTNC4xaZYj+mtP7rFUmXeAkrWbJeVie7J3+EKe/oSL1Oa07v2eW30yOtP4Ia92eFjYLjbpHknHeJ+kHUrmeCPaR1thqlpyHZf9UGakx/bWmF4KPEQV6p8oRZxl96LxPNj/seXX0yZY4QQP7JO+Rfvu2Ybk9Nm8ezSez/gWR7ucfRKVTQ6R1R1hrK3skWdsn4+24VvbJVXtkG5b52JwvGI9y/JFz4OemjT/jMmZoaY+kjtkn4+zblvbJxT3y5Nvwceyqj5oDPzet+gJMSIBAEwIXs6S5sGyiS4oYXeDiC2H0CU0QXkBYhm+hAggQqCEgLGsom4MAgfACwjJ8CxVAgEANAWFZQ9kcBAiEFxCW4VuoAAIEaggIyxrK5iBAILyAsAzfQgUQIFBDQFjWUDYHAQLhBYRl+BYqgACBGgLCsoayOQgQCC8gLMO3UAEECNQQEJY1lM1BgEB4AWEZvoUKIECghoCwrKFsDgIEwgsIy/AtVAABAjUEhGUNZXMQIBBeQFiGb6ECCBCoISAsayibgwCB8ALCMnwLFUCAQA0BYVlD2RwECIQXEJbhW6gAAgRqCAjLGsrmIEAgvICwDN9CBRAgUENAWNZQNgcBAuEFhGX4FiqAAIEaAsKyhrI5CBAILyAsw7dQAQQI1BAQljWUzUGAQHgBYRm+hQogQKCGgLCsoWwOAgTCCwjL8C1UAAECNQSEZQ1lcxAgEF5AWIZvoQIIEKghICxrKJuDAIHwAsIyfAsVQIBADQFhWUPZHAQIhBcQluFbqAACBGoICMsayuYgQCC8gLAM30IFECBQQ0BY1lA2BwEC4QWEZfgWKoAAgRoCwrKGsjkIEAgvICzDt1ABBAjUEBCWNZTNQYBAeAFhGb6FCiBAoIaAsKyhbA4CBMILCMvwLVQAAQI1BIRlDWVzECAQXkBYhm+hAggQqCEgLGsom4MAgfACwjJ8CxVAgEANAWFZQ9kcBAiEFxCW4VuoAAIEaggIyxrK5iBAILyAsAzfQgUQIFBDQFjWUDYHAQLhBYRl+BYqgACBGgLCsoayOQgQCC8gLMO3UAEECNQQEJY1lM1BgEB4AWEZvoUKIECghoCwrKFsDgIEwgsIy/AtVAABAjUEhGUNZXMQIBBeQFiGb6ECCBCoISAsayibgwCB8ALCMnwLFUCAQA0BYVlD2RwECIQXEJbhW6gAAgRqCAjLGsrmIEAgvICwDN9CBRAgUENAWNZQNgcBAuEFhGX4FiqAAIEaAsKyhrI5CBAILyAsw7dQAQQI1BAQljWUzUGAQHgBYRm+hQogQKCGgLCsoWwOAgTCCwjL8C1UAAECNQSEZQ1lcxAgEF5AWIZvoQIIEKghICxrKJuDAIHwAsIyfAsVQIBADQFhWUPZHAQIhBcQluFbqAACBGoICMsayuYgQCC8gLAM30IFECBQQ0BY1lA2BwEC4QWEZfgWKoAAgRoCwrKGsjkIEAgvICzDt1ABBAjUEBCWNZTNQYBAeAFhGb6FCiBAoIaAsKyhbA4CBMILCMvwLVQAAQI1BIRlDWVzECAQXkBYhm+hAggQqCEgLGsom4MAgfACwjJ8CxVAgEANAWFZQ9kcBAiEFxCW4VuoAAIEaggIyxrK5iBAILyAsAzfQgUQIFBDQFjWUDYHAQLhBYRl+BYqgACBGgLCsoayOQgQCC8gLMO3UAEECNQQEJY1lM1BgEB4AWEZvoUKIECghoCwrKFsDgIEwgsIy/AtVAABAjUEhGUNZXMQIBBeQFiGb6ECCBCoISAsayibgwCB8ALbsNw76Y5TJR/+9qPwBeUack3hC5pJAdkz+85kWc+9jJ//8Nfbz3381tHhcw/iEy8JtLRP8l7PNTV3svzwN6eB/+jh0fGlTnrguQWyZ/Z97oF8IoEgAnmv572/Dcv8i/RkTtMg9Ty1zHxaWHWnJ+WnnvSLWwtk18h7JCHYJ7feCjsHaDlLnpwsV92D/kbaqTHjJ/KLYL3q3p3xMsMuLbv+4J9+FraGFPT2ycjtazRLPpHZfvLjHx1//Zsv73/8X//7UnrshS98Oj8V4vYH//yzbrP2rts0yvtQ47Qs75HN6C+l0PnSVz43zkQjjZqD0j4ZCfhs2LxPWsuSzXdWTz7uvX6wf7Lq3kmPpLD86re+2L3w+XmHZnoB9E86j98+eqqmJ9W5V0rg7hsH6zxW2iPpv7l/9PfJZoMcf+ftoztzX3Pk9bWYJZeCJRW5+Xbr/ubVsJ+alUIzB+acTpv5W6n8Jmx6AazW3YP8nknkjRZh7XffPDjs1t39Oe+RtLar9omgrLPDWsuSS2GZGfsvhvzYbG996z1Ja0LtkY3Q3rq744tp/a0Sap9ckyU7wzKTbo/Te6enzM3J7dX8+NS36RSZ1mDzT92JrpvrHkky9sn0+yOvYK77xB7JHXJLgAABAgQIECBAgAABAgQIECBAgAABAgQIECBwe4H0U7Hbj2KE1gXsk9Y7fPv6au6RJ382/PbrvtEI6fdcpd/0fqOLXbRYAftksa2/ceG190j1sEx/6iP96aCaXxFurO/C+QjYJ/PpxVxXUnmPVA3L7e/kP4N3upzrDpx+XfbJ9D2Y+wqm2CNVwzL/WeLUCKfLuW/HCdd39mfO7ZMJezD3qSfYI9XCsv+VIPfB6TJLuM0C9kmWcLtLYKo9Ui0s+6fKjOB0mSXcngv0Tgz5MfskS7jdCky0R6qE5VVfCXLbnS6zhFv7xB54lsCUe6RKWF51qswoTg1Zwq19Yg88U+CKU2X+nLGzZPSwvO4rwXmRft9lpljsrX2y2NbfuPCp98joYXndaSErjf0VIc/jdsYC15wY8qrtkyyx0NuJ98ioYXmTrwS57d67zBLLu7VPltfzoRXPYY98cuiib3p9+hM669O/Wf04f046GeT76d/MyffdLlfAPllu729a+Vz2yDP/WYmbFnST67ZfHfJR+pp/6+ImY7mmXQH7pN3elqpsij0y6rfhpWCMQ4AAgakFhOXUHTA/AQIhBIRliDZZJAECUwsIy6k7YH4CBEIIVA3L/r87vnfip+EhdsgEi7RPJkAPNuUUe6RqWAbrh+USIEDgXEBYnlO4Q4AAgd0CwnK3jWcIECBwLiAszyncIUCAwG4BYbnbxjMECBA4FxCW5xTuECBAYLeAsNxt4xkCBAicCwjLcwp3CBAgsFtAWO628QwBAgTOBYTlOYU7BAgQ2C0gLHfbeIYAAQLnAlXDsv83pT96eHR8vgp3CPQE7JMehrtXCkyxR6qG5ZVVe5AAAQIBBIRlgCZZIgEC0wsIy+l7YAUECAQQEJYBmmSJBAhMLyAsp++BFRAgEEBAWAZokiUSIDC9gLCcvgdWQIBAAAFhGaBJlkiAwPQCwnL6HlgBAQIBBIRlgCZZIgEC0wsIy+l7YAUECAQQEJYBmmSJBAhMLyAsp++BFRAgEECgWljee/1gP3usuu4433dLoC9gn/Q13L9KYKo9Ui0sryraYwQIEIgiICyjdMo6CRCYVEBYTspvcgIEoggIyyidsk4CBCYVEJaT8pucAIEoAsIySqeskwCBSQWE5aT8JidAIIqAsIzSKeskQGBSAWE5Kb/JCRCIIiAso3TKOgkQmFRAWE7Kb3ICBKIICMsonbJOAgQmFagWlid73X6udL3q3s333RLoC9gnfQ33rxKYao9UC8urivYYAQIEoggIyyidsk4CBCYVEJaT8pucAIEoAsIySqeskwCBSQWE5aT8JidAIIqAsIzSKeskQGBSAWE5Kb/JCRCIIiAso3TKOgkQmFRg8w8t3uyj/y+q3ewznr5q+xtJ19399Gj61x1X6+7B01fc/FePHh4d3/xqV9YSuO0eSeu0T2p1a7p5brtPptoj14bla28cvJNI1123Px3t7plT6KY/DfT4raPD3Vd5ZkyBtPE3Pbg/1z2yrX11+oXZPhlzJ1w/dgtZcmVYphfAyarbBmUm+MznXsx3J7/9/QfvX17D5gXhxXCZZaxHrgrJOe2RVPfFfZK+uKbvaHxnMtauuDxuS1lyKSzvvnlwuDlK3k9lp83/pa+9sr29zDD9I+nFkP775U/fO12MwKzSlEh7JIH8arM/fne2V7ZA9ol9ckHgJlnyVFj2XwTf+OtvzzYkL9S5fTHkwNxbd3ecHC4Klft1f4+kL6Qvbf6L8pFC0z6p063+PmklS85/Gp6Oy/lEOefT5FWtTi/YtOb0cfHtg6uu99gtBM6+64gWlKni/j5J77PeQsGnXiPQapach2X+a48ivghS39ILIb9nlt9MvqafnnoOge1pYfN5UfdIKjnvk/Xmh5a5nueg8CnXCLSaJedhmU+VaTNF/ciny6jrn/26z06VkfdIMs77ZPPDnldnbx5xgQ3sk7xH+vzbsNwemzeP5pNZ/4JI9/P606kh0rojrLWVPZKs7ZPxdlwr++SqPbINy3xs/uyMfnvQ87YzF5mb9rzj+LynBVraI6ky++Tp/pb6VUv75OIeefJteCmticfJgZ+bNvFyTE+AQFCBi1nSXFgG7YtlVxa4+EKoPL3pAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2DRLJkCgvoCwrG9uRgIEAgoIy4BNs2QCBOoLCMv65mYkQCCggLAM2LRnLfnumweH6b9nXef5ZQvYJ8P6LyyHecW5et3dv/vGwVpoxmnZJCu1T27MLixvTBX0Qi+GoI2rvGz75JngwvKZRI1c4MXQSCNHLsM+2Qn8yZ3PNPbEvdcP9hsraWc5Jzuf2Txx+mK43626B4/fOjq87tIlPmefnHXdPrm0/RcTlier7p1L1bf6wPoGhXkxXIlkn1xgsU/OQXwbfk6x0DvpxeAn5wtt/oCy7ZNuMSfLVdcdD9gaoS/dHCz3b1SAb8UvMdknl0g6b9mcmiwmLL/z9tGdK7ZBkw9tT4qbk8DO4oTkThr7pEdjn/QwuuWcLJ+qeqm/sPmX2vlhddsnV3ot5mR5ZfVLedDmX0qnb1enfXKtn7C8lif4kzZ/8AZWWr59ciNoYXkjplgXnf3+ycNYq7ba2gL2yTBxv3VomJerCRBYqICwXGjjlU2AwDABYTnMy9UECCxUQFgutPHKJkBgmICwHOblagIEFiogLBfaeGUTIDBMQFgO83I1AQILFRCWC228sgkQGCYgLId5uZoAgYUKCMuFNl7ZBAgMExCWw7xcTYDAQgWE5UIbr2wCBIYJCMthXq4mQGChAsJyoY1XNgECwwSE5TAvVxMgsFABYbnQxiubAIFhAsJymJerCRBYqICwXGjjlU2AwDABYTnMy9UECCxUQFgutPHKJkBgmICwHOblagIEFiogLBfaeGUTIDBMQFgO83I1AQILFRCWC228sgkQGCYgLId5uZoAgYUKCMuFNl7ZBAgMExCWw7xcTYDAQgWE5UIbr2wCBIYJCMthXq4mQGChAsJyoY1XNgECwwSE5TAvVxMgsFABYbnQxiubAIFhAsJymJerCRBYqICwXGjjlU2AwDABYTnMy9UECCxUQFgutPHKJkBgmICwHOblagIEFiogLBfaeGUTIDBMQFgO83I1AQILFRCWC228sgkQGCYgLId5uZoAgYUKCMuFNl7ZBAgMExCWw7xcTYDAQgWE5UIbr2wCBIYJCMthXq4mQGChAsJyoY1XNgECwwSE5TAvVxMgsFABYbnQxiubAIFhAsJymJerCRBYqICwXGjjlU2AwDABYTnMy9UECCxUYBuWeyfdcar/dx+8H54h15BrCl/QTArIntl3Jst67mX88qfvbT/38VtHh889iE+8JNDSPsl7PdfU3Mny92eB/+jh0fGlTnrguQWyZ/Z97oF8IoEgAnmv572/Dcv8i/RkviBIPU8t81dnp4VVd3pSfupJv7i1QHaNvEcSgn1y662wc4CWs+TJyXLVPUgC+duTnRozfiKvfb3q3p3xMsMuLbv+6798N2wNKejtk5Hb12iWfCKz/eTHPzr++jdf3v+fjz96aXOC6D7zuRfzUyFu0wt4s/au2zTK+1DjtCzvkc3oL6XQ+ZM/+4txJhpp1ByU9slIwGfD5n3SWpakXDz/uPf6wf7JqnsnPZDC8ktfe2X2oZleAP2TzuO3j56q6bw4d4oJ3H3jYJ0HS3vkpc1/c//o75PNBjn+zttHd+a+5sjrazFLLgVLKnLz7db9zathPzUrheZnz06Zczpt5m+l0osgfaQXwGrdPcjvmWwf9L/RBO6+eXDYrbv7aYK57pG0tqv2iaBMMuN/tJYll8IyE/ZfDPmx2d761nuS1oTaIxuhvXV3xxfT+lsl1D65Jkt2hmUm3R6n905PmZuT26v58alv0ykyrcHmn7oTXTfXPZJk7JPp90dewVz3yWz3SPoq89obB+9kQLcECBCIIPDktw7VWu3mfa70fmj6KlNrSvMQIEDgtgJVw3L73sXZitMPkW67eJ9PgACBWgJVwzL/9DQV53RZq8XmIUCghEC1sOyfKvPCnS6zhFsCBOYuUC0s+6fKjOJ0mSXcEiAwd4EqYXnVqTLDOF1mCbcECMxZoEpYXnWqzChOl1nCLQECcxYYPSyvO1VmGKfLLOGWAIG5CoweltedKjOK02WWcEuAwFwFRg3Lm5wqM4zTZZZwS4DAHAU+Odai0p/QWZ/+WfLjPEc6Qeb76W8JyvfdEiBAYO4Cz/yLNEoWsD1pnv21Xv6S3pKyxiJAYGyBUb8NH3vxxidAgEAtAWFZS9o8BAiEFhCWodtn8QQI1BIQlrWkzUOAQGiBqmHZ/5vW9078NDz0zrF4AgsTqBqWC7NVLgECDQkIy4aaqRQCBMYTEJbj2RqZAIGGBIRlQ81UCgEC4wkIy/FsjUyAQEMCwrKhZiqFAIHxBITleLZGJkCgIQFh2VAzlUKAwHgCwnI8WyMTINCQgLBsqJlKIUBgPAFhOZ6tkQkQaEigalj2/6b0Rw+PjhtyVAoBAo0LVA3Lxi2VR4BAwwLCsuHmKo0AgXICwrKcpZEIEGhYQFg23FylESBQTkBYlrM0EgECDQsIy4abqzQCBMoJCMtylkYiQKBhAWHZcHOVRoBAOQFhWc7SSAQINCwgLBturtIIECgnICzLWRqJAIGGBYRlw81VGgEC5QSEZTlLIxEg0LBAtbC89/rBfnZcdd1xvu+WAAECEQSqhWUEDGskQIDALgFhuUvG4wQIEOgJCMsehrsECBDYJSAsd8l4nAABAj0BYdnDcJcAAQK7BITlLhmPEyBAoCcgLHsY7hIgQGCXgLDcJeNxAgQI9ASEZQ/DXQIECOwSEJa7ZDxOgACBnoCw7GG4S4AAgV0CwnKXjMcJECDQE6gWlid73X6ed73q3s333RIgQCCCQLWwjIBhjQQIENglICx3yXicAAECPQFh2cNwlwABArsEhOUuGY8TIECgJyAsexjuEiBAYJeAsNwl43ECBAj0BIRlD8NdAgQI7BIQlrtkPE6AAIGegLDsYbhLgACBXQLCcpeMxwkQIDC1QP/fEJ96LeYnQIAAAQIECBAgQIAAAQL1BFY3nWpO3zo/enh0fNN1u44AAQIlBK4Ny9feOHgnTbLuuv0Sk5UeY7P44/TXvT1+6+iw9NjGI0CAQF/gyrBMp8iTVbcNynzxl1/8w3x38ttfvP/x5TWsugdC8zKLRwgQKCNwKSzvvnlwuDlK3k/Dp4D821c+t70tM13ZUVJopv++994HpwMLzLLARiNA4FzgqbDsB+U/fPtPZxuS56s/u/P99z48D8y9dXfHe5oXhfyaAIHbCpz/pvTtD3DOTpRzPk1eVfDfvPLC9gScnrv49sFV13uMAAECQwXOwzL/GzkpKFP4RPtIa87vq+YfTEWrwXoJEJivwHlY5vcpIwZl5k1B74MAAQJjCGzDMv8eynwyG2OiGmPm9c/1tzrVMDAHAQLjCGzDMn8L/pUXPzXOLBVHzYGZvwBUnNpUBAg0LPDk2/BGisyBn78ANFKWMggQmFigubCc2NP0BAg0KiAsG22ssggQKCsgLMt6Go0AgUYFhGWjjVUWAQJlBYRlWU+jESDQqICwbLSxyiJAoKyAsCzraTQCBBoVEJaNNlZZBAiUFRCWZT2NRoBAowLCstHGKosAgbICwrKsp9EIEGhUQFg22lhlESBQVkBYlvU0GgECjQoIy0YbqywCBMoKCMuynkYjQKBRAWHZaGOVRYBAWQFhWdbTaAQINCogLBttrLIIECgrICzLehqNAIFGBYRlo41VFgECZQWEZVlPoxEg0KiAsGy0scoiQKCsgLAs62k0AgQaFRCWjTZWWQQIlBUQlmU9jUaAQKMCwrLRxiqLAIGyAsKyrKfRCBBoVEBYNtpYZREgUFZAWJb1NBoBAo0KCMtGG6ssAgTKCgjLsp5GI0CgUQFh2WhjlUWAQFkBYVnW02gECDQqICwbbayyCBAoKyAsy3oajQCBRgWEZaONVRYBAmUFhGVZT6MRINCogLBstLHKIkCgrICwLOtpNAIEGhUQlo02VlkECJQVEJZlPY1GgECjAsKy0cYqiwCBsgLCsqyn0QgQaFRAWDbaWGURIFBWQFiW9TQaAQKNCgjLRhurLAIEygoIy7KeRiNAoFEBYdloY5VFgEBZAWFZ1tNoBAg0KiAsG22ssggQKCsgLMt6Go0AgUYFhGWjjVUWAQJlBYRlWU+jESDQqICwbLSxyiJAoKyAsCzraTQCBBoVEJaNNlZZBAiUFRCWZT2NRoBAowLCstHGKosAgbICwrKsp9EIEGhUQFg22lhlESBQVkBYlvU0GgECjQoIy0YbqywCBMoKCMuynkYjQKBRAWHZaGOVRYBAWQFhWdbTaAQINCogLBttrLIIECgrICzLehqNAIFGBYRlo41VFgECZQWEZVlPoxEg0KiAsGy0scoiQKCsgLAs62k0AgQaFRCWjTZWWQQIlBUQlmU9jUaAQKMCwrLRxiqLAIGyAsKyrKfRCBBoVEBYNtpYZREgUFZAWJb1NBoBAo0KCMtGG6ssAgTKCgjLsp5GI0CgUQFh2WhjlUWAQFkBYVnW02gECDQqICwbbayyCBAoKyAsy3oajQCBRgWEZaONVRYBAmUFhGVZT6MRINCogLBstLHKIkCgrICwLOtpNAIEGhUQlo02VlkECJQVEJZlPY1GgECjAsKy0cYqiwCBsgLCsqyn0QgQaFRAWDbaWGURIFBWQFiW9TQaAQKNCgjLRhurLAIEygoIy7KeRiNAoFEBYdloY5VFgEBZAWFZ1tNoBAg0KiAsG22ssggQKCsgLMt6Go0AgUYFhGWjjVUWAQJlBYRlWU+jESDQqICwbLSxyiJAoKyAsCzraTQCBBoVEJaNNlZZBAiUFRCWZT2NRoBAowLCstHGKosAgbICwrKsp9EIEGhUQFg22lhlESBQVkBYlvU0GgECjQoIy0YbqywCBMoKCMuynkYjQKBRAWHZaGOVRYBAWQFhWdbTaAQINCogLBttrLIIECgrICzLehqNAIFGBYRlo41VFgECZQW2Ybl30h2nYf/t/f8uO/oEo+Uack0TLMGUBAg0KNDcyfIX73+8bdOjh0fHDfZLSQQITCSwDcscLClocthMtJ5bTfv99z7cfv6qOz0p32own0yAAIGewJOT5ap7kB7/3nsf9J6OdTevfb3q3o21cqslQGDuAp/IC/zJj390/PVvvrz/nx/930urbtV9+cU/zE+FuP3H7/5Ht1l7t1n6g8dvHR2GWLRFEiAQRuA8LNOKv/GXL7+/OZXdy9+O//Fn/qD7o81/c/5Ia/37o38/DcrNQh+/fXRnzuu1NgIEYgps3t57+uPe6wf7m8C8v+66/fRMOmF+5cVPbS+a02kzf8ud32PdFHK8WncP8vuv2wX7HwECBAoJXArLPO7dNw8Ou3V3P/961re+9Z51eyyOQAsC/w/47/vJh/bmSgAAAABJRU5ErkJggg==)" 361 | ] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "metadata": { 366 | "id": "kgAfWggrdvwa" 367 | }, 368 | "source": [ 369 | "# instantiate model\n", 370 | "o2m_model = Sequential()\n", 371 | "\n", 372 | "# time_steps is one in this case because the input consists of only one entity\n", 373 | "o2m_model.add(SimpleRNN(n_cells, input_shape=(1, features), return_sequences=True))\n", 374 | "\n", 375 | "# TimeDistributed(): multiple outputs at the output layer\n", 376 | "o2m_model.add(TimeDistributed(Dense(n_output, activation=output_activation)))" 377 | ], 378 | "execution_count": 26, 379 | "outputs": [] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "metadata": { 384 | "colab": { 385 | "base_uri": "https://localhost:8080/" 386 | }, 387 | "id": "qhA3xVaVd9av", 388 | "outputId": "04a3ed36-50a2-42e3-eda8-59e09f61ad42" 389 | }, 390 | "source": [ 391 | "o2m_model.summary()" 392 | ], 393 | "execution_count": 27, 394 | "outputs": [ 395 | { 396 | "output_type": "stream", 397 | "text": [ 398 | "Model: \"sequential_9\"\n", 399 | "_________________________________________________________________\n", 400 | "Layer (type) Output Shape Param # \n", 401 | "=================================================================\n", 402 | "simple_rnn_11 (SimpleRNN) (None, 1, 5) 50 \n", 403 | "_________________________________________________________________\n", 404 | "time_distributed_2 (TimeDist (None, 1, 4) 24 \n", 405 | "=================================================================\n", 406 | "Total params: 74\n", 407 | "Trainable params: 74\n", 408 | "Non-trainable params: 0\n", 409 | "_________________________________________________________________\n" 410 | ], 411 | "name": "stdout" 412 | } 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "metadata": { 418 | "id": "X4RG5w1-e8ng" 419 | }, 420 | "source": [ 421 | "#### Let's see how to build LSTM networks in Keras." 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "metadata": { 427 | "id": "4tk7mYa6emx_" 428 | }, 429 | "source": [ 430 | "# instantiate model\n", 431 | "lstm_model = Sequential()\n", 432 | "\n", 433 | "# replace the SimpleRNN() layer with LSTM() layer\n", 434 | "lstm_model.add(LSTM(n_cells, input_shape=(time_steps, features)))\n", 435 | "\n", 436 | "# output layer\n", 437 | "lstm_model.add(Dense(n_output, activation=output_activation))" 438 | ], 439 | "execution_count": 28, 440 | "outputs": [] 441 | }, 442 | { 443 | "cell_type": "code", 444 | "metadata": { 445 | "colab": { 446 | "base_uri": "https://localhost:8080/" 447 | }, 448 | "id": "3v_c2GNOfG3S", 449 | "outputId": "342c48ce-a6f4-43d6-d3b2-9c570d637a5d" 450 | }, 451 | "source": [ 452 | "lstm_model.summary()" 453 | ], 454 | "execution_count": 29, 455 | "outputs": [ 456 | { 457 | "output_type": "stream", 458 | "text": [ 459 | "Model: \"sequential_10\"\n", 460 | "_________________________________________________________________\n", 461 | "Layer (type) Output Shape Param # \n", 462 | "=================================================================\n", 463 | "lstm (LSTM) (None, 5) 200 \n", 464 | "_________________________________________________________________\n", 465 | "dense_9 (Dense) (None, 4) 24 \n", 466 | "=================================================================\n", 467 | "Total params: 224\n", 468 | "Trainable params: 224\n", 469 | "Non-trainable params: 0\n", 470 | "_________________________________________________________________\n" 471 | ], 472 | "name": "stdout" 473 | } 474 | ] 475 | }, 476 | { 477 | "cell_type": "markdown", 478 | "metadata": { 479 | "id": "I4CgVFjWfQAB" 480 | }, 481 | "source": [ 482 | "#### Let's see how to build GRU networks in Keras" 483 | ] 484 | }, 485 | { 486 | "cell_type": "code", 487 | "metadata": { 488 | "id": "uZX5eVEvfOvr" 489 | }, 490 | "source": [ 491 | "# instantiate model\n", 492 | "gru_model = Sequential()\n", 493 | "\n", 494 | "# replace the LSTM() layer with GRU() layer\n", 495 | "gru_model.add(GRU(n_cells, input_shape=(time_steps, features)))\n", 496 | "\n", 497 | "# output layer\n", 498 | "gru_model.add(Dense(n_output, activation=output_activation))" 499 | ], 500 | "execution_count": 30, 501 | "outputs": [] 502 | }, 503 | { 504 | "cell_type": "code", 505 | "metadata": { 506 | "colab": { 507 | "base_uri": "https://localhost:8080/" 508 | }, 509 | "id": "R8zqspSZffSm", 510 | "outputId": "2e6cab05-25c0-4166-e70e-67b8feece3c3" 511 | }, 512 | "source": [ 513 | "gru_model.summary()" 514 | ], 515 | "execution_count": 31, 516 | "outputs": [ 517 | { 518 | "output_type": "stream", 519 | "text": [ 520 | "Model: \"sequential_11\"\n", 521 | "_________________________________________________________________\n", 522 | "Layer (type) Output Shape Param # \n", 523 | "=================================================================\n", 524 | "gru (GRU) (None, 5) 165 \n", 525 | "_________________________________________________________________\n", 526 | "dense_10 (Dense) (None, 4) 24 \n", 527 | "=================================================================\n", 528 | "Total params: 189\n", 529 | "Trainable params: 189\n", 530 | "Non-trainable params: 0\n", 531 | "_________________________________________________________________\n" 532 | ], 533 | "name": "stdout" 534 | } 535 | ] 536 | }, 537 | { 538 | "cell_type": "code", 539 | "metadata": { 540 | "id": "IWCkmuW9fhfZ" 541 | }, 542 | "source": [ 543 | "" 544 | ], 545 | "execution_count": 31, 546 | "outputs": [] 547 | } 548 | ] 549 | } --------------------------------------------------------------------------------