├── .gitignore ├── LICENSE ├── README.md └── understanding-lstm-autoencoder.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | logs/ 2 | 3 | data/ 4 | *.DS_Store 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 cran2367 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # understanding-lstm-autoencoder 2 | Understanding an LSTM Autoencoder 3 | -------------------------------------------------------------------------------- /understanding-lstm-autoencoder.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Learning LSTM Autoencoder and LSTM Network on a simple Multivariate Timeseries Toy example" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 15, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# lstm autoencoder to recreate a timeseries\n", 17 | "import numpy as np\n", 18 | "from keras.models import Sequential\n", 19 | "from keras.layers import LSTM\n", 20 | "from keras.layers import Dense\n", 21 | "from keras.layers import RepeatVector\n", 22 | "from keras.layers import TimeDistributed" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 7, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "'''\n", 32 | "A UDF to convert input data into 3-D\n", 33 | "array as required for LSTM network.\n", 34 | "'''\n", 35 | "\n", 36 | "def temporalize(X, y, lookback):\n", 37 | " output_X = []\n", 38 | " output_y = []\n", 39 | " for i in range(len(X)-lookback-1):\n", 40 | " t = []\n", 41 | " for j in range(1,lookback+1):\n", 42 | " # Gather past records upto the lookback period\n", 43 | " t.append(X[[(i+j+1)], :])\n", 44 | " output_X.append(t)\n", 45 | " output_y.append(y[i+lookback+1])\n", 46 | " return output_X, output_y" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 8, 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "array([[0.1 , 0.001],\n", 58 | " [0.2 , 0.008],\n", 59 | " [0.3 , 0.027],\n", 60 | " [0.4 , 0.064],\n", 61 | " [0.5 , 0.125],\n", 62 | " [0.6 , 0.216],\n", 63 | " [0.7 , 0.343],\n", 64 | " [0.8 , 0.512],\n", 65 | " [0.9 , 0.729]])" 66 | ] 67 | }, 68 | "execution_count": 8, 69 | "metadata": {}, 70 | "output_type": "execute_result" 71 | } 72 | ], 73 | "source": [ 74 | "# define input timeseries\n", 75 | "timeseries = np.array([[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\n", 76 | " [0.1**3, 0.2**3, 0.3**3, 0.4**3, 0.5**3, 0.6**3, 0.7**3, 0.8**3, 0.9**3]]).transpose()\n", 77 | "\n", 78 | "timesteps = timeseries.shape[0]\n", 79 | "n_features = timeseries.shape[1]\n", 80 | "timeseries" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 9, 86 | "metadata": {}, 87 | "outputs": [ 88 | { 89 | "data": { 90 | "text/plain": [ 91 | "array([[[0.3 , 0.027],\n", 92 | " [0.4 , 0.064],\n", 93 | " [0.5 , 0.125]],\n", 94 | "\n", 95 | " [[0.4 , 0.064],\n", 96 | " [0.5 , 0.125],\n", 97 | " [0.6 , 0.216]],\n", 98 | "\n", 99 | " [[0.5 , 0.125],\n", 100 | " [0.6 , 0.216],\n", 101 | " [0.7 , 0.343]],\n", 102 | "\n", 103 | " [[0.6 , 0.216],\n", 104 | " [0.7 , 0.343],\n", 105 | " [0.8 , 0.512]],\n", 106 | "\n", 107 | " [[0.7 , 0.343],\n", 108 | " [0.8 , 0.512],\n", 109 | " [0.9 , 0.729]]])" 110 | ] 111 | }, 112 | "execution_count": 9, 113 | "metadata": {}, 114 | "output_type": "execute_result" 115 | } 116 | ], 117 | "source": [ 118 | "timesteps = 3\n", 119 | "X, y = temporalize(X = timeseries, y = np.zeros(len(timeseries)), lookback = timesteps)\n", 120 | "\n", 121 | "n_features = 2\n", 122 | "X = np.array(X)\n", 123 | "X = X.reshape(X.shape[0], timesteps, n_features)\n", 124 | "\n", 125 | "X" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "## LSTM Autoencoder" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 10, 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "_________________________________________________________________\n", 145 | "Layer (type) Output Shape Param # \n", 146 | "=================================================================\n", 147 | "lstm_5 (LSTM) (None, 3, 128) 67072 \n", 148 | "_________________________________________________________________\n", 149 | "lstm_6 (LSTM) (None, 64) 49408 \n", 150 | "_________________________________________________________________\n", 151 | "repeat_vector_1 (RepeatVecto (None, 3, 64) 0 \n", 152 | "_________________________________________________________________\n", 153 | "lstm_7 (LSTM) (None, 3, 64) 33024 \n", 154 | "_________________________________________________________________\n", 155 | "lstm_8 (LSTM) (None, 3, 128) 98816 \n", 156 | "_________________________________________________________________\n", 157 | "time_distributed_2 (TimeDist (None, 3, 2) 258 \n", 158 | "=================================================================\n", 159 | "Total params: 248,578\n", 160 | "Trainable params: 248,578\n", 161 | "Non-trainable params: 0\n", 162 | "_________________________________________________________________\n" 163 | ] 164 | } 165 | ], 166 | "source": [ 167 | "# define model\n", 168 | "model = Sequential()\n", 169 | "model.add(LSTM(128, activation='relu', input_shape=(timesteps,n_features), return_sequences=True))\n", 170 | "model.add(LSTM(64, activation='relu', return_sequences=False))\n", 171 | "model.add(RepeatVector(timesteps))\n", 172 | "model.add(LSTM(64, activation='relu', return_sequences=True))\n", 173 | "model.add(LSTM(128, activation='relu', return_sequences=True))\n", 174 | "model.add(TimeDistributed(Dense(n_features)))\n", 175 | "model.compile(optimizer='adam', loss='mse')\n", 176 | "model.summary()" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 11, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "name": "stdout", 186 | "output_type": "stream", 187 | "text": [ 188 | "WARNING:tensorflow:From /home/ubuntu/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", 189 | "Instructions for updating:\n", 190 | "Use tf.cast instead.\n", 191 | "---Predicted---\n", 192 | "[[[0.323 0.041]\n", 193 | " [0.423 0.069]\n", 194 | " [0.494 0.121]]\n", 195 | "\n", 196 | " [[0.391 0.069]\n", 197 | " [0.499 0.126]\n", 198 | " [0.587 0.209]]\n", 199 | "\n", 200 | " [[0.491 0.119]\n", 201 | " [0.596 0.216]\n", 202 | " [0.699 0.344]]\n", 203 | "\n", 204 | " [[0.596 0.203]\n", 205 | " [0.693 0.34 ]\n", 206 | " [0.808 0.513]]\n", 207 | "\n", 208 | " [[0.701 0.347]\n", 209 | " [0.798 0.509]\n", 210 | " [0.892 0.723]]]\n", 211 | "---Actual---\n", 212 | "[[[0.3 0.027]\n", 213 | " [0.4 0.064]\n", 214 | " [0.5 0.125]]\n", 215 | "\n", 216 | " [[0.4 0.064]\n", 217 | " [0.5 0.125]\n", 218 | " [0.6 0.216]]\n", 219 | "\n", 220 | " [[0.5 0.125]\n", 221 | " [0.6 0.216]\n", 222 | " [0.7 0.343]]\n", 223 | "\n", 224 | " [[0.6 0.216]\n", 225 | " [0.7 0.343]\n", 226 | " [0.8 0.512]]\n", 227 | "\n", 228 | " [[0.7 0.343]\n", 229 | " [0.8 0.512]\n", 230 | " [0.9 0.729]]]\n" 231 | ] 232 | } 233 | ], 234 | "source": [ 235 | "# fit model\n", 236 | "model.fit(X, X, epochs=300, batch_size=5, verbose=0)\n", 237 | "# demonstrate reconstruction\n", 238 | "yhat = model.predict(X, verbose=0)\n", 239 | "print('---Predicted---')\n", 240 | "print(np.round(yhat,3))\n", 241 | "print('---Actual---')\n", 242 | "print(np.round(X, 3))" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "## Regular LSTM Network" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 12, 255 | "metadata": {}, 256 | "outputs": [ 257 | { 258 | "name": "stdout", 259 | "output_type": "stream", 260 | "text": [ 261 | "_________________________________________________________________\n", 262 | "Layer (type) Output Shape Param # \n", 263 | "=================================================================\n", 264 | "lstm_9 (LSTM) (None, 3, 128) 67072 \n", 265 | "_________________________________________________________________\n", 266 | "lstm_10 (LSTM) (None, 3, 64) 49408 \n", 267 | "_________________________________________________________________\n", 268 | "lstm_11 (LSTM) (None, 3, 64) 33024 \n", 269 | "_________________________________________________________________\n", 270 | "lstm_12 (LSTM) (None, 3, 128) 98816 \n", 271 | "_________________________________________________________________\n", 272 | "time_distributed_3 (TimeDist (None, 3, 2) 258 \n", 273 | "=================================================================\n", 274 | "Total params: 248,578\n", 275 | "Trainable params: 248,578\n", 276 | "Non-trainable params: 0\n", 277 | "_________________________________________________________________\n" 278 | ] 279 | } 280 | ], 281 | "source": [ 282 | "# define model\n", 283 | "model = Sequential()\n", 284 | "model.add(LSTM(128, activation='relu', input_shape=(timesteps,n_features), return_sequences=True))\n", 285 | "model.add(LSTM(64, activation='relu', return_sequences=True))\n", 286 | "model.add(LSTM(64, activation='relu', return_sequences=True))\n", 287 | "model.add(LSTM(128, activation='relu', return_sequences=True))\n", 288 | "model.add(TimeDistributed(Dense(n_features)))\n", 289 | "model.compile(optimizer='adam', loss='mse')\n", 290 | "model.summary()" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": 14, 296 | "metadata": {}, 297 | "outputs": [ 298 | { 299 | "name": "stdout", 300 | "output_type": "stream", 301 | "text": [ 302 | "---Predicted---\n", 303 | "[[[0.306 0.026]\n", 304 | " [0.399 0.064]\n", 305 | " [0.5 0.124]]\n", 306 | "\n", 307 | " [[0.393 0.064]\n", 308 | " [0.502 0.124]\n", 309 | " [0.599 0.215]]\n", 310 | "\n", 311 | " [[0.502 0.126]\n", 312 | " [0.6 0.214]\n", 313 | " [0.7 0.343]]\n", 314 | "\n", 315 | " [[0.596 0.219]\n", 316 | " [0.699 0.344]\n", 317 | " [0.798 0.51 ]]\n", 318 | "\n", 319 | " [[0.703 0.34 ]\n", 320 | " [0.8 0.512]\n", 321 | " [0.899 0.727]]]\n", 322 | "---Actual---\n", 323 | "[[[0.3 0.027]\n", 324 | " [0.4 0.064]\n", 325 | " [0.5 0.125]]\n", 326 | "\n", 327 | " [[0.4 0.064]\n", 328 | " [0.5 0.125]\n", 329 | " [0.6 0.216]]\n", 330 | "\n", 331 | " [[0.5 0.125]\n", 332 | " [0.6 0.216]\n", 333 | " [0.7 0.343]]\n", 334 | "\n", 335 | " [[0.6 0.216]\n", 336 | " [0.7 0.343]\n", 337 | " [0.8 0.512]]\n", 338 | "\n", 339 | " [[0.7 0.343]\n", 340 | " [0.8 0.512]\n", 341 | " [0.9 0.729]]]\n" 342 | ] 343 | } 344 | ], 345 | "source": [ 346 | "# fit model\n", 347 | "model.fit(X, X, epochs=300, batch_size=5, verbose=0)\n", 348 | "# demonstrate reconstruction\n", 349 | "yhat = model.predict(X, verbose=0)\n", 350 | "print('---Predicted---')\n", 351 | "print(np.round(yhat,3))\n", 352 | "print('---Actual---')\n", 353 | "print(np.round(X, 3))" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [] 362 | } 363 | ], 364 | "metadata": { 365 | "kernelspec": { 366 | "display_name": "Python 3", 367 | "language": "python", 368 | "name": "python3" 369 | }, 370 | "language_info": { 371 | "codemirror_mode": { 372 | "name": "ipython", 373 | "version": 3 374 | }, 375 | "file_extension": ".py", 376 | "mimetype": "text/x-python", 377 | "name": "python", 378 | "nbconvert_exporter": "python", 379 | "pygments_lexer": "ipython3", 380 | "version": "3.7.1" 381 | } 382 | }, 383 | "nbformat": 4, 384 | "nbformat_minor": 2 385 | } 386 | --------------------------------------------------------------------------------