├── .gitignore ├── LICENSE ├── README.md ├── code.ipynb ├── images ├── pydata-annarbor-2017.jpg └── pydata-annarbor-aug.jpg └── pydata-annarbor2017-dl-tutorial.pdf /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Sebastian Raschka 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction to Deep Learning with TensorFlow 2 | 3 | Sebastian Raschka, 2017 4 | 5 | Code snippets for "Introduction to Deep Learning with TensorFlow" at PyData Ann Arbor Aug 2017 6 | 7 | Slides: https://speakerdeck.com/rasbt/introduction-to-deep-learning-with-tensorflow-at-pydata-ann-arbor 8 | 9 | 10 | 11 | - [Link to the recording](https://www.youtube.com/watch?v=vRF7ENlwD50&feature=youtu.be) 12 | 13 | [](https://www.youtube.com/watch?v=vRF7ENlwD50&feature=youtu.be) -------------------------------------------------------------------------------- /code.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Code Snippets for Intro to TensorFlow Talk @ PyData Ann Arbor Aug 2017\n", 8 | "\n", 9 | "\n", 10 | "GitHub Repo: https://github.com/rasbt/pydata-annarbor2017-dl-tutorial" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "outputs": [ 18 | { 19 | "name": "stdout", 20 | "output_type": "stream", 21 | "text": [ 22 | "Sebastian Raschka \n", 23 | "\n", 24 | "CPython 3.6.1\n", 25 | "IPython 6.1.0\n", 26 | "\n", 27 | "tensorflow 1.3.0\n", 28 | "numpy 1.12.1\n" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "%load_ext watermark\n", 34 | "%watermark -a 'Sebastian Raschka' -v -p tensorflow,numpy" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "## Vectorization" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 2, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "import numpy as np\n", 51 | "\n", 52 | "np.random.seed(123)\n", 53 | "\n", 54 | "\n", 55 | "num_train_examles = 150\n", 56 | "num_features = 50\n", 57 | "num_hidden = 20\n", 58 | "\n", 59 | "X = np.random.random((num_train_examles, num_features)) \n", 60 | "W = np.random.random((num_features, num_hidden))" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "logits = np.zeros([num_train_examles, num_hidden])\n", 70 | "\n", 71 | "for i, row in enumerate(X): # row = training_example\n", 72 | " \n", 73 | " for j, col in enumerate(W.T): # col = features\n", 74 | " \n", 75 | " vector_dot_product = 0\n", 76 | " for a, b in zip(row, col):\n", 77 | " vector_dot_product += a*b\n", 78 | " \n", 79 | " logits[i, j] = vector_dot_product" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 4, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "data": { 89 | "text/plain": [ 90 | "True" 91 | ] 92 | }, 93 | "execution_count": 4, 94 | "metadata": {}, 95 | "output_type": "execute_result" 96 | } 97 | ], 98 | "source": [ 99 | "np.allclose(logits, np.dot(X, W))" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "## Broadcasting" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 5, 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "(150, 20)\n", 119 | "(150, 20)\n" 120 | ] 121 | } 122 | ], 123 | "source": [ 124 | "X = np.random.random((num_train_examles, num_features)) \n", 125 | "W = np.random.random((num_features, num_hidden))\n", 126 | "b = np.random.random(num_hidden)\n", 127 | "\n", 128 | "print(np.dot(X, W).shape)\n", 129 | "print((np.dot(X, W) + b).shape)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "## Computation Graphs" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 6, 142 | "metadata": { 143 | "collapsed": true 144 | }, 145 | "outputs": [], 146 | "source": [ 147 | "%matplotlib inline\n", 148 | "import matplotlib.pyplot as plt" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 7, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "data": { 158 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG31JREFUeJzt3Xl81NW9//HXh5AQ9jXs+74oEIwIqHXBuuBaV/B6q1Uv\n1T5kURG3FrXe3rpUi1Jba6229WKQTUXrvteNCiRh3/c9bCEEQrbz+yPT+0OaMAOZmTPL+/l45MEk\n803m/TjDvOfkzJx8zTmHiIjEj1q+A4iIyPFRcYuIxBkVt4hInFFxi4jEGRW3iEicUXGLiMQZFbeI\nSJxRcYuIxBkVt4hInKkdiR/aokUL17lz50j8aBGRhDR//vxdzrmMUI6NSHF37tyZefPmReJHi4gk\nJDPbEOqxWioREYkzKm4RkTij4hYRiTMqbhGROKPiFhGJMypuEZE4o+IWEYkzKm4RkTD4du1u/vzl\nOqJxOkgVt4hIDe0tKmH8tFymfruBQ6XlEb89FbeISA0457hn5kL2FJXw7KhM6qVFZEP696i4RURq\n4G/fbOCjZTu496LenNSucVRuU8UtInKClm7dz6/eWca5vVty8+mdo3a7Km4RkRNwsKSMMdkLaFI3\nlSev7o+ZRe22I78YIyKSgB6Zs5S1u4qYestpNG9QJ6q3rRm3iMhxeitvK6/N28TPzu7GsO4ton77\nKm4RkeOwac9BHpi9iEEdmzD+vJ5eMqi4RURCVFpewZjsHDB4ZmQmqSl+KlRr3CIiIXr6w5XkbtrH\nc9cPokOzet5yaMYtIhKCL1ft4vnP1zBqcAcu7t/GaxYVt4hIELsOHObO6bl0y2jApEv6+Y6jpRIR\nkWOpqHBMmJFHwaFSXrllMHXTUnxH0oxbRORYXvpqHZ+tyOcXF/ehd+tGvuMAKm4RkWot2lzA4+8t\n5/y+rbhhSCffcf6PiltEpAoHDlduaW/RoA5PRHlLezBa4xYRqcKkNxazcc9Bpo0eSpN6ab7jfI9m\n3CIiR5m9YDOzc7YwdngPBndp5jvOv1Fxi4gcYd2uIn7xxmIGd2nGmHN7+I5TpZCK28zuNLMlZrbY\nzLLNLD3SwUREoq2krIKx2TnUTqnF5OsGklIrdta1jxS0uM2sHTAWyHLOnQSkACMjHUxEJNqeeG85\ni7YU8OTV/WnbpK7vONUKdamkNlDXzGoD9YCtkYskIhJ9n67YyYtfruPHQztxfr/WvuMcU9Dids5t\nAX4DbAS2AQXOuQ+OPs7MRpvZPDObl5+fH/6kIiIRsnN/MROm59G7dUMeGNHHd5ygQlkqaQpcDnQB\n2gL1zeyGo49zzr3gnMtyzmVlZGSEP6mISARUVDjump5HUUkZv7s+k/RU/1vagwllqeQ8YJ1zLt85\nVwrMBoZFNpaISHT88Yu1fLl6Fw9f2o/uLRv6jhOSUIp7IzDEzOpZ5dah4cCyyMYSEYm8nI17eeqD\nFVzcvw3XndrBd5yQhbLGPReYCSwAFgW+54UI5xIRiaj9xaWMyc6hVaN0/udHJ8fUlvZgQtry7px7\nCHgowllERKLCOccDsxexraCY6T8dSuO6qb4jHRftnBSRpDNj3mbeXriNu37Yk1M6NfUd57ipuEUk\nqazeWchDc5YwrFtzbjurm+84J0TFLSJJo7i0nDtezaFuWgq/jeEt7cHoz7qKSNJ47N3lLN9eyMs3\nnUqrRvH7J5c04xaRpPDh0h385ev13HJGF87p3dJ3nBpRcYtIwttWcIh7ZubRr20jJl7Yy3ecGlNx\ni0hCK69wjJ+WS0lZBVNGZVKnduxvaQ9Ga9wiktCe+3Q1c9ft4alrBtA1o4HvOGGhGbeIJKzv1u9h\n8kcruWJgW64c1M53nLBRcYtIQtp3sIRx2Tl0aFaP/46zLe3BaKlERBKOc477Zi1iZ+FhZv9sGA3q\nJFbVacYtIgln6tyNvLdkOxMv7EX/9k18xwk7FbeIJJQV2wt59O2l/KBnBree0dV3nIhQcYtIwjhU\nUs6Y7AU0TE/lqWsGUCtOt7QHk1gLPyKS1B79+1JW7jjAK7cMJqNhHd9xIkYzbhFJCO8u2sarczfy\n07O6cmaPxD7vrYpbROLe5r0HuXfWQgZ0aMKE8+N/S3swKm4RiWtl5RWMm5ZLhYMpIzNJTUn8WtMa\nt4jEtWc+XsX8DXt5ZuRAOjav5ztOVCT+U5OIJKyv1+zid5+u5ppT2nP5wMTZ0h6MiltE4tKeohLu\nfC2XLi3q88jl/XzHiSoVt4jEHecc98zIY29RKVNGZVIvLblWfVXcIhJ3/vL1ej5evpP7R/SmX9vG\nvuNEnYpbROLK4i0F/Pqd5Qzv3ZKbhnX2HccLFbeIxI2iw2WMzc6haf1UnrxmQEL9qdbjkVwLQyIS\n1x6es4R1u4uYeutpNKuf5juON5pxi0hceDN3CzPmb+aOc7ozrFsL33G8UnGLSMzbsLuIB19fzCmd\nmjJueA/fcbxTcYtITCspq2Bsdg61DJ4ZOZDaSbClPRitcYtITHvqwxXkbS7g9/8xiPZNk2NLezB6\n6hKRmPXFynz++PlaRg3uyIiT2/iOEzNU3CISk/ILD3PX9Dx6tmrApEv6+o4TU7RUIiIxp6LCcfeM\nPAqLS5l662nUTUvxHSmmaMYtIjHnxS/X8sXKfH5+SV96tW7oO07MCam4zayJmc00s+VmtszMhkY6\nmIgkp7xN+3jivRVc0K8VN5zW0XecmBTqUskzwHvOuavNLA3QS7siEnaFxaWMyc6hZcM6PH5V/6Td\n0h5M0OI2s8bAD4CbAJxzJUBJZGOJSLJxzvHzNxazee9Bpo0eSpN6ybulPZhQlkq6APnAy2aWY2Yv\nmln9COcSkSQza8EW3szdyrjhPRncpZnvODEtlOKuDQwC/uCcywSKgPuOPsjMRpvZPDObl5+fH+aY\nIpLI1uYfYNKbixncpRl3nNvdd5yYF0pxbwY2O+fmBj6fSWWRf49z7gXnXJZzLisjIyOcGUUkgR0u\nK2dMdg5ptWvxzMiBpNTSunYwQYvbObcd2GRmvQJfGg4sjWgqEUkaj7+7giVb9/PEVf1p07iu7zhx\nIdR3lYwBpgbeUbIW+EnkIolIsvhk+Q5e+modNw7txPn9WvuOEzdCKm7nXC6QFeEsIpJEduwvZsKM\nhfRu3ZD7R/TxHSeuaOekiERdeYVj/LRcDpWU87vrM0lP1Zb246G/VSIiUff852v4Zu1uHr/qZLq3\n1Jb246UZt4hE1fwNe3j6w5Vc0r8N12Z18B0nLqm4RSRqCg6VMjY7lzaN0/mfK0/WlvYTpKUSEYkK\n5xwPzF7E9v3FzLhtKI3SU31HiluacYtIVEz7bhN/X7SNu8/vyaCOTX3HiWsqbhGJuFU7CnnkrSWc\n0b0Ft/2gm+84cU/FLSIRVVxauaW9flptnr52ALW0pb3GtMYtIhH1q78vY/n2Ql7+yam0bJTuO05C\n0IxbRCLmvcXbeeXbDdx6RhfO6dXSd5yEoeIWkYjYuu8Q985ayMntGjPxwt6+4yQUFbeIhF1ZeQXj\np+VSVl7Bs6MySautqgknrXGLSNhN+WQ1/1y/h6evHUCXFjphVrjpaVBEwurbtbuZ8skqrsxsx5WD\n2vuOk5BU3CISNnuLSrjztVw6NqvHL684yXechKWlEhEJC+ccE2ctZNeBw8y+/XQa1FG9RIpm3CIS\nFq98u4EPl+7g3gt7c3L7xr7jJDQVt4jU2LJt+/nvvy/j7F4Z3Hx6F99xEp6KW0Rq5GBJGWOyc2hc\nN5XfXKMt7dGgRSgRqZFfvrWUNfkHeOXm02jRoI7vOElBM24ROWFvL9zKtO82cdtZ3TijRwvfcZKG\niltETsimPQe5f/YiBnZowl0/7Ok7TlJRcYvIcSstr2DstBxwMGVUJqkpqpJo0hq3iBy3yR+tJGfj\nPqaMyqRDs3q+4yQdPU2KyHH5evUufv/ZGq7L6sClA9r6jpOUVNwiErLdBw4z/rVcuraoz0OX9fUd\nJ2lpqUREQuKcY8KMPPYdKuUvPxlMvTTVhy+acYtISF76aj2frsjnwRF96Nu2ke84SU3FLSJBLd5S\nwGPvLuO8Pq348dBOvuMkPRW3iBxT0eHKLe3N69fhyav7Y6Yt7b5pkUpEjmnSm0vYsLuIV/9rCE3r\np/mOI2jGLSLH8EbOFmYt2Mwd5/ZgSNfmvuNIgIpbRKq0YXcRD76+iFM7N2Xsud19x5EjqLhF5N+U\nlFUwJjuH2im1mDwyk9ra0h5TtMYtIv/mNx+sYOHmAp6/4RTaNanrO44cJeSnUTNLMbMcM3s7koFE\nxK/PV+bzwhdruWFIRy48qbXvOFKF4/n9ZxywLFJBRMS/nYXF3D09l16tGvLzi7WlPVaFVNxm1h64\nGHgxsnFExJeKCsfd0/M4cLiMKddnkp6a4juSVCPUGfdkYCJQEcEsIuLRn/6xln+s2sWkS/rRs1VD\n33HkGIIWt5ldAux0zs0PctxoM5tnZvPy8/PDFlBEIi930z6efH8FF53UmlGDO/iOI0GEMuM+HbjM\nzNYD04Bzzex/jz7IOfeCcy7LOZeVkZER5pgiEimFxaWMzc6hVaN0HrtSW9rjQdDids7d75xr75zr\nDIwEPnHO3RDxZCIScc45Hnx9MVv2HeLZUQNpXC/VdyQJgd5VL5LEZs7fzJy8rdx5Xg9O6dTMdxwJ\n0XFtwHHOfQZ8FpEkIhJVa/IPMOnNJQzp2ozbz9aW9niiGbdIEjpcVs6YV3NIT63F5OsySamlde14\noi3vIknosXeXs3Tbfv58YxatG6f7jiPHSTNukSTz8bIdvPzVem4a1pnhfVr5jiMnQMUtkkS2FxQz\nYUYefds04v4RvX3HkROk4hZJEuUVjvGv5VBcWsGU6zOpU1tb2uOV1rhFksQfPlvNt2v38MTV/emW\n0cB3HKkBzbhFksD8DXv47UeruGxAW645pb3vOFJDKm6RBFdwsJSx2bm0a1KXX/3oJG1pTwBaKhFJ\nYM457pu9kB37i5l5+zAapmtLeyLQjFskgWX/cxPvLt7OhAt6MbBDE99xJExU3CIJauWOQh55awln\n9mjB6DO7+o4jYaTiFklAxaXl3PHqAhqm1+apawdQS1vaE4rWuEUS0KNvL2XljgP89ebBtGyoLe2J\nRjNukQTz3uJtTJ27kdE/6MpZPXVSk0Sk4hZJIFv2HWLizIUMaN+YCef38h1HIkTFLZIgysorGJed\nQ4WDZ0dlklZbD+9EpTVukQTx7MermLdhL5OvG0in5vV9x5EI0lOySAL4Zs1upny6mqsGteeKzHa+\n40iEqbhF4tzeohLufC2Xzs3r88vL+/mOI1GgpRKROOac456ZeewuOszrN55O/Tp6SCcDzbhF4tjf\nvtnAR8t2ct9FfTipXWPfcSRKVNwicWrp1v386p1lnNu7JTef3tl3HIkiFbdIHDpYUsYd2QtoUjeV\nJ6/urz/VmmS0ICYShx6Zs5R1u4qYestpNG9Qx3cciTLNuEXizFt5W3lt3iZ+dnY3hnVv4TuOeKDi\nFokjm/Yc5IHZixjUsQnjz+vpO454ouIWiROl5RWMyc4Bg2dGZpKaoodvstIat0icePrDleRu2sdz\n1w+iQ7N6vuOIR3rKFokDX67axfOfr2HU4A5c3L+N7zjimYpbJMbtOnCYO6fn0j2jAZMu0ZZ20VKJ\nSEyrqHDcPT2PgkOlvHLLYOqmpfiOJDFAM26RGPbSV+v4fGU+v7i4D71bN/IdR2KEilskRi3aXMDj\n7y3ngn6tuGFIJ99xJIaouEVi0IHDZYzJXkBGgzo8fpW2tMv3aY1bJAZNemMxG/ccZNrooTSpl+Y7\njsSYoDNuM+tgZp+a2VIzW2Jm46IRTCRZzV6wmdk5Wxg7vAeDuzTzHUdiUCgz7jLgbufcAjNrCMw3\nsw+dc0sjnE0k6azbVcTP31jM4C7NGHNuD99xJEYFnXE757Y55xYELhcCywCd1E4kzErKKhiTvYC0\n2rV4ZuRAUmppXVuqdlxr3GbWGcgE5kYijEgye+K95Szesp8X/vMU2jSu6zuOxLCQ31ViZg2AWcB4\n59z+Kq4fbWbzzGxefn5+ODOKJLxPV+zkxS/X8eOhnTi/X2vfcSTGhVTcZpZKZWlPdc7NruoY59wL\nzrks51xWRkZGODOKJLSd+4uZMD2P3q0b8sCIPr7jSBwIulRilW8g/TOwzDn3dOQjiSSPigrHndNz\nKSop47Xrh5Ceqi3tElwoM+7Tgf8EzjWz3MDHiAjnEkkKz3+xhq9W7+bhS/vRvWVD33EkTgSdcTvn\nvgT08rZImC3YuJenPljJxf3bcN2pHXzHkTiiLe8iHuwvLmVsdg5tGqfz6ytP1pZ2OS7a8i4SZc45\nHpi9iG0Fxcy4bSiN0lN9R5I4oxm3SJRNn7eJtxdu464f9mRQx6a+40gcUnGLRNHqnYU8PGcpp3dv\nzu1ndfMdR+KUilskSopLy7nj1RzqpqXw22sHUktb2uUEaY1bJEp+/c4ylm8v5OWbTqVlo3TfcSSO\nacYtEgUfLNnOX7/ZwC1ndOGc3i19x5E4p+IWibBtBYeYOGshJ7VrxMQLe/mOIwlAxS0SQeUVjvHT\ncikpq+DZkZnUqa0t7VJzWuMWiaDffbKauev28NQ1A+ia0cB3HEkQmnGLRMh36/fwzMcr+VFmO646\npb3vOJJAVNwiEbDvYAnjsnPo2Kwej15xku84kmC0VCISZs457p21kPwDh5l1+zAa1NHDTMJLM26R\nMPvfuRt5f8kOJl7Qm/7tm/iOIwlIxS0SRsu37+fRt5dyVs8Mbjmji+84kqBU3CJhcqiknDGv5tAo\nPZXfXDNAW9olYrT4JhImv3x7Kat2HuCVWwaT0bCO7ziSwDTjFgmDdxZtI/ufG7ntrG6c2UMny5bI\nUnGL1NDmvQe5b9ZCBnRowt3n9/QdR5KAilukBsrKKxg3LRfnYMrITFJT9JCSyNMat0gNTP5oFfM3\n7OXZUZl0bF7PdxxJEpoeiJygr9fs4rnPVnNtVnsuG9DWdxxJIipukROwp6iEO1/LpUuL+jx8WT/f\ncSTJqLhFjpNzjntm5LG3qJQpozKpl6YVR4kuFbfIcXr5q/V8vHwnD4zoTb+2jX3HkSSk4hY5Dou3\nFPDYu8s5r09LbhzW2XccSVIqbpEQFR0uY2x2Dk3rp/LE1QMw05Z28UOLcyIhemjOEtbtLuLVW4fQ\nrH6a7ziSxDTjFgnBm7lbmDl/M2PO6c7Qbs19x5Ekp+IWCWLD7iIefH0xWZ2aMnZ4D99xRFTcIsdS\nUlbB2OwcahlMHjmQ2trSLjFAa9wix/DUByvI21zAH/5jEO2baku7xAZNH0Sq8cXKfP74xVquP60j\nF53cxncckf+j4hapQn7hYe6ankfPVg2YdElf33FEvkdLJSJHqahw3DU9l8LiUl79r9NIT03xHUnk\ne0KacZvZhWa2wsxWm9l9kQ4l4tOf/rGWf6zaxaRL+9KzVUPfcUT+TdDiNrMU4DngIqAvMMrM9Luj\nJKS8Tft48v0VXHRSa64f3NF3HJEqhTLjHgysds6tdc6VANOAyyMbSyT6CotLGZOdQ6tG6Tx2ZX9t\naZeYFcoadztg0xGfbwZOi0SYS6d8SXFpeSR+tEhQhcVl7CwsZvpPh9K4XqrvOCLVCtuLk2Y2GhgN\n0LHjif2K2S2jPiXlFeGKJHLcLujXmqzOzXzHEDmmUIp7C9DhiM/bB772Pc65F4AXALKystyJhJk8\nMvNEvk1EJKmEssb9HdDDzLqYWRowEpgT2VgiIlKdoDNu51yZmd0BvA+kAC8555ZEPJmIiFQppDVu\n59w7wDsRziIiIiHQlncRkTij4hYRiTMqbhGROKPiFhGJMypuEZE4Y86d0F6ZY/9Qs3xgwwl+ewtg\nVxjjhJvy1Yzy1Yzy1Uws5+vknMsI5cCIFHdNmNk851yW7xzVUb6aUb6aUb6aifV8odJSiYhInFFx\ni4jEmVgs7hd8BwhC+WpG+WpG+Wom1vOFJObWuEVE5NhiccYtIiLH4KW4zewaM1tiZhVmlnXUdfcH\nTkq8wswuqOb7m5nZh2a2KvBv0whmfc3McgMf680st5rj1pvZosBx8yKVp4rbfdjMthyRcUQ1x3k5\n4bOZPWlmy81soZm9bmZNqjkuquMXbDys0rOB6xea2aBIZzritjuY2admtjTwOBlXxTFnm1nBEff7\npGjlC9z+Me8vz+PX64hxyTWz/WY2/qhjvI5fjTnnov4B9AF6AZ8BWUd8vS+QB9QBugBrgJQqvv8J\n4L7A5fuAx6OU+ylgUjXXrQdaeBjLh4EJQY5JCYxlVyAtMMZ9o5TvfKB24PLj1d1X0Ry/UMYDGAG8\nCxgwBJgbxfu0DTAocLkhsLKKfGcDb0f7/1uo95fP8avivt5O5XukY2b8avrhZcbtnFvmnFtRxVWX\nA9Occ4edc+uA1VSerLiq4/4auPxX4IrIJP3/rPLMsdcC2ZG+rQjwdsJn59wHzrmywKffUnkGJd9C\nGY/Lgb+5St8CTcysTTTCOee2OecWBC4XAsuoPPdrPPE2fkcZDqxxzp3ohsCYFGtr3FWdmLiq/7Ct\nnHPbApe3A60iHQw4E9jhnFtVzfUO+MjM5gfOvxlNYwK/jr5UzbJRqOMaaTdTOQurSjTHL5TxiIkx\nM7POQCYwt4qrhwXu93fNrF9UgwW/v2Ji/Kg8Y1d1ky2f41cjYTtZ8NHM7COgdRVXPeicezNct+Oc\nc2ZWo7fGhJh1FMeebZ/hnNtiZi2BD81suXPui5rkCiUf8AfgUSofSI9SuZxzczhuN1ShjJ+ZPQiU\nAVOr+TERG794ZWYNgFnAeOfc/qOuXgB0dM4dCLyu8QbQI4rxYv7+sspTLV4G3F/F1b7Hr0YiVtzO\nufNO4NtCOjExsMPM2jjntgV+/dp5Ihn/JVhWM6sNXAmccoyfsSXw704ze53KX8fD8h851LE0sz8B\nb1dxVajjekJCGL+bgEuA4S6wwFjFz4jY+FUhlPGI6JgFY2apVJb2VOfc7KOvP7LInXPvmNnvzayF\ncy4qf4cjhPvL6/gFXAQscM7tOPoK3+NXU7G2VDIHGGlmdcysC5XPgP+s5rgbA5dvBMI2g6/GecBy\n59zmqq40s/pm1vBfl6l8QW5xhDP967aPXDf8UTW36+2Ez2Z2ITARuMw5d7CaY6I9fqGMxxzgx4F3\nRwwBCo5YnouowOspfwaWOeeeruaY1oHjMLPBVD6Wd0cpXyj3l7fxO0K1vyX7HL+w8PGKKJUFsxk4\nDOwA3j/iugepfMV/BXDREV9/kcA7UIDmwMfAKuAjoFmE8/4FuO2or7UF3glc7krlOxPygCVULhFE\nayxfARYBC6l8sLQ5Ol/g8xFUvjthTZTzraZyrTM38PF8LIxfVeMB3Pav+5nKd0M8F7h+EUe8+ykK\n2c6gculr4RHjNuKofHcExiqPyhd9h0UxX5X3V6yMX+D261NZxI2P+FpMjF84PrRzUkQkzsTaUomI\niASh4hYRiTMqbhGROKPiFhGJMypuEZE4o+IWEYkzKm4RkTij4hYRiTP/D1hk/WpwEK2OAAAAAElF\nTkSuQmCC\n", 159 | "text/plain": [ 160 | "" 161 | ] 162 | }, 163 | "metadata": {}, 164 | "output_type": "display_data" 165 | } 166 | ], 167 | "source": [ 168 | "def relu(x):\n", 169 | " return x * (x > 0)\n", 170 | "\n", 171 | "x = np.arange(-10, 10)\n", 172 | "plt.plot(x, relu(x))\n", 173 | "plt.savefig('relu.pdf')\n", 174 | "plt.show()" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 8, 180 | "metadata": {}, 181 | "outputs": [ 182 | { 183 | "name": "stdout", 184 | "output_type": "stream", 185 | "text": [ 186 | "Tensor(\"x:0\", dtype=float32) Tensor(\"mul:0\", dtype=float32) Tensor(\"add:0\", dtype=float32) Tensor(\"Relu:0\", dtype=float32)\n" 187 | ] 188 | } 189 | ], 190 | "source": [ 191 | "import tensorflow as tf\n", 192 | "\n", 193 | "g = tf.Graph()\n", 194 | "with g.as_default() as g:\n", 195 | " \n", 196 | " x = tf.placeholder(dtype=tf.float32, shape=None, name='x')\n", 197 | " w = tf.Variable(initial_value=2, dtype=tf.float32, name='w')\n", 198 | " b = tf.Variable(initial_value=1, dtype=tf.float32, name='b')\n", 199 | " \n", 200 | " u = x * w\n", 201 | " v = u + b\n", 202 | " a = tf.nn.relu(v)\n", 203 | " \n", 204 | " init_op = tf.global_variables_initializer()\n", 205 | " \n", 206 | "print(x, w, b, u, v, a)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 9, 212 | "metadata": {}, 213 | "outputs": [ 214 | { 215 | "name": "stdout", 216 | "output_type": "stream", 217 | "text": [ 218 | "1.0\n" 219 | ] 220 | } 221 | ], 222 | "source": [ 223 | "with tf.Session(graph=g) as sess:\n", 224 | " sess.run(init_op)\n", 225 | " b_res = sess.run('b:0')\n", 226 | "\n", 227 | "print(b_res)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 10, 233 | "metadata": {}, 234 | "outputs": [ 235 | { 236 | "name": "stdout", 237 | "output_type": "stream", 238 | "text": [ 239 | "6.0 7.0 7.0\n" 240 | ] 241 | } 242 | ], 243 | "source": [ 244 | "with tf.Session(graph=g) as sess:\n", 245 | " sess.run(init_op)\n", 246 | " u_res, v_res, a_res = sess.run([u, v, a], feed_dict={'x:0': 3})\n", 247 | "\n", 248 | "print(u_res, v_res, a_res)" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": {}, 254 | "source": [ 255 | "### TensorBoard" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 11, 261 | "metadata": { 262 | "collapsed": true 263 | }, 264 | "outputs": [], 265 | "source": [ 266 | "with tf.Session(graph=g) as sess:\n", 267 | " \n", 268 | " sess.run(init_op)\n", 269 | " file_writer = tf.summary.FileWriter(logdir='logs/graph-1', graph=g)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "markdown", 274 | "metadata": {}, 275 | "source": [ 276 | "In your terminal\n", 277 | "\n", 278 | "```bash\n", 279 | "$ pip install tensorboard \n", 280 | "$ tensorboard --logdir logs/graph-1\n", 281 | "```" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": {}, 287 | "source": [ 288 | "### Gradients" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 12, 294 | "metadata": {}, 295 | "outputs": [ 296 | { 297 | "name": "stdout", 298 | "output_type": "stream", 299 | "text": [ 300 | "[3.0] [1.0]\n" 301 | ] 302 | } 303 | ], 304 | "source": [ 305 | "with g.as_default() as g:\n", 306 | " d_a_w = tf.gradients(a, w)\n", 307 | " d_b_w = tf.gradients(a, b)\n", 308 | " \n", 309 | " \n", 310 | "with tf.Session(graph=g) as sess:\n", 311 | " sess.run(init_op)\n", 312 | " dw, db = sess.run([d_a_w, d_b_w], feed_dict={'x:0': 3})\n", 313 | "\n", 314 | "print(dw, db)" 315 | ] 316 | }, 317 | { 318 | "cell_type": "markdown", 319 | "metadata": {}, 320 | "source": [ 321 | "# Multilayer Perceptrons" 322 | ] 323 | }, 324 | { 325 | "cell_type": "markdown", 326 | "metadata": {}, 327 | "source": [ 328 | "## Gradients by hand" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": 1, 334 | "metadata": {}, 335 | "outputs": [ 336 | { 337 | "name": "stdout", 338 | "output_type": "stream", 339 | "text": [ 340 | "Extracting ./train-images-idx3-ubyte.gz\n", 341 | "Extracting ./train-labels-idx1-ubyte.gz\n", 342 | "Extracting ./t10k-images-idx3-ubyte.gz\n", 343 | "Extracting ./t10k-labels-idx1-ubyte.gz\n", 344 | "Epoch: 001 | AvgCost: 0.786 | Train/Valid ACC: 0.886/0.895\n", 345 | "Epoch: 002 | AvgCost: 0.370 | Train/Valid ACC: 0.905/0.913\n", 346 | "Epoch: 003 | AvgCost: 0.317 | Train/Valid ACC: 0.912/0.919\n", 347 | "Epoch: 004 | AvgCost: 0.289 | Train/Valid ACC: 0.922/0.926\n", 348 | "Epoch: 005 | AvgCost: 0.268 | Train/Valid ACC: 0.928/0.932\n", 349 | "Epoch: 006 | AvgCost: 0.251 | Train/Valid ACC: 0.933/0.936\n", 350 | "Epoch: 007 | AvgCost: 0.235 | Train/Valid ACC: 0.936/0.936\n", 351 | "Epoch: 008 | AvgCost: 0.222 | Train/Valid ACC: 0.939/0.941\n", 352 | "Epoch: 009 | AvgCost: 0.210 | Train/Valid ACC: 0.944/0.944\n", 353 | "Epoch: 010 | AvgCost: 0.198 | Train/Valid ACC: 0.946/0.947\n", 354 | "Test ACC: 0.943\n" 355 | ] 356 | } 357 | ], 358 | "source": [ 359 | "import tensorflow as tf\n", 360 | "import numpy as np\n", 361 | "from tensorflow.examples.tutorials.mnist import input_data\n", 362 | "\n", 363 | "\n", 364 | "# Dataset\n", 365 | "mnist = input_data.read_data_sets(\"./\", one_hot=True)\n", 366 | "np.random.seed(123) # set seed for mnist shuffling\n", 367 | "\n", 368 | "##########################\n", 369 | "### SETTINGS\n", 370 | "##########################\n", 371 | "\n", 372 | "# Hyperparameters\n", 373 | "learning_rate = 0.1\n", 374 | "training_epochs = 10\n", 375 | "batch_size = 64\n", 376 | "\n", 377 | "# Architecture\n", 378 | "n_hidden_1 = 128\n", 379 | "n_input = 784\n", 380 | "n_classes = 10\n", 381 | "\n", 382 | "\n", 383 | "##########################\n", 384 | "### GRAPH DEFINITION\n", 385 | "##########################\n", 386 | "\n", 387 | "g = tf.Graph()\n", 388 | "with g.as_default():\n", 389 | " \n", 390 | " tf.set_random_seed(123)\n", 391 | "\n", 392 | " # Input data\n", 393 | " tf_x = tf.placeholder(tf.float32, [None, n_input], name='features')\n", 394 | " tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n", 395 | "\n", 396 | " # Model parameters\n", 397 | " weights = {\n", 398 | " 'h1': tf.Variable(tf.truncated_normal([n_input, n_hidden_1], stddev=0.1)),\n", 399 | " 'out': tf.Variable(tf.truncated_normal([n_hidden_1, n_classes], stddev=0.1))\n", 400 | " }\n", 401 | " biases = {\n", 402 | " 'b1': tf.Variable(tf.zeros([n_hidden_1])),\n", 403 | " 'out': tf.Variable(tf.zeros([n_classes]))\n", 404 | " }\n", 405 | "\n", 406 | " ######################\n", 407 | " # Forward Propagation\n", 408 | " ######################\n", 409 | " \n", 410 | " h1_z = tf.add(tf.matmul(tf_x, weights['h1']), biases['b1'])\n", 411 | " h1_act = tf.nn.sigmoid(h1_z)\n", 412 | " out_z = tf.matmul(h1_act, weights['out']) + biases['out']\n", 413 | " out_act = tf.nn.softmax(out_z, name='predicted_probabilities')\n", 414 | " out_labels = tf.argmax(out_z, axis=1, name='predicted_labels')\n", 415 | " \n", 416 | " # Loss\n", 417 | " loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_z, labels=tf_y)\n", 418 | " cost = tf.reduce_mean(loss, name='cost')\n", 419 | " \n", 420 | " ##################\n", 421 | " # Backpropagation\n", 422 | " ##################\n", 423 | " \n", 424 | " # Get Gradients\n", 425 | " \n", 426 | " # input/output dim: [n_samples, n_classlabels]\n", 427 | " sigma_out = (out_act - tf_y) / batch_size\n", 428 | " \n", 429 | " # input/output dim: [n_samples, n_hidden_1]\n", 430 | " softmax_derivative_h1 = h1_act * (1. - h1_act)\n", 431 | " \n", 432 | " # input dim: [n_samples, n_classlabels] dot [n_classlabels, n_hidden]\n", 433 | " # output dim: [n_samples, n_hidden]\n", 434 | " sigma_h = (tf.matmul(sigma_out, tf.transpose(weights['out'])) *\n", 435 | " softmax_derivative_h1)\n", 436 | " \n", 437 | " # input dim: [n_features, n_samples] dot [n_samples, n_hidden]\n", 438 | " # output dim: [n_features, n_hidden]\n", 439 | " grad_w_h1 = tf.matmul(tf.transpose(tf_x), sigma_h)\n", 440 | " grad_b_h1 = tf.reduce_sum(sigma_h, axis=0)\n", 441 | "\n", 442 | " # input dim: [n_hidden, n_samples] dot [n_samples, n_classlabels]\n", 443 | " # output dim: [n_hidden, n_classlabels]\n", 444 | " grad_w_out = tf.matmul(tf.transpose(h1_act), sigma_out)\n", 445 | " grad_b_out = tf.reduce_sum(sigma_out, axis=0)\n", 446 | " \n", 447 | " # Update weights\n", 448 | " upd_w_1 = tf.assign(weights['h1'], weights['h1'] - learning_rate * grad_w_h1)\n", 449 | " upd_b_1 = tf.assign(biases['b1'], biases['b1'] - learning_rate * grad_b_h1)\n", 450 | " upd_w_out = tf.assign(weights['out'], weights['out'] - learning_rate * grad_w_out)\n", 451 | " upd_b_out = tf.assign(biases['out'], biases['out'] - learning_rate * grad_b_out)\n", 452 | " \n", 453 | " train = tf.group(upd_w_1, upd_b_1, upd_w_out, upd_b_out, name='train')\n", 454 | " \n", 455 | " ##############\n", 456 | " # Prediction\n", 457 | " ##############\n", 458 | "\n", 459 | " correct_prediction = tf.equal(tf.argmax(tf_y, 1), out_labels)\n", 460 | " accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n", 461 | "\n", 462 | " \n", 463 | "##########################\n", 464 | "### TRAINING & EVALUATION\n", 465 | "##########################\n", 466 | "\n", 467 | "with tf.Session(graph=g) as sess:\n", 468 | " sess.run(tf.global_variables_initializer())\n", 469 | "\n", 470 | " for epoch in range(training_epochs):\n", 471 | " avg_cost = 0.\n", 472 | " total_batch = mnist.train.num_examples // batch_size\n", 473 | "\n", 474 | " for i in range(total_batch):\n", 475 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 476 | " _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n", 477 | " 'targets:0': batch_y})\n", 478 | " avg_cost += c\n", 479 | " \n", 480 | " train_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.train.images,\n", 481 | " 'targets:0': mnist.train.labels})\n", 482 | " valid_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.validation.images,\n", 483 | " 'targets:0': mnist.validation.labels}) \n", 484 | " \n", 485 | " print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / (i + 1)), end=\"\")\n", 486 | " print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n", 487 | " \n", 488 | " test_acc = sess.run(accuracy, feed_dict={'features:0': mnist.test.images,\n", 489 | " 'targets:0': mnist.test.labels})\n", 490 | " print('Test ACC: %.3f' % test_acc)" 491 | ] 492 | }, 493 | { 494 | "cell_type": "markdown", 495 | "metadata": {}, 496 | "source": [ 497 | "## Gradients still quite \"manually\" -- using tf.gradients" 498 | ] 499 | }, 500 | { 501 | "cell_type": "code", 502 | "execution_count": 2, 503 | "metadata": {}, 504 | "outputs": [ 505 | { 506 | "name": "stdout", 507 | "output_type": "stream", 508 | "text": [ 509 | "Extracting ./train-images-idx3-ubyte.gz\n", 510 | "Extracting ./train-labels-idx1-ubyte.gz\n", 511 | "Extracting ./t10k-images-idx3-ubyte.gz\n", 512 | "Extracting ./t10k-labels-idx1-ubyte.gz\n", 513 | "Epoch: 001 | AvgCost: 0.786 | Train/Valid ACC: 0.886/0.895\n", 514 | "Epoch: 002 | AvgCost: 0.370 | Train/Valid ACC: 0.905/0.913\n", 515 | "Epoch: 003 | AvgCost: 0.317 | Train/Valid ACC: 0.912/0.919\n", 516 | "Epoch: 004 | AvgCost: 0.289 | Train/Valid ACC: 0.922/0.926\n", 517 | "Epoch: 005 | AvgCost: 0.268 | Train/Valid ACC: 0.928/0.933\n", 518 | "Epoch: 006 | AvgCost: 0.251 | Train/Valid ACC: 0.933/0.936\n", 519 | "Epoch: 007 | AvgCost: 0.235 | Train/Valid ACC: 0.936/0.936\n", 520 | "Epoch: 008 | AvgCost: 0.222 | Train/Valid ACC: 0.939/0.941\n", 521 | "Epoch: 009 | AvgCost: 0.210 | Train/Valid ACC: 0.944/0.944\n", 522 | "Epoch: 010 | AvgCost: 0.198 | Train/Valid ACC: 0.946/0.947\n", 523 | "Test ACC: 0.943\n" 524 | ] 525 | } 526 | ], 527 | "source": [ 528 | "import tensorflow as tf\n", 529 | "import numpy as np\n", 530 | "from tensorflow.examples.tutorials.mnist import input_data\n", 531 | "\n", 532 | "\n", 533 | "# Dataset\n", 534 | "mnist = input_data.read_data_sets(\"./\", one_hot=True)\n", 535 | "np.random.seed(123) # set seed for mnist shuffling\n", 536 | "\n", 537 | "##########################\n", 538 | "### SETTINGS\n", 539 | "##########################\n", 540 | "\n", 541 | "# Hyperparameters\n", 542 | "learning_rate = 0.1\n", 543 | "training_epochs = 10\n", 544 | "batch_size = 64\n", 545 | "\n", 546 | "# Architecture\n", 547 | "n_hidden_1 = 128\n", 548 | "n_input = 784\n", 549 | "n_classes = 10\n", 550 | "\n", 551 | "\n", 552 | "\n", 553 | "##########################\n", 554 | "### GRAPH DEFINITION\n", 555 | "##########################\n", 556 | "\n", 557 | "g = tf.Graph()\n", 558 | "with g.as_default():\n", 559 | " \n", 560 | " tf.set_random_seed(123)\n", 561 | "\n", 562 | " # Input data\n", 563 | " tf_x = tf.placeholder(tf.float32, [None, n_input], name='features')\n", 564 | " tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n", 565 | "\n", 566 | " # Model parameters\n", 567 | " weights = {\n", 568 | " 'h1': tf.Variable(tf.truncated_normal([n_input, n_hidden_1], stddev=0.1)),\n", 569 | " 'out': tf.Variable(tf.truncated_normal([n_hidden_1, n_classes], stddev=0.1))\n", 570 | " }\n", 571 | " biases = {\n", 572 | " 'b1': tf.Variable(tf.zeros([n_hidden_1])),\n", 573 | " 'out': tf.Variable(tf.zeros([n_classes]))\n", 574 | " }\n", 575 | "\n", 576 | " ######################\n", 577 | " # Forward Propagation\n", 578 | " ######################\n", 579 | "\n", 580 | " h1_z = tf.add(tf.matmul(tf_x, weights['h1']), biases['b1'])\n", 581 | " h1_act = tf.nn.sigmoid(h1_z)\n", 582 | " out_z = tf.matmul(h1_act, weights['out']) + biases['out']\n", 583 | " out_act = tf.nn.softmax(out_z, name='predicted_probabilities')\n", 584 | " out_labels = tf.argmax(out_z, axis=1, name='predicted_labels')\n", 585 | " \n", 586 | " # Loss\n", 587 | " loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_z, labels=tf_y)\n", 588 | " cost = tf.reduce_mean(loss, name='cost')\n", 589 | " \n", 590 | " ##################\n", 591 | " # Backpropagation\n", 592 | " ##################\n", 593 | "\n", 594 | " # Get Gradients\n", 595 | " dc_dw_out, dc_db_out = tf.gradients(cost, [weights['out'], biases['out']])\n", 596 | " dc_dw_1, dc_db_1 = tf.gradients(cost, [weights['h1'], biases['b1']])\n", 597 | " \n", 598 | " # Update Weights\n", 599 | " upd_w_1 = tf.assign(weights['h1'], weights['h1'] - learning_rate * dc_dw_1)\n", 600 | " upd_b_1 = tf.assign(biases['b1'], biases['b1'] - learning_rate * dc_db_1)\n", 601 | " upd_w_out = tf.assign(weights['out'], weights['out'] - learning_rate * dc_dw_out)\n", 602 | " upd_b_out = tf.assign(biases['out'], biases['out'] - learning_rate * dc_db_out)\n", 603 | " \n", 604 | " train = tf.group(upd_w_1, upd_b_1, upd_w_out, upd_b_out, name='train')\n", 605 | "\n", 606 | " ##############\n", 607 | " # Prediction\n", 608 | " ##############\n", 609 | "\n", 610 | " correct_prediction = tf.equal(tf.argmax(tf_y, 1), out_labels)\n", 611 | " accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n", 612 | "\n", 613 | " \n", 614 | "##########################\n", 615 | "### TRAINING & EVALUATION\n", 616 | "##########################\n", 617 | "\n", 618 | "with tf.Session(graph=g) as sess:\n", 619 | " sess.run(tf.global_variables_initializer())\n", 620 | "\n", 621 | " for epoch in range(training_epochs):\n", 622 | " avg_cost = 0.\n", 623 | " total_batch = mnist.train.num_examples // batch_size\n", 624 | "\n", 625 | " for i in range(total_batch):\n", 626 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 627 | " _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n", 628 | " 'targets:0': batch_y})\n", 629 | " avg_cost += c\n", 630 | " \n", 631 | " train_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.train.images,\n", 632 | " 'targets:0': mnist.train.labels})\n", 633 | " valid_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.validation.images,\n", 634 | " 'targets:0': mnist.validation.labels}) \n", 635 | " \n", 636 | " print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / (i + 1)), end=\"\")\n", 637 | " print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n", 638 | " \n", 639 | " test_acc = sess.run(accuracy, feed_dict={'features:0': mnist.test.images,\n", 640 | " 'targets:0': mnist.test.labels})\n", 641 | " print('Test ACC: %.3f' % test_acc)" 642 | ] 643 | }, 644 | { 645 | "cell_type": "markdown", 646 | "metadata": {}, 647 | "source": [ 648 | "## Using TensorFlow's optimizers" 649 | ] 650 | }, 651 | { 652 | "cell_type": "code", 653 | "execution_count": 3, 654 | "metadata": {}, 655 | "outputs": [ 656 | { 657 | "name": "stdout", 658 | "output_type": "stream", 659 | "text": [ 660 | "Extracting ./train-images-idx3-ubyte.gz\n", 661 | "Extracting ./train-labels-idx1-ubyte.gz\n", 662 | "Extracting ./t10k-images-idx3-ubyte.gz\n", 663 | "Extracting ./t10k-labels-idx1-ubyte.gz\n", 664 | "Epoch: 001 | AvgCost: 0.786 | Train/Valid ACC: 0.886/0.895\n", 665 | "Epoch: 002 | AvgCost: 0.370 | Train/Valid ACC: 0.905/0.913\n", 666 | "Epoch: 003 | AvgCost: 0.317 | Train/Valid ACC: 0.912/0.919\n", 667 | "Epoch: 004 | AvgCost: 0.289 | Train/Valid ACC: 0.922/0.926\n", 668 | "Epoch: 005 | AvgCost: 0.268 | Train/Valid ACC: 0.928/0.932\n", 669 | "Epoch: 006 | AvgCost: 0.251 | Train/Valid ACC: 0.933/0.936\n", 670 | "Epoch: 007 | AvgCost: 0.235 | Train/Valid ACC: 0.936/0.936\n", 671 | "Epoch: 008 | AvgCost: 0.222 | Train/Valid ACC: 0.939/0.941\n", 672 | "Epoch: 009 | AvgCost: 0.210 | Train/Valid ACC: 0.944/0.944\n", 673 | "Epoch: 010 | AvgCost: 0.198 | Train/Valid ACC: 0.946/0.947\n", 674 | "Test ACC: 0.943\n" 675 | ] 676 | } 677 | ], 678 | "source": [ 679 | "import tensorflow as tf\n", 680 | "import numpy as np\n", 681 | "from tensorflow.examples.tutorials.mnist import input_data\n", 682 | "\n", 683 | "\n", 684 | "# Dataset\n", 685 | "mnist = input_data.read_data_sets(\"./\", one_hot=True)\n", 686 | "np.random.seed(123) # set seed for mnist shuffling\n", 687 | "\n", 688 | "##########################\n", 689 | "### SETTINGS\n", 690 | "##########################\n", 691 | "\n", 692 | "# Hyperparameters\n", 693 | "learning_rate = 0.1\n", 694 | "training_epochs = 10\n", 695 | "batch_size = 64\n", 696 | "\n", 697 | "# Architecture\n", 698 | "n_hidden_1 = 128\n", 699 | "n_input = 784\n", 700 | "n_classes = 10\n", 701 | "\n", 702 | "\n", 703 | "##########################\n", 704 | "### GRAPH DEFINITION\n", 705 | "##########################\n", 706 | "\n", 707 | "g = tf.Graph()\n", 708 | "with g.as_default():\n", 709 | " \n", 710 | " tf.set_random_seed(123)\n", 711 | "\n", 712 | " # Input data\n", 713 | " tf_x = tf.placeholder(tf.float32, [None, n_input], name='features')\n", 714 | " tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n", 715 | "\n", 716 | " # Model parameters\n", 717 | " weights = {\n", 718 | " 'h1': tf.Variable(tf.truncated_normal([n_input, n_hidden_1], stddev=0.1)),\n", 719 | " 'out': tf.Variable(tf.truncated_normal([n_hidden_1, n_classes], stddev=0.1))\n", 720 | " }\n", 721 | " biases = {\n", 722 | " 'b1': tf.Variable(tf.zeros([n_hidden_1])),\n", 723 | " 'out': tf.Variable(tf.zeros([n_classes]))\n", 724 | " }\n", 725 | "\n", 726 | " # Forward Propagation\n", 727 | " h1_z = tf.add(tf.matmul(tf_x, weights['h1']), biases['b1'])\n", 728 | " h1_act = tf.nn.sigmoid(h1_z)\n", 729 | " out_z = tf.matmul(h1_act, weights['out']) + biases['out']\n", 730 | " out_act = tf.nn.softmax(out_z, name='predicted_probabilities')\n", 731 | " out_labels = tf.argmax(out_z, axis=1, name='predicted_labels')\n", 732 | " \n", 733 | " ######################\n", 734 | " # Forward Propagation\n", 735 | " ######################\n", 736 | "\n", 737 | " loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_z, labels=tf_y)\n", 738 | " cost = tf.reduce_mean(loss, name='cost')\n", 739 | " \n", 740 | " ##################\n", 741 | " # Backpropagation\n", 742 | " ##################\n", 743 | "\n", 744 | " optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", 745 | " train = optimizer.minimize(cost, name='train')\n", 746 | "\n", 747 | " ##############\n", 748 | " # Prediction\n", 749 | " ##############\n", 750 | "\n", 751 | " correct_prediction = tf.equal(tf.argmax(tf_y, 1), out_labels)\n", 752 | " accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n", 753 | " init_op = tf.global_variables_initializer()\n", 754 | "\n", 755 | " \n", 756 | "##########################\n", 757 | "### TRAINING & EVALUATION\n", 758 | "##########################\n", 759 | "\n", 760 | "with tf.Session(graph=g) as sess:\n", 761 | " sess.run(init_op)\n", 762 | "\n", 763 | " for epoch in range(training_epochs):\n", 764 | " avg_cost = 0.\n", 765 | " total_batch = mnist.train.num_examples // batch_size\n", 766 | "\n", 767 | " for i in range(total_batch):\n", 768 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 769 | " _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n", 770 | " 'targets:0': batch_y})\n", 771 | " avg_cost += c\n", 772 | " \n", 773 | " train_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.train.images,\n", 774 | " 'targets:0': mnist.train.labels})\n", 775 | " valid_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.validation.images,\n", 776 | " 'targets:0': mnist.validation.labels}) \n", 777 | " \n", 778 | " print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / (i + 1)), end=\"\")\n", 779 | " print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n", 780 | " \n", 781 | " test_acc = sess.run(accuracy, feed_dict={'features:0': mnist.test.images,\n", 782 | " 'targets:0': mnist.test.labels})\n", 783 | " print('Test ACC: %.3f' % test_acc)" 784 | ] 785 | }, 786 | { 787 | "cell_type": "markdown", 788 | "metadata": { 789 | "collapsed": true 790 | }, 791 | "source": [ 792 | "# TensorFlow.layers" 793 | ] 794 | }, 795 | { 796 | "cell_type": "markdown", 797 | "metadata": {}, 798 | "source": [ 799 | "Defining functions for creating \"layers\" manually:" 800 | ] 801 | }, 802 | { 803 | "cell_type": "code", 804 | "execution_count": 17, 805 | "metadata": { 806 | "collapsed": true 807 | }, 808 | "outputs": [], 809 | "source": [ 810 | "def fully_connected(input_tensor, output_nodes,\n", 811 | " activation=None, seed=None,\n", 812 | " name='fully_connected'):\n", 813 | "\n", 814 | " with tf.variable_scope(name):\n", 815 | " input_nodes = input_tensor.get_shape().as_list()[1]\n", 816 | " weights = tf.Variable(tf.truncated_normal(shape=(input_nodes,\n", 817 | " output_nodes),\n", 818 | " mean=0.0,\n", 819 | " stddev=0.01,\n", 820 | " dtype=tf.float32,\n", 821 | " seed=seed),\n", 822 | " name='weights')\n", 823 | " biases = tf.Variable(tf.zeros(shape=[output_nodes]), name='biases')\n", 824 | "\n", 825 | " act = tf.matmul(input_tensor, weights) + biases\n", 826 | " if activation is not None:\n", 827 | " act = activation(act)\n", 828 | " return act" 829 | ] 830 | }, 831 | { 832 | "cell_type": "markdown", 833 | "metadata": {}, 834 | "source": [ 835 | "Using the `tensorflow.layers` API:" 836 | ] 837 | }, 838 | { 839 | "cell_type": "code", 840 | "execution_count": 18, 841 | "metadata": { 842 | "scrolled": true 843 | }, 844 | "outputs": [ 845 | { 846 | "name": "stdout", 847 | "output_type": "stream", 848 | "text": [ 849 | "Extracting ./train-images-idx3-ubyte.gz\n", 850 | "Extracting ./train-labels-idx1-ubyte.gz\n", 851 | "Extracting ./t10k-images-idx3-ubyte.gz\n", 852 | "Extracting ./t10k-labels-idx1-ubyte.gz\n", 853 | "Epoch: 001 | AvgCost: 0.350 | Train/Valid ACC: 0.949/0.953\n", 854 | "Epoch: 002 | AvgCost: 0.161 | Train/Valid ACC: 0.961/0.961\n", 855 | "Epoch: 003 | AvgCost: 0.116 | Train/Valid ACC: 0.972/0.969\n", 856 | "Epoch: 004 | AvgCost: 0.091 | Train/Valid ACC: 0.979/0.972\n", 857 | "Epoch: 005 | AvgCost: 0.074 | Train/Valid ACC: 0.980/0.970\n", 858 | "Epoch: 006 | AvgCost: 0.063 | Train/Valid ACC: 0.986/0.977\n", 859 | "Epoch: 007 | AvgCost: 0.052 | Train/Valid ACC: 0.989/0.978\n", 860 | "Epoch: 008 | AvgCost: 0.044 | Train/Valid ACC: 0.985/0.972\n", 861 | "Epoch: 009 | AvgCost: 0.038 | Train/Valid ACC: 0.987/0.976\n", 862 | "Epoch: 010 | AvgCost: 0.032 | Train/Valid ACC: 0.991/0.977\n", 863 | "Test ACC: 0.975\n" 864 | ] 865 | } 866 | ], 867 | "source": [ 868 | "import tensorflow as tf\n", 869 | "from tensorflow.examples.tutorials.mnist import input_data\n", 870 | "\n", 871 | "\n", 872 | "##########################\n", 873 | "### DATASET\n", 874 | "##########################\n", 875 | "\n", 876 | "mnist = input_data.read_data_sets(\"./\", one_hot=True)\n", 877 | "\n", 878 | "\n", 879 | "##########################\n", 880 | "### SETTINGS\n", 881 | "##########################\n", 882 | "\n", 883 | "# Hyperparameters\n", 884 | "learning_rate = 0.1\n", 885 | "training_epochs = 10\n", 886 | "batch_size = 64\n", 887 | "\n", 888 | "# Architecture\n", 889 | "n_hidden_1 = 128\n", 890 | "n_hidden_2 = 256\n", 891 | "n_input = 784\n", 892 | "n_classes = 10\n", 893 | "\n", 894 | "\n", 895 | "##########################\n", 896 | "### GRAPH DEFINITION\n", 897 | "##########################\n", 898 | "\n", 899 | "g = tf.Graph()\n", 900 | "with g.as_default():\n", 901 | "\n", 902 | " # Input data\n", 903 | " tf_x = tf.placeholder(tf.float32, [None, n_input], name='features')\n", 904 | " tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n", 905 | "\n", 906 | " # Multilayer perceptron\n", 907 | " layer_1 = tf.layers.dense(tf_x, n_hidden_1, \n", 908 | " activation=tf.nn.relu, \n", 909 | " kernel_initializer=tf.truncated_normal_initializer(stddev=0.1))\n", 910 | " layer_2 = tf.layers.dense(layer_1, n_hidden_2,\n", 911 | " activation=tf.nn.relu,\n", 912 | " kernel_initializer=tf.truncated_normal_initializer(stddev=0.1))\n", 913 | " out_layer = tf.layers.dense(layer_2, n_classes, activation=None)\n", 914 | "\n", 915 | " # Loss and optimizer\n", 916 | " loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_layer, labels=tf_y)\n", 917 | " cost = tf.reduce_mean(loss, name='cost')\n", 918 | " optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", 919 | " train = optimizer.minimize(cost, name='train')\n", 920 | "\n", 921 | " # Prediction\n", 922 | " correct_prediction = tf.equal(tf.argmax(tf_y, 1), tf.argmax(out_layer, 1))\n", 923 | " accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n", 924 | " init_op = tf.global_variables_initializer()\n", 925 | " \n", 926 | "\n", 927 | "##########################\n", 928 | "### TRAINING & EVALUATION\n", 929 | "##########################\n", 930 | " \n", 931 | "with tf.Session(graph=g) as sess:\n", 932 | " sess.run(init_op)\n", 933 | "\n", 934 | " for epoch in range(training_epochs):\n", 935 | " avg_cost = 0.\n", 936 | " total_batch = mnist.train.num_examples // batch_size\n", 937 | "\n", 938 | " for i in range(total_batch):\n", 939 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 940 | " _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n", 941 | " 'targets:0': batch_y})\n", 942 | " avg_cost += c\n", 943 | " \n", 944 | " train_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.train.images,\n", 945 | " 'targets:0': mnist.train.labels})\n", 946 | " valid_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.validation.images,\n", 947 | " 'targets:0': mnist.validation.labels}) \n", 948 | " \n", 949 | " print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / (i + 1)), end=\"\")\n", 950 | " print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n", 951 | " \n", 952 | " test_acc = sess.run('accuracy:0', feed_dict={'features:0': mnist.test.images,\n", 953 | " 'targets:0': mnist.test.labels})\n", 954 | " print('Test ACC: %.3f' % test_acc)" 955 | ] 956 | }, 957 | { 958 | "cell_type": "markdown", 959 | "metadata": {}, 960 | "source": [ 961 | "# Convolutional General Adversarial Networks" 962 | ] 963 | }, 964 | { 965 | "cell_type": "code", 966 | "execution_count": 1, 967 | "metadata": { 968 | "scrolled": false 969 | }, 970 | "outputs": [ 971 | { 972 | "name": "stdout", 973 | "output_type": "stream", 974 | "text": [ 975 | "Sebastian Raschka \n", 976 | "\n", 977 | "CPython 3.6.1\n", 978 | "IPython 6.1.0\n", 979 | "\n", 980 | "tensorflow 1.3.0\n" 981 | ] 982 | } 983 | ], 984 | "source": [ 985 | "%load_ext watermark\n", 986 | "%watermark -a 'Sebastian Raschka' -v -p tensorflow" 987 | ] 988 | }, 989 | { 990 | "cell_type": "markdown", 991 | "metadata": {}, 992 | "source": [ 993 | "Implementation of General Adversarial Nets (GAN) where both the discriminator and generator have convolutional and deconvolutional layers, respectively. In this example, the GAN generator was trained to generate MNIST images.\n", 994 | "\n", 995 | "Uses\n", 996 | "\n", 997 | "- samples from a random normal distribution (range [-1, 1])\n", 998 | "- dropout\n", 999 | "- leaky relus\n", 1000 | "- batch normalization\n", 1001 | "- separate batches for \"fake\" and \"real\" images (where the labels are 1 = real images, 0 = fake images)\n", 1002 | "- MNIST images normalized to [-1, 1] range\n", 1003 | "- generator with tanh output\n" 1004 | ] 1005 | }, 1006 | { 1007 | "cell_type": "code", 1008 | "execution_count": 2, 1009 | "metadata": {}, 1010 | "outputs": [ 1011 | { 1012 | "data": { 1013 | "text/plain": [ 1014 | "'/gpu:0'" 1015 | ] 1016 | }, 1017 | "execution_count": 2, 1018 | "metadata": {}, 1019 | "output_type": "execute_result" 1020 | } 1021 | ], 1022 | "source": [ 1023 | "import numpy as np\n", 1024 | "import tensorflow as tf\n", 1025 | "from tensorflow.examples.tutorials.mnist import input_data\n", 1026 | "import pickle as pkl\n", 1027 | "\n", 1028 | "tf.test.gpu_device_name()" 1029 | ] 1030 | }, 1031 | { 1032 | "cell_type": "code", 1033 | "execution_count": 3, 1034 | "metadata": {}, 1035 | "outputs": [ 1036 | { 1037 | "name": "stdout", 1038 | "output_type": "stream", 1039 | "text": [ 1040 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 1041 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 1042 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 1043 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" 1044 | ] 1045 | } 1046 | ], 1047 | "source": [ 1048 | "### Abbreviatiuons\n", 1049 | "# dis_*: discriminator network\n", 1050 | "# gen_*: generator network\n", 1051 | "\n", 1052 | "########################\n", 1053 | "### Helper functions\n", 1054 | "########################\n", 1055 | "\n", 1056 | "def leaky_relu(x, alpha=0.0001):\n", 1057 | " return tf.maximum(alpha * x, x)\n", 1058 | "\n", 1059 | "\n", 1060 | "########################\n", 1061 | "### DATASET\n", 1062 | "########################\n", 1063 | "\n", 1064 | "mnist = input_data.read_data_sets('MNIST_data')\n", 1065 | "\n", 1066 | "\n", 1067 | "#########################\n", 1068 | "### SETTINGS\n", 1069 | "#########################\n", 1070 | "\n", 1071 | "# Hyperparameters\n", 1072 | "learning_rate = 0.001\n", 1073 | "training_epochs = 50\n", 1074 | "batch_size = 64\n", 1075 | "dropout_rate = 0.5\n", 1076 | "\n", 1077 | "# Architecture\n", 1078 | "dis_input_size = 784\n", 1079 | "gen_input_size = 100\n", 1080 | "\n", 1081 | "# Other settings\n", 1082 | "print_interval = 200\n", 1083 | "\n", 1084 | "#########################\n", 1085 | "### GRAPH DEFINITION\n", 1086 | "#########################\n", 1087 | "\n", 1088 | "g = tf.Graph()\n", 1089 | "with g.as_default():\n", 1090 | " \n", 1091 | " # Placeholders for settings\n", 1092 | " dropout = tf.placeholder(tf.float32, shape=None, name='dropout')\n", 1093 | " is_training = tf.placeholder(tf.bool, shape=None, name='is_training')\n", 1094 | " \n", 1095 | " # Input data\n", 1096 | " dis_x = tf.placeholder(tf.float32, shape=[None, dis_input_size],\n", 1097 | " name='discriminator_inputs') \n", 1098 | " gen_x = tf.placeholder(tf.float32, [None, gen_input_size],\n", 1099 | " name='generator_inputs')\n", 1100 | "\n", 1101 | "\n", 1102 | " ##################\n", 1103 | " # Generator Model\n", 1104 | " ##################\n", 1105 | "\n", 1106 | " with tf.variable_scope('generator'):\n", 1107 | " \n", 1108 | " # 100 => 784 => 7x7x64\n", 1109 | " gen_fc = tf.layers.dense(inputs=gen_x, units=3136,\n", 1110 | " bias_initializer=None, # no bias required when using batch_norm\n", 1111 | " activation=None)\n", 1112 | " gen_fc = tf.layers.batch_normalization(gen_fc, training=is_training)\n", 1113 | " gen_fc = leaky_relu(gen_fc)\n", 1114 | " gen_fc = tf.reshape(gen_fc, (-1, 7, 7, 64))\n", 1115 | " \n", 1116 | " # 7x7x64 => 14x14x32\n", 1117 | " deconv1 = tf.layers.conv2d_transpose(gen_fc, filters=32, \n", 1118 | " kernel_size=(3, 3), strides=(2, 2), \n", 1119 | " padding='same',\n", 1120 | " bias_initializer=None,\n", 1121 | " activation=None)\n", 1122 | " deconv1 = tf.layers.batch_normalization(deconv1, training=is_training)\n", 1123 | " deconv1 = leaky_relu(deconv1) \n", 1124 | " deconv1 = tf.layers.dropout(deconv1, rate=dropout_rate)\n", 1125 | " \n", 1126 | " # 14x14x32 => 28x28x16\n", 1127 | " deconv2 = tf.layers.conv2d_transpose(deconv1, filters=16, \n", 1128 | " kernel_size=(3, 3), strides=(2, 2), \n", 1129 | " padding='same',\n", 1130 | " bias_initializer=None,\n", 1131 | " activation=None)\n", 1132 | " deconv2 = tf.layers.batch_normalization(deconv2, training=is_training)\n", 1133 | " deconv2 = leaky_relu(deconv2) \n", 1134 | " deconv2 = tf.layers.dropout(deconv2, rate=dropout_rate)\n", 1135 | " \n", 1136 | " # 28x28x16 => 28x28x8\n", 1137 | " deconv3 = tf.layers.conv2d_transpose(deconv2, filters=8, \n", 1138 | " kernel_size=(3, 3), strides=(1, 1), \n", 1139 | " padding='same',\n", 1140 | " bias_initializer=None,\n", 1141 | " activation=None)\n", 1142 | " deconv3 = tf.layers.batch_normalization(deconv3, training=is_training)\n", 1143 | " deconv3 = leaky_relu(deconv3) \n", 1144 | " deconv3 = tf.layers.dropout(deconv3, rate=dropout_rate)\n", 1145 | " \n", 1146 | " # 28x28x8 => 28x28x1\n", 1147 | " gen_logits = tf.layers.conv2d_transpose(deconv3, filters=1, \n", 1148 | " kernel_size=(3, 3), strides=(1, 1), \n", 1149 | " padding='same',\n", 1150 | " bias_initializer=None,\n", 1151 | " activation=None)\n", 1152 | " gen_out = tf.tanh(gen_logits, 'generator_outputs')\n", 1153 | "\n", 1154 | "\n", 1155 | " ######################\n", 1156 | " # Discriminator Model\n", 1157 | " ######################\n", 1158 | " \n", 1159 | " def build_discriminator_graph(input_x, reuse=None):\n", 1160 | "\n", 1161 | " with tf.variable_scope('discriminator', reuse=reuse):\n", 1162 | " \n", 1163 | " # 28x28x1 => 14x14x8\n", 1164 | " conv_input = tf.reshape(input_x, (-1, 28, 28, 1))\n", 1165 | " conv1 = tf.layers.conv2d(conv_input, filters=8, kernel_size=(3, 3),\n", 1166 | " strides=(2, 2), padding='same',\n", 1167 | " bias_initializer=None,\n", 1168 | " activation=None)\n", 1169 | " conv1 = tf.layers.batch_normalization(conv1, training=is_training)\n", 1170 | " conv1 = leaky_relu(conv1)\n", 1171 | " conv1 = tf.layers.dropout(conv1, rate=dropout_rate)\n", 1172 | " \n", 1173 | " # 14x14x8 => 7x7x32\n", 1174 | " conv2 = tf.layers.conv2d(conv1, filters=32, kernel_size=(3, 3),\n", 1175 | " strides=(2, 2), padding='same',\n", 1176 | " bias_initializer=None,\n", 1177 | " activation=None)\n", 1178 | " conv2 = tf.layers.batch_normalization(conv2, training=is_training)\n", 1179 | " conv2 = leaky_relu(conv2)\n", 1180 | " conv2 = tf.layers.dropout(conv2, rate=dropout_rate)\n", 1181 | "\n", 1182 | " # fully connected layer\n", 1183 | " fc_input = tf.reshape(conv2, (-1, 7*7*32))\n", 1184 | " logits = tf.layers.dense(inputs=fc_input, units=1, activation=None)\n", 1185 | " out = tf.sigmoid(logits)\n", 1186 | " \n", 1187 | " return logits, out \n", 1188 | "\n", 1189 | " # Create a discriminator for real data and a discriminator for fake data\n", 1190 | " dis_real_logits, dis_real_out = build_discriminator_graph(dis_x, reuse=False)\n", 1191 | " dis_fake_logits, dis_fake_out = build_discriminator_graph(gen_out, reuse=True)\n", 1192 | "\n", 1193 | "\n", 1194 | " #####################################\n", 1195 | " # Generator and Discriminator Losses\n", 1196 | " #####################################\n", 1197 | " \n", 1198 | " # Two discriminator cost components: loss on real data + loss on fake data\n", 1199 | " # Real data has class label 0, fake data has class label 1\n", 1200 | " dis_real_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=dis_real_logits, \n", 1201 | " labels=tf.zeros_like(dis_real_logits))\n", 1202 | " dis_fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=dis_fake_logits, \n", 1203 | " labels=tf.ones_like(dis_fake_logits))\n", 1204 | " dis_cost = tf.add(tf.reduce_mean(dis_fake_loss), \n", 1205 | " tf.reduce_mean(dis_real_loss), \n", 1206 | " name='discriminator_cost')\n", 1207 | " \n", 1208 | " # Generator cost: difference between dis. prediction and label \"0\" for real images\n", 1209 | " gen_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=dis_fake_logits,\n", 1210 | " labels=tf.zeros_like(dis_fake_logits))\n", 1211 | " gen_cost = tf.reduce_mean(gen_loss, name='generator_cost')\n", 1212 | " \n", 1213 | " \n", 1214 | " #########################################\n", 1215 | " # Generator and Discriminator Optimizers\n", 1216 | " #########################################\n", 1217 | " \n", 1218 | " dis_optimizer = tf.train.AdamOptimizer(learning_rate)\n", 1219 | " dis_train_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='discriminator')\n", 1220 | " dis_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='discriminator')\n", 1221 | " \n", 1222 | " with tf.control_dependencies(dis_update_ops): # required to upd. batch_norm params\n", 1223 | " dis_train = dis_optimizer.minimize(dis_cost, var_list=dis_train_vars,\n", 1224 | " name='train_discriminator')\n", 1225 | " \n", 1226 | " gen_optimizer = tf.train.AdamOptimizer(learning_rate)\n", 1227 | " gen_train_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='generator')\n", 1228 | " gen_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='generator')\n", 1229 | " \n", 1230 | " with tf.control_dependencies(gen_update_ops): # required to upd. batch_norm params\n", 1231 | " gen_train = gen_optimizer.minimize(gen_cost, var_list=gen_train_vars,\n", 1232 | " name='train_generator')\n", 1233 | " \n", 1234 | " # Saver to save session for reuse\n", 1235 | " saver = tf.train.Saver()" 1236 | ] 1237 | }, 1238 | { 1239 | "cell_type": "code", 1240 | "execution_count": 4, 1241 | "metadata": {}, 1242 | "outputs": [ 1243 | { 1244 | "name": "stdout", 1245 | "output_type": "stream", 1246 | "text": [ 1247 | "Minibatch: 0001 | Dis/Gen Cost: 1.657/0.917\n", 1248 | "Minibatch: 0201 | Dis/Gen Cost: 0.776/1.551\n", 1249 | "Minibatch: 0401 | Dis/Gen Cost: 0.827/1.839\n", 1250 | "Minibatch: 0601 | Dis/Gen Cost: 0.438/2.190\n", 1251 | "Minibatch: 0801 | Dis/Gen Cost: 1.583/0.872\n", 1252 | "Epoch: 0001 | Dis/Gen AvgCost: 0.934/1.593\n", 1253 | "Minibatch: 0001 | Dis/Gen Cost: 0.634/1.577\n", 1254 | "Minibatch: 0201 | Dis/Gen Cost: 0.598/2.277\n", 1255 | "Minibatch: 0401 | Dis/Gen Cost: 0.763/1.207\n", 1256 | "Minibatch: 0601 | Dis/Gen Cost: 0.524/2.216\n", 1257 | "Minibatch: 0801 | Dis/Gen Cost: 0.252/2.839\n", 1258 | "Epoch: 0002 | Dis/Gen AvgCost: 0.789/1.668\n", 1259 | "Minibatch: 0001 | Dis/Gen Cost: 0.726/1.502\n", 1260 | "Minibatch: 0201 | Dis/Gen Cost: 0.634/1.563\n", 1261 | "Minibatch: 0401 | Dis/Gen Cost: 0.956/1.716\n", 1262 | "Minibatch: 0601 | Dis/Gen Cost: 0.882/1.410\n", 1263 | "Minibatch: 0801 | Dis/Gen Cost: 0.861/1.835\n", 1264 | "Epoch: 0003 | Dis/Gen AvgCost: 0.783/1.663\n", 1265 | "Minibatch: 0001 | Dis/Gen Cost: 0.732/1.914\n", 1266 | "Minibatch: 0201 | Dis/Gen Cost: 1.240/1.239\n", 1267 | "Minibatch: 0401 | Dis/Gen Cost: 1.047/1.460\n", 1268 | "Minibatch: 0601 | Dis/Gen Cost: 0.749/1.630\n", 1269 | "Minibatch: 0801 | Dis/Gen Cost: 0.883/1.536\n", 1270 | "Epoch: 0004 | Dis/Gen AvgCost: 0.880/1.611\n", 1271 | "Minibatch: 0001 | Dis/Gen Cost: 0.726/1.820\n", 1272 | "Minibatch: 0201 | Dis/Gen Cost: 1.143/1.496\n", 1273 | "Minibatch: 0401 | Dis/Gen Cost: 0.925/1.249\n", 1274 | "Minibatch: 0601 | Dis/Gen Cost: 0.839/1.300\n", 1275 | "Minibatch: 0801 | Dis/Gen Cost: 1.318/0.955\n", 1276 | "Epoch: 0005 | Dis/Gen AvgCost: 1.018/1.426\n", 1277 | "Minibatch: 0001 | Dis/Gen Cost: 1.047/1.319\n", 1278 | "Minibatch: 0201 | Dis/Gen Cost: 0.982/1.724\n", 1279 | "Minibatch: 0401 | Dis/Gen Cost: 1.723/0.943\n", 1280 | "Minibatch: 0601 | Dis/Gen Cost: 1.036/1.284\n", 1281 | "Minibatch: 0801 | Dis/Gen Cost: 1.543/0.896\n", 1282 | "Epoch: 0006 | Dis/Gen AvgCost: 1.175/1.231\n", 1283 | "Minibatch: 0001 | Dis/Gen Cost: 1.128/1.209\n", 1284 | "Minibatch: 0201 | Dis/Gen Cost: 1.045/1.154\n", 1285 | "Minibatch: 0401 | Dis/Gen Cost: 1.449/0.896\n", 1286 | "Minibatch: 0601 | Dis/Gen Cost: 1.116/1.281\n", 1287 | "Minibatch: 0801 | Dis/Gen Cost: 1.380/0.949\n", 1288 | "Epoch: 0007 | Dis/Gen AvgCost: 1.244/1.107\n", 1289 | "Minibatch: 0001 | Dis/Gen Cost: 1.292/0.929\n", 1290 | "Minibatch: 0201 | Dis/Gen Cost: 1.295/0.918\n", 1291 | "Minibatch: 0401 | Dis/Gen Cost: 1.271/0.998\n", 1292 | "Minibatch: 0601 | Dis/Gen Cost: 1.078/1.300\n", 1293 | "Minibatch: 0801 | Dis/Gen Cost: 1.371/1.022\n", 1294 | "Epoch: 0008 | Dis/Gen AvgCost: 1.261/1.044\n", 1295 | "Minibatch: 0001 | Dis/Gen Cost: 1.352/1.008\n", 1296 | "Minibatch: 0201 | Dis/Gen Cost: 1.763/0.743\n", 1297 | "Minibatch: 0401 | Dis/Gen Cost: 1.040/1.291\n", 1298 | "Minibatch: 0601 | Dis/Gen Cost: 1.334/1.050\n", 1299 | "Minibatch: 0801 | Dis/Gen Cost: 1.214/1.039\n", 1300 | "Epoch: 0009 | Dis/Gen AvgCost: 1.283/1.015\n", 1301 | "Minibatch: 0001 | Dis/Gen Cost: 1.628/0.699\n", 1302 | "Minibatch: 0201 | Dis/Gen Cost: 1.204/1.033\n", 1303 | "Minibatch: 0401 | Dis/Gen Cost: 1.393/0.891\n", 1304 | "Minibatch: 0601 | Dis/Gen Cost: 1.176/1.043\n", 1305 | "Minibatch: 0801 | Dis/Gen Cost: 1.493/0.765\n", 1306 | "Epoch: 0010 | Dis/Gen AvgCost: 1.297/0.980\n", 1307 | "Minibatch: 0001 | Dis/Gen Cost: 1.421/0.793\n", 1308 | "Minibatch: 0201 | Dis/Gen Cost: 1.453/0.898\n", 1309 | "Minibatch: 0401 | Dis/Gen Cost: 1.090/1.107\n", 1310 | "Minibatch: 0601 | Dis/Gen Cost: 1.412/0.927\n", 1311 | "Minibatch: 0801 | Dis/Gen Cost: 1.319/0.813\n", 1312 | "Epoch: 0011 | Dis/Gen AvgCost: 1.309/0.944\n", 1313 | "Minibatch: 0001 | Dis/Gen Cost: 1.347/1.046\n", 1314 | "Minibatch: 0201 | Dis/Gen Cost: 1.255/1.034\n", 1315 | "Minibatch: 0401 | Dis/Gen Cost: 1.205/0.926\n", 1316 | "Minibatch: 0601 | Dis/Gen Cost: 1.191/0.935\n", 1317 | "Minibatch: 0801 | Dis/Gen Cost: 1.450/0.762\n", 1318 | "Epoch: 0012 | Dis/Gen AvgCost: 1.305/0.943\n", 1319 | "Minibatch: 0001 | Dis/Gen Cost: 1.375/0.727\n", 1320 | "Minibatch: 0201 | Dis/Gen Cost: 1.361/1.070\n", 1321 | "Minibatch: 0401 | Dis/Gen Cost: 1.020/0.943\n", 1322 | "Minibatch: 0601 | Dis/Gen Cost: 1.250/0.921\n", 1323 | "Minibatch: 0801 | Dis/Gen Cost: 1.461/0.996\n", 1324 | "Epoch: 0013 | Dis/Gen AvgCost: 1.303/0.927\n", 1325 | "Minibatch: 0001 | Dis/Gen Cost: 1.258/0.765\n", 1326 | "Minibatch: 0201 | Dis/Gen Cost: 1.177/1.066\n", 1327 | "Minibatch: 0401 | Dis/Gen Cost: 1.175/0.970\n", 1328 | "Minibatch: 0601 | Dis/Gen Cost: 1.213/0.845\n", 1329 | "Minibatch: 0801 | Dis/Gen Cost: 1.433/0.846\n", 1330 | "Epoch: 0014 | Dis/Gen AvgCost: 1.305/0.920\n", 1331 | "Minibatch: 0001 | Dis/Gen Cost: 1.654/0.708\n", 1332 | "Minibatch: 0201 | Dis/Gen Cost: 1.437/0.770\n", 1333 | "Minibatch: 0401 | Dis/Gen Cost: 1.487/0.740\n", 1334 | "Minibatch: 0601 | Dis/Gen Cost: 1.167/1.100\n", 1335 | "Minibatch: 0801 | Dis/Gen Cost: 1.342/0.854\n", 1336 | "Epoch: 0015 | Dis/Gen AvgCost: 1.319/0.893\n", 1337 | "Minibatch: 0001 | Dis/Gen Cost: 1.356/0.826\n", 1338 | "Minibatch: 0201 | Dis/Gen Cost: 1.161/1.101\n", 1339 | "Minibatch: 0401 | Dis/Gen Cost: 1.355/0.878\n", 1340 | "Minibatch: 0601 | Dis/Gen Cost: 1.281/1.022\n", 1341 | "Minibatch: 0801 | Dis/Gen Cost: 1.198/0.828\n", 1342 | "Epoch: 0016 | Dis/Gen AvgCost: 1.320/0.887\n", 1343 | "Minibatch: 0001 | Dis/Gen Cost: 1.197/0.808\n", 1344 | "Minibatch: 0201 | Dis/Gen Cost: 1.337/0.922\n", 1345 | "Minibatch: 0401 | Dis/Gen Cost: 1.223/0.934\n", 1346 | "Minibatch: 0601 | Dis/Gen Cost: 1.376/0.734\n", 1347 | "Minibatch: 0801 | Dis/Gen Cost: 1.334/0.806\n", 1348 | "Epoch: 0017 | Dis/Gen AvgCost: 1.338/0.865\n", 1349 | "Minibatch: 0001 | Dis/Gen Cost: 1.352/0.790\n", 1350 | "Minibatch: 0201 | Dis/Gen Cost: 1.391/0.910\n", 1351 | "Minibatch: 0401 | Dis/Gen Cost: 1.329/0.776\n", 1352 | "Minibatch: 0601 | Dis/Gen Cost: 1.445/0.681\n", 1353 | "Minibatch: 0801 | Dis/Gen Cost: 1.301/0.840\n", 1354 | "Epoch: 0018 | Dis/Gen AvgCost: 1.335/0.842\n", 1355 | "Minibatch: 0001 | Dis/Gen Cost: 1.377/0.781\n", 1356 | "Minibatch: 0201 | Dis/Gen Cost: 1.277/0.872\n", 1357 | "Minibatch: 0401 | Dis/Gen Cost: 1.351/0.767\n", 1358 | "Minibatch: 0601 | Dis/Gen Cost: 1.501/0.657\n", 1359 | "Minibatch: 0801 | Dis/Gen Cost: 1.343/0.797\n", 1360 | "Epoch: 0019 | Dis/Gen AvgCost: 1.331/0.850\n", 1361 | "Minibatch: 0001 | Dis/Gen Cost: 1.429/0.756\n", 1362 | "Minibatch: 0201 | Dis/Gen Cost: 1.341/0.840\n", 1363 | "Minibatch: 0401 | Dis/Gen Cost: 1.447/0.768\n", 1364 | "Minibatch: 0601 | Dis/Gen Cost: 1.284/0.909\n", 1365 | "Minibatch: 0801 | Dis/Gen Cost: 1.212/1.033\n", 1366 | "Epoch: 0020 | Dis/Gen AvgCost: 1.342/0.843\n", 1367 | "Minibatch: 0001 | Dis/Gen Cost: 1.332/0.827\n", 1368 | "Minibatch: 0201 | Dis/Gen Cost: 1.570/0.884\n", 1369 | "Minibatch: 0401 | Dis/Gen Cost: 1.455/0.659\n", 1370 | "Minibatch: 0601 | Dis/Gen Cost: 1.275/0.705\n", 1371 | "Minibatch: 0801 | Dis/Gen Cost: 1.288/0.851\n", 1372 | "Epoch: 0021 | Dis/Gen AvgCost: 1.343/0.832\n", 1373 | "Minibatch: 0001 | Dis/Gen Cost: 1.233/0.942\n", 1374 | "Minibatch: 0201 | Dis/Gen Cost: 1.375/0.816\n", 1375 | "Minibatch: 0401 | Dis/Gen Cost: 1.256/0.852\n", 1376 | "Minibatch: 0601 | Dis/Gen Cost: 1.320/0.970\n", 1377 | "Minibatch: 0801 | Dis/Gen Cost: 1.159/1.066\n", 1378 | "Epoch: 0022 | Dis/Gen AvgCost: 1.349/0.834\n", 1379 | "Minibatch: 0001 | Dis/Gen Cost: 1.429/0.885\n", 1380 | "Minibatch: 0201 | Dis/Gen Cost: 1.643/0.703\n", 1381 | "Minibatch: 0401 | Dis/Gen Cost: 1.471/0.893\n", 1382 | "Minibatch: 0601 | Dis/Gen Cost: 1.407/0.775\n", 1383 | "Minibatch: 0801 | Dis/Gen Cost: 1.364/0.728\n", 1384 | "Epoch: 0023 | Dis/Gen AvgCost: 1.331/0.850\n", 1385 | "Minibatch: 0001 | Dis/Gen Cost: 1.492/0.734\n", 1386 | "Minibatch: 0201 | Dis/Gen Cost: 1.354/0.808\n", 1387 | "Minibatch: 0401 | Dis/Gen Cost: 1.280/0.938\n", 1388 | "Minibatch: 0601 | Dis/Gen Cost: 1.545/0.723\n", 1389 | "Minibatch: 0801 | Dis/Gen Cost: 1.326/0.814\n", 1390 | "Epoch: 0024 | Dis/Gen AvgCost: 1.355/0.818\n", 1391 | "Minibatch: 0001 | Dis/Gen Cost: 1.293/0.903\n", 1392 | "Minibatch: 0201 | Dis/Gen Cost: 1.456/0.688\n", 1393 | "Minibatch: 0401 | Dis/Gen Cost: 1.466/0.781\n", 1394 | "Minibatch: 0601 | Dis/Gen Cost: 1.157/0.831\n", 1395 | "Minibatch: 0801 | Dis/Gen Cost: 1.445/0.715\n", 1396 | "Epoch: 0025 | Dis/Gen AvgCost: 1.350/0.811\n", 1397 | "Minibatch: 0001 | Dis/Gen Cost: 1.500/0.735\n", 1398 | "Minibatch: 0201 | Dis/Gen Cost: 1.589/0.799\n", 1399 | "Minibatch: 0401 | Dis/Gen Cost: 1.429/0.675\n", 1400 | "Minibatch: 0601 | Dis/Gen Cost: 1.329/0.673\n", 1401 | "Minibatch: 0801 | Dis/Gen Cost: 1.318/0.856\n", 1402 | "Epoch: 0026 | Dis/Gen AvgCost: 1.348/0.808\n", 1403 | "Minibatch: 0001 | Dis/Gen Cost: 1.207/0.994\n", 1404 | "Minibatch: 0201 | Dis/Gen Cost: 1.404/0.758\n", 1405 | "Minibatch: 0401 | Dis/Gen Cost: 1.410/0.788\n", 1406 | "Minibatch: 0601 | Dis/Gen Cost: 1.284/0.861\n", 1407 | "Minibatch: 0801 | Dis/Gen Cost: 1.397/0.760\n", 1408 | "Epoch: 0027 | Dis/Gen AvgCost: 1.349/0.798\n", 1409 | "Minibatch: 0001 | Dis/Gen Cost: 1.277/0.772\n", 1410 | "Minibatch: 0201 | Dis/Gen Cost: 1.252/0.962\n", 1411 | "Minibatch: 0401 | Dis/Gen Cost: 1.340/0.709\n", 1412 | "Minibatch: 0601 | Dis/Gen Cost: 1.322/0.947\n", 1413 | "Minibatch: 0801 | Dis/Gen Cost: 1.389/0.839\n", 1414 | "Epoch: 0028 | Dis/Gen AvgCost: 1.347/0.807\n", 1415 | "Minibatch: 0001 | Dis/Gen Cost: 1.485/0.634\n", 1416 | "Minibatch: 0201 | Dis/Gen Cost: 1.213/0.865\n", 1417 | "Minibatch: 0401 | Dis/Gen Cost: 1.316/0.836\n", 1418 | "Minibatch: 0601 | Dis/Gen Cost: 1.405/0.751\n", 1419 | "Minibatch: 0801 | Dis/Gen Cost: 1.453/0.704\n", 1420 | "Epoch: 0029 | Dis/Gen AvgCost: 1.350/0.801\n", 1421 | "Minibatch: 0001 | Dis/Gen Cost: 1.294/0.776\n", 1422 | "Minibatch: 0201 | Dis/Gen Cost: 1.321/0.800\n", 1423 | "Minibatch: 0401 | Dis/Gen Cost: 1.447/0.693\n", 1424 | "Minibatch: 0601 | Dis/Gen Cost: 1.305/0.809\n", 1425 | "Minibatch: 0801 | Dis/Gen Cost: 1.502/0.622\n", 1426 | "Epoch: 0030 | Dis/Gen AvgCost: 1.355/0.787\n", 1427 | "Minibatch: 0001 | Dis/Gen Cost: 1.369/0.679\n", 1428 | "Minibatch: 0201 | Dis/Gen Cost: 1.406/0.774\n", 1429 | "Minibatch: 0401 | Dis/Gen Cost: 1.424/0.804\n", 1430 | "Minibatch: 0601 | Dis/Gen Cost: 1.410/0.703\n", 1431 | "Minibatch: 0801 | Dis/Gen Cost: 1.273/0.876\n", 1432 | "Epoch: 0031 | Dis/Gen AvgCost: 1.347/0.796\n", 1433 | "Minibatch: 0001 | Dis/Gen Cost: 1.426/0.701\n", 1434 | "Minibatch: 0201 | Dis/Gen Cost: 1.494/0.801\n", 1435 | "Minibatch: 0401 | Dis/Gen Cost: 1.317/0.771\n", 1436 | "Minibatch: 0601 | Dis/Gen Cost: 1.404/0.819\n", 1437 | "Minibatch: 0801 | Dis/Gen Cost: 1.413/0.766\n", 1438 | "Epoch: 0032 | Dis/Gen AvgCost: 1.357/0.792\n", 1439 | "Minibatch: 0001 | Dis/Gen Cost: 1.348/0.782\n", 1440 | "Minibatch: 0201 | Dis/Gen Cost: 1.336/0.759\n", 1441 | "Minibatch: 0401 | Dis/Gen Cost: 1.470/0.683\n", 1442 | "Minibatch: 0601 | Dis/Gen Cost: 1.445/0.734\n", 1443 | "Minibatch: 0801 | Dis/Gen Cost: 1.332/0.863\n", 1444 | "Epoch: 0033 | Dis/Gen AvgCost: 1.350/0.780\n", 1445 | "Minibatch: 0001 | Dis/Gen Cost: 1.379/0.783\n", 1446 | "Minibatch: 0201 | Dis/Gen Cost: 1.392/0.876\n", 1447 | "Minibatch: 0401 | Dis/Gen Cost: 1.365/0.777\n", 1448 | "Minibatch: 0601 | Dis/Gen Cost: 1.497/0.734\n", 1449 | "Minibatch: 0801 | Dis/Gen Cost: 1.337/0.767\n", 1450 | "Epoch: 0034 | Dis/Gen AvgCost: 1.354/0.780\n", 1451 | "Minibatch: 0001 | Dis/Gen Cost: 1.340/0.795\n", 1452 | "Minibatch: 0201 | Dis/Gen Cost: 1.214/0.849\n", 1453 | "Minibatch: 0401 | Dis/Gen Cost: 1.240/0.846\n", 1454 | "Minibatch: 0601 | Dis/Gen Cost: 1.367/0.731\n", 1455 | "Minibatch: 0801 | Dis/Gen Cost: 1.368/0.680\n", 1456 | "Epoch: 0035 | Dis/Gen AvgCost: 1.351/0.786\n", 1457 | "Minibatch: 0001 | Dis/Gen Cost: 1.221/0.897\n", 1458 | "Minibatch: 0201 | Dis/Gen Cost: 1.242/0.850\n", 1459 | "Minibatch: 0401 | Dis/Gen Cost: 1.291/0.792\n", 1460 | "Minibatch: 0601 | Dis/Gen Cost: 1.264/0.818\n", 1461 | "Minibatch: 0801 | Dis/Gen Cost: 1.418/0.774\n", 1462 | "Epoch: 0036 | Dis/Gen AvgCost: 1.350/0.781\n", 1463 | "Minibatch: 0001 | Dis/Gen Cost: 1.446/0.740\n", 1464 | "Minibatch: 0201 | Dis/Gen Cost: 1.264/0.814\n", 1465 | "Minibatch: 0401 | Dis/Gen Cost: 1.398/0.859\n", 1466 | "Minibatch: 0601 | Dis/Gen Cost: 1.261/0.833\n", 1467 | "Minibatch: 0801 | Dis/Gen Cost: 1.409/0.786\n", 1468 | "Epoch: 0037 | Dis/Gen AvgCost: 1.350/0.797\n", 1469 | "Minibatch: 0001 | Dis/Gen Cost: 1.375/0.775\n", 1470 | "Minibatch: 0201 | Dis/Gen Cost: 1.558/0.715\n", 1471 | "Minibatch: 0401 | Dis/Gen Cost: 1.334/0.807\n", 1472 | "Minibatch: 0601 | Dis/Gen Cost: 1.445/0.734\n", 1473 | "Minibatch: 0801 | Dis/Gen Cost: 1.247/0.898\n", 1474 | "Epoch: 0038 | Dis/Gen AvgCost: 1.355/0.782\n", 1475 | "Minibatch: 0001 | Dis/Gen Cost: 1.386/0.795\n", 1476 | "Minibatch: 0201 | Dis/Gen Cost: 1.294/0.812\n", 1477 | "Minibatch: 0401 | Dis/Gen Cost: 1.395/0.805\n", 1478 | "Minibatch: 0601 | Dis/Gen Cost: 1.371/0.738\n", 1479 | "Minibatch: 0801 | Dis/Gen Cost: 1.346/0.752\n", 1480 | "Epoch: 0039 | Dis/Gen AvgCost: 1.346/0.784\n", 1481 | "Minibatch: 0001 | Dis/Gen Cost: 1.313/0.776\n", 1482 | "Minibatch: 0201 | Dis/Gen Cost: 1.300/0.861\n", 1483 | "Minibatch: 0401 | Dis/Gen Cost: 1.459/0.692\n", 1484 | "Minibatch: 0601 | Dis/Gen Cost: 1.310/0.822\n", 1485 | "Minibatch: 0801 | Dis/Gen Cost: 1.410/0.757\n", 1486 | "Epoch: 0040 | Dis/Gen AvgCost: 1.351/0.783\n", 1487 | "Minibatch: 0001 | Dis/Gen Cost: 1.253/0.860\n", 1488 | "Minibatch: 0201 | Dis/Gen Cost: 1.398/0.677\n", 1489 | "Minibatch: 0401 | Dis/Gen Cost: 1.373/0.787\n", 1490 | "Minibatch: 0601 | Dis/Gen Cost: 1.318/0.818\n", 1491 | "Minibatch: 0801 | Dis/Gen Cost: 1.306/0.757\n", 1492 | "Epoch: 0041 | Dis/Gen AvgCost: 1.350/0.773\n", 1493 | "Minibatch: 0001 | Dis/Gen Cost: 1.272/0.820\n", 1494 | "Minibatch: 0201 | Dis/Gen Cost: 1.237/0.793\n", 1495 | "Minibatch: 0401 | Dis/Gen Cost: 1.443/0.742\n", 1496 | "Minibatch: 0601 | Dis/Gen Cost: 1.406/0.774\n", 1497 | "Minibatch: 0801 | Dis/Gen Cost: 1.325/0.766\n", 1498 | "Epoch: 0042 | Dis/Gen AvgCost: 1.352/0.775\n", 1499 | "Minibatch: 0001 | Dis/Gen Cost: 1.314/0.775\n", 1500 | "Minibatch: 0201 | Dis/Gen Cost: 1.328/0.833\n", 1501 | "Minibatch: 0401 | Dis/Gen Cost: 1.404/0.679\n", 1502 | "Minibatch: 0601 | Dis/Gen Cost: 1.304/0.806\n", 1503 | "Minibatch: 0801 | Dis/Gen Cost: 1.358/0.687\n", 1504 | "Epoch: 0043 | Dis/Gen AvgCost: 1.352/0.775\n", 1505 | "Minibatch: 0001 | Dis/Gen Cost: 1.467/0.737\n", 1506 | "Minibatch: 0201 | Dis/Gen Cost: 1.378/0.694\n", 1507 | "Minibatch: 0401 | Dis/Gen Cost: 1.370/0.798\n", 1508 | "Minibatch: 0601 | Dis/Gen Cost: 1.244/0.857\n", 1509 | "Minibatch: 0801 | Dis/Gen Cost: 1.349/0.827\n", 1510 | "Epoch: 0044 | Dis/Gen AvgCost: 1.358/0.767\n", 1511 | "Minibatch: 0001 | Dis/Gen Cost: 1.368/0.737\n", 1512 | "Minibatch: 0201 | Dis/Gen Cost: 1.345/0.766\n", 1513 | "Minibatch: 0401 | Dis/Gen Cost: 1.378/0.760\n", 1514 | "Minibatch: 0601 | Dis/Gen Cost: 1.301/0.797\n", 1515 | "Minibatch: 0801 | Dis/Gen Cost: 1.356/0.789\n", 1516 | "Epoch: 0045 | Dis/Gen AvgCost: 1.356/0.757\n", 1517 | "Minibatch: 0001 | Dis/Gen Cost: 1.400/0.711\n", 1518 | "Minibatch: 0201 | Dis/Gen Cost: 1.311/0.829\n", 1519 | "Minibatch: 0401 | Dis/Gen Cost: 1.452/0.648\n", 1520 | "Minibatch: 0601 | Dis/Gen Cost: 1.365/0.765\n", 1521 | "Minibatch: 0801 | Dis/Gen Cost: 1.397/0.820\n", 1522 | "Epoch: 0046 | Dis/Gen AvgCost: 1.354/0.758\n", 1523 | "Minibatch: 0001 | Dis/Gen Cost: 1.385/0.723\n", 1524 | "Minibatch: 0201 | Dis/Gen Cost: 1.313/0.778\n", 1525 | "Minibatch: 0401 | Dis/Gen Cost: 1.318/0.773\n", 1526 | "Minibatch: 0601 | Dis/Gen Cost: 1.384/0.756\n", 1527 | "Minibatch: 0801 | Dis/Gen Cost: 1.435/0.718\n", 1528 | "Epoch: 0047 | Dis/Gen AvgCost: 1.351/0.771\n", 1529 | "Minibatch: 0001 | Dis/Gen Cost: 1.308/0.739\n", 1530 | "Minibatch: 0201 | Dis/Gen Cost: 1.384/0.739\n", 1531 | "Minibatch: 0401 | Dis/Gen Cost: 1.339/0.755\n", 1532 | "Minibatch: 0601 | Dis/Gen Cost: 1.339/0.801\n", 1533 | "Minibatch: 0801 | Dis/Gen Cost: 1.408/0.822\n", 1534 | "Epoch: 0048 | Dis/Gen AvgCost: 1.357/0.760\n", 1535 | "Minibatch: 0001 | Dis/Gen Cost: 1.324/0.782\n", 1536 | "Minibatch: 0201 | Dis/Gen Cost: 1.325/0.791\n", 1537 | "Minibatch: 0401 | Dis/Gen Cost: 1.394/0.731\n", 1538 | "Minibatch: 0601 | Dis/Gen Cost: 1.390/0.718\n", 1539 | "Minibatch: 0801 | Dis/Gen Cost: 1.374/0.772\n", 1540 | "Epoch: 0049 | Dis/Gen AvgCost: 1.360/0.755\n", 1541 | "Minibatch: 0001 | Dis/Gen Cost: 1.364/0.819\n", 1542 | "Minibatch: 0201 | Dis/Gen Cost: 1.384/0.759\n", 1543 | "Minibatch: 0401 | Dis/Gen Cost: 1.313/0.776\n", 1544 | "Minibatch: 0601 | Dis/Gen Cost: 1.351/0.744\n", 1545 | "Minibatch: 0801 | Dis/Gen Cost: 1.394/0.743\n", 1546 | "Epoch: 0050 | Dis/Gen AvgCost: 1.363/0.752\n" 1547 | ] 1548 | } 1549 | ], 1550 | "source": [ 1551 | "##########################\n", 1552 | "### TRAINING & EVALUATION\n", 1553 | "##########################\n", 1554 | "\n", 1555 | "with tf.Session(graph=g) as sess:\n", 1556 | " sess.run(tf.global_variables_initializer())\n", 1557 | " \n", 1558 | " avg_costs = {'discriminator': [], 'generator': []}\n", 1559 | "\n", 1560 | " for epoch in range(training_epochs):\n", 1561 | " dis_avg_cost, gen_avg_cost = 0., 0.\n", 1562 | " total_batch = mnist.train.num_examples // batch_size\n", 1563 | "\n", 1564 | " for i in range(total_batch):\n", 1565 | " \n", 1566 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 1567 | " batch_x = batch_x*2 - 1 # normalize\n", 1568 | " batch_randsample = np.random.uniform(-1, 1, size=(batch_size, gen_input_size))\n", 1569 | " \n", 1570 | " # Train\n", 1571 | " \n", 1572 | " _, dc = sess.run(['train_discriminator', 'discriminator_cost:0'],\n", 1573 | " feed_dict={'discriminator_inputs:0': batch_x, \n", 1574 | " 'generator_inputs:0': batch_randsample,\n", 1575 | " 'dropout:0': dropout_rate,\n", 1576 | " 'is_training:0': True})\n", 1577 | " \n", 1578 | " _, gc = sess.run(['train_generator', 'generator_cost:0'],\n", 1579 | " feed_dict={'generator_inputs:0': batch_randsample,\n", 1580 | " 'dropout:0': dropout_rate,\n", 1581 | " 'is_training:0': True})\n", 1582 | " \n", 1583 | " dis_avg_cost += dc\n", 1584 | " gen_avg_cost += gc\n", 1585 | "\n", 1586 | " if not i % print_interval:\n", 1587 | " print(\"Minibatch: %04d | Dis/Gen Cost: %.3f/%.3f\" % (i + 1, dc, gc))\n", 1588 | " \n", 1589 | "\n", 1590 | " print(\"Epoch: %04d | Dis/Gen AvgCost: %.3f/%.3f\" % \n", 1591 | " (epoch + 1, dis_avg_cost / total_batch, gen_avg_cost / total_batch))\n", 1592 | " \n", 1593 | " avg_costs['discriminator'].append(dis_avg_cost / total_batch)\n", 1594 | " avg_costs['generator'].append(gen_avg_cost / total_batch)\n", 1595 | " \n", 1596 | " \n", 1597 | " saver.save(sess, save_path='./gan-conv.ckpt')" 1598 | ] 1599 | }, 1600 | { 1601 | "cell_type": "code", 1602 | "execution_count": 5, 1603 | "metadata": {}, 1604 | "outputs": [ 1605 | { 1606 | "data": { 1607 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//HXd5Zksu8JIQmEfcvCEhBFFsUFtQK22kpp\n1V57adVee2837WpbtT9rW6utVWtbrt6KO4q4tbSAoqKyKHtAtkASQlayrzPz/f3xnYQA2Ugmmczk\n83w85jGZmTPnfE4I7znz/X7P9yitNUIIIQKLxdcFCCGE8D4JdyGECEAS7kIIEYAk3IUQIgBJuAsh\nRACScBdCiAAk4S6EEAFIwl0IIQKQhLsQQgQgm682HB8fr9PT0321eSGE8Evbt28v01ondLecz8I9\nPT2dbdu2+WrzQgjhl5RSx3qynDTLCCFEAJJwF0KIACThLoQQAchnbe5CiMGjpaWFgoICGhsbfV2K\n8HA4HKSmpmK323v1fgl3IQQFBQVERESQnp6OUsrX5Qx5WmvKy8spKChg1KhRvVqHNMsIIWhsbCQu\nLk6CfZBQShEXF9enb1IS7kIIAAn2Qaav/x6B0yzjdsP2/4XgCBiWCXHjwBo4uyeEEOcjcNLvyEZ4\n8zunH9sckDTFBP2wLMj4AoRE+64+IUSP/fznPyc8PJzq6mrmzZvHZZdd1qf1XX311Tz77LNER/cs\nA9auXcu+ffu4++67z3tblZWVPPvss9x+++3n/V5vCpxw37EKQmLgprVQsg9O7oainbB3DWx/Cva+\nCje/DvLVUwi/8ctf/rJP79dao7XmrbfeOq/3LV68mMWLF/dqm5WVlTz22GPnFe5OpxObzbtxHBht\n7g2nIPcNyPwiJGdB9o1w5f1wyxtwVx5c+SvIew8OrvN1pUKITtx///2MHz+eiy++mAMHDgBwyy23\n8PLLLwNw9913M3nyZLKysvje974HQHFxMddddx3Z2dlkZ2ezefNm8vLymDBhAjfddBMZGRnk5+eT\nnp5OWVkZeXl5TJw4kVtuuYXx48ezfPly/v3vfzNnzhzGjRvHli1bAHjqqaf41re+1VbDnXfeyUUX\nXcTo0aPb6qmtrWXhwoVMnz6dzMxMXnvttbY6Dx8+zNSpU/n+97+P1prvf//7ZGRkkJmZyQsvvADA\nO++8w9y5c1m8eDGTJ0/2+u8zMI7cd78MriaYtvzc15SCWStg699g3U9hzEJpixeiC794fS/7TlR7\ndZ2Th0dyz7VTOn19+/btPP/88+zYsQOn08n06dOZMWNG2+vl5eW8+uqr7N+/H6UUlZWVANx5553M\nnz+fV199FZfLRW1tLadOneLgwYM8/fTTzJ49+5xtHTp0iJdeeomVK1cyc+ZMnn32Wd5//33Wrl3L\nr371K9asWXPOe4qKinj//ffZv38/ixcv5vrrr8fhcPDqq68SGRlJWVkZs2fPZvHixTzwwAPs2bOH\nHTt2ALB69Wp27NjBzp07KSsrY+bMmcybNw+ATz75hD179vR6uGNXAuPI/dNnTNt6cnbHr1vtcPkv\noewAfPp/A1ubEKJb7733Htdddx2hoaFERkae0yQSFRWFw+Hg1ltv5ZVXXiE0NBSADRs2cNtttwFg\ntVqJiooCYOTIkR0GO8CoUaPIzMzEYrEwZcoUFi5ciFKKzMxM8vLyOnzP0qVLsVgsTJ48meLiYsA0\n+fzoRz8iKyuLyy67jMLCwrbX2nv//fdZtmwZVquVpKQk5s+fz9atWwGYNWtWvwQ7BMKR+8k9ULQD\nFv266+UmXgMjLoKNv4KM68EROTD1CeFnujrC9hWbzcaWLVtYv349L7/8Mo8++igbNmzodPmwsLBO\nXwsODm772WKxtD22WCw4nc5u36O1BmDVqlWUlpayfft27HY76enp5z0uvas6+8r/j9x3rAJrEGR9\nsevllIIr74O6UvjgkYGpTQjRI/PmzWPNmjU0NDRQU1PD66+/fsbrtbW1VFVVcfXVV/P73/+enTt3\nArBw4UIef/xxAFwuF1VVVQNWc1VVFYmJidjtdjZu3MixY2Ym3oiICGpqatqWmzt3Li+88AIul4vS\n0lI2bdrErFmz+r0+/z5ydzbDrhdgwlUQGtv98ikzIPMG+PBRyPkPiErp/xqFEN2aPn06X/rSl8jO\nziYxMZGZM2ee8XpNTQ1LliyhsbERrTUPPfQQAI888ggrVqzgb3/7G1arlccff5zk5OQBqXn58uVc\ne+21ZGZmkpOTw8SJEwGIi4tjzpw5ZGRkcNVVV/Hggw/y4Ycfkp2djVKKBx98kGHDhrF///5+rU+1\nfsUYaDk5ObrPF+vYtxZe/CosfxnGXd6z95w6Bo/ONOPer3u8b9sXIkDk5uYyadIkX5chztLRv4tS\narvWOqe79/p3s8yOVRCRDGMu7fl7YkbC7G/CzufMOHghhAhA/hvu1UVm3Hr2MrBYz++9F3/HnPC0\n7ifgo28uQgjRn7oNd6XUSqVUiVJqTxfLLFBK7VBK7VVKvevdEjux63nQbpjawdj27oREw4K74egm\nOPgv79cmhBA+1pMj96eARZ29qJSKBh4DFmutpwA3eKe0LmgNn66CERdC/NjerWPG1yB2NLz/kHdr\nE0KIQaDbcNdabwIquljky8ArWuvjnuVLvFRb5/K3QPnB3h21t7IFwdjLzTh5aZoRQgQYb7S5jwdi\nlFLvKKW2K6Vu8sI6u7bjGbCHwZSlfVtP/DhoroGak96pSwghBglvhLsNmAFcA1wJ/FQpNb6jBZVS\nK5RS25RS20pLS3u3teY62POKCfbgiN7WbMSPM/dln/VtPUKIgPHwww9TX1/v6zL6zBvhXgD8U2td\np7UuAzYBHU7yorV+Umudo7XOSUhI6N3W9q2F5lqY9pVeF9wm3vMZJOEuxJChtcbtdnf6em/C3eVy\n9bUsr/NGuL8GXKyUsimlQoELgFwvrLdjE6+Bz//FdKb2VUQyBIVD2cG+r0sI0Sf33nsvEyZM4OKL\nL2bZsmX89re/5fDhwyxatIgZM2Ywd+7ctrM6O5uGF+A3v/kNM2fOJCsri3vuuQegw2mAb7vtNnJy\ncpgyZUrbcn/4wx84ceIEl1xyCZdccgkAzz33HJmZmWRkZHDXXXe1bSc8PJzvfve7ZGdn8+GHHw7U\nr6nHup1+QCn1HLAAiFdKFQD3AHYArfUTWutcpdQ/gF2AG/ir1rrTYZN95ojsfh6ZnlLKNM2US7gL\n0ebtu83FbrxpWCZc9UCnL2/dupXVq1ezc+dOWlpa2qb8XbFiBU888QTjxo3j448/5vbbb2+bMKyj\naXjXrVvHwYMH2bJlC1prFi9ezKZNmxgxYsQ50wDff//9xMbG4nK5WLhwIbt27eLOO+/koYceYuPG\njcTHx3PixAnuuusutm/fTkxMDFdccQVr1qxh6dKl1NXVccEFF/C73/3Ou78rL+k23LXWy3qwzG+A\n33ilooEWPx6ObfZ1FUIMaR988AFLlizB4XDgcDi49tpraWxsZPPmzdxww+nR1U1NTW0/dzQN77p1\n61i3bh3Tpk0DzIRjBw8eZMSIEedMA/ziiy/y5JNP4nQ6KSoqYt++fWRlZZ1R19atW1mwYAGtzcjL\nly9n06ZNLF26FKvVyhe+8IV++530lX9PHOYN8ePM5GPNdRDUf9NvCuE3ujjCHkhut5vo6Oi2i16c\nraNpeLXW/PCHP+Qb3/jGGcvm5eWdMb3u0aNH+e1vf8vWrVuJiYnhlltuOe/peh0OB1breZ4dP4D8\nd/oBb4nzjJgpP+TbOoQYwubMmcPrr79OY2MjtbW1vPHGG4SGhjJq1CheeuklwAR361S/nbnyyitZ\nuXIltbW1ABQWFlJScu6pN9XV1YSFhREVFUVxcTFvv/1222vtp+ydNWsW7777LmVlZbhcLp577jnm\nz5/vrd3uV3Lk3jZi5mDnV3ISQvSrmTNnsnjxYrKyskhKSiIzM5OoqChWrVrFbbfdxn333UdLSws3\n3ngj2dmd/z+94ooryM3N5cILzYCL8PBwnnnmmXOOsLOzs5k2bRoTJ04kLS2NOXPmtL22YsUKFi1a\nxPDhw9m4cSMPPPAAl1xyCVprrrnmGpYsWdI/vwQv8+8pf72hpRF+lQzzfgCX/NDX1QjhE4Nhyt/a\n2lrCw8Opr69n3rx5PPnkk0yfPt2nNflaX6b8lSN3uwOiR8pYdyF8bMWKFezbt4/GxkZuvvnmIR/s\nfSXhDqZpRsa6C+FTzz77rK9LCCjSoQqnx7p3cdaaEIHOV020omN9/feQcAcT7s5GqMr3dSVC+ITD\n4aC8vFwCfpDQWlNeXo7D4ej1OqRZBs4cMRMz0re1COEDqampFBQU0OsJ/YTXORwOUlNTe/1+CXc4\nHe7lB2HcZb6tRQgfsNvtjBo1ytdlCC+SZhmA0DhzTVUZMSOECBAS7uCZQExGzAghAoeEe6u4cXLk\nLoQIGBLureLHQW0xNFb5uhIhhOgzCfdWbSNmZAIxIYT/k3BvJZfcE0IEEAn3VjEjwWKXcBdCBAQJ\n91ZWO8SOlnAXQgQECff24sfJcEghRECQcG8vfhxUHAGX09eVCCFEn0i4txc/HtwtUHnM15UIIUSf\nSLi3JyNmhBABQsK9vbix5l7CXQjh5yTc2wuJhrBECXchhN+TcD9b/Hg5S1UI4fck3M8WLxOICSH8\nn4T72eLHQ0MF1JX7uhIhhOg1CfezyYgZIUQAkHA/W/w4cy/hLoTwYxLuZ4tKA5tDwl0I4dck3M9m\nsZjx7uUyYkYI4b8k3DsiI2aEEH5Owr0j8ePhVB44m3xdiRBC9IqEe0fixoJ2m4AXQgg/1G24K6VW\nKqVKlFJ7ulluplLKqZS63nvl+UjcGHNffti3dQghRC/15Mj9KWBRVwsopazAr4F1XqjJ92Jbw106\nVYUQ/qnbcNdabwIqulnsv4DVQIk3ivK5kGgIjZdwF0L4rT63uSulUoDrgMf7Xs4gEjdWmmWEEH7L\nGx2qDwN3aa3d3S2olFqhlNqmlNpWWlrqhU33o7gxUCHhLoTwT94I9xzgeaVUHnA98JhSamlHC2qt\nn9Ra52itcxISEryw6X4UNwZqiqCp1teVCCHEebP1dQVa61GtPyulngLe0Fqv6et6fa71qkwVhyE5\n27e1CCHEeeo23JVSzwELgHilVAFwD2AH0Fo/0a/V+VJruJcfknAXQvidbsNda72spyvTWt/Sp2oG\nkxjPF5LyI76tQwghekHOUO1MUChEpspwSCGEX5Jw70rcGAl3IYRf6nOHakCLGwt7Vvu6CiGEn9Fa\nU9XQQlltMxV1zVTUNVFe10xFbTPldc1cOCaOK6cM69caJNy7EjcGGiuhvgJCY31djRAA5JXV0eJy\nMyYhHItF+bqcQafJ6WJPYTWfHDtF7slqIh12EiODSYpwmPtIB4kRwUQ47Fj78PtraHbxaf4p9hZW\nU1TVSHFNI8Wt99VNNDs7PvUnIthGbFiQhLtPtR8xEzrLt7WIPtFa89TmPFZ/UkBKdAhjEsIZkxDO\n6IQwxiSGE+mwD0gdTU4Xn52sZXdhFbsLq9hTWMXRsjoyUiK5ZEIil0xMZFxiOEqdGTpHSmt5c1cR\nb+4uYv/JGgCiQ+3kjIxhZnosOemxZKZEEWSz4HZrTlY3kl9RT/6pBvIr6imrbSI2LIiEiGASwoNJ\njAwmIdxBfEQQIXbrOdvritZm/blF1eQW1bCvqJr9RdXUN7tYOCmRqzOSmTUqFpu1762+WmsaWlxU\n1DVzqq6F+mZnh8udqm/h0+On2H7sFLsKq9qCNTEimPpmF7VNHb8v2GYhLNhGiN1KWLCV0CAb8eFB\npMaEkhoTQmpMKGmxIaTFhtLidLPt2Cm25VWwJe8UewurcLo1ACF2K8OiHCRFBjN9REzbB0h8eDCx\nYUHEhQcRFxZMTJidYJu1z7+XnlBa6wHZ0NlycnL0tm3bfLLtHis/DH+cDksfh6lf9nU1AvOfveBU\nA7sLq9h3opqs1Cgun5zUZTg1trj48at7WP1JAVOGR9LQ4uJ4eX3bf0yA+PBghkV5gi/CQUJEawAG\noz3raGxxm3uni6YWN0E2C5EhdqJC7EQ6bER5fgY4Wd1IcXUjJ6uaKPb8fLyins+Ka2hxme1GOmxk\npEQxMi6MT4+fagvtlOgQFkxIYO64eA6V1PLGrtOBnjMyhqszkwl32NiWV8G2vFMcKasDTFANi3JQ\nVNlIs+v0UaNSEBVip6qhhc7+uwfZLATbLDjsVoI9P3d2VFtS00RlfUvb49SYECYlR2KzKN45UEpD\ni8tzZJrEVRnJXDgmjoq6Zj4rruFgcS0HS2o5WFzD0bI6XFp7tmdtqyHIZqGxxc2pumYq6ps7PQI+\nZx+sFjJSIpkxMoYZI2OYPjKGxAgHAHVNTkpqmiipbqTYc1/b5KS+2UV9s5P6Jhf1zS7qmp2U1jRR\ncKqh0w+EIKuF7LQoZqbHMjM9lqlp0USH2s/rA7IvlFLbtdY53S4n4d4FVwvclwQX/zcs/Jmvqwlo\nWmuKqhqpa3LS2OKmyXk6TOuanRw4WdN2tNs+WAAWTEjgF4unMDIu7Jz1llQ3suLv29mRX8l/XzaO\nOy8dh8WiaHG5OV5Rz5HSOg6X1pJXVkdxdSMlNU2U1jRRVtuE20v/NSKCbSRGBpMSE8qU4ZFkDI8i\nMyWKtNiQMwKhqKqBdw6UsnF/Ce8fKqO+2QWcDvSrMoeRHBVyzvpLa5rYfqyCLUdPUVzTSGpMCGkx\noaTFhjIiNpTh0Q6CbVacLjcV9c2U1TRTWnt6PxuaXTQ5ze+6yWl+900tbtydZENUiJ1JyZFMSo5k\nYnLEGd96GppdvPtZCW/tPsn63GLqml1YLQpXu19mdKid8YkRjE4Iw2610OzZZrPLTVOLm2aXm2Cb\nhZjQIGLDgogJCyIm1E5MaBBhwTY6itCQICuTkiNx2L1zVNzaZp5f0UDBqXryT9Xj1ubfIiMlymvb\n6Q0Jd2/5w3QYlglffNrXlQxKJdWNNLa4SYwM7tUffE1jC2s+LWTVx8fbjk47YrMoxidFkJUaRUZK\nFFmpUYxNDOe5Lfk8tO4ATrfmjkvG8o35o9u+9u7Ir+Qbf99GTaOT392QzVWZyT2uy+XWlNc1UVbT\njFLgsFtx2C04bFYcdnOU2eJyU93YQnVDC1WeW3WDE7fWDIt0kBTlICnSQXjw+bd+Njld7CqoIjUm\npMNA9weNLS7eO1jGtrwKUmJCGJsYzvikCOLCggbsKDcQSbh7y6ovQvUJuO19X1cyqDS2uHh0wyGe\nePdwW/NGbFgQSZEOhkUGMywqhNSYENLjwkiPD2VUfBihQSbktNbsKqji2Y+Ps3bnCRpaXExOjuQL\nM1JJigwm2GaCNNhmmghCgqyMiA3t9MPjZFUj9765jzd3FTEqPoxfLplCaU0Td7+ym8SIYP5yUw6T\nkiMH7HcjRH/qabhLh2p34sZC3nvgdoNFTgsA2Hy4jB+/uoejZXV8fnoKs0fHUVzVSFG1GS1wsrqR\nXQVVlNc1n/G+pMhgRsWHUd3gZF9RNSF2K4uzh/PlC0aQlRrV66O5YVEO/vTl6Xwpp5SfvbaHr/5t\nCwCzR8fy2PIZxIYF9XmfhfA3Eu7diRsNLfVmhsioFF9X41On6pq5/61cXt5ewMi4UFZ9/QLmjI3v\ndPm6Jid55XXkldVztKyWo577IJuFe5dMYcm0FK+OUpk3PoF//Pc8/vb+URqaXXz7snHYvTBiQwh/\nJOHenfazQwZ4uGutKa5uoqHFRbPTTYvLTZPn/mhZHb/55wGqG1q4fcEY7lw4rts29rBgG1OGRzFl\neNQA7YFpG7/jkrEDtj0hBisJ9+60H+s+ap5va+knWmveOVDKHzcc5JPjlZ0uNzUtmv/3+UxpvxbC\nD0i4dydiONgcAXnJPbdbs27fSf644RB7T1STEh3CD6+aSGJkMEFWMyLEblUE2SyEBtnITInq0xl9\nQoiBI+HeHYsFYn07gdjO/Eqe3pzH23tOEh1qZ0RsKCPjQhkZF0ZabCgjY0NJiAgmOtTe7dmGLrem\nttHJO5+V8OiGQxwsqSU9LpQHr8/iumkp0kYtRICQcO+JuDFQkjugm2x2unl7TxFPbc7j0+OVhAVZ\nuTY7GadLc7yino0HSimtKTjnfUFWc9ZkdKidaM/ZktWNLdQ0OqluaKHOc2IMwPikcB65cSqfyxou\nR+RCBBgJ956IGwsH3gKXE6z9+ysrq23imY+Oserj45TWNDEqPox7rp3M9TNSiThrZEl9s5P8igaO\nV9RTUWdOCa9saKGyvoWqBjMXh1IwOj6cCIeNyBA7kQ47EQ4boxLCmD8uQSaeEiJASbj3RNwYcDuh\n8pj5uR/kV9Tzl/eO8MLWfJqcbhZMSOCWi9KZ10UAhwbZmDAsggnDIvqlJiGE/5Jw74m2ETOHvR7u\nB07W8MS7h1m78wQWBddNS2HFvDGMTQz36naEEEOLhHtPtB/r7iX7TlTzu3UHWL+/hNAgK1+7KJ1b\n547y23lEhBCDi4R7T4TGQXCU10bMvLD1OD99bS9hQVa+c/l4brpwJNGhcoq8EMJ7JNx7QimvXE+1\nscXFz17bw4vbCpg7Lp5Hbpwm854IIfqFhHtPxY2F4x/2+u3Hy+u5bdV29p6o5s5Lx/Lty8bL8EMh\nRL+RcO+puLGw+yVoaQD7+bWLr88t5n9e2AHAyltyuHRiUn9UKIQQbSTceypuDKCh4igkTe7RWxqa\nXTy8/jP+/O4RpgyP5ImvzCAtNrR/6xRCCCTce651CGT5oW7DXWvNm7uL+NWbuZyoauTGmWn8fPEU\nn16aSwgxtEi491Rsu3DvQm5RNT9fu5ePj1YwOTmSR5ZNY2Z67AAUKIQQp0m495QjEsKTOh3rXlnf\nzEP/+oxnPjpGZIid+5ZmsGzWCOk0FUL4hIT7+Ygd0+HUv9WNLVz58CZKa5r4yuyRfOfy8TJuXQjh\nUxLu5yNuDHz2j3OefmV7AcXVTTz79Qu4qIvLzgkhxECRybvPR9xYqCuFxqq2p7TW/P2jY2SnRUuw\nCyEGDQn389E6x0zZwbanPjxSzuHSOm6aPdJHRQkhxLkk3M/H8KnmvmBr21N///AY0aF2rslK9lFR\nQghxLgn38xGVClFpcPwjAE5WNbJuXzFfykmTMexCiEFFwv18pV0A+R+D1jy35ThurfnyBSN8XZUQ\nQpyh23BXSq1USpUopfZ08vpypdQupdRupdRmpVS298scREbMhpoiWsrzeG7LceaPT2BkXJivqxJC\niDP05Mj9KWBRF68fBeZrrTOBe4EnvVDX4DViNgB7P15HSU0TX5WOVCHEINRtuGutNwEVXby+WWt9\nyvPwIyDVS7UNTomTITiSsr3vkBIdwoIJib6uSAghzuHtNvdbgbc7e1EptUIptU0pta20tNTLmx4g\nFit1idNIrd3N8tkyvYAQYnDyWrgrpS7BhPtdnS2jtX5Sa52jtc5JSEjw1qYH3MfO8Uy05POljAhf\nlyKEEB3ySrgrpbKAvwJLtNbl3ljnYFXX5OSZE2ZMe1zFTh9XI4QQHetzuCulRgCvAF/VWn/W95IG\nt9d2nODDpnS0svbpsntCCNGfup04TCn1HLAAiFdKFQD3AHYArfUTwM+AOOAxpRSAU2ud018F+5LW\nmv/7MI/05EQIyTbj3YUQYhDqNty11su6ef3rwNe9VtEgtqugiv0na7j/ugzUqdmwbSU4m8Em0/sK\nIQYXOUP1PHx63Iz4XDgxyYx3dzZCkbS7CyEGHwn385BbVENMqJ2kyGBIMyczSbu7EGIwknA/D7kn\nq5mUHIlSCiKSIGaUtLsLIQYlCfcecrrcHDhZw6TkyNNPjphtZojU2neFCSFEByTceyivvI4mp/vc\ncK8v6/C6qkII4UsS7j20r6gGgEnJ7c5KbW13z//IBxUJIUTnJNx7KLeoGptFMTYx/PST8eMhJEY6\nVYUQg46Eew/lFlUzNjGcYFu7Ky5ZLObiHcelU1UIMbhIuPdQblH1me3trUbMhvKDUFc28EUJIUQn\nJNx7oKKumeLqpjPb21u1tbvL0bsQYvCQcO+B3KJqgI6P3IdPA2uQtLsLIQYVCfce6DLc7Q4T8NLu\nLoQYRCTce2BfUTUJEcHEhwd3vMCI2XDiU2hpGNjChBCiExLuPZBbVNPxUXurtNngbjEBL4QQg4CE\nezeanW4OldR03JnaasRsUBbY/+bAFSaEEF2QcO/G4dJaWlyayV0duYfGwpTrYPvT0Fg1cMUJIUQn\nJNy70WVnansX3QnNNbDtfwegKiGE6JqEezdyi6oJslkYHR/W9YLDp8LoBfDR4+BsGojShBCiUxLu\n3cgtqmF8Ujg2aw9+VRfdCbUnYfdL/V+YEEJ0QcK9C1prM+3AsG6aZFqNuRSSMuGDP4Db3b/FCSFE\nFyTcu1Ba00R5XXP37e2tlII534ayA3Dwn/1bnBBCdEHCvQv7etqZ2t6UpRA1Aj54pJ+qEkKI7km4\ndyHXc4GOLodBns1qhwvvMHPNyJQEQggfkXDvQm5RNcOjHESF2s/vjdO/ai7isfkP/VOYEEJ0Q8K9\nC53O4d6doDCY+Z/mjNWyg94vTAghuiHh3onGFhdHyup6F+4As1aALRg2/9G7hQkhRA9IuHfiYHEt\nLrfufbiHJ8DUL8PO56Cm2LvFCSFENyTcO3F62oEuJgzrzoXfAlcLvPsAaO2lyoQQonsS7p3YV1RN\niN3KyLhuph3oStwYuOCbsG0lvPP/vFecEEJ0w+brAgar3KJqJgyLwGpRfVvRlb8yE4q9+2uw2GH+\n971ToBBCdEHCvQOt0w5ckzW87yuzWODaP4DLCRvvA6sNLv6fvq9XCCG6IOHegRNVjVQ3Opncl/b2\n9ixWWPoYuJ3w75+bI/iLvuWddQshRAck3DuQe6IX0w50x2KF6/5sAn7dj82ZrBd8w3vrF0KIdiTc\nO9A6UmbCMC8dubey2uALfzUB//YPwGKDmbd6dxtCCEEPRssopVYqpUqUUns6eV0ppf6glDqklNql\nlJru/TIH1s6CKkYnhBHhOM9pB3rCaofr/xfGL4K3vgefrfP+NoQQQ15PhkI+BSzq4vWrgHGe2wrg\n8b6X5TtfLWivAAAURUlEQVRaa3bkVzI1Nbr/NmILgutXQlIGvPwfUJLbf9sSQgxJ3Ya71noTUNHF\nIkuA/9PGR0C0UirZWwUOtKKqRspqm8hO68dwBzP/zLLnISgUnv0S1JX37/aEEEOKN05iSgHy2z0u\n8Dx3DqXUCqXUNqXUttLSUi9s2vt2FVQCkJUa1f8bi0qBG5+D2mJ44SvgbO7/bQohhoQBPUNVa/2k\n1jpHa52TkJAwkJvusR35VdityrsjZbqSOgOW/AmOb4Y3/0emKRBCeIU3RssUAmntHqd6nvNLO/Mr\nmZQcicNuHbiNZl4PpQdg04OQMEnGwAsh+swbR+5rgZs8o2ZmA1Va6yIvrHfAud2a3YVVZPdnZ2pn\nFvwQJi2GdT+Bz+T6q0KIvun2yF0p9RywAIhXShUA9wB2AK31E8BbwNXAIaAe+Fp/FdvfjpTVUtvk\nHJj29rNZLHDdE1B5DF6+Ff5zPSRMGPg6hBABodtw11ov6+Z1DdzhtYp8aEd+FQBT+3ukTGeCwkwH\n65/nmQ7Wr68HxwC1/QshAopM+dvOzvxKwoNtjE4I910RUSlww1NQfhheu106WIUQvSLh3s6ugkoy\nUiL7Ps1vX42aC5f/EnJfhw8e9m0tQgi/JOHu0eR0sa+ouv9PXuqpC++AKZ+H9b+Ewxt8XY0Qws9I\nuHvkFtXQ4tL9O+3A+VAKljwKCRNNB+upY76uSAjhRyTcPdrOTB0sR+5gOli/9Ay4XfDiV6GlwdcV\nCSH8hIS7x478SuLDgxke5fB1KWeKGwOffxKKdsIb/wMVR6DyOFSfgNpSqK+AplpfVymEGGRkPneP\nnfmVTE2LQikfd6Z2ZMIimH83vPsA7Hyu42VGzYPL74XhUwe2NiHEoOSX4V7d2EJEsM1rQVzd2MKR\nsjqWTu1wvrPBYf5dkJoD9eXmYh+uFnPvdkFDBWz9Kzw5HzK/CJf+BGJG+rpiIYQP+V24v7ajkG8/\nv4N3v7+AkXFhXlnnnoIqtB5k7e1ns1hg3OWdv37hHfDBI/Dhn2DfGnMJv7nfhZCYgatRCDFo+F2b\n++h4c4LR7sIqr61zh6czNdsX0w54iyMKFv4M/usTyLwBNj8Kj0yFdT+FA/+AhlO+rlAIMYD87sh9\n/LBw7FbF7sIqPpc13Cvr3JlfSXpcKNGhQV5Zn09FpcDSx2D27WaM/EePweY/AAoSJ8PIC2HEhTBq\nPoQPzmmXhRB953fhHmyzMmFYBHu8eOS+q6CKmemxXlvfoDAsA5a/CM31ULgdjn8IxzbDzudN+3xw\nJCx/CUbM9nWlQoh+4HfhDpCZEsVbu0+ite5zp2pxdSNFVY2D58xUbwsKNdMZjJprHrucULQDXlkB\nf7/OXOpv9Hzf1iiE8Dq/a3MHyEiJoqqhhfyKvp/UszPftLdPTfPj9vbzYbWZUTdfexti0mHVDTJ/\nvBAByC/DPTPFBLE3OlV3FVRhtSgmJw+RcG8VkQS3vAmJk+D55bB3ja8rEkJ4kV+G+4RhEW2dqn21\ns6CSCUkRhAQN4GX1BovQWLh5LaRMh5e/Bjs6OUFKCOF3/DLcg21Wxif1vVPV7dbszK8M3Pb2nnBE\nwVdegfSLYc034eM/yxw2QgQAv+xQBdM08/aevnWq5pXXUd3oHDrt7Z0JDocvvwgv3gxv/wD++SPT\nXDN8ujmqHz7dPLbafV2pEKKH/DbcM1KieH5rPgWnGkiLDe3VOnYVmCP/rMEyza8v2UPgxlVwcB0U\nbIMTn5gzXT952vN6GFz6YzN+fjDOvyOEOIPfhntrp+qewqpeh/uO/EpC7FbGJfrwsnqDidUOE68x\nNzCX+Ks4AoWfwO4XzRH98Y9gyZ/k2q5CDHJ+2eYOplPVZulbp+pHR8rJTovCZvXbX0P/UspMOZx1\ng2m2ufxe2P8mPLkATu7xdXVCiC74bao57KZTtbfhXljZwP6TNVw6MdHLlQUopWDOnXDLG9BcB39d\nCJ+u8nVVQohO+G24g2ma2VNYhdb6vN+7YX8JAJdOTPJ2WYFt5EXwzfcgbRa8dju89i0ZXSPEIOTX\n4Z6RGsWp+hYKK88/XDbkFjMyLpQxCd6ZNnhICU+Er66Bed+HT/8OKxdBVYGvqxJCtOPX4d6+U/V8\n1Dc7+eBwOZdOTBycV17yBxaruSjIsheg/LBphz/2oa+rEkJ4+HW4T+xlp+rmQ+U0O90slCaZvpuw\nCP5zgzkZ6ulrYdtKX1ckhMCPh0KC6VQdlxTB7sLq83rf+v0lhAVZmTUqwKb59ZWE8fD19bD66+Yi\n3kW74KoHweaZH9/tNuPmP/snHPwnVBVC2gXmrNj0iyEpw1xpyhvKD8M7D5hvFov/KCdeiSHLr8Md\nIGN4JOv3l/T4TFWtNRv2FzNvfAJBNr/+4jK4hETDl1+ADffC+7+HklyY9Z9waL05Maq+DJQFUmeZ\nywUe/wgOvGne64g2HbWj5sGUz5tJzc5XVSG8+2v49BmwBoHT0w+z5DHvfXAI4Uf8PtwzU6N4aXsB\nJ6oaSYkO6Xb5vSeqKa5ukiGQ/cFihct+DsMyYc0dsPpWE9xjL4Pxi2DsQjNZWauqQjj2AeS9B3nv\nw4G34J8/hglXwfSbzfKWbiZ0qyuH9x+CLX8B7YaZX4d534PtT8PG+yAsHq64rz/3WohBye/DPaN1\n+t+Cqh6F+4b9JSgFCyZIuPebjC9ASg7UFpt5aayd/JlFpUDWF80NoOwgfPJ/sONZ2P8GRKbA1OUw\nbTkEhZv11RZDbYm5r8w3V5ZqqYOsG2HB3RAz0qxr3vfMMpv/CGGJZoy+EEOI34f75ORIrBbFnsIq\nFmUM63b59ftLyE6NJiEieACqG8JiRp4O2p6KHwdX3AuX/hQ++4eZ12bTb2DTgx0vbwsxR/eX/sRM\nbNaeUnDVr01z0L9+CmEJMHVZ7/ZFCD/k9+Hu8MwN05MRM6U1TezMr+S7l48fgMpEr9mCYPJic6vM\nh9y1YLGZ8fXhSZ5bojma76qfxWKF6/4M9RXw2h2mSWj8lQO3H0L4kN+HO5immY096FTdeMBzVuok\naZLxG9FpcOEdvX+/LdjMdvnUNWZK45tegxEXeK8+IQapgBhGkJkSRXldM0VVjV0utyG3hGGRDiYn\ny4yGQ0pwBCxfDZHJJuT/ernpuN33GtSc9HV1QvSLHh25K6UWAY8AVuCvWusHznp9BPA0EO1Z5m6t\n9VterrVTGe3OVB3eSadqk9PFewdLWTItRc5KHYrCE+DmN+Cjx6Bgqxld8+Gj5rWoEZA6A6JHQkQy\nRAxrdz/MHP0L4We6DXellBX4E3A5UABsVUqt1Vrva7fYT4AXtdaPK6UmA28B6f1Qb4cmJ0diUSbc\nr5jScafqlqMV1DW7WChDIIeuqBS48n7zs7PJnGxVsAXyt5g563PfAHfLue8bcSFk3gBTrjtzKGd7\nzmbI22TWUVdqrmCVOhOGTzPfHIQYYD05cp8FHNJaHwFQSj0PLAHah7sGWts6ooAT3iyyOyFBVsYl\ndj397/rcEoJtFi4aEz+AlYlByxYMaTPNrbVNX2vT+VpTZJpraorgVJ4Zlvnmd+Dtu8yY/awbYPxV\nZlz9oX+b1z9bB01V5opVEUnmOTAnbiVMgtQcc6LW5KVgd/hst8XQ0ZNwTwHy2z0uAM7ukfo5sE4p\n9V9AGHCZV6o7DxkpUbz7WWmHnarmrNQS5oyNJySom5NixNClFITFmduwjNPPX/oTOLnbXI1q92r4\n7G0zUsfVAq4mCI2DydfCxGth9HxzycL6CvNtoHCbaQba95oZ2vmvn5kPk5z/6PqI3u2Ck7vAGgyx\no8w6fU1rOLzeTPE84eruTzATPuWt0TLLgKe01r9TSl0I/F0plaG1drdfSCm1AlgBMGLECC9t2shM\niWT1JwUUVzcxLOrMI6PDpXUcr6hnxbzRXt2mGCKUguQsc7vsF+as2r2vgs1hLkmYNvvcE7VCY2Hc\nZeYGJhiPbjJn0/7rZ/DeQ3DBN+CCb55u6mk4dXq6hkP/hvry1gLMCV1xoyF2jLk6Vtps821goPqP\nTuyAdT8xZxMDJEw0J41NWiLTOwxSPQn3QiCt3eNUz3Pt3QosAtBaf6iUcgDxQEn7hbTWTwJPAuTk\n5Jz/FTa6kJlqOlV/sHoXF46OIyMlkinDo4gNC2LD/mIAmXJA9J3FaubAGTXv/N6nlDmqHz0fCreb\ncH/317D5Ucj4vJnwLP9j0C4IiTXz74y93Lyv/DBUHDb3+9aYDwEwzT0zboasL3XeF9Cd+gozo2dn\nR+HVJ2DDfeas4dBYuPq35pvKOw/AS7eYSd8W/NB8yMlAhUFFdXcVI6WUDfgMWIgJ9a3Al7XWe9st\n8zbwgtb6KaXUJGA9kKK7WHlOTo7etm2bF3bBaHa6uWv1LrYcrTjj4h3Doxw0Od0kRjp4+9tzvbY9\nIfqsJBfefxj2rDZn2I67wpxklTKj6yaPunLY/7qZP+fEJ6bpZvJiMx9P+sWdh2xLAxTtNM1EBVuh\nYBtUF5ozfRMnQdIUE9ZJUyB2tLkQywePgNsJs2+Dud81HwRgmo32rDYhX3EYkqdCztdMU1XDKWio\n9NyfguZaMzunLcT0ddhDzLeeoFAYeTGMufT0DKKiW0qp7VrrnG6X68kl6pRSVwMPY4Y5rtRa36+U\n+iWwTWu91jNC5i9AOKZz9Qda63VdrdPb4d5eZX0ze09Us/dEFXsKqzlwsoZb5qSzbJZ3m4KE8Aq3\nu/dNGyd3m5Df9aLp0HVEmU5dW5AJ/dZ7V5P5MHE7zfuiR5rRPMMyzRw8xXugeG+7piCPKdeZyeBi\n0jvevssJu14w30Iqj51+PigcQmLMxHHBEeBqBmejubU0mlk7m2pNXSExpqM583oYcZE083TDq+He\nH/oz3IUYcprrTadtwVYTmM7mM++VBYZlmUBPzTHTN5xNazMpW/EeKD1glkub1bPtu1rg1DHz4eKI\n6tmRuLMZjmyE3S/B/jehpd70LWR83nxzaKo1R/1NtdBc4/kwaDajlNpu2twHhcLIOTB6gekPCOAm\nIgl3IYT/aK6DA2/D7pfh0L9Of8MA800kONx8G7A5zJG98txQ5r6+zAxbBTP30Kj5JuhHLzDnN/SW\n1mYSu/d+Z6aozrrBNH/Fjen9OvtIwl0I4Z8aq0zYB4VDUFjPh1xWHocj78KRd8ytvsw8P3y6aV6a\nshSie9g063aZzuv3HjLfZKJHmm8Eh/5tOr3T58KMW2Di5wb8vAUJdyHE0OV2Q8k+E8b71sCJT83z\nqTNN0E9eAlGp5rnWph2tzRnKe1abq4mVH4L48aYjOeN6M9y1ugh2PGOuO1B53PQXZFwPw6ea0UsJ\n4/v9jGQJdyGEaFVxBPauMecnnNzV/fLDMmHu92DS4o47eN1uOPqO6cw+8Lbp12gVmQqJEyF+gvmm\nEDncNA1FpprrCvSxw1jCXQghOlJ+2FzSsbHadLy2td0rcz98qplmoqedsi6nGSlUkgul+01ndGmu\nubKY86yZai12MzvprBVw0X/1qvyehntAzOcuhBA9Fjem18HaIavNrDNuDEz63OnntTZDS6sKzMlg\n1YXmVlUI4d1fNa6vJNyFEKI/KGUu0B4Wb74NDDA5W0AIIQKQhLsQQgQgCXchhAhAEu5CCBGAJNyF\nECIASbgLIUQAknAXQogAJOEuhBAByGfTDyilSoFj3S7YsXigzIvl+JOhuu+y30OL7HfnRmqtE7pb\nkc/CvS+UUtt6MrdCIBqq+y77PbTIfvedNMsIIUQAknAXQogA5K/h/qSvC/Chobrvst9Di+x3H/ll\nm7sQQoiu+euRuxBCiC74XbgrpRYppQ4opQ4ppe72dT39RSm1UilVopTa0+65WKXUv5RSBz33Mb6s\nsT8opdKUUhuVUvuUUnuVUt/2PB/Q+66Uciiltiildnr2+xee50cppT72/L2/oJQK8nWt/UEpZVVK\nfaqUesPzOOD3WymVp5TarZTaoZTa5nnOa3/nfhXuSikr8CfgKmAysEwpNdm3VfWbp4BFZz13N7Be\naz0OWO95HGicwHe11pOB2cAdnn/jQN/3JuBSrXU2MBVYpJSaDfwa+L3WeixwCrjVhzX2p28Due0e\nD5X9vkRrPbXd8Eev/Z37VbgDs4BDWusjWutm4HlgiY9r6hda601AxVlPLwGe9vz8NLB0QIsaAFrr\nIq31J56fazD/4VMI8H3XRq3nod1z08ClwMue5wNuvwGUUqnANcBfPY8VQ2C/O+G1v3N/C/cUIL/d\n4wLPc0NFkta6yPPzSSDJl8X0N6VUOjAN+JghsO+epokdQAnwL+AwUKm1dnoWCdS/94eBHwBuz+M4\nhsZ+a2CdUmq7UmqF5zmv/Z3LNVT9lNZaK6UCdqiTUiocWA38t9a6WrW7En2g7rvW2gVMVUpFA68C\nE31cUr9TSn0OKNFab1dKLfB1PQPsYq11oVIqEfiXUmp/+xf7+nfub0fuhUBau8epnueGimKlVDKA\n577Ex/X0C6WUHRPsq7TWr3ieHhL7DqC1rgQ2AhcC0Uqp1oOwQPx7nwMsVkrlYZpZLwUeIfD3G611\noee+BPNhPgsv/p37W7hvBcZ5etKDgBuBtT6uaSCtBW72/Hwz8JoPa+kXnvbWvwG5WuuH2r0U0Puu\nlErwHLGjlAoBLsf0N2wErvcsFnD7rbX+odY6VWudjvn/vEFrvZwA32+lVJhSKqL1Z+AKYA9e/Dv3\nu5OYlFJXY9rorMBKrfX9Pi6pXyilngMWYGaJKwbuAdYALwIjMDNqflFrfXanq19TSl0MvAfs5nQb\n7I8w7e4Bu+9KqSxMB5oVc9D1otb6l0qp0Zgj2ljgU+ArWusm31XafzzNMt/TWn8u0Pfbs3+veh7a\ngGe11vcrpeLw0t+534W7EEKI7vlbs4wQQogekHAXQogAJOEuhBABSMJdCCECkIS7EEIEIAl3IYQI\nQBLuQggRgCTchRAiAP1/DjQBJxIUZnUAAAAASUVORK5CYII=\n", 1608 | "text/plain": [ 1609 | "" 1610 | ] 1611 | }, 1612 | "metadata": {}, 1613 | "output_type": "display_data" 1614 | } 1615 | ], 1616 | "source": [ 1617 | "%matplotlib inline\n", 1618 | "import matplotlib.pyplot as plt\n", 1619 | "\n", 1620 | "plt.plot(range(len(avg_costs['discriminator'])), \n", 1621 | " avg_costs['discriminator'], label='discriminator')\n", 1622 | "plt.plot(range(len(avg_costs['generator'])),\n", 1623 | " avg_costs['generator'], label='generator')\n", 1624 | "plt.legend()\n", 1625 | "plt.show()" 1626 | ] 1627 | }, 1628 | { 1629 | "cell_type": "code", 1630 | "execution_count": 6, 1631 | "metadata": { 1632 | "scrolled": true 1633 | }, 1634 | "outputs": [ 1635 | { 1636 | "name": "stdout", 1637 | "output_type": "stream", 1638 | "text": [ 1639 | "INFO:tensorflow:Restoring parameters from ./gan-conv.ckpt\n" 1640 | ] 1641 | }, 1642 | { 1643 | "data": { 1644 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAHVCAYAAADCVJOIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmgldP+P/D3EkmaR2nWzZCIHIoQEiEyT1eKyPwzK0Tm\n4Rq+uOE6t1IZSoQyplIy304JqVQilUY0mEo8vz86n8/5PJ292/vs4Xmevff79U+f1hn22uvss5+z\nPs9nreU8zwMRERGFZ5uwO0BERFToeDEmIiIKGS/GREREIePFmIiIKGS8GBMREYWMF2MiIqKQ8WJM\nREQUsrQuxs65bs65r51zC5xz/TPVKSIiokLiUt30wzlXCcA8AF0BLAEwDcBZnufNzlz3iIiI8t+2\naXztAQAWeJ63EACcc6MA9AAQ92Jcr149r0WLFmk8ZOGZPn36as/z6qfytRzviktnvAGOeSr4Gg8W\nxztYyY53OhfjxgAWm/8vAdBhy09yzvUF0BcAmjVrhpKSkjQesvA45xZV8PM53mmo6HiXfg3HPA18\njQeL4x2sZMc76wVcnucVe55X5HleUf36KU84KEkc7+BxzIPF8Q4WxzsY6cyMlwJoav7fpLSNiCLs\nzz//1Hi77bYLsSdEJNKZGU8D0No519I5VxnAmQDGZaZbREREhSPlmbHneZucc5cDGA+gEoChnud9\nlbGeUc655ZZbAABHH320th188MFhdYcAtGnTBgAwd+5cbdt11101tu1EFJ500tTwPO9NAG9mqC9E\nREQFiTtwERERhSytmTFlzqxZswAAe+65p7Y558LqTtL+/vtvjZ966ikAwCOPPKJtr776qsZdunQJ\nrmMFbPLkyRr//vvvAAC7uU/t2rUD7xNRvvvjjz8AAJUqVdK2ihRIcmZMREQUMl6MiYiIQlbQaWpJ\nsT7xxBPads4552hcq1atwPoiW8xt3LhR27bffvvAHj9VNv25ww47AADWrVunbQMHDtT4oIMO8n0e\nZUe9evW2+vGGDRsG1BOi4E2cOFHjESNGaPz+++8DAM444wxtu/nmmzWW9fd16tRJ6XG33Xbz5dTe\nuqsIzoyJiIhCVnAzY/tXy3777QcAmDlzprbdeeedGr/zzjsa77333gCyV1RVrVq1cv3LBbZY4ZRT\nTgEAjBo1Sts++ugjjZs3bw4AmDJlirbJOljKnPHjx2u8fPnych+/7rrrguxOTpHfvwkTJmjbxRdf\nrPGyZcsA+Hcxk/cGABgwYAAA4OSTT9a2XCjEzFVffPEFAH9Gc86cORpv2rSp3Nfcf//9Gj/++OMa\nP/vsswCA448/Xtu22Sb5+arMjFPFmTEREVHIeDEmIiIKWcGlqW3a4ocffij38ZUrV2rct29fjZ95\n5hkA/q0Es6EiaZGokfTPJ598om2S1gOA1atXAwDuuusubRs+fLjGPLQgM6ZPn66xrH2sUqWKtrVs\n2TLwPkXZhg0bNJbbJkuWLNE2+54R6zaSvc0lW8IeddRR2la9evXMdZZ8RaNyi6BJkybatmhR2YmF\ntpg0ll9++UXjK664AgBwzDHHaFvlypXT62wF5O47PxERUZ7gxZiIiChkBZemXrt2rcaJKpdnzJih\nsVQKn3XWWdpmq/aGDBkCINi0RtRImtmu47Mpa0kvjRkzRtukoh0Arr76ao1zOV0fts8++6xcm10z\nX7NmzSC7E3njxpWd/Lp48WIA/mrpipD3BPs+YbeErUjFraRb7e9C06ZN4316wYhVnf7GG29o3L59\ne40///xzjWXfBruXg0159+rVC0B4t8v4jkdERBSygpsZ21mBFFZIYdGW7Mz5q682H9V8++23a5td\nI2vX2xa6xo0ba2zHRQphbEHM0qVLg+tYgbDnSX/99dcA/IUqXPfqZ2fGO+64I4CywjcAaNSokcb1\n69cHAHz55ZfaJodxWHaPgp9++knjBg0abLUv9vvKTM2+z8haWIrPFt9ZtlBPdOrUSeMbb7wRQHi/\nH5wZExERhYwXYyIiopAVXJrapqQTFbLYFKtsV2lv/r/wwgsxP7fQrVmzRuO//vqr3MftWO2+++4a\ns2grM/bff/9ybTbtatdeSlq2kNntQQ899FAAwLHHHqttVatW1fi5554DALz88sva9uOPP2o8f/58\nAP7CIFvAZfcuEPa2zWOPPaaxrF/u0KFDsk+lYMnPBfCvqbc/B2HP87bFpPbnHIaE737OuaHOuZXO\nuVmmrY5zboJzbn7pvzytnIiIKEXJTEWGAei2RVt/AJM8z2sNYFLp/4mIiCgFCdPUnudNdc612KK5\nB4DDSuPhAKYA6JfBfmWNXT9o00vCrhN++umnNT777LOz27E8IGu4i4uLtS1WmsimRk877bTsd6zA\n2HWWwqZCp02bpvEJJ5wQSJ+irF27djHjWHr27AkA6N69u7bZ9ww5McuOcefOnWN+L/nduO+++7RN\ntt0Fym6N/fvf/976Eyhgc+fOBQA8+OCD2hZvhYakr++44w5tS1TdHqRUb9I19DxPNh1eDiDuaeXO\nub7OuRLnXMmqVatSfDhKFsc7eBzzYHG8g8XxDkbaBVye53nOufLTn7KPFwMoBoCioqK4nxcUuwPX\n+vXry318jz320DgXZ8Nhjrf8VR9vnZ+ws+EaNWpktU9BiNprvF69ehrLbMAWHto1x7kozPGWos94\nxZ/nnXee798t2Wzck08+CQC4++67tc3ubTB79mwA6Z+Tm66ovb6tm266CUDZucaAPxtni0JvvfVW\nAEDv3r2D6VwFpTozXuGcawQApf+uTPD5REREFEeqF+NxAHqVxr0AjM1Md4iIiApPwvyHc24kNhdr\n1XPOLQEwEMB9AEY75/oAWATg9Gx2MpNGjx6tsayHtamMyZMnB96nXDZp0iSNJd22fPnymJ8rG7W3\naNFC27g1Y+bFSmva9GfdunWD7E7Bs2u8H3nkEY0HDRpU7uP2gAl7Ri+VufDCCzUeO3bzPNCmpu3r\nf5dddtH4yCOPBFBWGBc1yVRTnxXnQ10y3BciIqKCxC2PiIiIQlZw22FOnTq1XNsOO+ygsa3Ki7c+\nsNDZtdoXX3yxxrHS07FSpm+99ZbGF1xwgcZRWvOXy+yaYjmb1aZCn3/+eY0ldQdwS9dMstvAPvzw\nwxrb9bDyMzn44IO1bfDgwQH0LvfYtdaxxsi+h9vbjq1atdK4bdu2WepdZnBmTEREFLKCmBnb4pWV\nK8uvwrLnkf7www+B9CkXSZGEXX/93XffaSyzYDsbtn+xyhpvOWMX8O9eZIswWNiVOrvDWawd0Ozu\nUL/++qvG+bDmOyrsQRJ2RzqboZBCIilCAsI/rCBqRo0aBcB/mIcVK/NmM0MffPCBxp9//jkAoGPH\njpnsYsZwZkxERBQyXoyJiIhCVhBpastuFRjr7NFYh0cUst9++03jPn36APCvLZa1w0BZymjDhg3a\nZlOfu+22GwDgq6++0rbDDz9cY1ukYYtaqGJs6l9SoXYLTBvbLWGZpk7fTz/9BAAYPny4ttmDC+zv\ny4QJEwAAderUCah3ueHtt9/W+JJLLgHg387Vvr5lW9J9991X2z7++GON7W2YM844A0DZYR6A/zz1\nsHFmTEREFDJejImIiEJWEGlqu+7MrrFs06YNAH8qY+HChcF1LAfYlJCcRDNr1ixtk1T/lp8r7HZ0\njz76KABgypQp2nbDDTdo3KVL2aZuUvUe73Qcis9Wk8r4rVixQtsklQoAP//8s8aNGzcOoHf57aOP\nPgLgT4XaNcf29st+++0XXMci7rPPPtPYnuomt1Tk9DEA2HnnnTXea6+9APjPKJY2wL+PwYgRIwAA\nTz31lLb93//9X9p9zxTOjImIiELGizEREVHICiJNbdlqxlq1agHwp6kbNmwYeJ+izC6qP+qoowAA\n7733nrbZ9NKcOXMAAMccc4y2tWzZUmO5XWDTqPZkmgULFmj87rvvAgBOOumk9J5AAbIbS9SvXx8A\nMG/ePG2zt23smEd9u8BcICsN7Gvcjvfpp5cdcCdblRYyWcly/vnna5ut9pcNbLp27apt3bt311hW\neMTzj3/8Q2PZSMhukRklnBkTERGFrOBmxm+++abGtpBF2JkcxWbXRdqiKxtvTbt27TSWmRvgn6XJ\nlo2cGVec3aLUzjKEXVdvC+woNXY8beGWsDPjWNuTFrJhw4YB8O89YIs2Zb32oYceqm21a9dO+vvb\nQyVklt2zZ8+U+pptnBkTERGFjBdjIiKikBVcmnry5Mkay7aNdg3bscceG3ifCo1N251wwgka223s\n9thjj0D7lE/s2mE5kcymR4uKijS2azIpNbZgTk4EkmJGwH9qnJwcVMjsa/Hpp58G4D8jvVmzZho3\nb94cQMVS03bs7S0bKdyK6ravCWfGzrmmzrnJzrnZzrmvnHNXlrbXcc5NcM7NL/03+dEiIiIilUya\nehOAaz3PawOgI4DLnHNtAPQHMMnzvNYAJpX+n4iIiCooYZra87xlAJaVxuudc3MANAbQA8BhpZ82\nHMAUAP2y0ssMeu211zSW9FGPHj20zR7MTtkXr5o3lw9Zt2k4WdsYJLtdqWyDafuxZs0ajcPuaz7Y\nYYcdyrXZsbRpaq4t9q/Btidaibp162qcyuqWm2++WWP7+v5//+//AYju67xCBVzOuRYA9gXwKYCG\npRdqAFgOIOZuGc65vs65EudcyapVq9LoKiWD4x08jnmwON7B4ngHI+kCLudcNQBjAFzled46+9eF\n53mecy7mAjrP84oBFANAUVFRKIvsbIGFPZ9XdOvWTeOo/tWUrCiMd0XE26jdrjmOuiiMuZ0BFBcX\nayy7y9md1OzMeNGiRRrnyhr7KIx3PLYYVNj3FLsDYK7I9HhXqlRJY9khyx7QU1JSorF93W7N66+/\nrvEnn3yisV2zfPHFF1e8swFKambsnNsOmy/Ez3me93Jp8wrnXKPSjzcCsDI7XSQiIspvyVRTOwBD\nAMzxPO9h86FxAHqVxr0AjM1894iIiPJfMjmATgB6AvjSOTeztO0mAPcBGO2c6wNgEYDT43x96GQt\nGwBs3LhRY0mBHHHEEYH3iTbbf//9NbbppR9//DGM7mREGLc6ZM084F9bKcUyNn0qa4+B3L8tEzWx\nUv32FsLxxx8fZHciye4zcNdddwEAJk6cqG12C1c5/9m+N6xdu1bjmTM3X5IGDBigbfb1PXLkSI2T\nTXmHJZlq6g8AxPuNTW4zYiIiIoqL22ESERGFLNrz9jTJ+j67BaYl5+7aM3UpWI0aNYrZLufC2jWa\nNr1Ffn/99VfMWNLQdm1ngwYNNLbtlD5bKSzk3HQA2G+//YLsTuTJ1qz29LaVK8tqgWX7UPs+YVfH\nyGvd7kswdOhQjeUM9lzAdzciIqKQ5fXMWHYfmjBhQsyPX3rppQA44wqTLbawvvnmGwD+DeQzuUZT\nimrypYBpxowZGttZxA8//ADAX7hoi70qsgE/xSavVQB44IEHyn28bdu2GufyznLZIL9/o0eP1rZz\nzz1X4+XLlwPwz4bt2mE5aObyyy/Xtlw9/IRXISIiopDxYkxERBSyvE5Ty5aKdgvMypUra3zAAQcE\n3ify69mzp8bDhw/X+OGHN+8vk63tA/MlPS3sgRt2TbGcDWtTe88//7zGdlN+Ss3dd9+tsayBtWta\n69Spo3G+ve4y5dBDD9XYrpMvJJwZExERhYwXYyIiopDlXZp63bp1Gt96660A/Kmhl156SWObPqJw\ntG7dWmN7ghBVTOPGjTWePXt2iD3Jb7Ku1a6FteMtt1VsxbrdypErNygevjKIiIhClncz4xo1amgs\nZ+Xus88+YXWHiPKI7LBl13Lbme/SpUsB+GfG3OGPksGZMRERUch4MSYiIgqZs2dtZv3BnFuFzWcf\n1wOwOrAHDk42nldzz/PqJ/608sx4A/k55pEabyDvX+PZek6ZeI3n43gDEXuN8z0lJUmNd6AXY31Q\n50o8zysK/IGzLMrPK8p9S1WUn1OU+5aqKD+nKPctHVF+XlHuW6rCfE5MUxMREYWMF2MiIqKQhXUx\nLg7pcbMtys8ryn1LVZSfU5T7lqooP6co9y0dUX5eUe5bqkJ7TqHcMyYiIqIyTFMTERGFjBdjIiKi\nkPFiTEREFDJejImIiELGizEREVHIeDEmIiIKGS/GREREIUvrYuyc6+ac+9o5t8A51z9TnSIiIiok\nKW/64ZyrBGAegK4AlgCYBuAsz/NmZ657RERE+W/bNL72AAALPM9bCADOuVEAegCIezGuV6+e16JF\nizQesvBMnz59darHnXG8Ky6d8QY45qngazxYHO9gJTve6VyMGwNYbP6/BECHLT/JOdcXQF8AaNas\nGUpKStJ4yMLjnFuU+LN8n8/xTkNFx7v0azjmaeBrPFgc72AlO95ZL+DyPK/Y87wiz/OK6tdPecJB\nSeJ4B49jHiyOd7A43sFI52K8FEBT8/8mpW1ERERUAelcjKcBaO2ca+mcqwzgTADjMtMtIiKiwpHy\nPWPP8zY55y4HMB5AJQBDPc/7KmM9o4Izfvx4jYcPHw4AePnll7Vt4sSJGh988MHBdYyIct6mTZsA\nADvuuKO2XXjhhRoPGjQo8D5Z6RRwwfO8NwG8maG+EBERFSTuwEVERBSytGbGROn64osvNP7ggw80\nnjlzJgBgm23K/l58882yJAzT1Lnn77//BuD/mRIF5eabbwZQ9joE/O8/YeNvBRERUch4MSYiIgoZ\n09QUuI0bN2o8atQojR988EGN//zzTwBA06ZlS9lvvPHGAHqXX/766y8AwKWXXqptl1xyicYtW7bU\nuGbNmlnti6Sn7X74zrmsPiYVtjVr1mj8zjvvACirqgaA2267LeguxcWZMRERUcg4M6bAyV+oAPD0\n009rLLNhy64trl69enY7lsPsbLNPnz4aP/fccwD82Yji4mKN7cy0f//Np6AOGDBA26pWrZrxvnI2\nvNngwYMB+GdnXbp00XjYsGEAOF7puPbaazWWYq0qVapoW4MGDQLvUzycGRMREYWMF2MiIqKQMU2d\nhpEjR2p87LHHAsh+EUw+sGnqlStXxvwcSdG1bt06iC7lvFdffVXj0aNHayzpabu216a0bXzvvfcC\n8G9L+v7772ucjZR1IbBj/Mwzz2g8duxYAMBPP/2kba+88orGkyZNAgBs2LBB2/bee+9yHyc/extm\nxIgRGsv64sMPP1zb2rZtG1zHEuDMmIiIKGS8GBMREYWMaWoA3377rcZ23aW1cOFCAECdOnW0Taoh\nAeD7778HAPTr1y8bXcwLa9euBeBPxdmt6Y4++miNzz33XACsJE1ExvSOO+7Qtl9//VXj7bffHgDQ\nvn17bZNbKoB/bff69esBAJ999pm22VsKJ554Yqa6HSlSRQ6UVTbbitt02bX0Q4cO1fjjjz8G4F9F\nYFPa8vOw7G0DqZq370OF+vuyZMkSjR9//HGN7ZpiGRs7XlHCmTEREVHICnpmLH+x2jNzDz30UI3t\njFkKYexZmLK7EQAsWLAAAPDuu+9q2xFHHJHhHuce+5fpFVdcAcD/V2yNGjU0HjdunMaF+hd+RT3y\nyCMAgDlz5mjbttuW/VrLTOyUU07RNlvM1aZNG43POeccAMDvv/+ubVJkBOTXzNgWTT322GMav/HG\nGwD8BwhU5LW4bt06AP6zce33X7FiRbmvsT8PyWQAZe85tnBun3320fjyyy9Pul/57sMPP9R4/vz5\nMT+nc+fOAIBatWoF0qeK4syYiIgoZLwYExERhSyv09R//PEHAOCjjz7SNps+mjx5MgDgt99+0zYp\nqgD8RRySXrIpvIYNG5b7eElJiba99957Gtt0bCGx42HT0MIeYFC5cuVA+pTrpGgLAB566CEA/lSn\nLco666yzAMRPtR511FEaSxrarp9/++23NZbfk3xYb2xTlbZoatasWQCAXXfdVdumTJmisfzO//zz\nz9om7zMAcPrpp/u+DwD88ssvMfsg7y9XXnmltjVp0kTjzz//HABw3nnnadtBBx0U9zkVIvnZ3XLL\nLdpm36O32247jWXr3ai+fhPOjJ1zQ51zK51zs0xbHefcBOfc/NJ/a2e3m0RERPkrmTT1MADdtmjr\nD2CS53mtAUwq/T8RERGlIGGa2vO8qc65Fls09wBwWGk8HMAUAIEssLXrUhctWgTAX/X89ddfayyV\noHatpE1JxWIrfROx6UI5/UPShkDhpqbtz6h3794ay3jZ9H+UzhONMvu6tuuEZS3q1VdfrW2XXXaZ\nxokqgatVq6Zxr169APjXxdoUq02F5zo7LnbLVUkvf/PNN9q25557atyoUSMAQKVKlbTNVmbLeNmx\nsmN8//33a3zxxReX+1yqGLkGLF68OObHDz74YI2jvlVxqq+Chp7nLSuNlwNoGO8TnXN9nXMlzrmS\nVatWpfhwlCyOd/A45sHieAeL4x2MtAu4PM/znHNxp5ue5xUDKAaAoqKirU9Lk3Drrbdq/O9//xuA\n/693OyvLBltkdOONN2p83XXXlft4GDI93qn48ccfNZ43b165j0vhHOBfV5mrsjnmMvMdOHCgttlZ\ngKx7t+eyprpGu169egD82aPTTjtN46j8rDIx3naM7N4ARx55JABg6dKl2mbPgp47d26572Vntu3a\ntQMA3HTTTdp26qmnptLFyIjCe0o8r732GgB/EZ1dZ3/NNddoHNX1xSLVmfEK51wjACj9N/bRO0RE\nRJRQqhfjcQB6lca9AIzdyucSERHRViRMUzvnRmJzsVY959wSAAMB3AdgtHOuD4BFAE7PZietE044\nQeMnn3wSALDDDjtom11jJuk2m3az6alExVyx2LTdDTfcoLFNjRQ6WwBkx1hSqR06dAi8T7nq4Ycf\nBgC8+OKL2mZvxVx11VUAgOuvvz7tx7r77rvLte27774a5+sWpZKeB4CZM2cC8BdnSioUKCuOk20v\nAf/P4/jjjweQ+6npKLP7Qsjvh2WvB3Xr1tU46q/fZKqpz4rzoS4Z7gsREVFBYk09ERFRyHIut3rA\nAQdovHr1agD+tcHjx4/XWKqt7RpNe9KSpDNslZ1UUwL+dKukv22lMFPTsdnKX0vWwkY9XRQlr7/+\nOgB/KtSuX7/22msB+Ne9VoT9vtOnTy/38fr166f0fXOdXZP6z3/+U+Pu3bsD8Kfvv/vuO41HjBgB\nwL89I98nMsueCW3HXnTrVrZHlf05RR1nxkRERCHL6T/ZZIbVtGlTbbvgggvKxc8++6y22XXAcpbr\nHnvsoW12zeBbb72lscyMZ8yYoW3xCsMKlWQN7Jo/y+50RPFJxgfwn1Ms7Jm2tkAlFfbcXnmN298R\n+7tRqOzvtmTRnn/+eW2z55bLjlByNjQADB48WGO7GxelJlahof0Z9e9ftjuz3e0v6jgzJiIiChkv\nxkRERCHL6TR1smzKKFFq2X7cnoUpmGaKTw7s2LBhg7bZ4rioniMaNbZI0I6lsLdi0i0OsgWP8lj2\n57T77run9f3zlX0fsK9x+dmNGTNG2yR1DQAffvihxjwgInm20DDW+dB2q9ZddtklkD5lGl8NRERE\nIePFmIiIKGQFkaa2ElU925TSn3/+We7j7du3T/p7FZqJEycC8Kf6f/31V41vv/12AMDHH3+sbT17\n9tTYVmE3b94cQNmpREDqa2lzjT1LV9jXmq22lvRdRVKemzZt0ri4uFhj2eJx77331raonNQUFQcd\ndBAA4Msvv9Q2+3qX16t93X/11Vca258tVxckb/bs2Rrb7TCFXQ+fq69ZzoyJiIhCxosxERFRyAJP\nU//8888aV69eHYA/xRZGhaHdInPSpEka23SgsKdGkV/btm0B+Ct8bapfNphYtWqVtsl2jwCw//77\na9yyZUsAwKWXXqptdhvIfGZfd7FuhUybNk1jSe3byv9YqwDisRt8yGPxNe5PPZ933nkaf/LJJwD8\nt0zsdply2+Woo47SNnv7xb7/UfJsFXqs0/bs74ndKCeXTh3jzJiIiChkgc+Ma9eurXGs84bD8MYb\nb2j82GOPaWwLXaQowG59R35dumw+VVNmtQAwb948jWW7Rbu5u/3Zz507V2NZx/nuu+9q20MPPaTx\nXnvtlaFeR499bvLXvP2rXtZzA2WFLTajY9chy3aZdpxldgf4fz4yoz7xxBPTewJ54PHHH9fYrhmW\ncWzVqpW2DRs2rNzX27XadmYs2UBKjhwCZM+UjnW9sGu9Bw0apPH8+fM1lm1J7fbJUdr7gDNjIiKi\nkPFiTEREFLJQ1xnHSsEFSQq37DmusdZ4AmXp6UaNGmW/YzlKzoe+7bbbtO3ss88u93l2aztL0thA\nWWpv6tSp2mbXa+Zzmtq+xiQlZ8ds7dq1Gkthy3XXXadtsdZj26+//vrrY7bL49q19IVGtrN85ZVX\ntM2ua5VUvi0osuTnZbcxtVuK8hSsilmwYAEA//uAJcWitvD3nXfe0dgWQ0oxly0EfeaZZzTu2rVr\nBnqcuoQzY+dcU+fcZOfcbOfcV865K0vb6zjnJjjn5pf+WzvR9yIiIqLykklTbwJwred5bQB0BHCZ\nc64NgP4AJnme1xrApNL/ExERUQUlTFN7nrcMwLLSeL1zbg6AxgB6ADis9NOGA5gCoF9WepklcvrH\niBEjtM1uY2fXa/brt/mp8aSVxA488ECNmzRporFURiZDbl3YymxZx5zv7IlAsg7Yrte221X26dMH\nQOKtQhcvXqyxrVq35GcV9fWY2SSnWE2ePFnbbPWurB+WKvUt7bPPPgD87xN33nlnxvuZz+ytk2uv\nvRYAsH79+pife8YZZwDwryDYaaedNLZrxD/77DMAwMqVK7Xt2GOP1bioqAgA8Pbbb2tbzZo1K/4E\nUlShK4tzrgWAfQF8CqBh6YUaAJYDaBjna/o650qccyV2swfKDo538DjmweJ4B4vjHYykC7icc9UA\njAFwled56+xfz57nec65mIuFPc8rBlAMAEVFReEuKN6CbPb+/fffa5t9XjvvvLPGdneoKIvCeNsC\npIsuukjj//znPwCAZcuWlfsawL/+r2/fvgCA7t27a1tUZ8aZHvMqVapoHCsT8+KLL2osZ3V36NBB\n21544QXL+cbtAAAgAElEQVSNn3/+eQBlswLAX1xkd0uTgzyiLpuv8QYNGshjxPy4ZC3kUA0AOPfc\nczWW95TOnTtrW48ePTLZxcAF/Z5iX5+xCmrtecVDhgwBEP9wCHsojWSETjrppHJtQNn6e7sXhi2+\nmzlzJgD/rnWZlNTM2Dm3HTZfiJ/zPO/l0uYVzrlGpR9vBGBlvK8nIiKi+JKppnYAhgCY43new+ZD\n4wD0Ko17ARib+e4RERHlv2TS1J0A9ATwpXNuZmnbTQDuAzDaOdcHwCIAp2eni9kjhxTYggGbghg1\napTGUdo2LepsymjAgAExY4rPFmPJmt/33ntP2+zr9fjjj6/w97c/n2uuuUZj2c60kB122GEAys7T\nBvxnnL/00ksA/Fvo2pS2FNe9+uqr2lbIBXGp+PTTTzWWvSDsrRubeq7I2cWScrYHSdjCSFl/vHHj\nRm2znytnJtvtfG1KO13JVFN/ACDeq4m/vURERGniOh0iIqKQhbodZthirQm0sV3PSRQUm9aUymi7\ndjveqVexyFp5m84bOXKkxscdd1xafc03Ul0uVbqAf0tXWaNqt2612yt+8MEHAPxrxalibAWzrAaw\n64ETramvCLuXhPxMZUtUwL81rLw27BabmUxTc2ZMREQUsoKeGXfs2BGAf32rFAwALLyg8DVsuHkv\nnYULF4bck8Jii9lWrFihscyMZ82apW084zyz7A5aqRQopsvurmaL94S9RtgdG3fccce0HpczYyIi\nopDxYkxERBSygk5Tt2jRAoC/IIaIKB7ZLpOp6cJlC8gyWUzGmTEREVHIeDEmIiIKWUGnqamw2DW5\nrJSnXGS3akx0epC83vlazx67TWe6ODMmIiIKGWfGVDA4Q6BcV5GzdPl6zy2cGRMREYWMF2MiIqKQ\nuUQbzWf0wZxbhc1nH9cDsDrBp+eibDyv5p7n1U/lC814A/k55pEabyDvX+PZek6ZeI3n43gDEXuN\n8z0lJUmNd6AXY31Q50o8zysK/IGzLMrPK8p9S1WUn1OU+5aqKD+nKPctHVF+XlHuW6rCfE5MUxMR\nEYWMF2MiIqKQhXUxLg7pcbMtys8ryn1LVZSfU5T7lqooP6co9y0dUX5eUe5bqkJ7TqHcMyYiIqIy\nTFMTERGFjBdjIiKikPFiTEREFDJejImIiELGizEREVHIeDEmIiIKGS/GREREIUvrYuyc6+ac+9o5\nt8A51z9TnSIiIiokKW/64ZyrBGAegK4AlgCYBuAsz/NmZ657RERE+W/bNL72AAALPM9bCADOuVEA\negCIezGuV6+e16JFizQesvBMnz59darHnXG8Ky6d8QY45qngazxYHO9gJTve6VyMGwNYbP6/BECH\nLT/JOdcXQF8AaNasGUpKStJ4yMLjnFuU+LN8n8/xTkNFx7v0azjmaeBrPFgc72AlO95ZL+DyPK/Y\n87wiz/OK6tdPecJBSeJ4B49jHiyOd7A43sFI52K8FEBT8/8mpW1ERERUAelcjKcBaO2ca+mcqwzg\nTADjMtMtIiKiwpHyPWPP8zY55y4HMB5AJQBDPc/7KmM9IyIiKhDpFHDB87w3AbyZob4QEREVJO7A\nRUREFLK0ZsZEiWzatEnjIUOGAADsOsUjjzxS40qVKgXWL6J8Jxs6OedC7gklgzNjIiKikPFiTERE\nFDKmqSljNmzYAAA44IADtG3hwoUa//XXXwCAatWqadupp56q8RNPPJHtLuasW265ReOnn35a4/Xr\n1wMA/vjjD23beeedNb7ooosAANtuW/arfvbZZ2tsbyPI19nPpcTs/v6HHnooAGD16tXa9umnn2pc\no0aNwPrF9HRu4cyYiIgoZAXxJ7D9619mZwCwYsUKAECtWrW0bdasWRrPnTtX4//+978AgJkzZ2qb\n/Yt4m202/13TrFkzbXvhhRc0bteuXepPIEfIjGvOnDna9ueff2rcvHlzAMAuu+yibeedd15AvctN\nixZt3tb2/vvv1zY7prF89913Gt94443lPn799ddrbGdPBx98MADghhtu0Lbu3btXrMMFyI7hjjvu\nCAD44IMPtK1z584az5gxI+bXUe5Lt2COM2MiIqKQ8WJMREQUsrxOU0t6etCgQdo2ffp0jffff38A\nQPXq1bXtX//6l8Y2TZ2sb775RuPbb79d46eeegoAkG+nnkyYMEHjN9/cvBmbvRXQqVMnjSdNmgQA\n2H777QPqXe778ssvASROTafK3mp5//33AQDz5s3TtmeeeUbjrl27ZqUP+STWa1tuhwHAmjVrNK5d\nu3YgfaJgpHvbgTNjIiKikPFiTEREFLK8TlPL9oojRozQNkn7AcBzzz0HwJ+qS0SqpgF/ClZS3kuW\nLNE2+31r1qyZ9GPkksGDB2u8ceNGAP51ruPHj9e4IunpWD+TQqw+3W233QD4n3tFXq+yZtiu7bav\n4V9//VXj7bbbDgDw+++/a9vff/9dwR4XtkaNGpVr++mnnzS2t7GKiooC6VM+kPcWAOjbt6/GckvF\nrt8eMGCAxrJCIBdwZkxERBSyvJ4Zy2zCziTsX/qxZhh296GqVatq3LJlSwD+ArBCO9hAiojat2+v\nbV99VXaEtRTCTZkyRdtk3WVFyc5SH330kba1atVKY1kbnm8FcVtq3bo1AP9zX7BgwVa/xs58v/32\nWwBAkyZNtO23337TeIcddtD4lFNOAQAsXbpU26pUqaKxFOYV2uu+Irp06QKgrGAT8L/nLFu2LPA+\n5Rr7vrxu3ToAwD333KNtb7zxhsay05nNHNnXb0lJicZRLxzlzJiIiChkvBgTERGFLK/T1OKOO+7Q\n2N78P+200wAA33//vbade+65Grdp00bj3XffPZtdzAmTJ08GAHz99dfaZlNKl112GQD/dpcV8eOP\nP2os24+OGTNG2+bPn6+xFBt98skn2mZTufnGpuvta9EWB4n99ttPY5ueFvb2yy+//KLxEUccAcCf\nBj/55JM1lp/r22+/rW1169ZN7gkUCJv2F3aN+OzZszWWrUYLoTDRpuovv/xyjd966y0A/i2LDz/8\ncI1XrVoFwL8NsT2EQ9hbJ/Y2jS3muvvuuwEAlStXrvgTCEDCmbFzbqhzbqVzbpZpq+Ocm+Ccm1/6\nL1evExERpSiZNPUwAN22aOsPYJLnea0BTCr9PxEREaUgYZra87ypzrkWWzT3AHBYaTwcwBQA/TLY\nr4w6/vjjNbbVjLFO2bBpV1u1J6km2fIR8FdeSzrFbnF3zTXXaCxp1Vwm6SWbdrPrp+Xs3FTTbvfd\nd5/GEydOBOBP69m1huKf//ynxjZlnW9s1fgrr7yicZ8+fQD4U8t2zCT1LOvgAX+abuzYsRrLbQD7\ncZv+kxOHGjdurG0tWrTQuLi4GEDZmb6FKFG19EEHHRRQT6Jl8eLFGr/44osa21tTYtSoUeXabBo7\nFrsFrz21zO6DMHr0aAD+k+IGDhyocdi3C1It4GroeZ686pYDaBjvE51zfZ1zJc65ErlgUfZwvIPH\nMQ8WxztYHO9gpF3A5Xme55yLuyWQ53nFAIoBoKioKPmtgwIgfwnZ4gK5yQ8ATz/9tMayXrMi+vcv\ny97LjNluwl+vXr0Kf89EMj3eds2enK1rnXPOORo3bdq0wt/f/sVs1wRKUZ392dgZm7TLOsQwBf0a\ntzNPmSXbophPP/1UYym6k3+3RtaP27X0NlP08ssvAwBuuukmbbOzkKuvvhoA8PHHH2tbNoplovye\nUqdOnXJtNitms0phz8SSlYnxtq8pWzQo7FjYbMyGDRukDzG/r6yDt2MsexRs+XVySIc9G/zSSy/V\nuEGDBgmeRXalOjNe4ZxrBACl/67MXJeIiIgKS6oX43EAepXGvQCM3crnEhER0VYkTFM750Zic7FW\nPefcEgADAdwHYLRzrg+ARQBOz2YnM8lujH/bbbcB8BcUxErFxmPXFEo65I8//tA2W+Al6amePXtq\n27BhwzRu2DDubfdQ2aIoeQ72edniOLu+b2tsgcazzz6rsS08klST3QDeFnFIqsumpGwKMB8K5pLR\ntm1bAP4tSGOxr/uKbFFq04eyXab8CwAdOnQo9xhRXccZBFkfb9kxjLVGthCMHDlSY/u7Ke+b7dq1\n07ZYr88ffvhBY1vI2a/f5rphu9Xlo48+qrHdrlfeHyT1DQCvvfaaxlIMGZZkqqnPivOhLhnuCxER\nUUHidphEREQhK4jtMNeuXavxkCFDNH788ccB+M9vjUcqWO32f3YdsaQ+brnlFm2zKZJYfcmFrQRf\nf/11jSWlZM/GtemlWOxWoy+88AIA4Oabb9a2eNWlcgKUTd/bE4SkKt2mrm36nPxSPT0rFluhaqvt\n7XalVMauCLDnqct2vLlSVZ2OXr16aWxvPcneAolOX7PvI/a89Fi/87K2HvDfRpOxt69fexsu7DQ1\nZ8ZEREQhy+upxP/+9z8AwPnnn69tsWarli0usDsNSeHVBRdcEPPrFi5cCAC49957tU02ggfK/hqz\n6z1zYSZnDx2QgrOddtpJ22KtlbZFVVJgtGV7LHbG/eqrrwIADjvsMG2zhRvy17WcawwUxgwjTPIa\ntpkNWxiWC6/nbEhUHGezNyeddJLGhfR6te+FNk5Ws2bNUvpcm4mU16fNxslOf0DZzyms1zFnxkRE\nRCHjxZiIiChkeZdXeueddzS+/vrrASROTdv1wra4wG5iHi89LSQlLmuXAX+hgKRr7Xq4XDB37lyN\nJa3WsWNHbbNb1wmbjrYpn06dOgHwHwhhDzCQgyYAf3panHrqqRpLqsseFEHZJbcJpBAPKKxUazz2\nkI5vvvmm3MftmdL2YA3KDruO2N52lPeqePsR2EK7MHBmTEREFDJejImIiEKWc2lqW7ko56s+/PDD\n2iZVuMk45JBDAAD33HOPttm1rFIhHc/KlWXnY9x1110AgCVLlmibTYG8//77SfcrSn766adybfbs\nUJuKl5SlXQdov17SQPG2zbSnZMnP+R//+Ie22dNeZOvMilRZUnLsyWL2ZyIrAezP3N4WKpQtSLe0\n2267aSwnA1n2PSuZPQ0oPYMGDdJ4xYoVGsttllj7GQDhv345MyYiIgoZL8ZEREQhy4k0ta1ye+CB\nBzSWQ6LtSUmJ2BTFtGnTAADdunXTNptiXbZsmcayVZpN0dmqPfm+VatW1TZ7CHvYKZBU2Up0ee42\nXRzv0O9YEp3qZE+8kc1Tfv75Z2077rjjNGZ6OjPs75ZUSd9+++3aZre7bNWqFQD/lo6Z3GYzH8Q6\nsapmzZoa2+0XTz75ZACsSM8EWyFtX7P2vUrG2W69ad+T5PYbN/0gIiIqUJGeGcusy67d/de//qWx\nnZlW9HsCsWfUiTa7t3/F2kMMDj/8cADAQw89pG2NGjWqcP+ixm4nJ+wWch9++KHGcphGRch5pAAw\nfPhwjaXwyx7GYdcn5wvZgs8WxU2dOlVjKSiUYsNU2dmCzS6NHz9eY9la1G5xagsid99997T6kK/s\n3gEydvZ9xI693RpT3os4M06fnQF//vnnGtute2VPAnsNsT+PsHFmTEREFDJejImIiEIW6TS1FE2N\nGjVK21JJTafKFmNISrp9+/baVlxcrHGDBg0C61eQ9t57b40XLVoEwL/dpb2FIGnmpk2bxvxekpaz\naaLHHntMY3vbYJ999gHgXwOeL+ytEhlf2/bdd99pLK/3WOu5t2xPlj3pqm/fvhoPGDAAgP/0LKZQ\nE7NjJK/tLl26aJstLrrhhhs0lvcSu5a+EMbbbjMsz92+r19yySUay62peNtWvvnmm77vA/i3a61T\np47GUqxlU9qTJk3SeM6cOQCAdevWaVuQ7z8JZ8bOuabOucnOudnOua+cc1eWttdxzk1wzs0v/bd2\n9rtLRESUf5JJU28CcK3neW0AdARwmXOuDYD+ACZ5ntcawKTS/xMREVEFJUxTe563DMCy0ni9c24O\ngMYAegA4rPTThgOYAqBfjG9RIbaq9JVXXgGQ2S3k7LoySUPbFIeN5dQnADjooIMy1odcYlPKUuVr\nK6zfe+89jQ888EAAwN13361tsjYVAB599FEAwGuvvaZtskUdAJx55pkaS1V6orXJucimlqUSd9as\nWdqWqMIzUWrajpmsmezRo4e2PfHEExrbymlKn/wO2PT/f/7zH42///57jSVl/eyzz2pbIazbrlu3\nrsa9evUCADz11FPa9vjjj2ssr9XWrVtr25FHHqmxpKxla+QtP9dudylbEtu9E2yaWthVMHfeeafG\nsU6oy6QKvdM551oA2BfApwAall6oAWA5gIZxvqavc67EOVeyatWqNLpKyeB4B49jHiyOd7A43sFw\nyRaAOOeqAXgPwN2e573snFvjeV4t8/GfPc/b6n3joqIir6SkJOnONW/eHACwePFibatIwYqcI3rd\ndddp29FHH61x7dqbu2vXCdrilihwzk33PK8ola+t6HgnImt+ZYYLxD4D1BZb2OKVWK666iqN7frX\nsHbBSWe8geTG3M585Yxmu6uVLSCRv+xtNsIWWEnW6JZbbtE2WfMOlP0ORO11bUXpNZ4pcr45AJxy\nyika24NkpCjUrvW2BZPZKuaK4nh/8cUXGvfvX3bH86OPPgLgz47a9xSZrdrfqXjjFuvaYXdHkxn1\nyJEjtc1mSlOV7HgnNTN2zm0HYAyA5zzPe7m0eYVzrlHpxxsBWBnv64mIiCi+ZKqpHYAhAOZ4nvew\n+dA4AL1K414Axma+e0RERPkvmVxgJwA9AXzpnJtZ2nYTgPsAjHbO9QGwCMDpme7cW2+9BQDo3Lmz\nttnzQqXYy278PXjwYI27d+8OIDfW7uXC1ngDBw4EAEyfPl3bJI0ElKWK4qWmJSVkizVOO+00jfOx\nWCsWm4K/6KKLyrV17do18D5RZu23334a270JbJpabj3IgTUAsNdee2kc5feCTLPpeVk7DAC//fYb\nAH8h1ZgxYzSW64E9UCZeyrpGjRoAgGOOOUbbbrzxRo3btWuX+hPIgGSqqT8AEO9V0SVOOxERESWp\nMKYiREREERbp7TDbtGkDwH+usD0xSNbs2TWU9iQlyixJM9u1xSeddJLGY8eWLxs48cQTNZZtTWOd\n+VqobMqM8oddk2rXdY8bN05jSafaE54KKTWdDDkf3p6xbbcUldPdHnzwQW3r1KmTxrKtLgDssssu\nAPyrZ7K9drgiODMmIiIKWaRnxsIWt3Tr1i3EntCWZJc0IPY6Pv6lT4WucePGGtvdo6ZMmQIA6N27\nt7bx9yU2m02zsewV8eSTTwbep0zjzJiIiChkvBgTERGFLCfS1IUgH9JT+fAciLJpwoQJYXeBIooz\nYyIiopDxYkxERBQypqlDZKuPmeIlIipcnBkTERGFjDPjEHE2TEREAGfGREREoePFmIiIKGQu1haG\nWXsw51Zh89nH9QCsDuyBg5ON59Xc87z6iT+tPDPeQH6OeaTGG8j713i2nlMmXuP5ON5AxF7jfE9J\nSVLjHejFWB/UuRLP84oCf+Asi/LzinLfUhXl5xTlvqUqys8pyn1LR5SfV5T7lqownxPT1ERERCHj\nxZiIiChkYV2Mi0N63GyL8vOKct9SFeXnFOW+pSrKzynKfUtHlJ9XlPuWqtCeUyj3jImIiKgM09RE\nREQh48WYiIgoZLwYExERhYwXYyIiopDxYkxERBQyXoyJiIhCxosxERFRyNK6GDvnujnnvnbOLXDO\n9c9Up4iIiApJypt+OOcqAZgHoCuAJQCmATjL87zZmeseERFR/ts2ja89AMACz/MWAoBzbhSAHgDi\nXozr1avntWjRIo2HLDzTp09fnepxZxzviktnvAGOeSr4Gg8WxztYyY53OhfjxgAWm/8vAdBhy09y\nzvUF0BcAmjVrhpKSkjQesvA45xYl/izf53O801DR8S79Go55GvgaDxbHO1jJjnfWC7g8zyv2PK/I\n87yi+vVTnnBQkjjeweOYB4vjHSyOdzDSuRgvBdDU/L9JaRsRERFVQDoX42kAWjvnWjrnKgM4E8C4\nzHSLiIiocKR8z9jzvE3OucsBjAdQCcBQz/O+yljPiIiIcsSvv/4KANhxxx1T+vp0Crjged6bAN5M\n53sQEREVOu7ARUREFLK0ZsZERERhmTJlCgBg7Nix2vbyyy9r3K1bN43799+8SWTLli2z0pclS5YA\nAHbddVdtc84l/fWcGRMREYWMF2MiIqKQMU1NRESR9NtvvwEA5syZo23PPfecxk8++SQA4I8//tA2\nmxp+6623NN5rr70AAH379tW2ypUrZ6yvrVq1AgD89ddf2rbttslfYjkzJiIiChlnxpRx559/vsb2\nr9hrr70WAHDzzTdrW5UqVTSuVKlSAL2jRHbffXeNq1atqvGMGTPC6A4VmNWrV2s8bNgwAMDtt9+u\nbb/88stWv75atWoaS1EVAFx99dUA/LNVO0veZpv05qby/vXnn3+m9PWcGRMREYWMF2MiIqKQMU0d\nsN9//13jadOmaXzIIYcAqNi6tKiRNX+vvfaatm3cuFHjRx55pNzXyNo/AKhRo0b2OkcJLVq0yPcv\nAFSvXl1jKaaxqet81bFjR41bt26tsZxaZE8vsmN06aWXAkg/5Vlo7PtE7969NZYCrL///lvbtttu\nO42lAGu//fbTtiOOOELjBx54QGPZrtK+59jve8oppwAAGjZsmNJzkPfuVIvC+IohIiIKGS/GRERE\nIWOaOsO++eYbjUtKSjQeOHBguY/b9Wgff/wxAKBDhw7Z7mJGeZ6nsTwHW+1oq6U3bdoEALj//vu1\nzY7BXXfdpbFNRVEwzjzzTAD+NZu2snr9+vUA8jtNvXbtWgDA999/r23Tp0/XWF7DtiLXvobvvPNO\nAMCgQYO07bDDDtO4Zs2aGmdyjWuusyss3n//fY3l/WWfffbRtiOPPFLjG264AQBQp04dbZOfEeBP\nSZ911lkAgAkTJmibVFgDZbdn7PtTkDgzJiIiChlnxlthiwo++eQTAP41ZKtWrdL4tttuAwB8/fXX\nW/2eTZo00fiCCy7Q+IADDkirr2GxM+PFixcD8M9qzznnHI1nzZoFwP+X79NPP63xscceq7EU0Gy/\n/fYZ7jFZb7/9tsaffvppuY8feOCBGtetWzeQPoVJZq52LNq0aaOxZH3s7MtauXIlAOD000/XNvsa\nPvvsszUuLi4GULFdmvLNunXrAAAvvvhiuTag7P3SFoXa99BY4u1X8MILLwAALrzwQm17/vnnNU71\nHOJM4cyYiIgoZLwYExERhSzv8iNSgAEAkyZNAgC89NJL2mZTIMuWLQMAfPvtt9rWuXNnjV9//XWN\n46WltmRTTrbQRVKzspk4EH5aJBNssY9sXVevXj1te+KJJzSeP38+AKBt27baZre+kzQSULamtWvX\nrtpWyOm8bFmzZo3GcsvBrpG1Z7MW0tpZmwq1WzHeeOONAPzvBzYNLWMka1oB/+0uuz3sZZddBsC/\nRrbQyHvv0qVLtc3utfDQQw8BAHbaaae0H+unn34CAEycODHmxxs3bpz2Y6Qj4W+Xc26oc26lc26W\naavjnJvgnJtf+m/t7HaTiIgofyXzp+4wAN22aOsPYJLnea0BTCr9PxEREaUgYd7P87ypzrkWWzT3\nAHBYaTwcwBQA/TLYr6RIauOpp57StnvvvVfjZFPL1quvvrrVj9sUiq0alvTUqFGjtK1Hjx4Vfvxc\n8+GHH2osW32edNJJMT+3WbNmAIBDDz1U22zKyP4c9913XwD+ivUGDRpozBOeUmd/L0aPHl3u43aL\nwO+++05jWzmf7+zv+R577KHxbrvtBsCfsrdrimW8Hn30UW2z+w3YlLVUCNs1tIXwut6wYYPGsprC\nviZr1aqlcffu3QFk5haVrDmWinfAP94nn3xy2o+RjlRvAjX0PG9ZabwcQNzNPJ1zfZ1zJc65EvvG\nStnB8Q4exzxYHO9gcbyDkfafG57nec65uH8ye55XDKAYAIqKilL60zrWLk8AcOqppwIoK8SKx/4V\na3e9kb+KbLGF/YvYPq6sP7Tf6/jjj9dY1h+HPRvOxHhX8PE0lrGR2cOWpNDFrm21OxLZn4PsxmX/\nSo434w5b0GOeLlu0tddee2ksPxd7mInNXMjvWaJ1ntkW9HjbHbRkp6ejjjpK2w4++OBysczoAKBR\no0Ya24LHkSNHAgCuuuoqbbOv96jI9HjbWa7s32DtueeeGqe7S5ndu2Dq1KkA/DumNW3aNGOPla5U\nZ8YrnHONAKD035UJPp+IiIjiSPViPA5Ar9K4F4CxmekOERFR4UmYpnbOjcTmYq16zrklAAYCuA/A\naOdcHwCLAJwe/zukTood3njjDW3r16+sTixWetqmmSXtYFOptnggVkGKbatWrZrGskm+veFvz30d\nP3781p5K3tpll100ljT1v//9b22z22HKecV2DA866CCN7QbusrWm3S4zqmnqoNm18nImtj3DNdGZ\n2HYduGzjCpQVH8p6cMC//vODDz4AUHagRCGS7W7tYQWx2HSzXSNrC+Ik/vnnn2N+Xb6yRWz29SV2\n3nlnjVMp3LJ7G9htTeX2iy1QtIfa2K2Ow5BMNfVZcT7UJcN9ISIiKkiFs6UOERFRREV6f0FZh3fN\nNddomz1nNJZ4KelknXDCCRrbSrvBgweX+562+q5QTxdq2bKlxu3btwfgX1dp1wZLBeqYMWO0zW63\naE9zkqrTd999V9vs151xxhlp9z0XSOVn7969tW3cuHEayzh06ZJaosqmtOX72nWvNo0n250WWpra\npkoljfz4449rm10RIFtc2lRovOVAcltnwYIF2mZ/n/KVvU0l2+HaVSqnnHJKhb+n3EIBgHvuuUdj\nuzJghx12AOBftWG3T5bbBvb1HyTOjImIiEIWuZmxXQMmm7Inmg3bv6rsDFX+ArMFA7Vrl22jLYdC\nDBkyRNts0dYVV1yhcayb+/as10Jl/8qVWawttJLDOoCyNeJ2nbEtWIm1+5DNRDz77LMa5/PM2Bad\nyPuhVqMAABzFSURBVIzYnpN96aWXavzII49k7HFlo3w7O5szZ47GK1asAODPPiUqFssHdqe9GTNm\nAPDPuOzOTeeddx4AYP/999c2OxOz5L3Ifm4hiPV7brOMdl13suxabykUBfxrvCVDIYfQAP73F1tI\nFwbOjImIiELGizEREVHIIpemtoUPidLTch6wPXjApvCqVKkCwL+u0qax7QbwsdjUoKSqbFoj3raP\nhUp+HvYc6Pvvv1/jQw45BADwr3/9S9tmzpypsU39CZsGtWuS842kgAHg8ssv11heg3fccYe23Xzz\nzRpnMk0s38v+jsQqXiq0NLUlRUDyLwBccMEFGt90000AgCVLlsT8ejue8nMuhLXFln39SErZvo7S\nPSxD1t4DZQV1APDOO++Ue3x7C6Jjx45pPW66ODMmIiIKGS/GREREIYtcmtqmDf7zn/8AAC666CJt\n++9//6uxpBXsWtVMst/XVgULOXOX/Gxl5IABAzSeN28eAOCLL77QNpuetbcoRNu2bTW+7rrrMtrP\nKJCzVW+99VZts+u0JX1nn3u2UsMvvvgiAP+tGHuL55lnngHgX71AQOvWrTWWFRh2fbZ19NFHa2xP\nayokNlUv4zV2bNnxBna717p16wKo2GtebpcB/tOzYp3d3aJFC43trYcw8LeKiIgoZLwYExERhSxy\naWpLNjw4++yztc2mOLKRLtu0aZPGNq0hKVSp0N4ypths6nn48OEAgB9//DHmx61YJ0DZWxj5Qio7\nZevXLckGCNl6rdnNbCRlbqvamzRporFsimBTsHaDhTp16mS1r1FiK3LtVqV2e0Vhq9MHDRqkcdWq\nVbPTuRwit7FuuOEGbbMbP6Vr5MiRGssWuzblfdxxx2XssdLFmTEREVHIIj0zFrYgKNs+//xzjWP9\nlWu3V5PtAyk+WdsHAC+//DKA+Ad42EyHrMHs1KlTFnsXPpkZ2LO5bQbAri/OFHtwgd3yVYpo7M/H\nrvVv165due9l14Sef/75AICnnnoqc52NGJkR2xnV3Llzt/o1dntYm2mgMpl8j7dZC3v4jLTbQq1+\n/fpl7HHTxZkxERFRyHgxJiIiCllOpKmDZE/0iHVSk03BBZk+zyU2TSRri4GyLQLtx62ioiKNH374\nYQDpb40XdbaYTdgCFtkO054QVpExmTVrFgD/SVk29b1+/fqtfn2sn5UtorRnft93331b/V5SrJfL\n65RHjRoFAPjyyy+3+nlSzAaU7ZcA+MeOssPuXRDrJKYePXpo3LBhw0D6lIyEvxXOuabOucnOudnO\nua+cc1eWttdxzk1wzs0v/bd2ou9FRERE5SXzJ+omANd6ntcGQEcAlznn2gDoD2CS53mtAUwq/T8R\nERFVUMKcied5ywAsK43XO+fmAGgMoAeAw0o/bTiAKQCiU5qWIpsitCk6qcDL9+reTLCpz9tuu03j\nWIes2zWpst0ikP/paSEV5rbCU9ZDAkDfvn0BAOPHj9c2SZUCwJQpU8p9vd26dciQIQD8VdHx1nbL\n+ku7SsAefH/uuecCAI4//viYX5/oZ5ar6Wn785DX6PLly7XNrluVNLS9nVWzZs1sd5GMk08+WWO7\nb4S4/fbbNY7SqWMV+u1wzrUAsC+ATwE0LL1QA8ByADGT7865vs65EudciV1SQdnB8Q4exzxYHO9g\ncbyDkXQ1gXOuGoAxAK7yPG+d/YvC8zzPORezKsfzvGIAxQBQVFQUu3InQuJt3t6+fXsA/t2JoriD\nThTG+9FHH9XYbvou7DpaewhIq1atstuxLMnEmMvuV4D/cIaNGzcCKDvEYcs4FfZ3t2XLlhqfd955\nvn+BaK6lD/o1/u2332osh3jYGZcdTzl44Igjjsh2twITxHjbbM0555yjcbVq1QDE34lP1sTbbNA3\n33wT8zHkYJ9mzZploMeZl9TM2Dm3HTZfiJ/zPO/l0uYVzrlGpR9vBGBlvK8nIiKi+JKppnYAhgCY\n43new+ZD4wD0Ko17ARi75dcSERFRYsmkqTsB6AngS+fczNK2mwDcB2C0c64PgEUATs9OF7MvVqHW\nliS1UatWrUD6lIs+++wzAMCwYcO0LVaxkC3akqIgoHCKtmKpXr26xt99953GUsD1ySefaJtd/y5p\nOltkZMlY29TcmDFjNLbnRVMZm5q2WyauWbMGgD81bd8/2rRpA4BFWxVl3yfsXg9vvfUWAODjjz/W\nNpuylt8b2ZcA8N/ysSltKZaM6kEmyVRTfwAgXslZl8x2h4iIqPDk5loDIiKiPMK92eBPOUkaakuL\nFy8GkLtrJbPFjtcbb7wBoGzby3hsappp0vLsyWByzrFd/24reeVsYZvif/311zXu1q0bAP/tFW7J\nGJ+k++32obFuEdj3AbstbocOHQAU9i2XVNjX5AsvvKCxnOc9Y8YMbTvqqKM0lt8Lm+a238tuxyuV\n7lHFKwsREVHI+CfyFjp37qyxPaf0hx9+AOAv1ojS7i1hsYUVd999N4CytbFb2n777QGUFSUB/gIL\nis/OtGwsY2rZdZpUMXKG+b333qttq1ev1liKf7p27aptkp0AgIsvvjjbXVSSIcm3TId9TcsZ2vac\nebvXg7wH16hRQ9vsbnX2IJNsS/fnwZkxERFRyHgxJiIiCll+5TcyINYZxgBQv359ACzgAvyp+qFD\nh2oca62rHa8jjzwSgH8bUab6KUp23HFHAMDuu++ubbVrl50O+9JLLwEAWrduHVifbPFerEMp8pmc\nBf3QQw9pm/yMgLL36yjc7kr3vYxXFiIiopDxYkxERBSy/M9zVJBdwyZnwZKfTcfEOyFF7LrrrhrL\nNoyxKoAzwabzuM6TUiGnBL3zzjsh96SM/X1LdJssX1d72NS0FYX0tEj3PYczYyIiopBxZryFM844\nI+wu5BS7M46s9dtnn320berUqYH1hbNhykcVKRrNp9lwoeHMmIiIKGS8GBMREYXM2Rv+WX8w51Zh\n89nH9QCsTvDpuSgbz6u553n1U/lCM95Afo55pMYbyPvXeLaeUyZe4/k43kDEXuN8T0lJUuMd6MVY\nH9S5Es/zigJ/4CyL8vOKct9SFeXnFOW+pSrKzynKfUtHlJ9XlPuWqjCfE9PUREREIePFmIiIKGRh\nXYyLQ3rcbIvy84py31IV5ecU5b6lKsrPKcp9S0eUn1eU+5aq0J5TKPeMiYiIqAzT1ERERCHjxZiI\niChkvBgTERGFjBdjIiKikPFiTEREFDJejImIiELGizEREVHI0roYO+e6Oee+ds4tcM71z1SniIiI\nCknKm3445yoBmAegK4AlAKYBOMvzvNmZ6x4REVH+2zaNrz0AwALP8xYCgHNuFIAeAOJejOvVq+e1\naNEijYcsPNOnT1+d6nFnHO+KS2e8AY55KvgaDxbHO1jJjnc6F+PGABab/y8B0GHLT3LO9QXQFwCa\nNWuGkpKSNB6y8DjnFiX+LN/nc7zTUNHxLv0ajnka+BoPFsc7WMmOd9YLuDzPK/Y8r8jzvKL69VOe\ncFCSON7B45gHi+MdLI53MNK5GC8F0NT8v0lpGxEREVVAOhfjaQBaO+daOucqAzgTwLjMdIuIiKhw\npHzP2PO8Tc65ywGMB1AJwFDP877KWM+IiIgKRDoFXPA8700Ab2aoL0RERAWJO3ARERGFjBdjIiKi\nkPFiTEREFDJejImIiEKWVgFXFH3zzTcaX3DBBQCA7bffXtuqV6+u8TvvvAMAqFy5srY1a9ZM4yOP\nPFLjW265BQBQtWpVbdtmG/4tQ9Fz9NFHa3zwwQcDKHv9ElE08WpCREQUsrybGb/yyisaT5s2DQDw\n22+/aVuiU6pWr16t8YwZMzR+6qmnAAAPPfSQtvXu3VvjSpUqpdbhPPTzzz9rbLMHX3zxBQDgr7/+\n0rbly5dr/Pfff2t89tlnZ7OLeWfu3Lkaf/755xpPnjwZAPD8889r28SJEzVu3LhxAL0rPL///rvG\nO+ywQ4g9ISHvO3vuuae2LVmyROP9998fADBuXNneVTaTmm2cGRMREYWMF2MiIqKQ5V2aukuXLhrf\ncccdABKnpp1zGsf73LVr1wIALr74Ym1btWqVxjfccAOAwi7quu+++wAAgwYN0jZ7i0AK6Wwae8OG\nDTG/13nnnQcA6NGjh7aNHj06c53NM8OGDdPYvi4l9W/TcXYcr7766ux3Lo98+umnAPy3AuyRgvPn\nzwcAfPzxx9pmX+NVqlTReOTIkQCAE088MTudJX1fBoAnnngCAPDrr7/G/Fy5rfnkk0/G/PpsK9wr\nBxERUUTwYkxERBSyvEtT77XXXhp36NABgL961Np22/JP36as7ZpiSfetX79e22xldc+ePQEUXnXq\nxo0bNb799tsBAH/88UfGvu97772nbT/++KPGdevWTfsx8snuu++ucaxbLb/88ovGL774osZXXnml\nxoV8i2VrFixYoLGMl6wMAIBNmzaV+5o///wz5veyvxtyK8a+Dx133HEa2/ci2jq7EmPvvffW+Kuv\ntn6QoB1j+ZnZ22zXXXedxtn+/eBvHxERUcjybmZs/8q88847AfiLKez6P1G7dm2NO3XqpPFLL72k\nsdzUv/7667XNztSWLl0KoPBmxt99953G8Yqxtsb+tWnXastfqXbdty1C4szYz66dtDvKyTjamcPM\nmTM1Xrhwocb/+Mc/stnFnDJ79myN+/btq7EUcMUj7z9FRUXaZl+3K1as0HjNmjUAgNNOO03b/ve/\n/2lss3wUm+wFYXeds+8ZsdSoUUPjfv36aTxgwAAAwMqVK7VNCvIAYLfddkuvswlwZkxERBQyXoyJ\niIhClndpaqtjx44AgAMPPFDbpk6dqrGk7mx61BZl2fZ58+YBABo0aKBtkmYCgDZt2mSq2zmlefPm\nGtevXx9A/O1HJXVnx238+PEay6EGQFnRTLw0NvnZQhSbppZ2Wzhk40WLFmnMNHWZOXPmaByrCMiO\nsS30POeccwD4C39s2vPYY4/VWH4PbBGkrD0GmKaOZ8KECRrLtrnxUtN2Xbfsc2CLFh999FGN5b3K\n/i4FeTss4czYOTfUObfSOTfLtNVxzk1wzs0v/bf21r4HERERxZdMmnoYgG5btPUHMMnzvNYAJpX+\nn4iIiFKQME3ted5U51yLLZp7ADisNB4OYAqAfoioc889V+NJkyZpLGlPux2drbSbNUuTAbqO84AD\nDtA2SYMDQLVq1TLY49xhz4qWSlG7HaMdl1in19g0tlSkWzY1XatWrfQ6m8d23XVXjW3aU9KpttI9\n0fawhUxS+HbbW3tbRXTu3Fnjxx57TONWrVoBALbbbjtts7dyZC0+AFxxxRUA/JXu77//fsp9z0ff\nf/89AGD48OHaJqtkgNjrue3Jfe3bt9e4UaNGAPzv8faWjaSnDz/8cG2rU6dOyn2vqFQLuBp6nres\nNF4OoGG8T3TO9XXOlTjnSuybNGUHxzt4HPNgcbyDxfEORtoFXJ7nec65uH9qe55XDKAYAIqKikL5\nk9wWQtib87Im0K79u/baazU+7LDDNO7Tp4/va4BoFhRFYbylkCsZdiZgzzmO9b3kL9uoicKY20IV\nK9buUDabUa9evaz1KVuyOd6SDbN7CFgyq7rwwgu1zRa+xdrVz7K7O9kZsbDvOVER9Ov7p59+0lhm\nucXFxdoWazZ86623amwzlrZgTgrxbObIatu2LQDg1Vdf1bYgd6VL9ZFWOOcaAUDpvysTfD4RERHF\nkerFeByAXqVxLwBjM9MdIiKiwpMwTe2cG4nNxVr1nHNLAAwEcB+A0c65PgAWATg9m51M10477aSx\nTTtIuuKtt97Stu7du2t88skna2xTe5QeWzRxzz33aBwr/XT55ZdrHMXbAlFhC4bsbRnZLtAWbdlC\nOrtunhIXUMlr197uWrduncaxCn7suvtly5aV+7jdjteuTy5U9lbhiBEjAPhT+nbs5XaBPfxEzlUH\n4qekhf1deOONNwD415AHKZlq6rPifKhLhvtCRERUkLgdJhERUcjyejtMYdObttpRUk624tSe2lSz\nZs0Aeld4SkpKNLan4NhUqqRd7c+D4rOpu0MOOURjO9bCnnAT5DrKXCApY/s+YW+fSNrzrLPKEoa2\nyv/MM88EAFSvXl3bhg4dqrFdsyw/s969e2ubXQNbSAYPHqyx3RI0VsW5vWX4ww8/AAAuvfRSbXvg\ngQc0Xrx4scax1tfLmfdA+Ks1ODMmIiIKWUHMjO1fRLHWXdpdomwBF6VP/nIFgFGjRgEAnnnmGW1b\nu3ZtzK+TtZv77rtvVvoVa1P4fGHPz43lmGOO0ZiFiX6nn765FnXKlCnaNnHiRI1l0wtbGGTHWw6a\nse858XY8k139Bg4cqG35+HrcGslOXn311doW61x0W3hrd+KTTIOdGdvC2xYtWmgcq0DU/mzCLhDl\nzJiIiChkvBgTERGFrCDS1DvuuKPGsdJADRuWba1t1yRTxXzxxRcAgAcffFDbXnvttXKfZ7e9tIUy\n9ucgaadYh0tkQj6nA+2tAWGf72677aaxTdPl85gkS84mtkVXNr0pB83Yc3Dnz5+vcbt27QD4U9u/\n/vqrxna8L7nkEgCFXSgqZxPbtdix2PcBm3p+9913y33utGnTNE70mrZrvMN+/XNmTEREFDJejImI\n6P+3d2+hVWV3HMe/f8qIg6NQYw1e4qUwD8YLWKVU65uiMoIOiGNBikhFH4p2oA8GfRF8sT5E+qCC\n4EPE8VIddcQHtfWCFHSsDE5LlVbHWyteKhQ6ilAKqw85a+WfutPEJGfvfXZ+Hwizsk7Gs/Y/55yV\n9d/rIgUbEmnq3s5y9WfB+pS2ZPNpZn+W6/bt24Hu2wNm8TMj/QxGnyaK2zj67QPHjx+f+bMCL150\nndVy69attx5vbW1N5VmzZqWy4pjNv0b9jPN4CpA/Dci7d+8eAOvXr091V69ezfzZnlYSVJ3//Ii3\no3q6XdLS0gLA5cuXU50/Hzp+Pty4cSPVtbe3p3LWqUu+LqtvKOo9oZGxiIhIwdQZi4iIFGxIpKn9\nTL2sFITfMvDly5ep7A+2l640zr59+1JdW1tbKmedkOLjnbXBhE9Z+a3rDh8+DMC5c+dS3eLFi1M5\nzmYdyjNRPZ/Of/PmzVuPr1mzJpV9zGVwTZo0Ceh+ilZPW2vOmzcvv4aVyI4dO1L52bNnQM9paj9z\nOvKf0fHEvUWLFqU6v5oga5Mnf2ts+vTpqaw0tYiIyBA3JEbG/vzcrAlcz58/T2U/2WLlypX1bViD\n2bNnDwCbN29OdVnx9NvV+b9Mm5qagK6/hiF7izromljhR3wdHR2pfOrUKQDu37//1r8/lMRshJ/g\nkrW5/sKFXSee+u1f5d3E2PqJP8OHD0/l8+fPA/Dw4cNU5zMRftQ1bdq0ejWz1OJabOgapfrPCf+Z\nEidy+ve+z8bFLUVPnDiR6p48eZLKWSNjv5/BqlWrUjn+nrImfeVBI2MREZGCqTMWEREpWK9pajNr\nAQ4CzUAA9ocQfm1mo4FjwBTgIfBJCOGf9Wtq//kzW3tLQfgUq3RPGR0/fvytuix+/aRPy8WJdFlp\n1HcV1zL79ZwnT57MfN6h4Nq1a6nsJw9NmDABgGHDhqW6oRabwRRvC/hUqJ9E+OrVK6DrdCfo/nqP\naVUo/vzcovgzhFevXg3A6dOnU12MoS8fOHAg1cVtd6Fr++KYzobs1DR0ffbv3bs31c2ZMyeVs05t\n8pNS/XuoHvoyMv4P8MsQQivwI+DnZtYKtAEXQwgfAhdr34uIiMg76rUzDiE8DSF8VSt/C9wBJgAr\ngHhXvQP4uF6NFBERqbJ3mk1tZlOA2cCXQHMIIU51fUZnGruUfJooK03t03oTJ07MpU2Nwscupmz8\nusmslFBfDlbP4tNAcb2mv23g1yFHDx48SOWeZrhW2fXr14Huqbtt27alcpxFPXPmzHwbVlFxu8sr\nV66kuhUrVqTyrl27gJ63hPUrEYbq7QK/rW08HWvjxo2pbtmyZakcb3n52wK+HGPY0zplv1fEwYMH\nAVi6dGmf21rv1LTX5wlcZvYB8DnwaQih2ystdEYi81PXzDaY2U0zu+nvo0h9KN75U8zzpXjnS/HO\nR59Gxmb2Hp0d8WchhDhL5rmZjQshPDWzccCLrP83hLAf2A8wd+7cvg+TBpFfq+pHdZEfLY8cOTKX\nNtXLYMfbT2o4cuRIfI5U50cIGzZsAHqeQBH/YvW/gwULFqTyzp07UzmO5Pzv5vbt26kcsxl+rWbW\nBIw85P0a99mKuObS77q1ZcuWVK5ihiDvePvYxlHwjBkzUp2fPPTo0aPYxlTnX+9Zu9CVXT3jHd+z\n8+fPT3W+w4/vbz8a9nsTxM8BH++jR4+m8pIlS1K57O+FXkfG1vkJegC4E0Jodw+dAdbWymuBLwa/\neSIiItXXl5Hxj4GfAn8ys3g221ZgJ/AbM/sZ8Aj4pD5NFBERqbZeO+MQwu+BnmYaLOyhvhRiOu/S\npUupzh8aEcXJQgCjRo2qf8MaVNam7VOnTk3ldevW1fX5Z8+eXdd/v1H47UTjWtV4CwHKn45rNP4g\nmdevXwNw4cKFVOdvG2Rt7+oPI1i+fHk9mlgpPq1/9+7dAluSL+3AJSIiUjB1xiIiIgWr9KlNcSbu\n48ePU13WmmM/404n2kjZ+e0Xjx07BmSvEpDBMXny5FSOKwL8mvastfT+rOJ4whh035pXxNPIWERE\npGCV/nM6/sXa00L1uDvL1q1bc2uTyECNGDGi6CYMKe+//34qxx3Nzp49m+r8pNCxY8cCsG/fvlTn\nMxlFnZUr5adXhoiISMHUGYuIiBSs0mnqONli9+7dqW7MmDGpvGnTJgCam0t7xoWIFMwfNnDo0KEC\nWyJlFm+L9vcAEI2MRURECqbOWEREpGCVTlNHTU1Nqdze3v5/flJEBlPWCTsiVTTQ86k1MhYRESnY\nkBgZi0gxNBoW6RuNjEVERAqmzlhERKRglrXJed2ezOwfwCNgDPAytyfOTz2ua3II4Xu9/9jbXLyh\nmjEvVbyh8q/xel3TYLzGqxhvKNlrXJ8p/dKneOfaGacnNbsZQpib+xPXWZmvq8xt668yX1OZ29Zf\nZb6mMrdtIMp8XWVuW38VeU1KU4uIiBRMnbGIiEjBiuqM9xf0vPVW5usqc9v6q8zXVOa29VeZr6nM\nbRuIMl9XmdvWX4VdUyH3jEVERKSL0tQiIiIFU2csIiJSsNw7YzNbamZ/MbN7ZtaW9/MPBjNrMbPL\nZnbbzP5sZr+o1Y82s9+a2d3af79bgrY2fLyhcWKueBfS1oaPueKdv9LFPISQ2xfwHeAb4PvAMOBr\noDXPNgzSdYwDflArjwT+CrQCu4C2Wn0b8KuC21mJeDdKzBVvxVzxbpyvssU875HxD4F7IYT7IYR/\nA0eBFTm3YcBCCE9DCF/Vyt8Cd4AJdF5LR+3HOoCPi2lhUol4Q8PEXPHOXyVirnjnr2wxz7szngD8\nzX3/91pdwzKzKcBs4EugOYTwtPbQM6C5oGZFlYs3lDrminf+KhdzxTt/ZYi5JnANgJl9AHwOfBpC\n+Jd/LHTmOLRubJAp5vlSvPOleOevLDHPuzN+ArS47yfW6hqOmb1H5y/wsxDCyVr1czMbV3t8HPCi\nqPbVVCbe0BAxV7zzV5mYK975K1PM8+6M/wB8aGZTzWwY8BPgTM5tGDAzM+AAcCeE0O4eOgOsrZXX\nAl/k3bb/UYl4Q8PEXPHOXyVirnjnr3QxL2AG20d0zlr7BtiW9/MP0jUsoDN18UfgVu3rI6AJuAjc\nBX4HjC5BWxs+3o0Uc8VbMVe8G+OrbDHXdpgiIiIF0wQuERGRgqkzFhERKZg6YxERkYKpMxYRESmY\nOmMREZGCqTMWEREpmDpjERGRgv0X+ZeOJyTJitcAAAAASUVORK5CYII=\n", 1645 | "text/plain": [ 1646 | "" 1647 | ] 1648 | }, 1649 | "metadata": {}, 1650 | "output_type": "display_data" 1651 | } 1652 | ], 1653 | "source": [ 1654 | "####################################\n", 1655 | "### RELOAD & GENERATE SAMPLE IMAGES\n", 1656 | "####################################\n", 1657 | "\n", 1658 | "\n", 1659 | "n_examples = 25\n", 1660 | "\n", 1661 | "with tf.Session(graph=g) as sess:\n", 1662 | " saver.restore(sess, save_path='./gan-conv.ckpt')\n", 1663 | "\n", 1664 | " batch_randsample = np.random.uniform(-1, 1, size=(n_examples, gen_input_size))\n", 1665 | " new_examples = sess.run('generator/generator_outputs:0',\n", 1666 | " feed_dict={'generator_inputs:0': batch_randsample,\n", 1667 | " 'dropout:0': 0.0,\n", 1668 | " 'is_training:0': False})\n", 1669 | "\n", 1670 | "fig, axes = plt.subplots(nrows=5, ncols=5, figsize=(8, 8),\n", 1671 | " sharey=True, sharex=True)\n", 1672 | "\n", 1673 | "for image, ax in zip(new_examples, axes.flatten()):\n", 1674 | " ax.imshow(image.reshape((dis_input_size // 28, dis_input_size // 28)), cmap='binary')\n", 1675 | "\n", 1676 | "plt.show()" 1677 | ] 1678 | } 1679 | ], 1680 | "metadata": { 1681 | "kernelspec": { 1682 | "display_name": "Python 3", 1683 | "language": "python", 1684 | "name": "python3" 1685 | }, 1686 | "language_info": { 1687 | "codemirror_mode": { 1688 | "name": "ipython", 1689 | "version": 3 1690 | }, 1691 | "file_extension": ".py", 1692 | "mimetype": "text/x-python", 1693 | "name": "python", 1694 | "nbconvert_exporter": "python", 1695 | "pygments_lexer": "ipython3", 1696 | "version": "3.6.1" 1697 | } 1698 | }, 1699 | "nbformat": 4, 1700 | "nbformat_minor": 2 1701 | } 1702 | -------------------------------------------------------------------------------- /images/pydata-annarbor-2017.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/pydata-annarbor2017-dl-tutorial/3fa6a630a86be73269fb6c6d4cc97ccc1bc79c3a/images/pydata-annarbor-2017.jpg -------------------------------------------------------------------------------- /images/pydata-annarbor-aug.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/pydata-annarbor2017-dl-tutorial/3fa6a630a86be73269fb6c6d4cc97ccc1bc79c3a/images/pydata-annarbor-aug.jpg -------------------------------------------------------------------------------- /pydata-annarbor2017-dl-tutorial.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/pydata-annarbor2017-dl-tutorial/3fa6a630a86be73269fb6c6d4cc97ccc1bc79c3a/pydata-annarbor2017-dl-tutorial.pdf --------------------------------------------------------------------------------